From aa21c7fa268d08cb2e946f79f5ad81f84550a7c5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 23 Jan 2026 20:30:30 +0000 Subject: [PATCH 01/98] wip it works --- examples/llama/llama.zig | 18 +- examples/llama/main.zig | 20 +- zml/attention/flashattn.zig | 512 +++++++++++++++++++++++++++++++++++- zml/slice.zig | 3 +- 4 files changed, 533 insertions(+), 20 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index fa567455..352dc01c 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -113,8 +113,8 @@ pub const LlamaLM = struct { token_index: Tensor, kv_cache: KvCache, rng: Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); @@ -227,8 +227,8 @@ pub const Llama = struct { tokens: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { const embeds = embed(self.embed_tokens, tokens); var hidden = embeds; @@ -274,8 +274,8 @@ pub const TransformerLayer = struct { x0: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); @@ -409,8 +409,8 @@ pub const SelfAttn = struct { x: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); @@ -436,7 +436,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index b23e6601..aa048103 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -25,7 +25,7 @@ const CliArgs = struct { reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, async_limit: ?u32 = null, - backend: ?zml.attention.Backend = null, + backend: ?zml.attention.attention.Backend = null, pub const help = \\Usage: llama --model= [options] @@ -103,7 +103,7 @@ pub fn main(init: std.process.Init) !void { const llama_options: llama.LlamaLM.Options = .{ .max_seq_len = args.seqlen, .sampling_strategy = .{ - .topk = 2, + .topk = 1, .temperature = 1.0, }, }; @@ -121,7 +121,7 @@ pub fn main(init: std.process.Init) !void { defer llama_model.deinit(allocator); const backend = args.backend orelse b: { - const selected = zml.attention.Backend.auto(platform); + const selected = zml.attention.attention.Backend.auto(platform); log.info("Selected backend: {}", .{selected}); break :b selected; }; @@ -187,8 +187,8 @@ pub fn main(init: std.process.Init) !void { var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); - defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); + defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); @@ -269,8 +269,8 @@ const LlamaParameters = struct { token_index: zml.Tensor, kv_cache: llama.KvCache, rng: zml.Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, }; const CompileModelResult = struct { @@ -435,7 +435,7 @@ pub fn generateText( prefill_exe: zml.exe.Exe, decode_exe: zml.exe.Exe, kv_cache_buffers: *zml.Bufferized(llama.KvCache), - attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + attention_metadata_buffers: *zml.Bufferized(zml.attention.attention.Metadata), tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, options: LlamaLM.Options, @@ -483,6 +483,10 @@ pub fn generateText( prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; + + std.log.info("\n\ngenerated tokens: {any}\n\n", .{prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]}); + const out = try tokenizer_decoder.decode(prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]); + std.log.info("\n\ngenerated: {s}\n\n", .{out}); } // Prepare for token-by-token generation, diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 8673bc4e..3bf1d95b 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -52,6 +52,13 @@ const Buffer = struct { flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), ); } + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("{f}@{*}", .{ self.shape, self.ptr }); + } }; fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { @@ -121,8 +128,8 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { - pub fn register(platform: *zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); + pub fn register(platform: zml.Platform) !void { + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = false }); } pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { @@ -469,3 +476,504 @@ pub const fa3 = struct { return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); } }; + +pub const paged_fa2 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(options_: Options) usize { + return switch (options_) { + .decode => |decode_options| DecodeParameters.allocationSize(decode_options), + .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: DecodeOptions) usize { + const shapes_ = DecodeParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table.byteSize(); + allocation_size += shapes_.cu_seqlens_q.byteSize(); + allocation_size += shapes_.seqused_k.byteSize(); + allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: MixedOptions) usize { + const shapes_ = MixedParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); + allocation_size += shapes_.seqused_k_prefill.byteSize(); + + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); + allocation_size += shapes_.seqused_k_decode.byteSize(); + allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + allocation_size += shapes_.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + //pub fn init(options: Options) zml.ShapeOf(Context) { + // const max_seqlen_k = switch (options) { + // inline else => |v| v.max_seqlen_k, + // }; + // const decode_offset: ?zml.Shape = switch (options) { + // .mixed => zml.Shape.init(.{1}, .i32), + // .decode => null, + // }; + // return .{ + // .max_seqlen_k = max_seqlen_k, + // .decode_offset = decode_offset, + // }; + //} + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa2_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa2_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); + + const original_tot = q.dim(.b); + const num_heads = q.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + var q2 = q; + if (seqlenq_ngroups_swapped) { + q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q2, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + dummy_cu_seqlens_k, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + layer_index, + }, + .{q2.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = num_heads, + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o = o.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + dummy_cu_seqlens_k_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = false, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = q.dim(.h), + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const original_tot = q_decode.dim(.b); + const num_heads = q_decode.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q_decode.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + if (seqlenq_ngroups_swapped) { + q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + dummy_cu_seqlens_k_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = q.dim(.h), + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o_decode = o_decode.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; + +pub const paged_fa3 = struct {}; diff --git a/zml/slice.zig b/zml/slice.zig index d9fb9956..e33316db 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -1,9 +1,9 @@ const std = @import("std"); const stdx = @import("stdx"); -const floats = @import("floats.zig"); const DataType = @import("dtype.zig").DataType; +const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; pub fn isBytes(comptime T: type) bool { @@ -25,6 +25,7 @@ pub const Slice = struct { pub fn init(shape: Shape, bytes: anytype) Slice { stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); + stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); const type_info = @typeInfo(@TypeOf(bytes)); return if (type_info.pointer.is_const) .{ .inner_data = .{ .immutable = bytes }, .shape = shape } From eb4ab8853a24bde2825efb17e25151ce295b7557 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:51:15 +0000 Subject: [PATCH 02/98] wip: argument baking --- zml/exe.zig | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/zml/exe.zig b/zml/exe.zig index f5503d65..67493887 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -90,6 +90,7 @@ pub const Exe = struct { pub const Arguments = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, + baked_count: usize = 0, pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); @@ -118,7 +119,7 @@ pub const Exe = struct { self: *Arguments, current_index: usize = 0, }; - var context: LocalContext = .{ .self = self, .current_index = offset }; + var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const Buffer) void { stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); @@ -130,6 +131,30 @@ pub const Exe = struct { } }.cb, &context, &v); } + + pub fn bake(self: *Arguments, v: anytype) void { + const LocalContext = struct { + self: *Arguments, + + current_index: usize = 0, + }; + + var context: LocalContext = .{ .self = self, .current_index = self.baked_count }; + + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + + self.baked_count = context.current_index; + } }; pub const Results = struct { From 47132159ae72a260a3e75f5cd4165877ba4907b9 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:51:54 +0000 Subject: [PATCH 03/98] Added paged attention FA3 --- zml/attention/flashattn.zig | 491 +++++++++++++++++++++++++++++- zml/attention/paged_attention.zig | 63 ++++ 2 files changed, 538 insertions(+), 16 deletions(-) create mode 100644 zml/attention/paged_attention.zig diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 3bf1d95b..ec402dc7 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -17,6 +17,10 @@ pub fn register(platform: *zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try fa2.register(platform); try fa3.register(platform); + try paged_fa2.Decode.register(platform); + try paged_fa2.Prefill.register(platform); + try paged_fa3.Decode.register(platform); + try paged_fa3.Prefill.register(platform); } } @@ -129,7 +133,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { pub fn register(platform: zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = false }); + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); } pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { @@ -670,20 +674,6 @@ pub const paged_fa2 = struct { max_seqlen_k: usize, decode_offset: ?zml.Tensor = null, - //pub fn init(options: Options) zml.ShapeOf(Context) { - // const max_seqlen_k = switch (options) { - // inline else => |v| v.max_seqlen_k, - // }; - // const decode_offset: ?zml.Shape = switch (options) { - // .mixed => zml.Shape.init(.{1}, .i32), - // .decode => null, - // }; - // return .{ - // .max_seqlen_k = max_seqlen_k, - // .decode_offset = decode_offset, - // }; - //} - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { _ = num_heads; // autofix _ = num_kv_heads; // autofix @@ -976,4 +966,473 @@ pub const paged_fa2 = struct { } }; -pub const paged_fa3 = struct {}; +pub const paged_fa3 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(options_: Options) usize { + return switch (options_) { + .decode => |decode_options| DecodeParameters.allocationSize(decode_options), + .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: DecodeOptions) usize { + const shapes_ = DecodeParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table.byteSize(); + allocation_size += shapes_.cu_seqlens_q.byteSize(); + allocation_size += shapes_.seqused_k.byteSize(); + allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{16 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: MixedOptions) usize { + const shapes_ = MixedParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); + allocation_size += shapes_.seqused_k_prefill.byteSize(); + + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); + allocation_size += shapes_.seqused_k_decode.byteSize(); + allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + allocation_size += shapes_.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa3_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa3_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = decode_parameters.block_table.dim(0) + 1 }, .i32)); + + const o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + scheduler_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_prefill.dim(0) + 1 }, .i32)); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + scheduler_metadata_prefill, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = false, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_decode.dim(0) + 1 }, .i32)); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + scheduler_metadata_decode, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig new file mode 100644 index 00000000..767ac8ee --- /dev/null +++ b/zml/attention/paged_attention.zig @@ -0,0 +1,63 @@ +const std = @import("std"); + +const zml = @import("../zml.zig"); +const flashattn = @import("flashattn.zig"); + +const PagedAttention = @This(); + +pub const Backend = enum { + cuda_fa2, + cuda_fa3, +}; + +pub const Options = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Options, + cuda_fa3: flashattn.paged_fa3.Options, + + pub fn isPrefill(self: Options) bool { + return switch (self) { + inline else => |v| v.isPrefill(), + }; + } +}; + +pub const Parameters = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Parameters, + cuda_fa3: flashattn.paged_fa3.Parameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, + .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, + .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, + }; + } +}; + +/// Internal state that can be used inside the model code. It's derived from Parameters and Option. +pub const Context = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Context, + cuda_fa3: flashattn.paged_fa3.Context, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + }; + } +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + _ = k; // autofix + _ = v; // autofix + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index), + }; +} From 790dc9838478de8d9deb0f77208b3540ab77b17b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:52:14 +0000 Subject: [PATCH 04/98] zml: added simple tracer support back --- zml/BUILD.bazel | 1 + zml/tracer.zig | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ zml/zml.zig | 1 + 3 files changed, 79 insertions(+) create mode 100644 zml/tracer.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 497bc834..f69b1067 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -41,6 +41,7 @@ zig_library( "slice.zig", "tensor.zig", "testing.zig", + "tracer.zig", "zml.zig", ], main = "zml.zig", diff --git a/zml/tracer.zig b/zml/tracer.zig new file mode 100644 index 00000000..e2e173ef --- /dev/null +++ b/zml/tracer.zig @@ -0,0 +1,77 @@ +const builtin = @import("builtin"); + +const c = @import("c"); + +pub const Tracer = switch (builtin.os.tag) { + // TODO(cerisier): fix MacOsTracer + // .macos => MacOsTracer, + .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + else => FakeTracer, +}; + +const CudaTracer = struct { + + // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so + // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). + const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; + const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; + + // Those symbols are defined in nvToolsExt.h which we don't want to provide. + // However, we link with libnvToolsExt.so which provides them. + // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). + const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; + const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; + const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; + + pub fn event(message: [:0]const u8) void { + nvtxMarkA(message.ptr); + } + + pub fn frameStart(message: [:0]const u8) u64 { + return @intCast(nvtxRangeStartA(message.ptr)); + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = message; + nvtxRangeEnd(@intCast(interval_id)); + return; + } +}; + +const MacOsTracer = struct { + logger: c.os_log_t, + + pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_event(self.logger, interval_id, message); + } + + pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_interval_begin(self.logger, interval_id, message); + return interval_id; + } + + pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { + c.zml_os_signpost_interval_end(self.logger, interval_id, message); + } +}; + +/// Mock tracer for OS which don't have an impl. +const FakeTracer = struct { + pub fn event(message: [:0]const u8) void { + _ = message; + return; + } + + pub fn frameStart(message: [:0]const u8) u64 { + _ = message; + return 0; + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = interval_id; + _ = message; + return; + } +}; diff --git a/zml/zml.zig b/zml/zml.zig index 4d576110..7453674f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -43,6 +43,7 @@ pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); +pub const tracer = @import("tracer.zig"); test "zml" { std.testing.refAllDecls(@This()); From 6962d13aaec4acdac5494c2e39a5b35b21620636 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:52:36 +0000 Subject: [PATCH 05/98] flashattn lib override --- third_party/flashattn/repo.bzl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index 84781cb6..a500ec9f 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -2,12 +2,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): - http_archive( - name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", - sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", - build_file = "//:third_party/flashattn/BUILD.bazel", - ) + #http_archive( + # name = "flashattn", + # url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", + # sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", + # build_file = "//:third_party/flashattn/BUILD.bazel", + #) - #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") From b2d60e0be62a43ac2bc93277b973d1a4e2c74e17 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 10:41:10 +0000 Subject: [PATCH 06/98] small fixes --- zml/attention/flashattn.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index ec402dc7..4f4aa26e 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -1051,7 +1051,7 @@ pub const paged_fa3 = struct { pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { return .{ - .out_accum = .init(.{16 * max_token_count * num_heads * head_dim * 4}, .i8), + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), }; } @@ -1416,8 +1416,8 @@ pub const paged_fa3 = struct { mixed_parameters.block_table_decode, softmax_lse_decode, softmax_lse_accum_decode, - scheduler_metadata_decode, mixed_parameters.metadata.out_accum, + scheduler_metadata_decode, layer_index, }, .{q_decode.shape()}, From fef994659d4248c68caab71b99386b52016dd78a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 16:17:36 +0000 Subject: [PATCH 07/98] third_party/flashattn: bump libflashattn.so --- third_party/flashattn/repo.bzl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index a500ec9f..654620cc 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -2,12 +2,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): - #http_archive( - # name = "flashattn", - # url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", - # sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", - # build_file = "//:third_party/flashattn/BUILD.bazel", - #) + http_archive( + name = "flashattn", + url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", + sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", + build_file = "//:third_party/flashattn/BUILD.bazel", + ) - new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") From d6ac5ac59059e8194e504a2774c77877e9949a30 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:07:07 +0000 Subject: [PATCH 08/98] zml/attention: fix wrong causal option --- zml/attention/flashattn.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 4f4aa26e..f7d1f38b 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -906,7 +906,7 @@ pub const paged_fa2 = struct { }, .{q.shape()}, .{ - .is_causal = false, + .is_causal = true, .max_seqlen_k = context.max_seqlen_k, .num_heads = q.dim(.h), }, From b4fd283abe27abd410a977a669c942104ea27d5a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:07:33 +0000 Subject: [PATCH 09/98] zml/nn: removed commented code --- zml/nn.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index cdc2b66b..c53b840b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -29,7 +29,6 @@ pub const Linear = struct { pub fn forward(self: Linear, x: Tensor) Tensor { var y = x.dot(self.weight.convert(x.dtype()), self.tag); - // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } }; From b563bbed5e3620c66e67908bfcb55406ad533cf5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:08:05 +0000 Subject: [PATCH 10/98] zml/nn: Use higher precision for rmsNorm --- zml/nn.zig | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index c53b840b..0a53de0e 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -90,12 +90,13 @@ pub const LayerNorm = struct { } }; -pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { - const ax = x.axis(axis); +pub fn rmsNorm(x_: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x_.axis(axis); // upcast to improve precision - const variance = x.convert(.f32).powByConst(2).mean(ax); - const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); - return x.mul(rsqrt.broad(x.shape())); + var x = x_.convert(.f32); + const variance = x.powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); + return x.mul(rsqrt.broad(x.shape())).convert(x_.dtype()); } /// Center and scale by the variance. From 9b6388079e887390ec9728686ad7f622189d7f63 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:18:41 +0000 Subject: [PATCH 11/98] zml/io: added TensorStore.hasKey() to check key existence It's a bit hacky since it does an array traversal to see if there is a match. --- zml/io.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zml/io.zig b/zml/io.zig index 8b8703cf..404843d2 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -120,6 +120,14 @@ pub const TensorStore = struct { return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; } + pub fn hasKey(self: *const View, subkey: []const u8) bool { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + return for (self.store.registry.tensors.keys()) |k| { + if (std.mem.startsWith(u8, k, key)) break true; + } else false; + } + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { var buffer: [256]u8 = undefined; const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; From f3333fd65f3000b269dde4f58f85ac03ccbbd61b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:21:53 +0000 Subject: [PATCH 12/98] zml/attention: Re-introduced allocationSize --- zml/attention/flashattn.zig | 98 +++++++++++++++---------------- zml/attention/paged_attention.zig | 6 ++ 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index f7d1f38b..354252e4 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -507,10 +507,10 @@ pub const paged_fa2 = struct { }; } - pub fn allocationSize(options_: Options) usize { - return switch (options_) { - .decode => |decode_options| DecodeParameters.allocationSize(decode_options), - .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), }; } @@ -549,13 +549,13 @@ pub const paged_fa2 = struct { }; } - pub fn allocationSize(options: DecodeOptions) usize { - const shapes_ = DecodeParameters.shapes(options); + pub fn allocationSize(self: DecodeParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table.byteSize(); - allocation_size += shapes_.cu_seqlens_q.byteSize(); - allocation_size += shapes_.seqused_k.byteSize(); - allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -580,10 +580,9 @@ pub const paged_fa2 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: DecodeMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); + allocation_size += self.out_accum.byteSize(); return allocation_size; } }; @@ -623,17 +622,16 @@ pub const paged_fa2 = struct { }; } - pub fn allocationSize(options: MixedOptions) usize { - const shapes_ = MixedParameters.shapes(options); + pub fn allocationSize(self: MixedParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); - allocation_size += shapes_.seqused_k_prefill.byteSize(); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); - allocation_size += shapes_.seqused_k_decode.byteSize(); - allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -661,11 +659,10 @@ pub const paged_fa2 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: MixedMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); - allocation_size += shapes_.host_metadata.byteSize(); + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); return allocation_size; } }; @@ -992,10 +989,10 @@ pub const paged_fa3 = struct { }; } - pub fn allocationSize(options_: Options) usize { - return switch (options_) { - .decode => |decode_options| DecodeParameters.allocationSize(decode_options), - .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), }; } @@ -1034,13 +1031,13 @@ pub const paged_fa3 = struct { }; } - pub fn allocationSize(options: DecodeOptions) usize { - const shapes_ = DecodeParameters.shapes(options); + pub fn allocationSize(self: DecodeParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table.byteSize(); - allocation_size += shapes_.cu_seqlens_q.byteSize(); - allocation_size += shapes_.seqused_k.byteSize(); - allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -1065,10 +1062,9 @@ pub const paged_fa3 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: DecodeMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); + allocation_size += self.out_accum.byteSize(); return allocation_size; } }; @@ -1108,17 +1104,16 @@ pub const paged_fa3 = struct { }; } - pub fn allocationSize(options: MixedOptions) usize { - const shapes_ = MixedParameters.shapes(options); + pub fn allocationSize(self: MixedParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); - allocation_size += shapes_.seqused_k_prefill.byteSize(); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); - allocation_size += shapes_.seqused_k_decode.byteSize(); - allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -1146,11 +1141,10 @@ pub const paged_fa3 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: MixedMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); - allocation_size += shapes_.host_metadata.byteSize(); + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); return allocation_size; } }; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 767ac8ee..f07eb289 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -38,6 +38,12 @@ pub const Parameters = union(Backend) { .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, }; } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + inline else => |v| v.allocationSize(), + }; + } }; /// Internal state that can be used inside the model code. It's derived from Parameters and Option. From 3306cc2b4965628747f3b5eeb2224d4416d03763 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:22:36 +0000 Subject: [PATCH 13/98] zml/nn: Improved Yarn logic --- zml/nn.zig | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 0a53de0e..f6a9617f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -169,35 +169,64 @@ pub const RopeOpts = struct { pub const Yarn = struct { beta_fast: f32 = 32.0, beta_slow: f32 = 1.0, - factor: f32, + factor: ?f32 = null, + mscale: ?f32 = null, + mscale_all_dim: ?f32 = null, truncate: bool = true, original_max_position_embeddings: u32, + rope_theta: f32, + llama_4_scaling_beta: ?f32 = null, + attention_factor: ?f32 = null, }; /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { - const content = try std.json.Value.jsonParse(allocator, source, options); - if (content == .null) return .default; + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } - if (content != .object) return error.InvalidEnumTag; + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { + _ = allocator; // autofix + _ = options; // autofix + if (source == .null) return .default; - const obj = content.object; + if (source != .object) return error.InvalidEnumTag; + + const obj = source.object; const impl = obj.get("rope_type") orelse return error.MissingField; if (impl != .string) return error.InvalidEnumTag; if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { - return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; } } + fn getMScale(factor: f32, mscale: f32) f32 { + if (mscale < 1.0) { + return 1.0; + } + return 0.1 * mscale * @log(factor) + 1.0; + } + pub fn attentionScaling(scaling: Scaling) f32 { return switch (scaling) { - .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, + .yarn => |yarn| yarn_scaling: { + const attention_factor = if (yarn.attention_factor) |af| af else b: { + break :b if (yarn.mscale != null and yarn.mscale_all_dim != null) + getMScale(yarn.factor.?, yarn.mscale.?) / getMScale(yarn.factor.?, yarn.mscale_all_dim.?) + else + getMScale(yarn.factor.?, 1.0); + }; + break :yarn_scaling attention_factor; + }, else => 1.0, }; } @@ -317,7 +346,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { const M: f64 = @floatFromInt(s.original_max_position_embeddings); const f_high = s.beta_fast * (2 * std.math.pi) / M; const f_low = s.beta_slow * (2 * std.math.pi) / M; - const downscaling = 1.0 / s.factor; + const downscaling = 1.0 / s.factor.?; // This isn't a typo: low n have a high frequency, high n have a low frequency. var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; From 4e079c58df8051496cac356c44a4dc7a96c681ed Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 17:05:11 +0000 Subject: [PATCH 14/98] zml/testing: Made testlayer take a runtime name --- zml/testing.zig | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/zml/testing.zig b/zml/testing.zig index f84d9f62..6fda9241 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -287,10 +287,13 @@ pub fn testLayer( layer: anytype, comptime func: std.meta.DeclEnum(@TypeOf(layer)), activation_store: zml.io.TensorStore.View, - comptime name: []const u8, + name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), opts: CompareOpts, ) !void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const forward = @field(@TypeOf(layer), @tagName(func)); const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); @@ -301,13 +304,14 @@ pub fn testLayer( index: usize = 0, }; + const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); const input_count = zml.meta.count(zml.Tensor, &args); - const expected_input_count = countWithPrefix(activation_store, name ++ ".in"); + const expected_input_count = countWithPrefix(activation_store, input_name); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } - const store_input = activation_store.withPrefix(name ++ ".in"); + const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { @@ -322,9 +326,10 @@ pub fn testLayer( const exe = try platform.compile(allocator, io, layer, func, args); defer exe.deinit(); + const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); const output_count = exe.output_shapes.len; - const store_output = activation_store.withPrefix(name ++ ".out"); - const expected_output_count = countWithPrefix(store_output, name ++ ".out"); + const store_output = activation_store.withPrefix(output_name); + const expected_output_count = countWithPrefix(store_output, output_name); if (output_count != expected_output_count) { log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); } @@ -365,7 +370,7 @@ pub fn testLayer( try reader.interface.readSliceAll(expected_slice.data()); break :b expected_slice; }; - defer allocator.free(expected_slice.data()); + defer expected_slice.free(allocator); const output_slice = try results[i].toSliceAlloc(allocator, io); defer output_slice.free(allocator); @@ -375,7 +380,7 @@ pub fn testLayer( const stderr = std.debug.lockStderr(&.{}); defer std.debug.unlockStderr(); const w = &stderr.file_writer.interface; - try w.print("{s}.{d} doesn't match !", .{ name ++ ".out", i }); + try w.print("{s}.{d} doesn't match !", .{ output_name, i }); failed = true; continue; }, From c46989654b2a6176932e7ecd11bf4af961861672 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 17:10:11 +0000 Subject: [PATCH 15/98] zml: added ops.scaledDot() It implements a scaled dot product --- zml/ops.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zml/ops.zig b/zml/ops.zig index 757949cd..81b5022a 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1063,3 +1063,10 @@ fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } + +pub fn scaled_dot(lhs: Tensor, rhs: Tensor, lhs_scale: Tensor, rhs_scale: Tensor, args: anytype) Tensor { + const real_lhs = lhs.convert(.bf16).mul(lhs_scale.convert(.bf16)); + const real_rhs = rhs.convert(.bf16).mul(rhs_scale.convert(.bf16)); + + return real_lhs.dot(real_rhs, args).convert(.bf16); +} From cdd49268ade180f7a2e0b2216b468809fa882d44 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 17:29:31 +0000 Subject: [PATCH 16/98] zml: added some convenience to paged_attention --- zml/attention/paged_attention.zig | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index f07eb289..77f49125 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const stdx = @import("stdx"); const zml = @import("../zml.zig"); const flashattn = @import("flashattn.zig"); @@ -8,12 +9,82 @@ const PagedAttention = @This(); pub const Backend = enum { cuda_fa2, cuda_fa3, + + pub fn auto(platform: zml.Platform) Backend { + return switch (platform.target) { + .cuda => b: { + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + if (std.mem.eql(u8, cc, "9.0")) { + break :b .cuda_fa3; + } + } + + break :b .cuda_fa2; + }, + else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } }; pub const Options = union(Backend) { cuda_fa2: flashattn.paged_fa2.Options, cuda_fa3: flashattn.paged_fa3.Options, + pub fn fromBackend(backend: Backend, is_prefill: bool, batch_size: u32, seq_len: u32, page_chunk_size: u32, max_token_count: u32, num_attention_heads: u32, head_dim: u32) Options { + return switch (backend) { + .cuda_fa2 => if (is_prefill) .{ + .cuda_fa2 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa2 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + .cuda_fa3 => if (is_prefill) .{ + .cuda_fa3 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa3 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + }; + } + pub fn isPrefill(self: Options) bool { return switch (self) { inline else => |v| v.isPrefill(), From 4f1000cf11ea9f434f5a48f742262fafd9fddecf Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:50:17 +0000 Subject: [PATCH 17/98] zml/attention: update to latest zml/v2 changes --- zml/attention/flashattn.zig | 4 ++-- zml/attention/paged_attention.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 354252e4..0a11629b 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -13,7 +13,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { } } -pub fn register(platform: *zml.Platform) !void { +pub fn register(platform: *const zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try fa2.register(platform); try fa3.register(platform); @@ -132,7 +132,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { - pub fn register(platform: zml.Platform) !void { + pub fn register(platform: *const zml.Platform) !void { try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); } diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 77f49125..29f05876 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -10,7 +10,7 @@ pub const Backend = enum { cuda_fa2, cuda_fa3, - pub fn auto(platform: zml.Platform) Backend { + pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { .cuda => b: { const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; From f6b1c107ff0fdbcd2250f53f5b121a718fd5bf89 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:54:11 +0000 Subject: [PATCH 18/98] zml/nn: added common `rope_theta` parameter to rope variants --- examples/llama/llama.zig | 2 +- zml/nn.zig | 34 +++++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 352dc01c..0968b83e 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -35,7 +35,7 @@ pub const LlamaLM = struct { rms_norm_eps: f32, hf_rope_impl: bool = true, tie_word_embeddings: bool = false, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, }; pub const Options = struct { diff --git a/zml/nn.zig b/zml/nn.zig index f6a9617f..167925c6 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -143,7 +143,7 @@ test normalizeL2 { pub const RopeOpts = struct { layout: Layout = .sequential, freq_base: f32 = 10_000, - scaling: Scaling = .default, + scaling: Scaling = .{ .default = .{} }, /// There are two layouts corresponding to how to split `x` in real/imag parts. /// * Interleaved means that the real/imag of each scalar is contiguous. @@ -153,10 +153,14 @@ pub const RopeOpts = struct { /// There are several ways to init the scaling aka "inv_freq" pub const Scaling = union(enum) { - default: void, - custom: []const f32, + default: Default, llama3: Llama3, yarn: Yarn, + linear: Scaling.Linear, + + pub const Default = struct { + rope_theta: f32 = 10000, + }; pub const Llama3 = struct { factor: f32, @@ -164,6 +168,7 @@ pub const RopeOpts = struct { low_freq_factor: f32, original_max_position_embeddings: u32, truncate: bool = true, + rope_theta: f32 = 10000, }; pub const Yarn = struct { @@ -179,6 +184,11 @@ pub const RopeOpts = struct { attention_factor: ?f32 = null, }; + pub const Linear = struct { + factor: f32, + rope_theta: f32, + }; + /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { return jsonParseFromValue( @@ -191,7 +201,7 @@ pub const RopeOpts = struct { pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { _ = allocator; // autofix _ = options; // autofix - if (source == .null) return .default; + if (source == .null) return .{ .default = .{} }; if (source != .object) return error.InvalidEnumTag; @@ -203,9 +213,11 @@ pub const RopeOpts = struct { return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "linear")) { + return .{ .linear = try std.json.parseFromValueLeaky(Scaling.Linear, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); - return .{ .default = {} }; + return .{ .default = .{} }; } } @@ -230,6 +242,13 @@ pub const RopeOpts = struct { else => 1.0, }; } + + pub fn setRopeTheta(self: *Scaling, rope_theta: f32) void { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |*s| s.rope_theta = rope_theta, + }; + } }; }; @@ -314,10 +333,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, - .custom => { - stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); - @memcpy(inv_freq, opts.scaling.custom); - }, + .linear => @panic("Not implemented"), .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. From 35f184ab66fcd615afe39aef7aa29c25d4a5124c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:54:56 +0000 Subject: [PATCH 19/98] zml/testing: formatting fix --- zml/testing.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/testing.zig b/zml/testing.zig index 6fda9241..8e33ed58 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -380,7 +380,7 @@ pub fn testLayer( const stderr = std.debug.lockStderr(&.{}); defer std.debug.unlockStderr(); const w = &stderr.file_writer.interface; - try w.print("{s}.{d} doesn't match !", .{ output_name, i }); + try w.print("{s}.{d} doesn't match !\n", .{ output_name, i }); failed = true; continue; }, From 1b562d93021f0ae8822949dc06a8b1d371cd7e97 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 16 Feb 2026 09:05:21 +0000 Subject: [PATCH 20/98] zml/attention: added options for paged_attention --- zml/attention/flashattn.zig | 26 ++++++++++++++++---------- zml/attention/paged_attention.zig | 13 +++++++++---- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 0a11629b..111b7b1c 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -7,6 +7,8 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const ffi = zml.pjrt.ffi; +const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { if (comptime platforms.isEnabled(.cuda)) { try flashattn.load(allocator, io); @@ -829,7 +831,7 @@ pub const paged_fa2 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); @@ -837,7 +839,7 @@ pub const paged_fa2 = struct { const o = switch (parameters) { .decode => |decode_parameters| b: { const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); const original_tot = q.dim(.b); @@ -845,7 +847,7 @@ pub const paged_fa2 = struct { const num_heads_k = k_cache.dim(.h); const head_size = q.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; var q2 = q; if (seqlenq_ngroups_swapped) { q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); @@ -868,9 +870,10 @@ pub const paged_fa2 = struct { }, .{q2.shape()}, .{ - .is_causal = true, + .is_causal = opts.is_causal, .max_seqlen_k = context.max_seqlen_k, .num_heads = num_heads, + .window_size_left = opts.sliding_window, }, .{ .has_side_effect = false }, ); @@ -883,7 +886,7 @@ pub const paged_fa2 = struct { }, .mixed => |mixed_parameters| b: { const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * q.dim(.b) * q.dim(.h) * 4}, .i8)); const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); var o = zml.ops.customCall( Prefill.custom_call_name, @@ -903,16 +906,17 @@ pub const paged_fa2 = struct { }, .{q.shape()}, .{ - .is_causal = true, + .is_causal = opts.is_causal, .max_seqlen_k = context.max_seqlen_k, .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, }, .{ .has_side_effect = false }, ); const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); - const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); @@ -921,7 +925,7 @@ pub const paged_fa2 = struct { const num_heads_k = k_cache.dim(.h); const head_size = q_decode.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; if (seqlenq_ngroups_swapped) { q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); } @@ -943,9 +947,10 @@ pub const paged_fa2 = struct { }, .{q_decode.shape()}, .{ - .is_causal = true, + .is_causal = opts.is_causal, .max_seqlen_k = context.max_seqlen_k, .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, }, .{ .has_side_effect = false }, ); @@ -1329,7 +1334,8 @@ pub const paged_fa3 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = opts; // autofix stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 29f05876..deb63ec7 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -13,7 +13,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { .cuda => b: { - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + const first_device = platform.devices[0].pjrt_device; if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { if (std.mem.eql(u8, cc, "9.0")) { @@ -130,11 +130,16 @@ pub const Context = union(Backend) { } }; -pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { +pub const AttentionOptions = struct { + is_causal: bool = true, + sliding_window: i32 = -1, +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { _ = k; // autofix _ = v; // autofix return switch (parameters) { - .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index), - .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index), + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index, opts), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index, opts), }; } From 7a5b371ea1da04e1e5bc1197eb86a246978865c8 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 16 Feb 2026 09:05:48 +0000 Subject: [PATCH 21/98] zml/nn: added default value for rope_theta + linear scaling --- zml/nn.zig | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 167925c6..ee79e9a9 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -186,7 +186,7 @@ pub const RopeOpts = struct { pub const Linear = struct { factor: f32, - rope_theta: f32, + rope_theta: f32 = 10000, }; /// Read a Rope scaling config from HF config.json format. @@ -249,6 +249,13 @@ pub const RopeOpts = struct { inline else => |*s| s.rope_theta = rope_theta, }; } + + pub fn getRopeTheta(self: *const Scaling) f32 { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |s| s.rope_theta, + }; + } }; }; @@ -333,7 +340,9 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, - .linear => @panic("Not implemented"), + .linear => |l| { + for (inv_freq) |*f| f.* /= l.factor; + }, .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. From d03c92950bfd6cf5ee03694605114ca91b467d1c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 16 Feb 2026 09:06:06 +0000 Subject: [PATCH 22/98] zml/nn: remove useless convert --- zml/nn.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index ee79e9a9..a903c712 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -27,7 +27,7 @@ pub const Linear = struct { } pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight.convert(x.dtype()), self.tag); + var y = x.dot(self.weight, self.tag); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } From ef2ae9253bef75705a436340eecfe7d40227b319 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 20:28:17 +0000 Subject: [PATCH 23/98] examples/llama: fix new flashattn --- examples/llama/llama.zig | 18 +++++++++--------- examples/llama/main.zig | 14 +++++++------- zml/BUILD.bazel | 1 + 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 0968b83e..f4f3095b 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -113,8 +113,8 @@ pub const LlamaLM = struct { token_index: Tensor, kv_cache: KvCache, rng: Tensor.Rng, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); @@ -227,8 +227,8 @@ pub const Llama = struct { tokens: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { Tensor, KvCache } { const embeds = embed(self.embed_tokens, tokens); var hidden = embeds; @@ -274,8 +274,8 @@ pub const TransformerLayer = struct { x0: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { Tensor, KvCache } { // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); @@ -409,8 +409,8 @@ pub const SelfAttn = struct { x: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); @@ -436,7 +436,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index aa048103..ee685a8c 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -25,7 +25,7 @@ const CliArgs = struct { reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, async_limit: ?u32 = null, - backend: ?zml.attention.attention.Backend = null, + backend: ?zml.attention.Backend = null, pub const help = \\Usage: llama --model= [options] @@ -121,7 +121,7 @@ pub fn main(init: std.process.Init) !void { defer llama_model.deinit(allocator); const backend = args.backend orelse b: { - const selected = zml.attention.attention.Backend.auto(platform); + const selected: zml.attention.Backend = .auto(platform); log.info("Selected backend: {}", .{selected}); break :b selected; }; @@ -187,8 +187,8 @@ pub fn main(init: std.process.Init) !void { var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); - defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); @@ -269,8 +269,8 @@ const LlamaParameters = struct { token_index: zml.Tensor, kv_cache: llama.KvCache, rng: zml.Tensor.Rng, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, }; const CompileModelResult = struct { @@ -435,7 +435,7 @@ pub fn generateText( prefill_exe: zml.exe.Exe, decode_exe: zml.exe.Exe, kv_cache_buffers: *zml.Bufferized(llama.KvCache), - attention_metadata_buffers: *zml.Bufferized(zml.attention.attention.Metadata), + attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, options: LlamaLM.Options, diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f69b1067..0b730d0f 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -22,6 +22,7 @@ zig_library( srcs = [ "attention/attention.zig", "attention/flashattn.zig", + "attention/paged_attention.zig", "buffer.zig", "constants.zig", "dtype.zig", From 798a6c281934c8a9de4e6a18f920922c74c7aa3f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 20:28:59 +0000 Subject: [PATCH 24/98] third_party/zls: allow using outside of ZML repo --- third_party/zls/zls_completion.bzl | 2 +- third_party/zls/zls_toolchain.bzl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index eac7dbd3..9028c386 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -61,7 +61,7 @@ def zls_completion(name, deps, **kwargs): main = name + ".runner", data = [ "@rules_zig//zig:resolved_toolchain", - "@@//third_party/zls:resolved_toolchain", + "@zml//third_party/zls:resolved_toolchain", name + ".build_runner", ], deps = [ diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index 5ca5fde0..d83e8f1a 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -80,7 +80,7 @@ def _zls_toolchains_repos_impl(mctx): ), sha256 = _ZLS_VERSIONS[platform], build_file_content = """\ -load("@@//third_party/zls:zls_toolchain.bzl", "zls_toolchain") +load("@zml//third_party/zls:zls_toolchain.bzl", "zls_toolchain") zls_toolchain(name = "toolchain", bin = "zls") """, ) @@ -98,7 +98,7 @@ zls_toolchains = module_extension( ) def _zls_toolchains_repo_impl(rctx): - build_content = """# @generated by @@//third_party/zls:zls_toolchain.bzl""" + build_content = """# @generated by @zml//third_party/zls:zls_toolchain.bzl""" for platform, config in _ZLS_PLATFORMS.items(): build_content += """ @@ -107,7 +107,7 @@ toolchain( exec_compatible_with = {compatible_with}, target_compatible_with = {compatible_with}, toolchain = "@zls_{platform}//:toolchain", - toolchain_type = "@@//third_party/zls:toolchain_type", + toolchain_type = "@zml//third_party/zls:toolchain_type", ) """.format( platform = platform, From fe5777db1c974085adb0e9c6669da4a736582f33 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 17 Feb 2026 11:07:01 +0000 Subject: [PATCH 25/98] zml: added paged_attention to zml --- zml/zml.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/zml/zml.zig b/zml/zml.zig index 7453674f..58444934 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -14,6 +14,7 @@ pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); pub const attention = @import("attention/attention.zig"); +pub const paged_attention = @import("attention/paged_attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const constants = @import("constants.zig"); pub const dtype = @import("dtype.zig"); From cce0923490f0ebc6dcfa840126a1ec4e0f6ab71b Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 10 Feb 2026 12:06:06 +0100 Subject: [PATCH 26/98] voxtral: support wav inputs --- examples/voxtral/BUILD.bazel | 16 ++++ examples/voxtral/python/main.py | 156 ++++++++++++++++++++++++++++++++ examples/voxtral/voxtral.zig | 65 +++++++++++++ examples/voxtral/wav.zig | 72 +++++++++++++++ 4 files changed, 309 insertions(+) create mode 100644 examples/voxtral/BUILD.bazel create mode 100644 examples/voxtral/python/main.py create mode 100644 examples/voxtral/voxtral.zig create mode 100644 examples/voxtral/wav.zig diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel new file mode 100644 index 00000000..3fcd130d --- /dev/null +++ b/examples/voxtral/BUILD.bazel @@ -0,0 +1,16 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "voxtral", + args = [], + data = [], + main = "voxtral.zig", + srcs = ["wav.zig"], + deps = ["//zml"], +) + +build_test( + name = "test", + targets = [":voxtral"], +) diff --git a/examples/voxtral/python/main.py b/examples/voxtral/python/main.py new file mode 100644 index 00000000..79f1841d --- /dev/null +++ b/examples/voxtral/python/main.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +""" +Reference mel spectrogram implementation for Voxtral. +Outputs intermediate values for comparison with the Zig implementation. + +Usage: + pip install numpy soundfile + python main.py +""" + +import sys + +import numpy as np +import soundfile as sf +import torch + +# Audio parameters (from Voxtral config / params.json) +SAMPLE_RATE = 16000 +NUM_MEL_BINS = 128 +HOP_LENGTH = 160 +WINDOW_SIZE = 400 # also used as n_fft +GLOBAL_LOG_MEL_MAX = 1.5 + + +# ========================================================================== +# Mel filter bank (Slaney-style, from mistral_common/audio.py) +# ========================================================================== +def hertz_to_mel(freq): + min_log_hertz = 1000.0 + min_log_mel = 15.0 + logstep = 27.0 / np.log(6.4) + mels = 3.0 * freq / 200.0 + if isinstance(freq, np.ndarray): + log_region = freq >= min_log_hertz + mels[log_region] = ( + min_log_mel + np.log(freq[log_region] / min_log_hertz) * logstep + ) + elif freq >= min_log_hertz: + mels = min_log_mel + np.log(freq / min_log_hertz) * logstep + return mels + + +def mel_to_hertz(mels): + min_log_hertz = 1000.0 + min_log_mel = 15.0 + logstep = np.log(6.4) / 27.0 + freq = 200.0 * mels / 3.0 + log_region = mels >= min_log_mel + freq[log_region] = min_log_hertz * np.exp( + logstep * (mels[log_region] - min_log_mel) + ) + return freq + + +def compute_mel_filters(): + num_frequency_bins = 1 + WINDOW_SIZE // 2 # 201 + fft_freqs = np.linspace(0, SAMPLE_RATE // 2, num_frequency_bins) + mel_min = hertz_to_mel(0.0) + mel_max = hertz_to_mel(8000.0) + mel_freqs = np.linspace(mel_min, mel_max, NUM_MEL_BINS + 2) + filter_freqs = mel_to_hertz(mel_freqs) + filter_diff = np.diff(filter_freqs) + slopes = np.expand_dims(filter_freqs, 0) - np.expand_dims(fft_freqs, 1) + down_slopes = -slopes[:, :-2] / filter_diff[:-1] + up_slopes = slopes[:, 2:] / filter_diff[1:] + fb = np.maximum(np.zeros(1), np.minimum(down_slopes, up_slopes)) + enorm = 2.0 / (filter_freqs[2 : NUM_MEL_BINS + 2] - filter_freqs[:NUM_MEL_BINS]) + fb *= np.expand_dims(enorm, 0) + return fb # [201, 128] + + +# ============================================================================ +# Mel spectrogram (from vLLM voxtral.py compute_whisper_melspec) +# ============================================================================ + + +def compute_mel_spectrogram(audio, mel_filters): + """audio: 1D tensor, mel_filters: [freq_bins, mel_bins] tensor""" + window = torch.hann_window(WINDOW_SIZE) + stft = torch.stft( + audio, WINDOW_SIZE, HOP_LENGTH, window=window, return_complex=True + ) + magnitudes = stft[..., :-1].abs() ** 2 + mel_spec = mel_filters.T @ magnitudes # [mel_bins, frames] + log_spec = torch.clamp(mel_spec, min=1e-10).log10() + # Use fixed global_log_mel_max from params.json (matching vLLM compute_whisper_melspec + # which uses config.global_log_mel_max when set, falling back to dynamic max otherwise) + log_spec = torch.maximum(log_spec, torch.tensor(GLOBAL_LOG_MEL_MAX) - 8.0) + log_spec = (log_spec + 4.0) / 4.0 + return log_spec # [128, frames] + + +# ========================================================================== +# Main +# ========================================================================== + + +def execute(path): + # Load audio + audio, sr = sf.read(path, dtype="float32") + if audio.ndim > 1: + audio = audio.mean(axis=1) + if sr != SAMPLE_RATE: + import soxr + + audio = soxr.resample(audio, sr, SAMPLE_RATE, quality="HQ") + + print( + f"Audio: {len(audio)} samples, {len(audio) / SAMPLE_RATE:.2f}s", file=sys.stderr + ) + + # Compute mel filters (numpy) and convert to torch + mel_filters_np = compute_mel_filters() + mel_filters = torch.tensor(mel_filters_np, dtype=torch.float32) + print(f"Mel filters: {mel_filters.shape}", file=sys.stderr) + print( + f" min={mel_filters.min():.6f}, max={mel_filters.max():.6f}", file=sys.stderr + ) + + # Compute mel spectrogram + audio_tensor = torch.tensor(audio, dtype=torch.float32) + mel = compute_mel_spectrogram(audio_tensor, mel_filters) + print(f"Mel spectrogram: {mel.shape}", file=sys.stderr) + print( + f" min={mel.min():.6f}, max={mel.max():.6f}, mean={mel.mean():.6f}", + file=sys.stderr, + ) + + # Dump first few values for comparison + print("\n# First 10 mel filter values (row 0):", file=sys.stderr) + print(mel_filters[0, :10].tolist(), file=sys.stderr) + + print("\n# First frame mel spectrogram (all 128 bins):", file=sys.stderr) + print(mel[:, 0].tolist(), file=sys.stderr) + + print("\n# Mel spectrogram corner [0:4, 0:4]:", file=sys.stderr) + print(mel[:4, :4].tolist(), file=sys.stderr) + + # Save as raw f32 binary (row-major) for loading in Zig + # Layout: [num_mel_bins, num_frames] contiguous float32 + output_path = path.rsplit(".", 1)[0] + "_mel.bin" + mel.numpy().astype(np.float32).tofile(output_path) + print(f"\nSaved mel spectrogram ({mel.shape[0]}x{mel.shape[1]}, f32) to {output_path}", file=sys.stderr) + + +# def main(): +# if len(sys.argv) < 2: +# print(f"Usage: {sys.argv[0]} ", file=sys.stderr) +# sys.exit(1) + +# wav_path = sys.argv[1] +# execute(wav_path) + + +# if __name__ == "__main__": +# main() diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig new file mode 100644 index 00000000..14dba906 --- /dev/null +++ b/examples/voxtral/voxtral.zig @@ -0,0 +1,65 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const log = std.log; +const wav = @import("wav.zig"); + +const zml = @import("zml"); + +pub fn main() !void { + log.info("Start of Voxtral", .{}); + + var dbg = std.heap.DebugAllocator(.{}).init; + defer if (builtin.mode == .Debug) std.debug.assert(dbg.deinit() == .ok); + + const allocator = switch (builtin.mode) { + .Debug => dbg.allocator(), + else => std.heap.c_allocator, + }; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const io = threaded.io(); + + const arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const file = try std.Io.Dir.openFile(.cwd(), io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/inputs/harvard.wav", .{}); + defer file.close(io); + + var wav_buffer: [4096]u8 = undefined; + var reader = file.reader(io, &wav_buffer); + + const wav_file = try loadWav(allocator, &reader.interface); + defer allocator.free(wav_file); + std.debug.print("{any}\n", .{wav_file[0..100]}); +} + +fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + + const arena = arena_state.allocator(); + var sample_list: std.ArrayList(u8) = .empty; + + const wav_fmt = try wav.readPcmWav(arena, reader, &sample_list); + const byte_per_sample = wav_fmt.bits_per_sample / 8; + const sample_count = sample_list.items.len / (byte_per_sample * wav_fmt.num_channels); + + const samples = try allocator.alloc(f32, sample_count); + for (0..sample_count) |i| { + const offset = i * byte_per_sample * wav_fmt.num_channels; + + const sample = switch(byte_per_sample) { + 1 => (@as(f32, @floatFromInt(std.mem.bytesToValue(u8, sample_list.items[offset .. offset + 1]))) - 128.0) / 128.0, + 2 => @as(f32, @floatFromInt(std.mem.bytesToValue(i16, sample_list.items[offset .. offset + 2]))) / 32768.0, + 3 => @as(f32, @floatFromInt(std.mem.bytesToValue(i24, sample_list.items[offset .. offset + 3]))) / 8388608.0, + 4 => @as(f32, @floatFromInt(std.mem.bytesToValue(i32, sample_list.items[offset .. offset + 4]))) / 2147483648.0, + else => unreachable, + }; + + samples[i] = sample; + } + + return samples; +} diff --git a/examples/voxtral/wav.zig b/examples/voxtral/wav.zig new file mode 100644 index 00000000..a4683939 --- /dev/null +++ b/examples/voxtral/wav.zig @@ -0,0 +1,72 @@ +const std = @import("std"); + +pub const Chunk = struct { + pub const Id = enum(u32) { + riff = @bitCast(@as([4]u8, "RIFF".*)), + fmt = @bitCast(@as([4]u8, "fmt ".*)), + data = @bitCast(@as([4]u8, "data".*)), + _, + }; + + pub const Header = extern struct { + id: Id align(1), + size: u32 align(1), + }; + + pub const Riff = extern struct { + pub const Format = enum(u32) { + wave = @bitCast(@as([4]u8, "WAVE".*)), + _, + }; + + format: Format align(1), + }; + + pub const Fmt = extern struct { + pub const AudioFormat = enum(u16) { + pcm = 1, + _, + }; + + audio_format: AudioFormat align(1), + num_channels: u16 align(1), + sample_rate: u32 align(1), + byte_rate: u32 align(1), + block_align: u16 align(1), + bits_per_sample: u16 align(1), + }; +}; + +pub fn readPcmWav(allocator: std.mem.Allocator, reader: *std.Io.Reader, out: *std.ArrayList(u8)) !Chunk.Fmt { + std.debug.assert((try reader.takeStruct(Chunk.Header, .little)).id == .riff); + std.debug.assert((try reader.takeStruct(Chunk.Riff, .little)).format == .wave); + var fmt: ?Chunk.Fmt = null; + var data_read: bool = false; + while (fmt == null or !data_read) { + const hdr = try reader.takeStruct(Chunk.Header, .little); + switch (hdr.id) { + .fmt => { + fmt = try reader.takeStruct(Chunk.Fmt, .little); + if (hdr.size > @sizeOf(Chunk.Fmt)) { + try reader.discardAll(hdr.size - @sizeOf(Chunk.Fmt)); + } + }, + .data => { + try out.resize(allocator, hdr.size); + try reader.readSliceAll(out.items); + data_read = true; + }, + else => try reader.discardAll(hdr.size), + } + } + return fmt.?; +} + +// pub fn writePcmWav(writer: anytype, fmt: Chunk.Fmt, data: []const u8) !void { +// try writer.writeStruct(Chunk.Header{ .id = .riff, .size = @sizeOf(Chunk.Riff) }); +// try writer.writeStruct(Chunk.Riff{ .format = .wave }); +// try writer.writeStruct(Chunk.Header{ .id = .fmt, .size = @sizeOf(Chunk.Fmt) }); +// try writer.writeStruct(fmt); +// try writer.writeStruct(Chunk.Header{ .id = .data, .size = @intCast(data.len) }); +// try writer.writeAll(data); +// } From 742622d9792ea060e1d88fec5b0433c9b73eef2d Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 10 Feb 2026 14:56:46 +0100 Subject: [PATCH 27/98] Compile model and load buffers --- examples/voxtral/BUILD.bazel | 5 +- examples/voxtral/config.zig | 5 ++ examples/voxtral/voxtral.zig | 155 ++++++++++++++++++++++++++++++++++- 3 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 examples/voxtral/config.zig diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel index 3fcd130d..2e027b37 100644 --- a/examples/voxtral/BUILD.bazel +++ b/examples/voxtral/BUILD.bazel @@ -6,7 +6,10 @@ zig_binary( args = [], data = [], main = "voxtral.zig", - srcs = ["wav.zig"], + srcs = [ + "wav.zig", + "config.zig", + ], deps = ["//zml"], ) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig new file mode 100644 index 00000000..d9839305 --- /dev/null +++ b/examples/voxtral/config.zig @@ -0,0 +1,5 @@ +pub const MelSpectrumConfig = struct { + hop_length: u32 = 160, + n_fft: u32 = 400, + global_log_mel_max: f32 = 1.5, +}; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 14dba906..7c3980cc 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -1,9 +1,121 @@ const std = @import("std"); const builtin = @import("builtin"); const log = std.log; -const wav = @import("wav.zig"); +const wav_utils = @import("wav.zig"); +const cfg = @import("config.zig"); +const MelSpectrumConfig = cfg.MelSpectrumConfig; const zml = @import("zml"); +const Tensor = zml.Tensor; + +pub const LogMelSpectrogram = struct { + window: AudioWindow, + + n_fft: u63 = 400, // Must match window size + mel_filters: Tensor, // Shape must match (201, 128) + + hop_len: u63 = 160, + global_log_mel_max: f32 = 1.5, + + mel_floor: f32 = 1e-10, + force_num_frames: u32 = 3000, + + precision: zml.DataType = .f32, + + pub fn init(config: MelSpectrumConfig) LogMelSpectrogram { + return .{ + .window = .hann, + .mel_filters = Tensor.init(.{201, 128}, .f32).withTags(.{.freq, ._ }), + .hop_len = config.hop_length, + .n_fft = config.n_fft, + .global_log_mel_max = 1.5, + }; + } + + pub fn forward(self: LogMelSpectrogram, waveform: Tensor) Tensor { + const dtype = waveform.dtype(); + const rank: u63 = @intCast(waveform.shape().rank()); + const window_weight = self.window.getWeights(self.n_fft, dtype); + const fft_len = window_weight.dim(rank - 1); + var num_frames: u63 = @intCast(@divFloor(waveform.dim(rank - 1), self.hop_len)); + + var wav = waveform; + const force_num_frames = self.force_num_frames; + const force_num_samples = force_num_frames * self.hop_len; + if (num_frames > force_num_frames) { + wav = wav.slice1d(-1, .{ .end = force_num_samples }); + num_frames = force_num_frames; + } else if (num_frames < force_num_frames) { + const tagged = wav.withTags(.{ .t, ._ }); + wav = tagged.pad(0.0, .{ .t = Tensor.Pad{ .high = force_num_samples - tagged.dim(.t) } }); + // wav = wav.pad(0.0, .{ .samples = .{ .high = force_num_samples - wav.dim(rank - 1) } }); + } + + num_frames = @intCast(@divFloor(wav.dim(0), self.hop_len)); + std.debug.assert(num_frames == force_num_frames); + + // Reflect padding + const padded_wav = blk: { + const l = wav.slice1d(-1, .{ .start = 1, .end = @divExact(fft_len, 2) + 1 }).reverse(.{-1}); + const r = wav.slice1d(-1, .{ .start = -@divExact(fft_len, 2) - 1, .end = -1 }).reverse(.{-1}); + break :blk zml.Tensor.concatenate(&.{ l, wav, r }, -1); + }; + + // Use Short Time Fourier Transform to compute features. + var spectrogram = stft(padded_wav, window_weight, num_frames, self.hop_len, self.precision); + spectrogram = spectrogram.convert(dtype); + // Re-weight frequencies for speech + spectrogram = spectrogram.dot(self.mel_filters, .freq); + + // Convert to log scale + var log_spec = spectrogram.log().scale(1.0 / @log(10.0)); + + // Cut out noise below self.mel_floor or below max(log_spec) - 8 + var log_spec_min = Tensor.constant(dtype.constant(@log10(self.mel_floor))); + const log_spec_max = log_spec.flatten().max(0).squeeze(0); + log_spec_min = log_spec_min.maximum(log_spec_max.addConstant(-8)); + + log_spec = log_spec.maximum(log_spec_min); + const log_spec_rank = log_spec.shape().rank(); + // "center" the distribution + return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ log_spec_rank - 1, log_spec_rank - 2 }); + } + + pub fn load(self: *LogMelSpectrogram, io: std.Io, platform: *zml.Platform) !zml.Bufferized(LogMelSpectrogram) { + const mel_filters_data = @embedFile("assets/voxtral_mel_filter.data"); + const slice = zml.Slice.init(self.mel_filters.shape(), mel_filters_data); + + return .{ + .mel_filters = try zml.Buffer.fromSlice(io, platform, slice), + }; + } + + pub fn unload(self: *zml.Bufferized(LogMelSpectrogram)) void { + self.mel_filters.deinit(); + } +}; + +pub fn stft(waveform: Tensor, weight: Tensor, num_frames: usize, stride: u63, precision: zml.DataType) Tensor { + const fft_len = weight.dim(0); + + const num_samples = waveform.dim(0); + + // const num_frames = 3000; + const indices = Tensor.arange(.{ .end = @intCast(num_frames * stride), .step = stride }, .i32); + std.log.warn("num_samples: {d}, num_frames: {d}, end: {d}, stride: {d}, indices: {}", .{ num_samples, num_frames, num_frames * stride, stride, indices }); + var windows = waveform.gatherSlices(.{fft_len}, indices.appendAxes(.{.coord }), .{ .indices_are_sorted = true }); + + windows = windows.mul(weight.broadcastLeft(windows.shape())); + + var fft = windows.convert(precision).fft(.{ .kind = .RFFT, .length = &.{fft_len} }); + const spectrogram = fft.abs(); + + const ret = spectrogram.mul(spectrogram).convert(waveform.dtype()).renameAxis(0, .temp); + log.info("{f}", .{ret.shape()}); + + return ret; +} + pub fn main() !void { log.info("Start of Voxtral", .{}); @@ -33,6 +145,21 @@ pub fn main() !void { const wav_file = try loadWav(allocator, &reader.interface); defer allocator.free(wav_file); std.debug.print("{any}\n", .{wav_file[0..100]}); + + const melspectrum_config = MelSpectrumConfig{}; + + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform .deinit(allocator); + log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); + + var melspectro_model: LogMelSpectrogram = .init(melspectrum_config); + var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model}); + var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); + + var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); + defer compiled_mel_spectrum.deinit(); + var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); + defer LogMelSpectrogram.unload(&mel_spectrum_buffers); } fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { @@ -42,7 +169,7 @@ fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { const arena = arena_state.allocator(); var sample_list: std.ArrayList(u8) = .empty; - const wav_fmt = try wav.readPcmWav(arena, reader, &sample_list); + const wav_fmt = try wav_utils.readPcmWav(arena, reader, &sample_list); const byte_per_sample = wav_fmt.bits_per_sample / 8; const sample_count = sample_list.items.len / (byte_per_sample * wav_fmt.num_channels); @@ -63,3 +190,27 @@ fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { return samples; } + +pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram) !zml.Exe { + return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{model.force_num_frames * model.hop_len}, .f32).withTags(.{.freq})}); +} + +pub const AudioWindow = enum { + /// https://numpy.org/doc/stable/reference/generated/numpy.hanning.html + hann, + boxcar, + // Other possible windows: https://en.wikipedia.org/wiki/Window_function + + pub fn getWeights(self: AudioWindow, len: i64, dtype: zml.DataType) Tensor { + return switch (self) { + .boxcar => Tensor.constant(dtype.one()), + .hann => { + if (len <= 1) return Tensor.constant(dtype.one()); + const flen: f64 = @floatFromInt(len); + const freq = Tensor.constant(dtype.constant(std.math.pi / flen)); + const steps = Tensor.arange(.{ .start = -len, .end = len, .step = 2 }, dtype); + return steps.mul(freq).cos().scale(0.5).addConstant(0.5).convert(dtype); + }, + }; + } +}; From d18dd5e1dd7abc7be47ffb57893cd7e67ad7a7d2 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 10 Feb 2026 17:19:12 +0100 Subject: [PATCH 28/98] Call model --- examples/voxtral/voxtral.zig | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 7c3980cc..ccf7cbe9 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -48,7 +48,6 @@ pub const LogMelSpectrogram = struct { } else if (num_frames < force_num_frames) { const tagged = wav.withTags(.{ .t, ._ }); wav = tagged.pad(0.0, .{ .t = Tensor.Pad{ .high = force_num_samples - tagged.dim(.t) } }); - // wav = wav.pad(0.0, .{ .samples = .{ .high = force_num_samples - wav.dim(rank - 1) } }); } num_frames = @intCast(@divFloor(wav.dim(0), self.hop_len)); @@ -143,6 +142,7 @@ pub fn main() !void { var reader = file.reader(io, &wav_buffer); const wav_file = try loadWav(allocator, &reader.interface); + std.debug.print("LEN: {d}\n", .{wav_file.len}); defer allocator.free(wav_file); std.debug.print("{any}\n", .{wav_file[0..100]}); @@ -160,6 +160,35 @@ pub fn main() !void { defer compiled_mel_spectrum.deinit(); var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); defer LogMelSpectrogram.unload(&mel_spectrum_buffers); + + // Execute + var args = try compiled_mel_spectrum.args(allocator); + defer args.deinit(allocator); + var results = try compiled_mel_spectrum.results(allocator); + defer results.deinit(allocator); + + const first: zml.Slice = try zml.Slice.alloc(allocator, .init(.{ .freq = wav_file.len }, .f32)); + defer first.free(allocator); + var first_buffer: zml.Buffer = try .fromSlice(io, platform, first); + defer first_buffer.deinit(); + + args.set(.{mel_spectrum_buffers, first_buffer}); + compiled_mel_spectrum.call(args, &results); + results.fill(.{&first_buffer}); + + + try first_buffer.toSlice(io, first); + + const outfile = try std.Io.Dir.createFile(.cwd(), io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/outputs/out.bin", .{}); + defer outfile.close(io); + + var outbuff: [4096]u8 = undefined; + var writer = outfile.writer(io, &outbuff); + const writer_interface = &writer.interface; + + try writer_interface.writeAll(first.data()); + + // std.debug.print("Here is the results: {f}\n", .{first}); } fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { @@ -192,7 +221,8 @@ fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { } pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram) !zml.Exe { - return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{model.force_num_frames * model.hop_len}, .f32).withTags(.{.freq})}); + // return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{model.force_num_frames * model.hop_len}, .f32).withTags(.{.freq})}); + return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{809508}, .f32).withTags(.{.freq})}); } pub const AudioWindow = enum { From 5101eb261278af66b865b432354a8ac878ee57e4 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 10 Feb 2026 18:28:48 +0100 Subject: [PATCH 29/98] correct output --- examples/voxtral/voxtral.zig | 46 ++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index ccf7cbe9..daa98536 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -16,7 +16,6 @@ pub const LogMelSpectrogram = struct { hop_len: u63 = 160, global_log_mel_max: f32 = 1.5, - mel_floor: f32 = 1e-10, force_num_frames: u32 = 3000, @@ -46,7 +45,7 @@ pub const LogMelSpectrogram = struct { wav = wav.slice1d(-1, .{ .end = force_num_samples }); num_frames = force_num_frames; } else if (num_frames < force_num_frames) { - const tagged = wav.withTags(.{ .t, ._ }); + const tagged = wav.withTags(.{.t}); wav = tagged.pad(0.0, .{ .t = Tensor.Pad{ .high = force_num_samples - tagged.dim(.t) } }); } @@ -61,21 +60,21 @@ pub const LogMelSpectrogram = struct { }; // Use Short Time Fourier Transform to compute features. - var spectrogram = stft(padded_wav, window_weight, num_frames, self.hop_len, self.precision); + // Generate num_frames+1 (matching torch.stft center=True frame count), + // then drop the last frame to match Whisper convention (stft[..., :-1]). + var spectrogram = stft(padded_wav, window_weight, num_frames + 1, self.hop_len, self.precision); + spectrogram = spectrogram.slice1d(0, .{ .end = -1 }); spectrogram = spectrogram.convert(dtype); // Re-weight frequencies for speech spectrogram = spectrogram.dot(self.mel_filters, .freq); - // Convert to log scale + spectrogram = spectrogram.maximum(Tensor.constant(dtype.constant(self.mel_floor))); var log_spec = spectrogram.log().scale(1.0 / @log(10.0)); - // Cut out noise below self.mel_floor or below max(log_spec) - 8 - var log_spec_min = Tensor.constant(dtype.constant(@log10(self.mel_floor))); - const log_spec_max = log_spec.flatten().max(0).squeeze(0); - log_spec_min = log_spec_min.maximum(log_spec_max.addConstant(-8)); - + const log_spec_min = Tensor.constant(dtype.constant(self.global_log_mel_max - 8.0)); log_spec = log_spec.maximum(log_spec_min); const log_spec_rank = log_spec.shape().rank(); + // "center" the distribution return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ log_spec_rank - 1, log_spec_rank - 2 }); } @@ -109,7 +108,7 @@ pub fn stft(waveform: Tensor, weight: Tensor, num_frames: usize, stride: u63, pr var fft = windows.convert(precision).fft(.{ .kind = .RFFT, .length = &.{fft_len} }); const spectrogram = fft.abs(); - const ret = spectrogram.mul(spectrogram).convert(waveform.dtype()).renameAxis(0, .temp); + const ret = spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{.temp, .freq}); log.info("{f}", .{ret.shape()}); return ret; @@ -135,7 +134,7 @@ pub fn main() !void { const arena: std.heap.ArenaAllocator = .init(allocator); defer arena.deinit(); - const file = try std.Io.Dir.openFile(.cwd(), io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/inputs/harvard.wav", .{}); + const file = try std.Io.Dir.openFile(.cwd(), io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/inputs/harvard_16k.wav", .{}); defer file.close(io); var wav_buffer: [4096]u8 = undefined; @@ -167,17 +166,22 @@ pub fn main() !void { var results = try compiled_mel_spectrum.results(allocator); defer results.deinit(allocator); - const first: zml.Slice = try zml.Slice.alloc(allocator, .init(.{ .freq = wav_file.len }, .f32)); - defer first.free(allocator); - var first_buffer: zml.Buffer = try .fromSlice(io, platform, first); - defer first_buffer.deinit(); + const input_slice = zml.Slice.init(.init(.{ .freq = wav_file.len }, .f32), std.mem.sliceAsBytes(wav_file)); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); + defer input_buffer.deinit(); - args.set(.{mel_spectrum_buffers, first_buffer}); + const output_shape = zml.Shape.init(.{ 128, melspectro_model.force_num_frames }, .f32); + const output_slice: zml.Slice = try zml.Slice.alloc(allocator, output_shape); + defer output_slice.free(allocator); + var output_buffer: zml.Buffer = try .fromSlice(io, platform, output_slice); + defer output_buffer.deinit(); + + args.set(.{mel_spectrum_buffers, input_buffer}); compiled_mel_spectrum.call(args, &results); - results.fill(.{&first_buffer}); + results.fill(.{&output_buffer}); - - try first_buffer.toSlice(io, first); + + try output_buffer.toSlice(io, output_slice); const outfile = try std.Io.Dir.createFile(.cwd(), io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/outputs/out.bin", .{}); defer outfile.close(io); @@ -186,7 +190,7 @@ pub fn main() !void { var writer = outfile.writer(io, &outbuff); const writer_interface = &writer.interface; - try writer_interface.writeAll(first.data()); + try writer_interface.writeAll(output_slice.data()); // std.debug.print("Here is the results: {f}\n", .{first}); } @@ -222,7 +226,7 @@ fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram) !zml.Exe { // return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{model.force_num_frames * model.hop_len}, .f32).withTags(.{.freq})}); - return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{809508}, .f32).withTags(.{.freq})}); + return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{293699}, .f32).withTags(.{.freq})}); } pub const AudioWindow = enum { From 0179c3b884d1da0e6d23ece8d81e5bc46118b85f Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 13:20:50 +0100 Subject: [PATCH 30/98] voxtral: convert old code to use tags --- examples/voxtral/config.zig | 4 +- examples/voxtral/voxtral.zig | 74 ++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index d9839305..b4af02aa 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -1,5 +1,5 @@ pub const MelSpectrumConfig = struct { - hop_length: u32 = 160, - n_fft: u32 = 400, + hop_length: u63 = 160, + window_size: u63 = 400, global_log_mel_max: f32 = 1.5, }; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index daa98536..a50d582a 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -11,72 +11,72 @@ const Tensor = zml.Tensor; pub const LogMelSpectrogram = struct { window: AudioWindow, - n_fft: u63 = 400, // Must match window size + n_fft: u63, // Must match window size mel_filters: Tensor, // Shape must match (201, 128) - - hop_len: u63 = 160, - global_log_mel_max: f32 = 1.5, mel_floor: f32 = 1e-10, - force_num_frames: u32 = 3000, + + hop_len: u63, + global_log_mel_max: f32, + + // force_num_frames: u32 = 3000, precision: zml.DataType = .f32, pub fn init(config: MelSpectrumConfig) LogMelSpectrogram { return .{ .window = .hann, - .mel_filters = Tensor.init(.{201, 128}, .f32).withTags(.{.freq, ._ }), + .mel_filters = Tensor.init(.{201, 128}, .f32).withTags(.{.freq_bins, .mel}), .hop_len = config.hop_length, - .n_fft = config.n_fft, + .n_fft = config.window_size, .global_log_mel_max = 1.5, }; } pub fn forward(self: LogMelSpectrogram, waveform: Tensor) Tensor { const dtype = waveform.dtype(); - const rank: u63 = @intCast(waveform.shape().rank()); - const window_weight = self.window.getWeights(self.n_fft, dtype); - const fft_len = window_weight.dim(rank - 1); - var num_frames: u63 = @intCast(@divFloor(waveform.dim(rank - 1), self.hop_len)); - - var wav = waveform; - const force_num_frames = self.force_num_frames; - const force_num_samples = force_num_frames * self.hop_len; - if (num_frames > force_num_frames) { - wav = wav.slice1d(-1, .{ .end = force_num_samples }); - num_frames = force_num_frames; - } else if (num_frames < force_num_frames) { - const tagged = wav.withTags(.{.t}); - wav = tagged.pad(0.0, .{ .t = Tensor.Pad{ .high = force_num_samples - tagged.dim(.t) } }); - } - num_frames = @intCast(@divFloor(wav.dim(0), self.hop_len)); - std.debug.assert(num_frames == force_num_frames); + const window_weight = self.window.getWeights(self.n_fft, dtype); + const fft_len = window_weight.dim(.samples); + const num_frames: u63 = @intCast(@divFloor(waveform.dim(.samples), self.hop_len)); + + // var wav = waveform; + // const force_num_frames = self.force_num_frames; + // const force_num_samples = force_num_frames * self.hop_len; + // if (num_frames > force_num_frames) { + // wav = wav.slice1d(-1, .{ .end = force_num_samples }); + // num_frames = force_num_frames; + // } else if (num_frames < force_num_frames) { + // const tagged = wav.withTags(.{.t}); + // wav = tagged.pad(0.0, .{ .t = Tensor.Pad{ .high = force_num_samples - tagged.dim(.t) } }); + // } + + // num_frames = @intCast(@divFloor(wav.dim(0), self.hop_len)); + // std.debug.assert(num_frames == force_num_frames); // Reflect padding const padded_wav = blk: { - const l = wav.slice1d(-1, .{ .start = 1, .end = @divExact(fft_len, 2) + 1 }).reverse(.{-1}); - const r = wav.slice1d(-1, .{ .start = -@divExact(fft_len, 2) - 1, .end = -1 }).reverse(.{-1}); - break :blk zml.Tensor.concatenate(&.{ l, wav, r }, -1); + const l = waveform.slice1d(.samples, .{ .start = 1, .end = @divExact(fft_len, 2) + 1 }).reverse(.{.samples}); + const r = waveform.slice1d(.samples, .{ .start = -@divExact(fft_len, 2) - 1, .end = -1 }).reverse(.{.samples}); + break :blk zml.Tensor.concatenate(&.{ l, waveform, r }, .samples); }; // Use Short Time Fourier Transform to compute features. // Generate num_frames+1 (matching torch.stft center=True frame count), // then drop the last frame to match Whisper convention (stft[..., :-1]). var spectrogram = stft(padded_wav, window_weight, num_frames + 1, self.hop_len, self.precision); - spectrogram = spectrogram.slice1d(0, .{ .end = -1 }); + spectrogram = spectrogram.slice1d(.frames, .{ .end = -1 }); spectrogram = spectrogram.convert(dtype); // Re-weight frequencies for speech - spectrogram = spectrogram.dot(self.mel_filters, .freq); + spectrogram = spectrogram.dot(self.mel_filters, .freq_bins); spectrogram = spectrogram.maximum(Tensor.constant(dtype.constant(self.mel_floor))); var log_spec = spectrogram.log().scale(1.0 / @log(10.0)); const log_spec_min = Tensor.constant(dtype.constant(self.global_log_mel_max - 8.0)); log_spec = log_spec.maximum(log_spec_min); - const log_spec_rank = log_spec.shape().rank(); // "center" the distribution - return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ log_spec_rank - 1, log_spec_rank - 2 }); + return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ .mel, .frames }); } pub fn load(self: *LogMelSpectrogram, io: std.Io, platform: *zml.Platform) !zml.Bufferized(LogMelSpectrogram) { @@ -108,7 +108,7 @@ pub fn stft(waveform: Tensor, weight: Tensor, num_frames: usize, stride: u63, pr var fft = windows.convert(precision).fft(.{ .kind = .RFFT, .length = &.{fft_len} }); const spectrogram = fft.abs(); - const ret = spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{.temp, .freq}); + const ret = spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{.frames, .freq_bins}); log.info("{f}", .{ret.shape()}); return ret; @@ -170,7 +170,9 @@ pub fn main() !void { var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); - const output_shape = zml.Shape.init(.{ 128, melspectro_model.force_num_frames }, .f32); + // const output_shape = zml.Shape.init(.{ 128, melspectro_model.force_num_frames }, .f32); + const num_frames = wav_file.len / melspectrum_config.hop_length; + const output_shape = zml.Shape.init(.{ 128, num_frames }, .f32); const output_slice: zml.Slice = try zml.Slice.alloc(allocator, output_shape); defer output_slice.free(allocator); var output_buffer: zml.Buffer = try .fromSlice(io, platform, output_slice); @@ -226,7 +228,7 @@ fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram) !zml.Exe { // return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{model.force_num_frames * model.hop_len}, .f32).withTags(.{.freq})}); - return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{293699}, .f32).withTags(.{.freq})}); + return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{293699}, .f32).withTags(.{.samples})}); } pub const AudioWindow = enum { @@ -237,13 +239,13 @@ pub const AudioWindow = enum { pub fn getWeights(self: AudioWindow, len: i64, dtype: zml.DataType) Tensor { return switch (self) { - .boxcar => Tensor.constant(dtype.one()), + .boxcar => Tensor.constant(dtype.one()).withTags(.{.samples}), .hann => { if (len <= 1) return Tensor.constant(dtype.one()); const flen: f64 = @floatFromInt(len); const freq = Tensor.constant(dtype.constant(std.math.pi / flen)); const steps = Tensor.arange(.{ .start = -len, .end = len, .step = 2 }, dtype); - return steps.mul(freq).cos().scale(0.5).addConstant(0.5).convert(dtype); + return steps.mul(freq).cos().scale(0.5).addConstant(0.5).convert(dtype).withTags(.{.samples}); }, }; } From e72e979c9c5c36e3daed81d18ef179e001e8ad82 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 13:22:39 +0100 Subject: [PATCH 31/98] fix pad function args order --- mlir/dialects/stablehlo/stablehlo.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index a21eec76..3412b3c5 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -461,7 +461,7 @@ pub const PadOpts = struct { interior: []const i64, }; -pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, location: *const mlir.Location, opts: PadOpts) *mlir.Operation { +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.pad", .{ .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, From 8bdc64b2eda12fe976015612a0789fc5eb689c49 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 13:23:39 +0100 Subject: [PATCH 32/98] add pre-computed mel filter as an asset --- examples/voxtral/assets/BUILD.bazel | 5 +++++ .../voxtral/assets/voxtral_mel_filter.data | Bin 0 -> 102912 bytes 2 files changed, 5 insertions(+) create mode 100644 examples/voxtral/assets/BUILD.bazel create mode 100644 examples/voxtral/assets/voxtral_mel_filter.data diff --git a/examples/voxtral/assets/BUILD.bazel b/examples/voxtral/assets/BUILD.bazel new file mode 100644 index 00000000..c60a233b --- /dev/null +++ b/examples/voxtral/assets/BUILD.bazel @@ -0,0 +1,5 @@ +filegroup( + name = "voxtral_mel_filter", + srcs = ["voxtral_mel_filter.data"], + visibility = ["//visibility:public"], +) diff --git a/examples/voxtral/assets/voxtral_mel_filter.data b/examples/voxtral/assets/voxtral_mel_filter.data new file mode 100644 index 0000000000000000000000000000000000000000..7e8a85ebd1e619aff36a270d5e668336492bc333 GIT binary patch literal 102912 zcmeI5i(AiUAIE>5MbjjtM�-5Gy%WzwhV%x#*zik<*MTr48gonbr`lVf7lGkT1@$>-PgMwvU zEy43rcwPkdz}ecQ2KKSz-T>GE9X)@>F4c5*#F+r@!GEW>v;_6*MGt^Ic;wkuUI+E$ zOaOL3N4EqmQdb8)U>AUU(DY0}6;-XE2jCv`f7;!Wd2}Yvi@+W@oonB8cVxr80k8u) zdai%23bfDXOaS*_&c)VEcZd%30N8`-zzGt4)rB(w*a02g*?PIUHl~_g0Pew%uv)dK zHk=-Sdyqe)hhn13LPzj^!%2R}^Ogum>{fhA}AIkzD}n zfR3KMeW&_gA%A~{tidnKdf#WSgr8296 z4gfo#qxbCoyGqM+H^7$xI=*|*300djgU=FyEpV>MQTtQ!=m4+-lRn5$A!V280I&mH zrfpI?y5FS(zz(c>vO$Hr|3(LZ9oRc4Sv44CbO6|ag8yt$A)lww0bmDOpV_V^t{6=R zfE~!}mZy5v|G+yUPzSyrw$lh*u$Onmp!{|0iujeX{p@-=0PH}2r-!n5+blW&>_B0^ z-fHgJK6C)sfw%2usC}kuJQDyr5TBT!th@E41E3C!n}6I`JvfJpYS)uzzUeI3=UTOkqkc!_T@mU94rM~ z;)dCC4uJC4vC9`4D%8r%y&$Lqod<`C`nxYX0F=Luy&=&_CEQ%k4ght)#`%F^-s;N^ z0OhY^Ur+vrN^Nt19RTWp^GFZF`AG;n0F=LuT{ru#jM;I99RTdWz$fEXjO__;;3a^L zo;~%vjI^s_2LM};Qa?hqnzM_)5dd|dBIl%3?%;Q4LGd@oz7rInZhNG$13)F1;#el7 zTX+@##orwJR!D%l{z`Zj0F@wX(RuN0U(B-rD*xB*4Zl0yyb}cFuVbh0s*!K))7b%_ z4yfP)BOs|KI{=iwj_q~)g^X~EVh4aaaHyoSr1kH>4glq^V?TRrqwdx6`;w>%EVh~@ z^JX`22A~Silza2KuIlBV1$ydsZ%= zOkf9qI&jhTo^j6BjvWBXU&kJ%-bzBF8#@5hfuen%Nttyu`GfM;v7h&LQk(Mvcoqb8 zpvo>+98RUO13>xPdS)117jPd4ioZGbkNY0W`=y@j08k08E}kegW&E8uDE{WyV<$MM zJxd(94+NE<)3JPsu3Wa)b+vZm3?KlHK`RmyGv(8D*Ulw)%r~^IjS>%A-ujCKPU&l_G(ngh}KjJ&%pbpfT z^JH}Je0BgR|F4$5Co?l|kv}MZ9XlfJja+e9#k&Ad2dW*m%DvgK>;O>y9n4pZaq+d} z56WN1KHK&WS#mm;GXT_qKYSA;e=wg3g5o!M-8yLa9lFmMfGR*!?i;Rm<(uJ0I0HZ( z$hsLP@7Be!13>v-8c}G3+uSC9Q2sjhRFrtp!^3EKR5b$o+N)z{yO%T-j8M3Mi{qI@rJt80I{;LINdx`F zEhU-!LHW1eIl%C-<2S=V@i)hQHK$aQP*Yp8{P$?D)80wWJ!AJ!43cwz}w43j^}S7e^CCdyPh%yMH#`OxREUp!{|0xE`0}ap!t|D+bhob`B}h?!x=* z08st|Tlbal=+)#8%HJ_F&? z(;02~JQ&o0D9a{kk#(JW08suOF2lvDWDI8jDF3p=V}}3ut>h2NU&r=;Rwh?7PVgHM zpbmI>q{xz;XSfFdBA+nVGLHX<0D-$=% z-J^TC2LN>-#{DDd)Ttjk0F?iWh&rQY&MNW;<)1&f(p*?pNdBPwb?nJEGGupS1or@- z4lHz?EwNvAWCwuqcTH|9Y4_)mKPdmt?IszMbABX$Q2sjh`sBUx%Q_SH0H6-^`E6ZZh1 z4s<*=T0FBmvjaf+x5;{A7#?%TAC!Mhh8X{xm`eVj{B`VM{@dlJS&e)z2-Jb#`w Date: Wed, 11 Feb 2026 13:31:02 +0100 Subject: [PATCH 33/98] separate mel spectrogram to different file --- examples/voxtral/BUILD.bazel | 4 + examples/voxtral/mel_spectrogram.zig | 133 ++++++++++++++++++++++++ examples/voxtral/voxtral.zig | 150 +++------------------------ 3 files changed, 150 insertions(+), 137 deletions(-) create mode 100644 examples/voxtral/mel_spectrogram.zig diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel index 2e027b37..19be53cf 100644 --- a/examples/voxtral/BUILD.bazel +++ b/examples/voxtral/BUILD.bazel @@ -7,9 +7,13 @@ zig_binary( data = [], main = "voxtral.zig", srcs = [ + "mel_spectrogram.zig", "wav.zig", "config.zig", ], + extra_srcs = [ + "//examples/voxtral/assets:voxtral_mel_filter", + ], deps = ["//zml"], ) diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig new file mode 100644 index 00000000..ac193ac8 --- /dev/null +++ b/examples/voxtral/mel_spectrogram.zig @@ -0,0 +1,133 @@ +const std = @import("std"); +const log = std.log; +const cfg = @import("config.zig"); +const MelSpectrumConfig = cfg.MelSpectrumConfig; + +const zml = @import("zml"); +const Tensor = zml.Tensor; + +pub const LogMelSpectrogram = struct { + window: AudioWindow, + + n_fft: u63, // Must match window size + mel_filters: Tensor, // Shape must match (201, 128) + mel_floor: f32 = 1e-10, + + hop_len: u63, + global_log_mel_max: f32, + + // force_num_frames: u32 = 3000, + + precision: zml.DataType = .f32, + + pub fn init(config: MelSpectrumConfig) LogMelSpectrogram { + return .{ + .window = .hann, + .mel_filters = Tensor.init(.{201, 128}, .f32).withTags(.{.freq_bins, .mel}), + .hop_len = config.hop_length, + .n_fft = config.window_size, + .global_log_mel_max = 1.5, + }; + } + + pub fn forward(self: LogMelSpectrogram, waveform: Tensor) Tensor { + const dtype = waveform.dtype(); + + const window_weight = self.window.getWeights(self.n_fft, dtype); + const fft_len = window_weight.dim(.samples); + const num_frames: u63 = @intCast(@divFloor(waveform.dim(.samples), self.hop_len)); + + // var wav = waveform; + // const force_num_frames = self.force_num_frames; + // const force_num_samples = force_num_frames * self.hop_len; + // if (num_frames > force_num_frames) { + // wav = wav.slice1d(-1, .{ .end = force_num_samples }); + // num_frames = force_num_frames; + // } else if (num_frames < force_num_frames) { + // const tagged = wav.withTags(.{.t}); + // wav = tagged.pad(0.0, .{ .t = Tensor.Pad{ .high = force_num_samples - tagged.dim(.t) } }); + // } + + // num_frames = @intCast(@divFloor(wav.dim(0), self.hop_len)); + // std.debug.assert(num_frames == force_num_frames); + + // Reflect padding + const padded_wav = blk: { + const l = waveform.slice1d(.samples, .{ .start = 1, .end = @divExact(fft_len, 2) + 1 }).reverse(.{.samples}); + const r = waveform.slice1d(.samples, .{ .start = -@divExact(fft_len, 2) - 1, .end = -1 }).reverse(.{.samples}); + break :blk zml.Tensor.concatenate(&.{ l, waveform, r }, .samples); + }; + + // Use Short Time Fourier Transform to compute features. + // Generate num_frames+1 (matching torch.stft center=True frame count), + // then drop the last frame to match Whisper convention (stft[..., :-1]). + var spectrogram = stft(padded_wav, window_weight, num_frames + 1, self.hop_len, self.precision); + spectrogram = spectrogram.slice1d(.frames, .{ .end = -1 }); + spectrogram = spectrogram.convert(dtype); + // Re-weight frequencies for speech + spectrogram = spectrogram.dot(self.mel_filters, .freq_bins); + + spectrogram = spectrogram.maximum(Tensor.constant(dtype.constant(self.mel_floor))); + var log_spec = spectrogram.log().scale(1.0 / @log(10.0)); + + const log_spec_min = Tensor.constant(dtype.constant(self.global_log_mel_max - 8.0)); + log_spec = log_spec.maximum(log_spec_min); + + // "center" the distribution + return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ .mel, .frames }); + } + + pub fn load(self: *LogMelSpectrogram, io: std.Io, platform: *zml.Platform) !zml.Bufferized(LogMelSpectrogram) { + const mel_filters_data = @embedFile("assets/voxtral_mel_filter.data"); + const slice = zml.Slice.init(self.mel_filters.shape(), mel_filters_data); + + return .{ + .mel_filters = try zml.Buffer.fromSlice(io, platform, slice), + }; + } + + pub fn unload(self: *zml.Bufferized(LogMelSpectrogram)) void { + self.mel_filters.deinit(); + } +}; + +pub fn stft(waveform: Tensor, weight: Tensor, num_frames: usize, stride: u63, precision: zml.DataType) Tensor { + const fft_len = weight.dim(0); + + const num_samples = waveform.dim(0); + + // const num_frames = 3000; + const indices = Tensor.arange(.{ .end = @intCast(num_frames * stride), .step = stride }, .i32); + std.log.warn("num_samples: {d}, num_frames: {d}, end: {d}, stride: {d}, indices: {}", .{ num_samples, num_frames, num_frames * stride, stride, indices }); + var windows = waveform.gatherSlices(.{fft_len}, indices.appendAxes(.{.coord }), .{ .indices_are_sorted = true }); + + windows = windows.mul(weight.broadcastLeft(windows.shape())); + + var fft = windows.convert(precision).fft(.{ .kind = .RFFT, .length = &.{fft_len} }); + const spectrogram = fft.abs(); + + const ret = spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{.frames, .freq_bins}); + log.info("{f}", .{ret.shape()}); + + return ret; +} + +pub const AudioWindow = enum { + /// https://numpy.org/doc/stable/reference/generated/numpy.hanning.html + hann, + boxcar, + // Other possible windows: https://en.wikipedia.org/wiki/Window_function + + pub fn getWeights(self: AudioWindow, len: i64, dtype: zml.DataType) Tensor { + return switch (self) { + .boxcar => Tensor.constant(dtype.one()).withTags(.{.samples}), + .hann => { + if (len <= 1) return Tensor.constant(dtype.one()); + const flen: f64 = @floatFromInt(len); + const freq = Tensor.constant(dtype.constant(std.math.pi / flen)); + const steps = Tensor.arange(.{ .start = -len, .end = len, .step = 2 }, dtype); + return steps.mul(freq).cos().scale(0.5).addConstant(0.5).convert(dtype).withTags(.{.samples}); + }, + }; + } +}; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index a50d582a..0f77b62d 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -8,112 +8,8 @@ const MelSpectrumConfig = cfg.MelSpectrumConfig; const zml = @import("zml"); const Tensor = zml.Tensor; -pub const LogMelSpectrogram = struct { - window: AudioWindow, - - n_fft: u63, // Must match window size - mel_filters: Tensor, // Shape must match (201, 128) - mel_floor: f32 = 1e-10, - - hop_len: u63, - global_log_mel_max: f32, - - // force_num_frames: u32 = 3000, - - precision: zml.DataType = .f32, - - pub fn init(config: MelSpectrumConfig) LogMelSpectrogram { - return .{ - .window = .hann, - .mel_filters = Tensor.init(.{201, 128}, .f32).withTags(.{.freq_bins, .mel}), - .hop_len = config.hop_length, - .n_fft = config.window_size, - .global_log_mel_max = 1.5, - }; - } - - pub fn forward(self: LogMelSpectrogram, waveform: Tensor) Tensor { - const dtype = waveform.dtype(); - - const window_weight = self.window.getWeights(self.n_fft, dtype); - const fft_len = window_weight.dim(.samples); - const num_frames: u63 = @intCast(@divFloor(waveform.dim(.samples), self.hop_len)); - - // var wav = waveform; - // const force_num_frames = self.force_num_frames; - // const force_num_samples = force_num_frames * self.hop_len; - // if (num_frames > force_num_frames) { - // wav = wav.slice1d(-1, .{ .end = force_num_samples }); - // num_frames = force_num_frames; - // } else if (num_frames < force_num_frames) { - // const tagged = wav.withTags(.{.t}); - // wav = tagged.pad(0.0, .{ .t = Tensor.Pad{ .high = force_num_samples - tagged.dim(.t) } }); - // } - - // num_frames = @intCast(@divFloor(wav.dim(0), self.hop_len)); - // std.debug.assert(num_frames == force_num_frames); - - // Reflect padding - const padded_wav = blk: { - const l = waveform.slice1d(.samples, .{ .start = 1, .end = @divExact(fft_len, 2) + 1 }).reverse(.{.samples}); - const r = waveform.slice1d(.samples, .{ .start = -@divExact(fft_len, 2) - 1, .end = -1 }).reverse(.{.samples}); - break :blk zml.Tensor.concatenate(&.{ l, waveform, r }, .samples); - }; - - // Use Short Time Fourier Transform to compute features. - // Generate num_frames+1 (matching torch.stft center=True frame count), - // then drop the last frame to match Whisper convention (stft[..., :-1]). - var spectrogram = stft(padded_wav, window_weight, num_frames + 1, self.hop_len, self.precision); - spectrogram = spectrogram.slice1d(.frames, .{ .end = -1 }); - spectrogram = spectrogram.convert(dtype); - // Re-weight frequencies for speech - spectrogram = spectrogram.dot(self.mel_filters, .freq_bins); - - spectrogram = spectrogram.maximum(Tensor.constant(dtype.constant(self.mel_floor))); - var log_spec = spectrogram.log().scale(1.0 / @log(10.0)); - - const log_spec_min = Tensor.constant(dtype.constant(self.global_log_mel_max - 8.0)); - log_spec = log_spec.maximum(log_spec_min); - - // "center" the distribution - return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ .mel, .frames }); - } - - pub fn load(self: *LogMelSpectrogram, io: std.Io, platform: *zml.Platform) !zml.Bufferized(LogMelSpectrogram) { - const mel_filters_data = @embedFile("assets/voxtral_mel_filter.data"); - const slice = zml.Slice.init(self.mel_filters.shape(), mel_filters_data); - - return .{ - .mel_filters = try zml.Buffer.fromSlice(io, platform, slice), - }; - } - - pub fn unload(self: *zml.Bufferized(LogMelSpectrogram)) void { - self.mel_filters.deinit(); - } -}; - -pub fn stft(waveform: Tensor, weight: Tensor, num_frames: usize, stride: u63, precision: zml.DataType) Tensor { - const fft_len = weight.dim(0); - - const num_samples = waveform.dim(0); - - // const num_frames = 3000; - const indices = Tensor.arange(.{ .end = @intCast(num_frames * stride), .step = stride }, .i32); - std.log.warn("num_samples: {d}, num_frames: {d}, end: {d}, stride: {d}, indices: {}", .{ num_samples, num_frames, num_frames * stride, stride, indices }); - var windows = waveform.gatherSlices(.{fft_len}, indices.appendAxes(.{.coord }), .{ .indices_are_sorted = true }); - - windows = windows.mul(weight.broadcastLeft(windows.shape())); - - var fft = windows.convert(precision).fft(.{ .kind = .RFFT, .length = &.{fft_len} }); - const spectrogram = fft.abs(); - - const ret = spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{.frames, .freq_bins}); - log.info("{f}", .{ret.shape()}); - - return ret; -} - +const mel = @import("mel_spectrogram.zig"); +const LogMelSpectrogram = mel.LogMelSpectrogram; pub fn main() !void { log.info("Start of Voxtral", .{}); @@ -128,9 +24,9 @@ pub fn main() !void { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); - + const io = threaded.io(); - + const arena: std.heap.ArenaAllocator = .init(allocator); defer arena.deinit(); @@ -139,22 +35,22 @@ pub fn main() !void { var wav_buffer: [4096]u8 = undefined; var reader = file.reader(io, &wav_buffer); - + const wav_file = try loadWav(allocator, &reader.interface); std.debug.print("LEN: {d}\n", .{wav_file.len}); defer allocator.free(wav_file); std.debug.print("{any}\n", .{wav_file[0..100]}); const melspectrum_config = MelSpectrumConfig{}; - + var platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform .deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); - + var melspectro_model: LogMelSpectrogram = .init(melspectrum_config); var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model}); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); - + var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); defer compiled_mel_spectrum.deinit(); var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); @@ -162,10 +58,10 @@ pub fn main() !void { // Execute var args = try compiled_mel_spectrum.args(allocator); - defer args.deinit(allocator); + defer args.deinit(allocator); var results = try compiled_mel_spectrum.results(allocator); defer results.deinit(allocator); - + const input_slice = zml.Slice.init(.init(.{ .freq = wav_file.len }, .f32), std.mem.sliceAsBytes(wav_file)); var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); @@ -184,16 +80,16 @@ pub fn main() !void { try output_buffer.toSlice(io, output_slice); - + const outfile = try std.Io.Dir.createFile(.cwd(), io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/outputs/out.bin", .{}); defer outfile.close(io); - + var outbuff: [4096]u8 = undefined; var writer = outfile.writer(io, &outbuff); const writer_interface = &writer.interface; try writer_interface.writeAll(output_slice.data()); - + // std.debug.print("Here is the results: {f}\n", .{first}); } @@ -230,23 +126,3 @@ pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *c // return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{model.force_num_frames * model.hop_len}, .f32).withTags(.{.freq})}); return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{293699}, .f32).withTags(.{.samples})}); } - -pub const AudioWindow = enum { - /// https://numpy.org/doc/stable/reference/generated/numpy.hanning.html - hann, - boxcar, - // Other possible windows: https://en.wikipedia.org/wiki/Window_function - - pub fn getWeights(self: AudioWindow, len: i64, dtype: zml.DataType) Tensor { - return switch (self) { - .boxcar => Tensor.constant(dtype.one()).withTags(.{.samples}), - .hann => { - if (len <= 1) return Tensor.constant(dtype.one()); - const flen: f64 = @floatFromInt(len); - const freq = Tensor.constant(dtype.constant(std.math.pi / flen)); - const steps = Tensor.arange(.{ .start = -len, .end = len, .step = 2 }, dtype); - return steps.mul(freq).cos().scale(0.5).addConstant(0.5).convert(dtype).withTags(.{.samples}); - }, - }; - } -}; From fbe5970f7aab805d50e42e3e30255c639499e64c Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 14:30:21 +0100 Subject: [PATCH 34/98] use vfs, tensor store and registry to test --- examples/voxtral/python/main.py | 156 -------------------------------- examples/voxtral/voxtral.zig | 17 +++- 2 files changed, 16 insertions(+), 157 deletions(-) delete mode 100644 examples/voxtral/python/main.py diff --git a/examples/voxtral/python/main.py b/examples/voxtral/python/main.py deleted file mode 100644 index 79f1841d..00000000 --- a/examples/voxtral/python/main.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 -""" -Reference mel spectrogram implementation for Voxtral. -Outputs intermediate values for comparison with the Zig implementation. - -Usage: - pip install numpy soundfile - python main.py -""" - -import sys - -import numpy as np -import soundfile as sf -import torch - -# Audio parameters (from Voxtral config / params.json) -SAMPLE_RATE = 16000 -NUM_MEL_BINS = 128 -HOP_LENGTH = 160 -WINDOW_SIZE = 400 # also used as n_fft -GLOBAL_LOG_MEL_MAX = 1.5 - - -# ========================================================================== -# Mel filter bank (Slaney-style, from mistral_common/audio.py) -# ========================================================================== -def hertz_to_mel(freq): - min_log_hertz = 1000.0 - min_log_mel = 15.0 - logstep = 27.0 / np.log(6.4) - mels = 3.0 * freq / 200.0 - if isinstance(freq, np.ndarray): - log_region = freq >= min_log_hertz - mels[log_region] = ( - min_log_mel + np.log(freq[log_region] / min_log_hertz) * logstep - ) - elif freq >= min_log_hertz: - mels = min_log_mel + np.log(freq / min_log_hertz) * logstep - return mels - - -def mel_to_hertz(mels): - min_log_hertz = 1000.0 - min_log_mel = 15.0 - logstep = np.log(6.4) / 27.0 - freq = 200.0 * mels / 3.0 - log_region = mels >= min_log_mel - freq[log_region] = min_log_hertz * np.exp( - logstep * (mels[log_region] - min_log_mel) - ) - return freq - - -def compute_mel_filters(): - num_frequency_bins = 1 + WINDOW_SIZE // 2 # 201 - fft_freqs = np.linspace(0, SAMPLE_RATE // 2, num_frequency_bins) - mel_min = hertz_to_mel(0.0) - mel_max = hertz_to_mel(8000.0) - mel_freqs = np.linspace(mel_min, mel_max, NUM_MEL_BINS + 2) - filter_freqs = mel_to_hertz(mel_freqs) - filter_diff = np.diff(filter_freqs) - slopes = np.expand_dims(filter_freqs, 0) - np.expand_dims(fft_freqs, 1) - down_slopes = -slopes[:, :-2] / filter_diff[:-1] - up_slopes = slopes[:, 2:] / filter_diff[1:] - fb = np.maximum(np.zeros(1), np.minimum(down_slopes, up_slopes)) - enorm = 2.0 / (filter_freqs[2 : NUM_MEL_BINS + 2] - filter_freqs[:NUM_MEL_BINS]) - fb *= np.expand_dims(enorm, 0) - return fb # [201, 128] - - -# ============================================================================ -# Mel spectrogram (from vLLM voxtral.py compute_whisper_melspec) -# ============================================================================ - - -def compute_mel_spectrogram(audio, mel_filters): - """audio: 1D tensor, mel_filters: [freq_bins, mel_bins] tensor""" - window = torch.hann_window(WINDOW_SIZE) - stft = torch.stft( - audio, WINDOW_SIZE, HOP_LENGTH, window=window, return_complex=True - ) - magnitudes = stft[..., :-1].abs() ** 2 - mel_spec = mel_filters.T @ magnitudes # [mel_bins, frames] - log_spec = torch.clamp(mel_spec, min=1e-10).log10() - # Use fixed global_log_mel_max from params.json (matching vLLM compute_whisper_melspec - # which uses config.global_log_mel_max when set, falling back to dynamic max otherwise) - log_spec = torch.maximum(log_spec, torch.tensor(GLOBAL_LOG_MEL_MAX) - 8.0) - log_spec = (log_spec + 4.0) / 4.0 - return log_spec # [128, frames] - - -# ========================================================================== -# Main -# ========================================================================== - - -def execute(path): - # Load audio - audio, sr = sf.read(path, dtype="float32") - if audio.ndim > 1: - audio = audio.mean(axis=1) - if sr != SAMPLE_RATE: - import soxr - - audio = soxr.resample(audio, sr, SAMPLE_RATE, quality="HQ") - - print( - f"Audio: {len(audio)} samples, {len(audio) / SAMPLE_RATE:.2f}s", file=sys.stderr - ) - - # Compute mel filters (numpy) and convert to torch - mel_filters_np = compute_mel_filters() - mel_filters = torch.tensor(mel_filters_np, dtype=torch.float32) - print(f"Mel filters: {mel_filters.shape}", file=sys.stderr) - print( - f" min={mel_filters.min():.6f}, max={mel_filters.max():.6f}", file=sys.stderr - ) - - # Compute mel spectrogram - audio_tensor = torch.tensor(audio, dtype=torch.float32) - mel = compute_mel_spectrogram(audio_tensor, mel_filters) - print(f"Mel spectrogram: {mel.shape}", file=sys.stderr) - print( - f" min={mel.min():.6f}, max={mel.max():.6f}, mean={mel.mean():.6f}", - file=sys.stderr, - ) - - # Dump first few values for comparison - print("\n# First 10 mel filter values (row 0):", file=sys.stderr) - print(mel_filters[0, :10].tolist(), file=sys.stderr) - - print("\n# First frame mel spectrogram (all 128 bins):", file=sys.stderr) - print(mel[:, 0].tolist(), file=sys.stderr) - - print("\n# Mel spectrogram corner [0:4, 0:4]:", file=sys.stderr) - print(mel[:4, :4].tolist(), file=sys.stderr) - - # Save as raw f32 binary (row-major) for loading in Zig - # Layout: [num_mel_bins, num_frames] contiguous float32 - output_path = path.rsplit(".", 1)[0] + "_mel.bin" - mel.numpy().astype(np.float32).tofile(output_path) - print(f"\nSaved mel spectrogram ({mel.shape[0]}x{mel.shape[1]}, f32) to {output_path}", file=sys.stderr) - - -# def main(): -# if len(sys.argv) < 2: -# print(f"Usage: {sys.argv[0]} ", file=sys.stderr) -# sys.exit(1) - -# wav_path = sys.argv[1] -# execute(wav_path) - - -# if __name__ == "__main__": -# main() diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 0f77b62d..1e75a045 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -25,7 +25,14 @@ pub fn main() !void { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); - const io = threaded.io(); + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + defer vfs.deinit(); + + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); + defer vfs_file.deinit(); + try vfs.register("file", vfs_file.io()); + + const io = vfs.io(); const arena: std.heap.ArenaAllocator = .init(allocator); defer arena.deinit(); @@ -91,6 +98,14 @@ pub fn main() !void { try writer_interface.writeAll(output_slice.data()); // std.debug.print("Here is the results: {f}\n", .{first}); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/outputs/voxtral_activations.safetensors"); + defer registry.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + // try zml.testing.testlayer(allocator, io, platform.*, melspectro_model, .forward, store.view(), "mel", mel_spectrum_buffers, 1e-3); } fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { From e9456b806ca8e8116d127407bb0c2dc1e44ab2fd Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 14:54:34 +0100 Subject: [PATCH 35/98] harness to tag mel test tensors --- examples/voxtral/voxtral.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 1e75a045..529ff2ec 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -97,17 +97,23 @@ pub fn main() !void { try writer_interface.writeAll(output_slice.data()); - // std.debug.print("Here is the results: {f}\n", .{first}); - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/outputs/voxtral_activations.safetensors"); defer registry.deinit(); var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); - // try zml.testing.testlayer(allocator, io, platform.*, melspectro_model, .forward, store.view(), "mel", mel_spectrum_buffers, 1e-3); + try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, store.view(), "mel", .{ .inner = mel_spectrum_buffers }, 1e-3); } +const MelTestHarness = struct { + inner: LogMelSpectrogram, + + pub fn forward(self: MelTestHarness, waveform: Tensor) Tensor { + return self.inner.forward(waveform.withTags(.{.samples})); + } +}; + fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { var arena_state: std.heap.ArenaAllocator = .init(allocator); defer arena_state.deinit(); From 82c82ed8b0bcd7ee53c4efefecd7d5b3b3ebbeed Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 15:11:44 +0100 Subject: [PATCH 36/98] separate test file --- examples/voxtral/BUILD.bazel | 20 ++++++++--- examples/voxtral/test.zig | 69 ++++++++++++++++++++++++++++++++++++ examples/voxtral/voxtral.zig | 55 +++++++++++----------------- 3 files changed, 106 insertions(+), 38 deletions(-) create mode 100644 examples/voxtral/test.zig diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel index 19be53cf..601aff1b 100644 --- a/examples/voxtral/BUILD.bazel +++ b/examples/voxtral/BUILD.bazel @@ -3,8 +3,6 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( name = "voxtral", - args = [], - data = [], main = "voxtral.zig", srcs = [ "mel_spectrogram.zig", @@ -12,12 +10,26 @@ zig_binary( "config.zig", ], extra_srcs = [ - "//examples/voxtral/assets:voxtral_mel_filter", + "//examples/voxtral/assets:voxtral_mel_filter", + ], + deps = ["//zml"], +) + +zig_binary( + name = "voxtral_test", + main = "test.zig", + srcs = [ + "mel_spectrogram.zig", + "wav.zig", + "config.zig", + ], + extra_srcs = [ + "//examples/voxtral/assets:voxtral_mel_filter", ], deps = ["//zml"], ) build_test( name = "test", - targets = [":voxtral"], + targets = [":voxtral_test"], ) diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig new file mode 100644 index 00000000..65bad9f5 --- /dev/null +++ b/examples/voxtral/test.zig @@ -0,0 +1,69 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const log = std.log; +const cfg = @import("config.zig"); +const MelSpectrumConfig = cfg.MelSpectrumConfig; + +const zml = @import("zml"); +const stdx = zml.stdx; +const Tensor = zml.Tensor; + +const mel = @import("mel_spectrogram.zig"); +const LogMelSpectrogram = mel.LogMelSpectrogram; + +const CliArgs = struct { + reference: []const u8, +}; + +pub fn main() !void { + log.info("Start of Voxtral test", .{}); + + var dbg = std.heap.DebugAllocator(.{}).init; + defer if (builtin.mode == .Debug) std.debug.assert(dbg.deinit() == .ok); + + const allocator = switch (builtin.mode) { + .Debug => dbg.allocator(), + else => std.heap.c_allocator, + }; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const args = stdx.flags.parseProcessArgs(CliArgs); + + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + defer vfs.deinit(); + + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); + defer vfs_file.deinit(); + try vfs.register("file", vfs_file.io()); + + const io = vfs.io(); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.reference); + defer registry.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + const melspectrum_config = MelSpectrumConfig{}; + + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); + + var melspectro_model: LogMelSpectrogram = .init(melspectrum_config); + var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); + var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); + defer LogMelSpectrogram.unload(&mel_spectrum_buffers); + + try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, store.view(), "mel", .{ .inner = mel_spectrum_buffers }, 1e-3); +} + +const MelTestHarness = struct { + inner: LogMelSpectrogram, + + pub fn forward(self: MelTestHarness, waveform: Tensor) Tensor { + return self.inner.forward(waveform.withTags(.{.samples})); + } +}; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 529ff2ec..4236e98e 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -6,11 +6,17 @@ const cfg = @import("config.zig"); const MelSpectrumConfig = cfg.MelSpectrumConfig; const zml = @import("zml"); +const stdx = zml.stdx; const Tensor = zml.Tensor; const mel = @import("mel_spectrogram.zig"); const LogMelSpectrogram = mel.LogMelSpectrogram; +const CliArgs = struct { + input: []const u8, + // output: []const u8, +}; + pub fn main() !void { log.info("Start of Voxtral", .{}); @@ -25,9 +31,11 @@ pub fn main() !void { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); + const args = stdx.flags.parseProcessArgs(CliArgs); + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); defer vfs.deinit(); - + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); defer vfs_file.deinit(); try vfs.register("file", vfs_file.io()); @@ -37,21 +45,19 @@ pub fn main() !void { const arena: std.heap.ArenaAllocator = .init(allocator); defer arena.deinit(); - const file = try std.Io.Dir.openFile(.cwd(), io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/inputs/harvard_16k.wav", .{}); + const file = try std.Io.Dir.openFile(.cwd(), io, args.input, .{}); defer file.close(io); var wav_buffer: [4096]u8 = undefined; var reader = file.reader(io, &wav_buffer); const wav_file = try loadWav(allocator, &reader.interface); - std.debug.print("LEN: {d}\n", .{wav_file.len}); defer allocator.free(wav_file); - std.debug.print("{any}\n", .{wav_file[0..100]}); const melspectrum_config = MelSpectrumConfig{}; var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform .deinit(allocator); + defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); var melspectro_model: LogMelSpectrogram = .init(melspectrum_config); @@ -64,8 +70,8 @@ pub fn main() !void { defer LogMelSpectrogram.unload(&mel_spectrum_buffers); // Execute - var args = try compiled_mel_spectrum.args(allocator); - defer args.deinit(allocator); + var exec_args = try compiled_mel_spectrum.args(allocator); + defer exec_args.deinit(allocator); var results = try compiled_mel_spectrum.results(allocator); defer results.deinit(allocator); @@ -73,7 +79,6 @@ pub fn main() !void { var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); - // const output_shape = zml.Shape.init(.{ 128, melspectro_model.force_num_frames }, .f32); const num_frames = wav_file.len / melspectrum_config.hop_length; const output_shape = zml.Shape.init(.{ 128, num_frames }, .f32); const output_slice: zml.Slice = try zml.Slice.alloc(allocator, output_shape); @@ -81,39 +86,22 @@ pub fn main() !void { var output_buffer: zml.Buffer = try .fromSlice(io, platform, output_slice); defer output_buffer.deinit(); - args.set(.{mel_spectrum_buffers, input_buffer}); - compiled_mel_spectrum.call(args, &results); + exec_args.set(.{mel_spectrum_buffers, input_buffer}); + compiled_mel_spectrum.call(exec_args, &results); results.fill(.{&output_buffer}); - try output_buffer.toSlice(io, output_slice); - const outfile = try std.Io.Dir.createFile(.cwd(), io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/outputs/out.bin", .{}); - defer outfile.close(io); + // const outfile = try std.Io.Dir.createFile(.cwd(), io, args.output, .{}); + // defer outfile.close(io); - var outbuff: [4096]u8 = undefined; - var writer = outfile.writer(io, &outbuff); - const writer_interface = &writer.interface; + // var outbuff: [4096]u8 = undefined; + // var writer = outfile.writer(io, &outbuff); + // const writer_interface = &writer.interface; - try writer_interface.writeAll(output_slice.data()); - - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, "/Users/raph/Documents/Git-Repos/zml/examples/voxtral/outputs/voxtral_activations.safetensors"); - defer registry.deinit(); - - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - - try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, store.view(), "mel", .{ .inner = mel_spectrum_buffers }, 1e-3); + // try writer_interface.writeAll(output_slice.data()); } -const MelTestHarness = struct { - inner: LogMelSpectrogram, - - pub fn forward(self: MelTestHarness, waveform: Tensor) Tensor { - return self.inner.forward(waveform.withTags(.{.samples})); - } -}; - fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { var arena_state: std.heap.ArenaAllocator = .init(allocator); defer arena_state.deinit(); @@ -144,6 +132,5 @@ fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { } pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram) !zml.Exe { - // return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{model.force_num_frames * model.hop_len}, .f32).withTags(.{.freq})}); return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{293699}, .f32).withTags(.{.samples})}); } From 567ec83e9964dfc72137f580487d494fec6edf1d Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 16:43:48 +0100 Subject: [PATCH 37/98] encoder conv stem --- examples/voxtral/BUILD.bazel | 2 + examples/voxtral/encoder.zig | 83 ++++++++++++++++++++++++++++++++++++ examples/voxtral/test.zig | 44 ++++++++++++++++--- examples/voxtral/voxtral.zig | 2 +- 4 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 examples/voxtral/encoder.zig diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel index 601aff1b..03505def 100644 --- a/examples/voxtral/BUILD.bazel +++ b/examples/voxtral/BUILD.bazel @@ -8,6 +8,7 @@ zig_binary( "mel_spectrogram.zig", "wav.zig", "config.zig", + "encoder.zig", ], extra_srcs = [ "//examples/voxtral/assets:voxtral_mel_filter", @@ -22,6 +23,7 @@ zig_binary( "mel_spectrogram.zig", "wav.zig", "config.zig", + "encoder.zig", ], extra_srcs = [ "//examples/voxtral/assets:voxtral_mel_filter", diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig new file mode 100644 index 00000000..5ca25875 --- /dev/null +++ b/examples/voxtral/encoder.zig @@ -0,0 +1,83 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const log = std.log; + +const zml = @import("zml"); +const Tensor = zml.Tensor; + + +pub const Encoder = struct { + conv0: CausalConv1d, + conv1: CausalConv1d, + + pub fn init(store: zml.io.TensorStore.View) Encoder { + const conv_store = store.withPrefix("conv_layers"); + return .{ + .conv0 = CausalConv1d.init(conv_store.withLayer(0).withPrefix("conv"), 1), + .conv1 = CausalConv1d.init(conv_store.withLayer(1).withPrefix("conv"), 2), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Encoder)) void { + CausalConv1d.unloadBuffers(&self.conv0); + CausalConv1d.unloadBuffers(&self.conv1); + } + + /// mel: [channels=128, time=frames] + /// Returns: [seq, d=1280] + pub fn forward(self: Encoder, mel: Tensor) Tensor { + // [channels, time] -> [batch=1, channels, time] + var h = mel.insertAxes(.channels, .{.batch}); + + h = self.conv0.forward(h).gelu(); + h = self.conv1.forward(h).gelu(); + + // [batch=1, channels=1280, time'] -> [time', channels=1280] + h = h.squeeze(.batch); + return h.transpose(.{ .time, .channels }); + } +}; + +/// Expects input layout [batch, channels, time] and kernel [cout, cin, k]. +pub const CausalConv1d = struct { + weight: Tensor, + bias: Tensor, + stride: i64, + + pub fn init(store: zml.io.TensorStore.View, stride: i64) CausalConv1d { + return .{ + .weight = store.createTensorWithTags("weight", .{ .cout, .cin, .k }), + .bias = store.createTensorWithTags("bias", .{.channels}), + .stride = stride, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(CausalConv1d)) void { + self.weight.deinit(); + self.bias.deinit(); + } + + pub fn forward(self: CausalConv1d, input: Tensor) Tensor { + const kernel_size: i64 = @intCast(self.weight.dim(.k)); + const stride = self.stride; + const input_len: i64 = @intCast(input.dim(.time)); + + // Causal padding: padding_total goes on the left. + // Right padding is computed to align the output length with stride. + const padding_left = kernel_size - stride; + const numerator = input_len - kernel_size + padding_left; + const n_frames = @divTrunc(numerator + stride - 1, stride) + 1; + const target_length = (n_frames - 1) * stride + kernel_size - padding_left; + const padding_right = target_length - input_len; + + const dtype = input.dtype(); + var h = input.conv1d(self.weight.convert(dtype), .{ + .window_strides = stride, + .padding = &.{ padding_left, padding_right }, + }); + + // Bias: [channels] broadcasts to [batch, channels, time] + h = h.add(self.bias.convert(dtype).broad(h.shape())); + return h; + } +}; diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 65bad9f5..03fc2228 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -11,8 +11,12 @@ const Tensor = zml.Tensor; const mel = @import("mel_spectrogram.zig"); const LogMelSpectrogram = mel.LogMelSpectrogram; +const enc = @import("encoder.zig"); +const Encoder = enc.Encoder; + const CliArgs = struct { reference: []const u8, + model: []const u8, }; pub fn main() !void { @@ -40,11 +44,19 @@ pub fn main() !void { const io = vfs.io(); - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.reference); - defer registry.deinit(); + var ref_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.reference); + defer ref_registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); + var ref_store: zml.io.TensorStore = .fromRegistry(allocator, &ref_registry); + defer ref_store.deinit(); + + // const repo = try zml.safetensors.resolveModelRepo(io, args.model); + // var model_registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); + defer model_registry.deinit(); + + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); + defer model_store.deinit(); const melspectrum_config = MelSpectrumConfig{}; @@ -52,12 +64,26 @@ pub fn main() !void { defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); + // Mel spectrogram test var melspectro_model: LogMelSpectrogram = .init(melspectrum_config); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); defer LogMelSpectrogram.unload(&mel_spectrum_buffers); - try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, store.view(), "mel", .{ .inner = mel_spectrum_buffers }, 1e-3); + try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, ref_store.view(), "mel", .{ .inner = mel_spectrum_buffers }, 1e-3); + + // Encoder conv stem test + const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; + const encoder_model = Encoder.init(model_store.view().withPrefix(encoder_prefix)); + var encoder_buffers = try zml.io.load(Encoder, &encoder_model, allocator, io, platform, .{ + .store = &model_store, + .parallelism = 16, + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + }); + defer Encoder.unloadBuffers(&encoder_buffers); + + try zml.testing.testLayer(allocator, io, platform, EncoderTestHarness{ .inner = encoder_model }, .forward, ref_store.view(), "conv_stem", .{ .inner = encoder_buffers }, 1e-3); } const MelTestHarness = struct { @@ -67,3 +93,11 @@ const MelTestHarness = struct { return self.inner.forward(waveform.withTags(.{.samples})); } }; + +const EncoderTestHarness = struct { + inner: Encoder, + + pub fn forward(self: EncoderTestHarness, mel_input: Tensor) Tensor { + return self.inner.forward(mel_input.withTags(.{ .channels, .time })); + } +}; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 4236e98e..434c5dc7 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -14,7 +14,7 @@ const LogMelSpectrogram = mel.LogMelSpectrogram; const CliArgs = struct { input: []const u8, - // output: []const u8, + model: []const u8, }; pub fn main() !void { From 450abf35c17cdb9177f7bf6336f489c7ee00227c Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 16:56:46 +0100 Subject: [PATCH 38/98] Encoder seems to be working --- examples/voxtral/config.zig | 13 ++ examples/voxtral/encoder.zig | 235 ++++++++++++++++++++++++++++++++++- examples/voxtral/test.zig | 93 +++++++++++--- 3 files changed, 316 insertions(+), 25 deletions(-) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index b4af02aa..f7d2d25e 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -3,3 +3,16 @@ pub const MelSpectrumConfig = struct { window_size: u63 = 400, global_log_mel_max: f32 = 1.5, }; + +pub const EncoderConfig = struct { + dim: usize = 1280, + n_layers: usize = 32, + n_heads: usize = 32, + head_dim: usize = 64, + hidden_dim: usize = 5120, + n_kv_heads: usize = 32, + window_size: u32 = 750, + norm_eps: f32 = 1e-5, + rope_theta: f32 = 1_000_000.0, + downsample_factor: usize = 4, +}; diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 5ca25875..bac3b0c4 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -4,37 +4,77 @@ const log = std.log; const zml = @import("zml"); const Tensor = zml.Tensor; +const Shape = zml.Shape; +const cfg = @import("config.zig"); +pub const EncoderConfig = cfg.EncoderConfig; pub const Encoder = struct { conv0: CausalConv1d, conv1: CausalConv1d, + layers: []TransformerLayer, + norm: RmsNorm, + config: EncoderConfig, - pub fn init(store: zml.io.TensorStore.View) Encoder { + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: EncoderConfig) Encoder { const conv_store = store.withPrefix("conv_layers"); + const transformer_store = store.withPrefix("transformer"); + + const layers = allocator.alloc(TransformerLayer, config.n_layers) catch unreachable; + for (layers, 0..) |*layer, i| { + layer.* = TransformerLayer.init(transformer_store.withPrefix("layers").withLayer(i), config); + } + return .{ .conv0 = CausalConv1d.init(conv_store.withLayer(0).withPrefix("conv"), 1), .conv1 = CausalConv1d.init(conv_store.withLayer(1).withPrefix("conv"), 2), + .layers = layers, + .norm = RmsNorm.init(transformer_store.withPrefix("norm"), config.norm_eps), + .config = config, }; } - pub fn unloadBuffers(self: *zml.Bufferized(Encoder)) void { + pub fn deinit(self: Encoder, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Encoder), allocator: std.mem.Allocator) void { CausalConv1d.unloadBuffers(&self.conv0); CausalConv1d.unloadBuffers(&self.conv1); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); } /// mel: [channels=128, time=frames] - /// Returns: [seq, d=1280] + /// Returns: [s, d=1280] pub fn forward(self: Encoder, mel: Tensor) Tensor { - // [channels, time] -> [batch=1, channels, time] + // Conv stem: [channels, time] -> [batch=1, channels, time] var h = mel.insertAxes(.channels, .{.batch}); h = self.conv0.forward(h).gelu(); h = self.conv1.forward(h).gelu(); - // [batch=1, channels=1280, time'] -> [time', channels=1280] + // [batch=1, channels=1280, time'] -> [s, d=1280] h = h.squeeze(.batch); - return h.transpose(.{ .time, .channels }); + h = h.transpose(.{ .time, .channels }).rename(.{ .time = .s, .channels = .d }); + + // Left-truncate to multiple of downsample_factor + const seq_len = h.dim(.s); + const trunc: i64 = @intCast(@as(usize, @intCast(seq_len)) % self.config.downsample_factor); + if (trunc > 0) { + h = h.slice1d(.s, .{ .start = trunc }); + } + + // Transformer layers + for (self.layers) |layer| { + h = layer.forward(h, self.config); + } + + // Final norm + return self.norm.forward(h); } }; @@ -81,3 +121,186 @@ pub const CausalConv1d = struct { return h; } }; + +pub const TransformerLayer = struct { + attention_norm: RmsNorm, + attention: SelfAttention, + ffn_norm: RmsNorm, + feed_forward: SwiGluFfn, + + pub fn init(store: zml.io.TensorStore.View, config: EncoderConfig) TransformerLayer { + return .{ + .attention_norm = RmsNorm.init(store.withPrefix("attention_norm"), config.norm_eps), + .attention = SelfAttention.init(store.withPrefix("attention"), config), + .ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), config.norm_eps), + .feed_forward = SwiGluFfn.init(store.withPrefix("feed_forward")), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.attention_norm); + SelfAttention.unloadBuffers(&self.attention); + RmsNorm.unloadBuffers(&self.ffn_norm); + SwiGluFfn.unloadBuffers(&self.feed_forward); + } + + /// h: [s, d] -> [s, d] + pub fn forward(self: TransformerLayer, h: Tensor, config: EncoderConfig) Tensor { + // Pre-attention norm + attention + residual + const attn_out = self.attention.forward(self.attention_norm.forward(h), config); + var out = h.add(attn_out); + + // Pre-FFN norm + FFN + residual + const ffn_out = self.feed_forward.forward(self.ffn_norm.forward(out)); + out = out.add(ffn_out); + + return out; + } +}; + +pub const SelfAttention = struct { + wq: Linear, + wk: Linear, + wv: Linear, + wo: Linear, + + pub fn init(store: zml.io.TensorStore.View, config: EncoderConfig) SelfAttention { + _ = config; + return .{ + .wq = Linear.initWithBias(store.withPrefix("wq")), + .wk = Linear.initNoBias(store.withPrefix("wk")), + .wv = Linear.initWithBias(store.withPrefix("wv")), + .wo = Linear.initWithBias(store.withPrefix("wo")), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttention)) void { + Linear.unloadBuffers(&self.wq); + Linear.unloadBuffers(&self.wk); + Linear.unloadBuffers(&self.wv); + Linear.unloadBuffers(&self.wo); + } + + /// x: [s, d] -> [s, d] + pub fn forward(self: SelfAttention, x: Tensor, config: EncoderConfig) Tensor { + const dtype = x.dtype(); + + // Q, K, V projections + // Encoder: Q and V have biases, K does not + var q = self.wq.forward(x); // [s, n_heads * head_dim] + var k = self.wk.forward(x); // [s, n_kv_heads * head_dim] + var v = self.wv.forward(x); // [s, n_kv_heads * head_dim] + + // Split into heads: [s, d] -> [s, h, hd] + q = q.splitAxis(.d, .{ .h = config.n_heads, .hd = config.head_dim }); + k = k.splitAxis(.d, .{ .h = config.n_kv_heads, .hd = config.head_dim }); + v = v.splitAxis(.d, .{ .h = config.n_kv_heads, .hd = config.head_dim }); + + // RoPE (interleaved, matching is_neox_style=False in Python) + const rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = config.rope_theta, + }; + q = zml.nn.rope(q, null, rope_opts); + k = zml.nn.rope(k, null, rope_opts); + + // Rename seq dim for sdpa: .s -> .q/.k + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + // Causal attention with sliding window + const seq_len = q.dim(.q); + const attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, config.window_size); + const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); + + // Merge heads: [s, h, hd] -> [s, d] + const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + + // Output projection + return self.wo.forward(merged); + } +}; + +pub const SwiGluFfn = struct { + w1: Linear, + w2: Linear, + w3: Linear, + + pub fn init(store: zml.io.TensorStore.View) SwiGluFfn { + return .{ + .w1 = Linear.initNoBias(store.withPrefix("w1")), + .w2 = Linear.initWithBias(store.withPrefix("w2")), + .w3 = Linear.initNoBias(store.withPrefix("w3")), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SwiGluFfn)) void { + Linear.unloadBuffers(&self.w1); + Linear.unloadBuffers(&self.w2); + Linear.unloadBuffers(&self.w3); + } + + /// x: [s, d] -> [s, d] + pub fn forward(self: SwiGluFfn, x: Tensor) Tensor { + // SwiGLU: silu(gate) * up, then down + const gate = self.w1.forward(x).silu(); + const up = self.w3.forward(x); + return self.w2.forward(gate.mul(up)); + } +}; + +pub const Linear = struct { + weight: Tensor, + bias: ?Tensor, + + pub fn initWithBias(store: zml.io.TensorStore.View) Linear { + return .{ + .weight = store.createTensorWithTags("weight", .{ .out, .d }), + .bias = store.createTensorWithTags("bias", .{.d}), + }; + } + + pub fn initNoBias(store: zml.io.TensorStore.View) Linear { + return .{ + .weight = store.createTensorWithTags("weight", .{ .out, .d }), + .bias = null, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { + self.weight.deinit(); + if (self.bias) |*b| b.deinit(); + } + + /// x: [..., d] -> [..., out] (renamed back to d) + pub fn forward(self: Linear, x: Tensor) Tensor { + const dtype = x.dtype(); + var y = x.dot(self.weight.convert(dtype), .d).rename(.{ .out = .d }); + if (self.bias) |bias| { + y = y.add(bias.convert(dtype).broad(y.shape())); + } + return y; + } +}; + +pub const RmsNorm = struct { + weight: Tensor, + eps: f32, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ + .weight = store.createTensorWithTags("weight", .{.d}), + .eps = eps, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNorm, x: Tensor) Tensor { + const normalized = zml.nn.rmsNorm(x, .d, self.eps); + return normalized.mul(self.weight.convert(x.dtype()).broad(x.shape())); + } +}; diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 03fc2228..3ebf54c2 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); const log = std.log; const cfg = @import("config.zig"); const MelSpectrumConfig = cfg.MelSpectrumConfig; +const EncoderConfig = cfg.EncoderConfig; const zml = @import("zml"); const stdx = zml.stdx; @@ -50,8 +51,6 @@ pub fn main() !void { var ref_store: zml.io.TensorStore = .fromRegistry(allocator, &ref_registry); defer ref_store.deinit(); - // const repo = try zml.safetensors.resolveModelRepo(io, args.model); - // var model_registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); defer model_registry.deinit(); @@ -59,33 +58,58 @@ pub fn main() !void { defer model_store.deinit(); const melspectrum_config = MelSpectrumConfig{}; + const encoder_config = EncoderConfig{}; + const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; + const load_opts: zml.io.LoadOpts = .{ + .store = &model_store, + .parallelism = 16, + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + }; var platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); - // Mel spectrogram test - var melspectro_model: LogMelSpectrogram = .init(melspectrum_config); - var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); - var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); - defer LogMelSpectrogram.unload(&mel_spectrum_buffers); + // --- Mel spectrogram test --- + { + var melspectro_model: LogMelSpectrogram = .init(melspectrum_config); + var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); + var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); + defer LogMelSpectrogram.unload(&mel_spectrum_buffers); - try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, ref_store.view(), "mel", .{ .inner = mel_spectrum_buffers }, 1e-3); + try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, ref_store.view(), "mel", .{ .inner = mel_spectrum_buffers }, 1e-3); + } - // Encoder conv stem test - const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; - const encoder_model = Encoder.init(model_store.view().withPrefix(encoder_prefix)); - var encoder_buffers = try zml.io.load(Encoder, &encoder_model, allocator, io, platform, .{ - .store = &model_store, - .parallelism = 16, - .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, - }); - defer Encoder.unloadBuffers(&encoder_buffers); + // --- Conv stem test --- + { + const conv_store = model_store.view().withPrefix(encoder_prefix).withPrefix("conv_layers"); + const conv_stem = ConvStemTestHarness{ + .conv0 = enc.CausalConv1d.init(conv_store.withLayer(0).withPrefix("conv"), 1), + .conv1 = enc.CausalConv1d.init(conv_store.withLayer(1).withPrefix("conv"), 2), + }; + var conv_buffers = try zml.io.load(ConvStemTestHarness, &conv_stem, allocator, io, platform, load_opts); + defer ConvStemTestHarness.unloadBuffers(&conv_buffers); - try zml.testing.testLayer(allocator, io, platform, EncoderTestHarness{ .inner = encoder_model }, .forward, ref_store.view(), "conv_stem", .{ .inner = encoder_buffers }, 1e-3); + try zml.testing.testLayer(allocator, io, platform, conv_stem, .forward, ref_store.view(), "conv_stem", conv_buffers, 1e-3); + } + + // --- Encoder test --- + { + const encoder_model = Encoder.init(allocator, model_store.view().withPrefix(encoder_prefix), encoder_config); + defer encoder_model.deinit(allocator); + var encoder_buffers = try zml.io.load(Encoder, &encoder_model, allocator, io, platform, load_opts); + defer Encoder.unloadBuffers(&encoder_buffers, allocator); + + const harness = EncoderTestHarness{ .inner = encoder_model }; + try zml.testing.testLayer(allocator, io, platform, harness, .forward, ref_store.view(), "encoder", .{ .inner = encoder_buffers }, 1e-3); + } } +// ============================================================================ +// Test harnesses +// ============================================================================ + const MelTestHarness = struct { inner: LogMelSpectrogram, @@ -94,6 +118,37 @@ const MelTestHarness = struct { } }; +const ConvStemTestHarness = struct { + conv0: enc.CausalConv1d, + conv1: enc.CausalConv1d, + + pub fn unloadBuffers(self: *zml.Bufferized(ConvStemTestHarness)) void { + enc.CausalConv1d.unloadBuffers(&self.conv0); + enc.CausalConv1d.unloadBuffers(&self.conv1); + } + + pub fn forward(self: ConvStemTestHarness, mel_input: Tensor) Tensor { + var h = mel_input.withTags(.{ .channels, .time }).insertAxes(.channels, .{.batch}); + h = self.conv0.forward(h).gelu(); + h = self.conv1.forward(h).gelu(); + h = h.squeeze(.batch); + return h.transpose(.{ .time, .channels }); + } +}; + +const TransformerLayerTestHarness = struct { + inner: enc.TransformerLayer, + config: EncoderConfig, + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayerTestHarness)) void { + enc.TransformerLayer.unloadBuffers(&self.inner); + } + + pub fn forward(self: TransformerLayerTestHarness, h: Tensor) Tensor { + return self.inner.forward(h.withTags(.{ .s, .d }), self.config); + } +}; + const EncoderTestHarness = struct { inner: Encoder, From 7c632808334255f9199a69a7fc95cff7d1dcca78 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 17:28:22 +0100 Subject: [PATCH 39/98] parse whole config --- examples/voxtral/config.zig | 94 +++++++++++++++++++++++----- examples/voxtral/mel_spectrogram.zig | 11 ++-- examples/voxtral/voxtral.zig | 11 ++-- 3 files changed, 91 insertions(+), 25 deletions(-) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index f7d2d25e..b852342a 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -1,18 +1,80 @@ -pub const MelSpectrumConfig = struct { - hop_length: u63 = 160, - window_size: u63 = 400, - global_log_mel_max: f32 = 1.5, +const std = @import("std"); + +pub const Config = struct { + dim: u32, + n_layers: u32, + head_dim: u32, + hidden_dim: u32, + n_heads: u32, + n_kv_heads: u32, + use_biases: bool = false, + causal: bool = true, + rope_theta: f32, + norm_eps: f32, + vocab_size: u32, + sliding_window: u32, + tied_embeddings: bool = true, + ada_rms_norm_t_cond: bool = true, + ada_rms_norm_t_cond_dim: u32 = 32, + multimodal: Multimodal, + + pub const Multimodal = struct { + whisper_model_args: WhisperModelArgs, + }; + + pub const WhisperModelArgs = struct { + encoder_args: EncoderArgs, + downsample_args: DownsampleArgs, + }; + + pub const DownsampleArgs = struct { + downsample_factor: u32, + }; + + pub const EncoderArgs = struct { + audio_encoding_args: AudioEncodingArgs, + dim: u32, + n_layers: u32, + head_dim: u32, + hidden_dim: u32, + n_heads: u32, + n_kv_heads: u32, + use_biases: bool = true, + rope_theta: f32, + norm_eps: f32, + sliding_window: u32, + }; + + pub const AudioEncodingArgs = struct { + sampling_rate: u32 = 16000, + frame_rate: f32 = 12.5, + num_mel_bins: u32 = 128, + hop_length: u32 = 160, + window_size: u32 = 400, + global_log_mel_max: f32 = 1.5, + }; + + // -- Shortcuts + + pub fn encoder(self: Config) EncoderArgs { + return self.multimodal.whisper_model_args.encoder_args; + } + + pub fn audio(self: Config) AudioEncodingArgs { + return self.multimodal.whisper_model_args.encoder_args.audio_encoding_args; + } + + pub fn downsample_factor(self: Config) u32 { + return self.multimodal.whisper_model_args.downsample_args.downsample_factor; + } }; -pub const EncoderConfig = struct { - dim: usize = 1280, - n_layers: usize = 32, - n_heads: usize = 32, - head_dim: usize = 64, - hidden_dim: usize = 5120, - n_kv_heads: usize = 32, - window_size: u32 = 750, - norm_eps: f32 = 1e-5, - rope_theta: f32 = 1_000_000.0, - downsample_factor: usize = 4, -}; +pub fn parseConfig(allocator: std.mem.Allocator, io: std.Io, model_dir: std.Io.Dir) !std.json.Parsed(Config) { + const config_file = try model_dir.openFile(io, "params.json", .{}); // it is named params not config here + defer config_file.close(io); + var buffer: [4096]u8 = undefined; + var file_reader = config_file.reader(io, &buffer); + var reader: std.json.Reader = .init(allocator, &file_reader.interface); + defer reader.deinit(); + return try std.json.parseFromTokenSource(Config, allocator, &reader, .{ .ignore_unknown_fields = true }); +} diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig index ac193ac8..844956df 100644 --- a/examples/voxtral/mel_spectrogram.zig +++ b/examples/voxtral/mel_spectrogram.zig @@ -1,7 +1,7 @@ const std = @import("std"); const log = std.log; const cfg = @import("config.zig"); -const MelSpectrumConfig = cfg.MelSpectrumConfig; +const Config = cfg.Config; const zml = @import("zml"); const Tensor = zml.Tensor; @@ -20,13 +20,14 @@ pub const LogMelSpectrogram = struct { precision: zml.DataType = .f32, - pub fn init(config: MelSpectrumConfig) LogMelSpectrogram { + pub fn init(config: Config) LogMelSpectrogram { + const audio = config.audio(); return .{ .window = .hann, .mel_filters = Tensor.init(.{201, 128}, .f32).withTags(.{.freq_bins, .mel}), - .hop_len = config.hop_length, - .n_fft = config.window_size, - .global_log_mel_max = 1.5, + .hop_len = audio.hop_length, + .n_fft = audio.window_size, + .global_log_mel_max = audio.global_log_mel_max, }; } diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 434c5dc7..c018199a 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -3,7 +3,7 @@ const builtin = @import("builtin"); const log = std.log; const wav_utils = @import("wav.zig"); const cfg = @import("config.zig"); -const MelSpectrumConfig = cfg.MelSpectrumConfig; +const Config = cfg.Config; const zml = @import("zml"); const stdx = zml.stdx; @@ -54,13 +54,16 @@ pub fn main() !void { const wav_file = try loadWav(allocator, &reader.interface); defer allocator.free(wav_file); - const melspectrum_config = MelSpectrumConfig{}; + const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); + var parsed_config = try cfg.parseConfig(allocator, io, model_dir); + defer parsed_config.deinit(); + const config = parsed_config.value; var platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); - var melspectro_model: LogMelSpectrogram = .init(melspectrum_config); + var melspectro_model: LogMelSpectrogram = .init(config); var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model}); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); @@ -79,7 +82,7 @@ pub fn main() !void { var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); - const num_frames = wav_file.len / melspectrum_config.hop_length; + const num_frames = wav_file.len / config.audio().hop_length; const output_shape = zml.Shape.init(.{ 128, num_frames }, .f32); const output_slice: zml.Slice = try zml.Slice.alloc(allocator, output_shape); defer output_slice.free(allocator); From 6308e40431f023f02a7f55f3399840a9664fac9b Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 11 Feb 2026 17:28:33 +0100 Subject: [PATCH 40/98] add transformer layers to encoder --- examples/voxtral/encoder.zig | 71 ++++++++++++++++++------------------ examples/voxtral/test.zig | 27 ++++++++------ 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index bac3b0c4..20e2378f 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -7,20 +7,21 @@ const Tensor = zml.Tensor; const Shape = zml.Shape; const cfg = @import("config.zig"); -pub const EncoderConfig = cfg.EncoderConfig; +const Config = cfg.Config; pub const Encoder = struct { conv0: CausalConv1d, conv1: CausalConv1d, layers: []TransformerLayer, norm: RmsNorm, - config: EncoderConfig, + config: Config, - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: EncoderConfig) Encoder { + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Encoder { const conv_store = store.withPrefix("conv_layers"); const transformer_store = store.withPrefix("transformer"); - const layers = allocator.alloc(TransformerLayer, config.n_layers) catch unreachable; + const enc = config.encoder(); + const layers = allocator.alloc(TransformerLayer, enc.n_layers) catch unreachable; for (layers, 0..) |*layer, i| { layer.* = TransformerLayer.init(transformer_store.withPrefix("layers").withLayer(i), config); } @@ -29,7 +30,7 @@ pub const Encoder = struct { .conv0 = CausalConv1d.init(conv_store.withLayer(0).withPrefix("conv"), 1), .conv1 = CausalConv1d.init(conv_store.withLayer(1).withPrefix("conv"), 2), .layers = layers, - .norm = RmsNorm.init(transformer_store.withPrefix("norm"), config.norm_eps), + .norm = RmsNorm.init(transformer_store.withPrefix("norm"), enc.norm_eps), .config = config, }; } @@ -41,9 +42,11 @@ pub const Encoder = struct { pub fn unloadBuffers(self: *zml.Bufferized(Encoder), allocator: std.mem.Allocator) void { CausalConv1d.unloadBuffers(&self.conv0); CausalConv1d.unloadBuffers(&self.conv1); + for (self.layers) |*layer| { TransformerLayer.unloadBuffers(layer); } + allocator.free(self.layers); RmsNorm.unloadBuffers(&self.norm); } @@ -63,7 +66,8 @@ pub const Encoder = struct { // Left-truncate to multiple of downsample_factor const seq_len = h.dim(.s); - const trunc: i64 = @intCast(@as(usize, @intCast(seq_len)) % self.config.downsample_factor); + const trunc: i64 = @intCast(@as(usize, @intCast(seq_len)) % self.config.downsample_factor()); + if (trunc > 0) { h = h.slice1d(.s, .{ .start = trunc }); } @@ -128,11 +132,12 @@ pub const TransformerLayer = struct { ffn_norm: RmsNorm, feed_forward: SwiGluFfn, - pub fn init(store: zml.io.TensorStore.View, config: EncoderConfig) TransformerLayer { + pub fn init(store: zml.io.TensorStore.View, config: Config) TransformerLayer { + const enc = config.encoder(); return .{ - .attention_norm = RmsNorm.init(store.withPrefix("attention_norm"), config.norm_eps), - .attention = SelfAttention.init(store.withPrefix("attention"), config), - .ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), config.norm_eps), + .attention_norm = RmsNorm.init(store.withPrefix("attention_norm"), enc.norm_eps), + .attention = SelfAttention.init(store.withPrefix("attention")), + .ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), enc.norm_eps), .feed_forward = SwiGluFfn.init(store.withPrefix("feed_forward")), }; } @@ -145,7 +150,7 @@ pub const TransformerLayer = struct { } /// h: [s, d] -> [s, d] - pub fn forward(self: TransformerLayer, h: Tensor, config: EncoderConfig) Tensor { + pub fn forward(self: TransformerLayer, h: Tensor, config: Config) Tensor { // Pre-attention norm + attention + residual const attn_out = self.attention.forward(self.attention_norm.forward(h), config); var out = h.add(attn_out); @@ -164,13 +169,12 @@ pub const SelfAttention = struct { wv: Linear, wo: Linear, - pub fn init(store: zml.io.TensorStore.View, config: EncoderConfig) SelfAttention { - _ = config; + pub fn init(store: zml.io.TensorStore.View) SelfAttention { return .{ - .wq = Linear.initWithBias(store.withPrefix("wq")), - .wk = Linear.initNoBias(store.withPrefix("wk")), - .wv = Linear.initWithBias(store.withPrefix("wv")), - .wo = Linear.initWithBias(store.withPrefix("wo")), + .wq = Linear.init(store.withPrefix("wq")), + .wk = Linear.init(store.withPrefix("wk")), + .wv = Linear.init(store.withPrefix("wv")), + .wo = Linear.init(store.withPrefix("wo")), }; } @@ -182,7 +186,8 @@ pub const SelfAttention = struct { } /// x: [s, d] -> [s, d] - pub fn forward(self: SelfAttention, x: Tensor, config: EncoderConfig) Tensor { + pub fn forward(self: SelfAttention, x: Tensor, config: Config) Tensor { + const enc = config.encoder(); const dtype = x.dtype(); // Q, K, V projections @@ -192,15 +197,16 @@ pub const SelfAttention = struct { var v = self.wv.forward(x); // [s, n_kv_heads * head_dim] // Split into heads: [s, d] -> [s, h, hd] - q = q.splitAxis(.d, .{ .h = config.n_heads, .hd = config.head_dim }); - k = k.splitAxis(.d, .{ .h = config.n_kv_heads, .hd = config.head_dim }); - v = v.splitAxis(.d, .{ .h = config.n_kv_heads, .hd = config.head_dim }); + q = q.splitAxis(.d, .{ .h = enc.n_heads, .hd = enc.head_dim }); + k = k.splitAxis(.d, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); + v = v.splitAxis(.d, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); // RoPE (interleaved, matching is_neox_style=False in Python) const rope_opts: zml.nn.RopeOpts = .{ .layout = .interleaved, - .freq_base = config.rope_theta, + .freq_base = enc.rope_theta, }; + q = zml.nn.rope(q, null, rope_opts); k = zml.nn.rope(k, null, rope_opts); @@ -211,7 +217,7 @@ pub const SelfAttention = struct { // Causal attention with sliding window const seq_len = q.dim(.q); - const attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, config.window_size); + const attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, enc.sliding_window); const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); // Merge heads: [s, h, hd] -> [s, d] @@ -229,9 +235,9 @@ pub const SwiGluFfn = struct { pub fn init(store: zml.io.TensorStore.View) SwiGluFfn { return .{ - .w1 = Linear.initNoBias(store.withPrefix("w1")), - .w2 = Linear.initWithBias(store.withPrefix("w2")), - .w3 = Linear.initNoBias(store.withPrefix("w3")), + .w1 = Linear.init(store.withPrefix("w1")), + .w2 = Linear.init(store.withPrefix("w2")), + .w3 = Linear.init(store.withPrefix("w3")), }; } @@ -254,17 +260,10 @@ pub const Linear = struct { weight: Tensor, bias: ?Tensor, - pub fn initWithBias(store: zml.io.TensorStore.View) Linear { + pub fn init(store: zml.io.TensorStore.View) Linear { return .{ .weight = store.createTensorWithTags("weight", .{ .out, .d }), - .bias = store.createTensorWithTags("bias", .{.d}), - }; - } - - pub fn initNoBias(store: zml.io.TensorStore.View) Linear { - return .{ - .weight = store.createTensorWithTags("weight", .{ .out, .d }), - .bias = null, + .bias = store.maybeCreateTensorWithTags("bias", .{.d}), }; } @@ -277,9 +276,11 @@ pub const Linear = struct { pub fn forward(self: Linear, x: Tensor) Tensor { const dtype = x.dtype(); var y = x.dot(self.weight.convert(dtype), .d).rename(.{ .out = .d }); + if (self.bias) |bias| { y = y.add(bias.convert(dtype).broad(y.shape())); } + return y; } }; diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 3ebf54c2..3289f3ea 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -2,8 +2,7 @@ const std = @import("std"); const builtin = @import("builtin"); const log = std.log; const cfg = @import("config.zig"); -const MelSpectrumConfig = cfg.MelSpectrumConfig; -const EncoderConfig = cfg.EncoderConfig; +const Config = cfg.Config; const zml = @import("zml"); const stdx = zml.stdx; @@ -57,8 +56,12 @@ pub fn main() !void { var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); defer model_store.deinit(); - const melspectrum_config = MelSpectrumConfig{}; - const encoder_config = EncoderConfig{}; + // Parse config from model directory + const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); + var parsed_config = try cfg.parseConfig(allocator, io, model_dir); + defer parsed_config.deinit(); + const config = parsed_config.value; + const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; const load_opts: zml.io.LoadOpts = .{ .store = &model_store, @@ -71,9 +74,9 @@ pub fn main() !void { defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); - // --- Mel spectrogram test --- + // -- Mel spectrogram test { - var melspectro_model: LogMelSpectrogram = .init(melspectrum_config); + var melspectro_model: LogMelSpectrogram = .init(config); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); defer LogMelSpectrogram.unload(&mel_spectrum_buffers); @@ -81,7 +84,7 @@ pub fn main() !void { try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, ref_store.view(), "mel", .{ .inner = mel_spectrum_buffers }, 1e-3); } - // --- Conv stem test --- + // -- Conv stem test { const conv_store = model_store.view().withPrefix(encoder_prefix).withPrefix("conv_layers"); const conv_stem = ConvStemTestHarness{ @@ -94,9 +97,9 @@ pub fn main() !void { try zml.testing.testLayer(allocator, io, platform, conv_stem, .forward, ref_store.view(), "conv_stem", conv_buffers, 1e-3); } - // --- Encoder test --- + // -- Encoder test { - const encoder_model = Encoder.init(allocator, model_store.view().withPrefix(encoder_prefix), encoder_config); + const encoder_model = Encoder.init(allocator, model_store.view().withPrefix(encoder_prefix), config); defer encoder_model.deinit(allocator); var encoder_buffers = try zml.io.load(Encoder, &encoder_model, allocator, io, platform, load_opts); defer Encoder.unloadBuffers(&encoder_buffers, allocator); @@ -106,9 +109,9 @@ pub fn main() !void { } } -// ============================================================================ +// ================================================================ // Test harnesses -// ============================================================================ +// ================================================================ const MelTestHarness = struct { inner: LogMelSpectrogram, @@ -138,7 +141,7 @@ const ConvStemTestHarness = struct { const TransformerLayerTestHarness = struct { inner: enc.TransformerLayer, - config: EncoderConfig, + config: Config, pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayerTestHarness)) void { enc.TransformerLayer.unloadBuffers(&self.inner); From 11d1f62d90c5834ec3fe0f95d321df669cd76f8b Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 09:57:51 +0100 Subject: [PATCH 41/98] Base encoder --- examples/voxtral/decoder.zig | 354 +++++++++++++++++++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 examples/voxtral/decoder.zig diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig new file mode 100644 index 00000000..895f0d3e --- /dev/null +++ b/examples/voxtral/decoder.zig @@ -0,0 +1,354 @@ +const std = @import("std"); +const log = std.log; + +const zml = @import("zml"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; + +const cfg = @import("config.zig"); +const Config = cfg.Config; + +const enc = @import("encoder.zig"); +const Linear = enc.Linear; +const RmsNorm = enc.RmsNorm; +const SwiGluFfn = enc.SwiGluFfn; + +/// Adapter: encoder→decoder projection. +/// Reshapes encoder output [s, d=1280] → [s/4, d=5120] (concatenate 4 timesteps), +/// then projects through a 2-layer MLP: Linear(5120→3072) → GELU → Linear(3072→3072). +pub const Adapter = struct { + proj0: Linear, + proj1: Linear, + + pub fn init(store: zml.io.TensorStore.View) Adapter { + const proj_store = store.withPrefix("mm_streams_embeddings.embedding_module.audio_language_projection"); + return .{ + .proj0 = Linear.init(proj_store.withLayer(0)), + .proj1 = Linear.init(proj_store.withLayer(2)), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Adapter)) void { + Linear.unloadBuffers(&self.proj0); + Linear.unloadBuffers(&self.proj1); + } + + /// encoder_out: [s, d=1280] → [s/4, d=3072] + pub fn forward(self: Adapter, encoder_out: Tensor) Tensor { + // Reshape [s, d=1280] → [s/4, d=5120] by concatenating 4 consecutive timesteps. + // Split s into (s_new, group=4), then merge (group, d) → d_new. + const h = encoder_out + .splitAxis(.s, .{ .s = .auto, .group = 4 }) + .merge(.{ .d = .{ .group, .d } }); + + return self.proj1.forward(self.proj0.forward(h).gelu()); + } +}; + +/// AdaRmsNorm: per-layer time conditioning. +/// Computes ada_scale = up(gelu(down(t_cond))), then returns h * (1 + ada_scale). +pub const AdaRmsNorm = struct { + down: Linear, + up: Linear, + + pub fn init(store: zml.io.TensorStore.View) AdaRmsNorm { + const ada_store = store.withPrefix("ada_rms_norm_t_cond"); + return .{ + .down = Linear.init(ada_store.withLayer(0)), + .up = Linear.init(ada_store.withLayer(2)), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(AdaRmsNorm)) void { + Linear.unloadBuffers(&self.down); + Linear.unloadBuffers(&self.up); + } + + /// Apply adaptive conditioning: h * (1 + ada_scale). + /// h: [s, d], t_cond: [d] → [s, d] + pub fn forward(self: AdaRmsNorm, h: Tensor, t_cond: Tensor) Tensor { + const ada_scale = self.up.forward(self.down.forward(t_cond).gelu()); + const one_plus_scale = ada_scale.broad(h.shape()).add(Tensor.scalar(1.0, h.dtype()).broad(h.shape())); + return h.mul(one_plus_scale); + } +}; + +/// KV cache for all decoder layers. +/// Stores K/V tensors with shape {layer, k=max_seq_len, h, hd}. +pub const KvCache = struct { + k: Tensor, + v: Tensor, + layer_index: Tensor, + + pub fn init(kv_shape: Shape) KvCache { + return .{ + .k = .fromShape(kv_shape), + .v = .fromShape(kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initShape(kv_shape: Shape) zml.ShapeOf(KvCache) { + return .{ + .k = kv_shape, + .v = kv_shape, + .layer_index = Shape.init(.{}, .u32), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + self.k.deinit(); + self.v.deinit(); + self.layer_index.deinit(); + } + + pub fn keys(self: KvCache) Tensor { + return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache) Tensor { + return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .k = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .k = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } +}; + +/// Decoder self-attention with KV cache. +pub const SelfAttention = struct { + wq: Linear, + wk: Linear, + wv: Linear, + wo: Linear, + + pub fn init(store: zml.io.TensorStore.View) SelfAttention { + return .{ + .wq = Linear.init(store.withPrefix("wq")), + .wk = Linear.init(store.withPrefix("wk")), + .wv = Linear.init(store.withPrefix("wv")), + .wo = Linear.init(store.withPrefix("wo")), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttention)) void { + Linear.unloadBuffers(&self.wq); + Linear.unloadBuffers(&self.wk); + Linear.unloadBuffers(&self.wv); + Linear.unloadBuffers(&self.wo); + } + + /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) + pub fn forward(self: SelfAttention, x: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config) struct { Tensor, KvCache } { + const dtype = x.dtype(); + + // Q, K, V projections + var q = self.wq.forward(x); + var k = self.wk.forward(x); + var v = self.wv.forward(x); + + // Split into heads: [s, d] → [s, h, hd] + q = q.splitAxis(.d, .{ .h = config.n_heads, .hd = config.head_dim }); + k = k.splitAxis(.d, .{ .h = config.n_kv_heads, .hd = config.head_dim }); + v = v.splitAxis(.d, .{ .h = config.n_kv_heads, .hd = config.head_dim }); + + // Position indices for RoPE + const pos_index = b: { + const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) + .withTags(.{.s}).broad(Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + // RoPE (interleaved) + const rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = config.rope_theta, + }; + q = zml.nn.rope(q, pos_index, rope_opts); + k = zml.nn.rope(k, pos_index, rope_opts); + + // Rename for attention: .s → .q/.k + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + // Update KV cache and retrieve cached K/V + const new_kv_cache = kv_cache.update(k, v, token_index); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + // Causal attention with sliding window + const full_seq_len = k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = full_seq_len, .k = full_seq_len }, dtype, config.sliding_window); + attn_mask = attn_mask.gatherSlices( + Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), + token_index.reshape(.{ .coord = 1 }), + .{}, + ); + const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); + + // Merge heads: [q, h, hd] → [s, d] + const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + + return .{ self.wo.forward(merged), new_kv_cache }; + } +}; + +/// Single decoder transformer layer with AdaRmsNorm time conditioning. +pub const DecoderLayer = struct { + attention_norm: RmsNorm, + attention: SelfAttention, + ffn_norm: RmsNorm, + feed_forward: SwiGluFfn, + ada_norm: AdaRmsNorm, + + pub fn init(store: zml.io.TensorStore.View, config: Config) DecoderLayer { + return .{ + .attention_norm = RmsNorm.init(store.withPrefix("attention_norm"), config.norm_eps), + .attention = SelfAttention.init(store.withPrefix("attention")), + .ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), config.norm_eps), + .feed_forward = SwiGluFfn.init(store.withPrefix("feed_forward")), + .ada_norm = AdaRmsNorm.init(store), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { + RmsNorm.unloadBuffers(&self.attention_norm); + SelfAttention.unloadBuffers(&self.attention); + RmsNorm.unloadBuffers(&self.ffn_norm); + SwiGluFfn.unloadBuffers(&self.feed_forward); + AdaRmsNorm.unloadBuffers(&self.ada_norm); + } + + /// h: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] → ([s, d], KvCache) + pub fn forward(self: DecoderLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, config: Config) struct { Tensor, KvCache } { + // Pre-attention norm → attention (with KV cache) → residual + const attn_out, const updated_kv_cache = self.attention.forward( + self.attention_norm.forward(h), + token_index, + kv_cache, + config, + ); + var out = h.add(attn_out); + + // Pre-FFN norm → ada conditioning → FFN → residual + const normed = self.ffn_norm.forward(out); + const conditioned = self.ada_norm.forward(normed, t_cond); + const ffn_out = self.feed_forward.forward(conditioned); + out = out.add(ffn_out); + + return .{ out, updated_kv_cache }; + } +}; + +/// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. +pub const Decoder = struct { + tok_embeddings: Tensor, + layers: []DecoderLayer, + norm: RmsNorm, + config: Config, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Decoder { + const layers = allocator.alloc(DecoderLayer, config.n_layers) catch unreachable; + for (layers, 0..) |*layer, i| { + layer.* = DecoderLayer.init(store.withPrefix("layers").withLayer(i), config); + } + + return .{ + .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .vocab, .d }), + .layers = layers, + .norm = RmsNorm.init(store.withPrefix("norm"), config.norm_eps), + .config = config, + }; + } + + pub fn deinit(self: Decoder, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { + self.tok_embeddings.deinit(); + for (self.layers) |*layer| { + DecoderLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); + } + + /// Run all decoder layers on input embeddings. + /// input_embeds: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] + /// Returns: ([s, d], KvCache) + pub fn forward(self: Decoder, input_embeds: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor) struct { Tensor, KvCache } { + var h = input_embeds; + var cache = kv_cache; + + for (self.layers, 0..) |layer, i| { + const layer_cache = cache.atLayer(i); + const result = layer.forward(h, token_index, layer_cache, t_cond, self.config); + h = result[0]; + const updated_layer_cache = result[1]; + cache = updated_layer_cache.reuseBuffer(layer_cache); + } + + h = self.norm.forward(h); + return .{ h, cache }; + } + + /// Compute output logits via tied embeddings: hidden.dot(tok_embeddings, .d) + /// hidden: [s, d] → [s, vocab] + pub fn logits(self: Decoder, hidden: Tensor) Tensor { + return hidden.dot(self.tok_embeddings.convert(hidden.dtype()), .d); + } +}; From 7cd2745353198cf2bfc1eff487fda0d186941890 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 10:56:33 +0100 Subject: [PATCH 42/98] Assert same type for linear forward for input and weights and fix format for tensor dot --- zml/nn.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zml/nn.zig b/zml/nn.zig index a903c712..4f75931f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -27,6 +27,8 @@ pub const Linear = struct { } pub fn forward(self: Linear, x: Tensor) Tensor { + stdx.debug.assert(self.weight.dtype() == x.dtype(), "Linear expects input and weight to have the same dtype, got {} and {}", .{ x.dtype(), self.weight.dtype() }); + var y = x.dot(self.weight, self.tag); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; From 313c46191ed0b3fb1ff86ca4b705854355d695b3 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 10:58:00 +0100 Subject: [PATCH 43/98] simplify encoder layout --- examples/voxtral/BUILD.bazel | 2 + examples/voxtral/decoder.zig | 116 +++++++++++++----------- examples/voxtral/encoder.zig | 171 +++++++++++++++-------------------- 3 files changed, 138 insertions(+), 151 deletions(-) diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel index 03505def..049e2f8a 100644 --- a/examples/voxtral/BUILD.bazel +++ b/examples/voxtral/BUILD.bazel @@ -9,6 +9,7 @@ zig_binary( "wav.zig", "config.zig", "encoder.zig", + "decoder.zig", ], extra_srcs = [ "//examples/voxtral/assets:voxtral_mel_filter", @@ -24,6 +25,7 @@ zig_binary( "wav.zig", "config.zig", "encoder.zig", + "decoder.zig", ], extra_srcs = [ "//examples/voxtral/assets:voxtral_mel_filter", diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 895f0d3e..6f9a40e9 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -9,65 +9,62 @@ const cfg = @import("config.zig"); const Config = cfg.Config; const enc = @import("encoder.zig"); -const Linear = enc.Linear; -const RmsNorm = enc.RmsNorm; const SwiGluFfn = enc.SwiGluFfn; +const rmsNorm = enc.rmsNorm; /// Adapter: encoder→decoder projection. /// Reshapes encoder output [s, d=1280] → [s/4, d=5120] (concatenate 4 timesteps), /// then projects through a 2-layer MLP: Linear(5120→3072) → GELU → Linear(3072→3072). pub const Adapter = struct { - proj0: Linear, - proj1: Linear, + proj0: zml.nn.Linear, + proj1: zml.nn.Linear, pub fn init(store: zml.io.TensorStore.View) Adapter { const proj_store = store.withPrefix("mm_streams_embeddings.embedding_module.audio_language_projection"); return .{ - .proj0 = Linear.init(proj_store.withLayer(0)), - .proj1 = Linear.init(proj_store.withLayer(2)), + .proj0 = linear(proj_store.withLayer(0)), + .proj1 = linear(proj_store.withLayer(2)), }; } pub fn unloadBuffers(self: *zml.Bufferized(Adapter)) void { - Linear.unloadBuffers(&self.proj0); - Linear.unloadBuffers(&self.proj1); + deinitLinear(&self.proj0); + deinitLinear(&self.proj1); } /// encoder_out: [s, d=1280] → [s/4, d=3072] pub fn forward(self: Adapter, encoder_out: Tensor) Tensor { // Reshape [s, d=1280] → [s/4, d=5120] by concatenating 4 consecutive timesteps. - // Split s into (s_new, group=4), then merge (group, d) → d_new. const h = encoder_out .splitAxis(.s, .{ .s = .auto, .group = 4 }) .merge(.{ .d = .{ .group, .d } }); - return self.proj1.forward(self.proj0.forward(h).gelu()); + return self.proj1.forward(self.proj0.forward(h).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); } }; /// AdaRmsNorm: per-layer time conditioning. /// Computes ada_scale = up(gelu(down(t_cond))), then returns h * (1 + ada_scale). pub const AdaRmsNorm = struct { - down: Linear, - up: Linear, + down: zml.nn.Linear, + up: zml.nn.Linear, pub fn init(store: zml.io.TensorStore.View) AdaRmsNorm { const ada_store = store.withPrefix("ada_rms_norm_t_cond"); return .{ - .down = Linear.init(ada_store.withLayer(0)), - .up = Linear.init(ada_store.withLayer(2)), + .down = linear(ada_store.withLayer(0)), + .up = linear(ada_store.withLayer(2)), }; } pub fn unloadBuffers(self: *zml.Bufferized(AdaRmsNorm)) void { - Linear.unloadBuffers(&self.down); - Linear.unloadBuffers(&self.up); + deinitLinear(&self.down); + deinitLinear(&self.up); } - /// Apply adaptive conditioning: h * (1 + ada_scale). /// h: [s, d], t_cond: [d] → [s, d] pub fn forward(self: AdaRmsNorm, h: Tensor, t_cond: Tensor) Tensor { - const ada_scale = self.up.forward(self.down.forward(t_cond).gelu()); + const ada_scale = self.up.forward(self.down.forward(t_cond).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); const one_plus_scale = ada_scale.broad(h.shape()).add(Tensor.scalar(1.0, h.dtype()).broad(h.shape())); return h.mul(one_plus_scale); } @@ -169,40 +166,39 @@ pub const KvCache = struct { /// Decoder self-attention with KV cache. pub const SelfAttention = struct { - wq: Linear, - wk: Linear, - wv: Linear, - wo: Linear, + wq: zml.nn.Linear, + wk: zml.nn.Linear, + wv: zml.nn.Linear, + wo: zml.nn.Linear, pub fn init(store: zml.io.TensorStore.View) SelfAttention { return .{ - .wq = Linear.init(store.withPrefix("wq")), - .wk = Linear.init(store.withPrefix("wk")), - .wv = Linear.init(store.withPrefix("wv")), - .wo = Linear.init(store.withPrefix("wo")), + .wq = linear(store.withPrefix("wq")), + .wk = linear(store.withPrefix("wk")), + .wv = linear(store.withPrefix("wv")), + .wo = linear(store.withPrefix("wo")), }; } pub fn unloadBuffers(self: *zml.Bufferized(SelfAttention)) void { - Linear.unloadBuffers(&self.wq); - Linear.unloadBuffers(&self.wk); - Linear.unloadBuffers(&self.wv); - Linear.unloadBuffers(&self.wo); + deinitLinear(&self.wq); + deinitLinear(&self.wk); + deinitLinear(&self.wv); + deinitLinear(&self.wo); } /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) pub fn forward(self: SelfAttention, x: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config) struct { Tensor, KvCache } { const dtype = x.dtype(); - // Q, K, V projections var q = self.wq.forward(x); var k = self.wk.forward(x); var v = self.wv.forward(x); - // Split into heads: [s, d] → [s, h, hd] - q = q.splitAxis(.d, .{ .h = config.n_heads, .hd = config.head_dim }); - k = k.splitAxis(.d, .{ .h = config.n_kv_heads, .hd = config.head_dim }); - v = v.splitAxis(.d, .{ .h = config.n_kv_heads, .hd = config.head_dim }); + // Split into heads: [s, dout] → [s, h, hd] + q = q.splitAxis(.dout, .{ .h = config.n_heads, .hd = config.head_dim }); + k = k.splitAxis(.dout, .{ .h = config.n_kv_heads, .hd = config.head_dim }); + v = v.splitAxis(.dout, .{ .h = config.n_kv_heads, .hd = config.head_dim }); // Position indices for RoPE const pos_index = b: { @@ -211,7 +207,6 @@ pub const SelfAttention = struct { break :b temp.add(token_index.broad(temp.shape())); }; - // RoPE (interleaved) const rope_opts: zml.nn.RopeOpts = .{ .layout = .interleaved, .freq_base = config.rope_theta, @@ -219,7 +214,6 @@ pub const SelfAttention = struct { q = zml.nn.rope(q, pos_index, rope_opts); k = zml.nn.rope(k, pos_index, rope_opts); - // Rename for attention: .s → .q/.k q = q.rename(.{ .s = .q }); k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); @@ -239,35 +233,36 @@ pub const SelfAttention = struct { ); const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); - // Merge heads: [q, h, hd] → [s, d] + // Merge heads and output projection: [q, h, hd] → [s, d] const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - - return .{ self.wo.forward(merged), new_kv_cache }; + return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; } }; /// Single decoder transformer layer with AdaRmsNorm time conditioning. pub const DecoderLayer = struct { - attention_norm: RmsNorm, + attention_norm: Tensor, attention: SelfAttention, - ffn_norm: RmsNorm, + ffn_norm: Tensor, feed_forward: SwiGluFfn, ada_norm: AdaRmsNorm, + norm_eps: f32, pub fn init(store: zml.io.TensorStore.View, config: Config) DecoderLayer { return .{ - .attention_norm = RmsNorm.init(store.withPrefix("attention_norm"), config.norm_eps), + .attention_norm = store.withPrefix("attention_norm").createTensorWithTags("weight", .{.d}), .attention = SelfAttention.init(store.withPrefix("attention")), - .ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), config.norm_eps), + .ffn_norm = store.withPrefix("ffn_norm").createTensorWithTags("weight", .{.d}), .feed_forward = SwiGluFfn.init(store.withPrefix("feed_forward")), .ada_norm = AdaRmsNorm.init(store), + .norm_eps = config.norm_eps, }; } pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { - RmsNorm.unloadBuffers(&self.attention_norm); + self.attention_norm.deinit(); SelfAttention.unloadBuffers(&self.attention); - RmsNorm.unloadBuffers(&self.ffn_norm); + self.ffn_norm.deinit(); SwiGluFfn.unloadBuffers(&self.feed_forward); AdaRmsNorm.unloadBuffers(&self.ada_norm); } @@ -276,7 +271,7 @@ pub const DecoderLayer = struct { pub fn forward(self: DecoderLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, config: Config) struct { Tensor, KvCache } { // Pre-attention norm → attention (with KV cache) → residual const attn_out, const updated_kv_cache = self.attention.forward( - self.attention_norm.forward(h), + rmsNorm(h, self.attention_norm, self.norm_eps), token_index, kv_cache, config, @@ -284,7 +279,7 @@ pub const DecoderLayer = struct { var out = h.add(attn_out); // Pre-FFN norm → ada conditioning → FFN → residual - const normed = self.ffn_norm.forward(out); + const normed = rmsNorm(out, self.ffn_norm, self.norm_eps); const conditioned = self.ada_norm.forward(normed, t_cond); const ffn_out = self.feed_forward.forward(conditioned); out = out.add(ffn_out); @@ -297,7 +292,8 @@ pub const DecoderLayer = struct { pub const Decoder = struct { tok_embeddings: Tensor, layers: []DecoderLayer, - norm: RmsNorm, + norm: Tensor, + norm_eps: f32, config: Config, pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Decoder { @@ -309,7 +305,8 @@ pub const Decoder = struct { return .{ .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .vocab, .d }), .layers = layers, - .norm = RmsNorm.init(store.withPrefix("norm"), config.norm_eps), + .norm = store.withPrefix("norm").createTensorWithTags("weight", .{.d}), + .norm_eps = config.norm_eps, .config = config, }; } @@ -324,7 +321,7 @@ pub const Decoder = struct { DecoderLayer.unloadBuffers(layer); } allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); + self.norm.deinit(); } /// Run all decoder layers on input embeddings. @@ -342,7 +339,7 @@ pub const Decoder = struct { cache = updated_layer_cache.reuseBuffer(layer_cache); } - h = self.norm.forward(h); + h = rmsNorm(h, self.norm, self.norm_eps); return .{ h, cache }; } @@ -352,3 +349,18 @@ pub const Decoder = struct { return hidden.dot(self.tok_embeddings.convert(hidden.dtype()), .d); } }; + +// -- Helpers + +fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init( + store.createTensorWithTags("weight", .{ .dout, .d }), + store.maybeCreateTensorWithTags("bias", .{.dout}), + .d, + ); +} + +fn deinitLinear(l: *zml.Bufferized(zml.nn.Linear)) void { + l.weight.deinit(); + if (l.bias) |*b| b.deinit(); +} diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 20e2378f..f4cc3e07 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -13,7 +13,8 @@ pub const Encoder = struct { conv0: CausalConv1d, conv1: CausalConv1d, layers: []TransformerLayer, - norm: RmsNorm, + norm: Tensor, + norm_eps: f32, config: Config, pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Encoder { @@ -30,7 +31,8 @@ pub const Encoder = struct { .conv0 = CausalConv1d.init(conv_store.withLayer(0).withPrefix("conv"), 1), .conv1 = CausalConv1d.init(conv_store.withLayer(1).withPrefix("conv"), 2), .layers = layers, - .norm = RmsNorm.init(transformer_store.withPrefix("norm"), enc.norm_eps), + .norm = transformer_store.withPrefix("norm").createTensorWithTags("weight", .{.d}), + .norm_eps = enc.norm_eps, .config = config, }; } @@ -42,13 +44,13 @@ pub const Encoder = struct { pub fn unloadBuffers(self: *zml.Bufferized(Encoder), allocator: std.mem.Allocator) void { CausalConv1d.unloadBuffers(&self.conv0); CausalConv1d.unloadBuffers(&self.conv1); - + for (self.layers) |*layer| { TransformerLayer.unloadBuffers(layer); } - + allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); + self.norm.deinit(); } /// mel: [channels=128, time=frames] @@ -67,18 +69,21 @@ pub const Encoder = struct { // Left-truncate to multiple of downsample_factor const seq_len = h.dim(.s); const trunc: i64 = @intCast(@as(usize, @intCast(seq_len)) % self.config.downsample_factor()); - + if (trunc > 0) { h = h.slice1d(.s, .{ .start = trunc }); } + // Convert to weight dtype (bf16) before transformer layers. + // Conv stem operates in f32; transformer weights are stored in bf16. + h = h.convert(self.layers[0].attention.wq.weight.dtype()); + // Transformer layers for (self.layers) |layer| { h = layer.forward(h, self.config); } - // Final norm - return self.norm.forward(h); + return rmsNorm(h, self.norm, self.norm_eps).convert(.f32); } }; @@ -127,36 +132,38 @@ pub const CausalConv1d = struct { }; pub const TransformerLayer = struct { - attention_norm: RmsNorm, + attention_norm: Tensor, attention: SelfAttention, - ffn_norm: RmsNorm, + ffn_norm: Tensor, feed_forward: SwiGluFfn, + norm_eps: f32, pub fn init(store: zml.io.TensorStore.View, config: Config) TransformerLayer { const enc = config.encoder(); return .{ - .attention_norm = RmsNorm.init(store.withPrefix("attention_norm"), enc.norm_eps), + .attention_norm = store.withPrefix("attention_norm").createTensorWithTags("weight", .{.d}), .attention = SelfAttention.init(store.withPrefix("attention")), - .ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), enc.norm_eps), + .ffn_norm = store.withPrefix("ffn_norm").createTensorWithTags("weight", .{.d}), .feed_forward = SwiGluFfn.init(store.withPrefix("feed_forward")), + .norm_eps = enc.norm_eps, }; } pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.attention_norm); + self.attention_norm.deinit(); SelfAttention.unloadBuffers(&self.attention); - RmsNorm.unloadBuffers(&self.ffn_norm); + self.ffn_norm.deinit(); SwiGluFfn.unloadBuffers(&self.feed_forward); } /// h: [s, d] -> [s, d] pub fn forward(self: TransformerLayer, h: Tensor, config: Config) Tensor { // Pre-attention norm + attention + residual - const attn_out = self.attention.forward(self.attention_norm.forward(h), config); + const attn_out = self.attention.forward(rmsNorm(h, self.attention_norm, self.norm_eps), config); var out = h.add(attn_out); // Pre-FFN norm + FFN + residual - const ffn_out = self.feed_forward.forward(self.ffn_norm.forward(out)); + const ffn_out = self.feed_forward.forward(rmsNorm(out, self.ffn_norm, self.norm_eps)); out = out.add(ffn_out); return out; @@ -164,49 +171,49 @@ pub const TransformerLayer = struct { }; pub const SelfAttention = struct { - wq: Linear, - wk: Linear, - wv: Linear, - wo: Linear, + wq: zml.nn.Linear, + wk: zml.nn.Linear, + wv: zml.nn.Linear, + wo: zml.nn.Linear, pub fn init(store: zml.io.TensorStore.View) SelfAttention { return .{ - .wq = Linear.init(store.withPrefix("wq")), - .wk = Linear.init(store.withPrefix("wk")), - .wv = Linear.init(store.withPrefix("wv")), - .wo = Linear.init(store.withPrefix("wo")), + .wq = linear(store.withPrefix("wq")), + .wk = linear(store.withPrefix("wk")), + .wv = linear(store.withPrefix("wv")), + .wo = linear(store.withPrefix("wo")), }; } pub fn unloadBuffers(self: *zml.Bufferized(SelfAttention)) void { - Linear.unloadBuffers(&self.wq); - Linear.unloadBuffers(&self.wk); - Linear.unloadBuffers(&self.wv); - Linear.unloadBuffers(&self.wo); + deinitLinear(&self.wq); + deinitLinear(&self.wk); + deinitLinear(&self.wv); + deinitLinear(&self.wo); } /// x: [s, d] -> [s, d] pub fn forward(self: SelfAttention, x: Tensor, config: Config) Tensor { const enc = config.encoder(); const dtype = x.dtype(); + // const converted_x = x.convert(self.wq.weight.dtype()); - // Q, K, V projections - // Encoder: Q and V have biases, K does not - var q = self.wq.forward(x); // [s, n_heads * head_dim] - var k = self.wk.forward(x); // [s, n_kv_heads * head_dim] - var v = self.wv.forward(x); // [s, n_kv_heads * head_dim] + // Q, K, V projections (encoder: Q and V have biases, K does not) + var q = self.wq.forward(x); + var k = self.wk.forward(x); + var v = self.wv.forward(x); - // Split into heads: [s, d] -> [s, h, hd] - q = q.splitAxis(.d, .{ .h = enc.n_heads, .hd = enc.head_dim }); - k = k.splitAxis(.d, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); - v = v.splitAxis(.d, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); + // Split into heads: [s, dout] -> [s, h, hd] + q = q.splitAxis(.dout, .{ .h = enc.n_heads, .hd = enc.head_dim }); + k = k.splitAxis(.dout, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); + v = v.splitAxis(.dout, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); // RoPE (interleaved, matching is_neox_style=False in Python) const rope_opts: zml.nn.RopeOpts = .{ .layout = .interleaved, .freq_base = enc.rope_theta, }; - + q = zml.nn.rope(q, null, rope_opts); k = zml.nn.rope(k, null, rope_opts); @@ -220,88 +227,54 @@ pub const SelfAttention = struct { const attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, enc.sliding_window); const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); - // Merge heads: [s, h, hd] -> [s, d] + // Merge heads and output projection: [q, h, hd] -> [s, d] const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - - // Output projection - return self.wo.forward(merged); + return self.wo.forward(merged).rename(.{ .dout = .d }); } }; pub const SwiGluFfn = struct { - w1: Linear, - w2: Linear, - w3: Linear, + w1: zml.nn.Linear, + w2: zml.nn.Linear, + w3: zml.nn.Linear, pub fn init(store: zml.io.TensorStore.View) SwiGluFfn { return .{ - .w1 = Linear.init(store.withPrefix("w1")), - .w2 = Linear.init(store.withPrefix("w2")), - .w3 = Linear.init(store.withPrefix("w3")), + .w1 = linear(store.withPrefix("w1")), + .w2 = linear(store.withPrefix("w2")), + .w3 = linear(store.withPrefix("w3")), }; } pub fn unloadBuffers(self: *zml.Bufferized(SwiGluFfn)) void { - Linear.unloadBuffers(&self.w1); - Linear.unloadBuffers(&self.w2); - Linear.unloadBuffers(&self.w3); + deinitLinear(&self.w1); + deinitLinear(&self.w2); + deinitLinear(&self.w3); } /// x: [s, d] -> [s, d] pub fn forward(self: SwiGluFfn, x: Tensor) Tensor { - // SwiGLU: silu(gate) * up, then down const gate = self.w1.forward(x).silu(); const up = self.w3.forward(x); - return self.w2.forward(gate.mul(up)); + return self.w2.forward(gate.mul(up).rename(.{ .dout = .d })).rename(.{ .dout = .d }); } }; -pub const Linear = struct { - weight: Tensor, - bias: ?Tensor, +// -- Helpers - pub fn init(store: zml.io.TensorStore.View) Linear { - return .{ - .weight = store.createTensorWithTags("weight", .{ .out, .d }), - .bias = store.maybeCreateTensorWithTags("bias", .{.d}), - }; - } +fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init( + store.createTensorWithTags("weight", .{ .dout, .d }), + store.maybeCreateTensorWithTags("bias", .{.dout}), + .d, + ); +} - pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { - self.weight.deinit(); - if (self.bias) |*b| b.deinit(); - } +fn deinitLinear(l: *zml.Bufferized(zml.nn.Linear)) void { + l.weight.deinit(); + if (l.bias) |*b| b.deinit(); +} - /// x: [..., d] -> [..., out] (renamed back to d) - pub fn forward(self: Linear, x: Tensor) Tensor { - const dtype = x.dtype(); - var y = x.dot(self.weight.convert(dtype), .d).rename(.{ .out = .d }); - - if (self.bias) |bias| { - y = y.add(bias.convert(dtype).broad(y.shape())); - } - - return y; - } -}; - -pub const RmsNorm = struct { - weight: Tensor, - eps: f32, - - pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ - .weight = store.createTensorWithTags("weight", .{.d}), - .eps = eps, - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - - pub fn forward(self: RmsNorm, x: Tensor) Tensor { - const normalized = zml.nn.rmsNorm(x, .d, self.eps); - return normalized.mul(self.weight.convert(x.dtype()).broad(x.shape())); - } -}; +pub fn rmsNorm(x: Tensor, weight: Tensor, eps: f32) Tensor { + return zml.nn.rmsNorm(x, .d, eps).mul(weight.convert(x.dtype()).broad(x.shape())); +} From 839d78ad89b02c08d081ce6ab0cb5b079e002547 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 12:01:49 +0100 Subject: [PATCH 44/98] type conversion --- examples/voxtral/encoder.zig | 31 +++++++++++++++---------------- examples/voxtral/test.zig | 8 +++++--- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index f4cc3e07..524b4eab 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -56,12 +56,14 @@ pub const Encoder = struct { /// mel: [channels=128, time=frames] /// Returns: [s, d=1280] pub fn forward(self: Encoder, mel: Tensor) Tensor { - // Conv stem: [channels, time] -> [batch=1, channels, time] - var h = mel.insertAxes(.channels, .{.batch}); + // Standardize precision to match model weights (usually bf16) + const dtype = self.conv0.weight.dtype(); + var h = mel.convert(dtype).insertAxes(.channels, .{.batch}); h = self.conv0.forward(h).gelu(); h = self.conv1.forward(h).gelu(); + // [batch=1, channels=1280, time'] -> [s, d=1280] h = h.squeeze(.batch); h = h.transpose(.{ .time, .channels }).rename(.{ .time = .s, .channels = .d }); @@ -74,10 +76,6 @@ pub const Encoder = struct { h = h.slice1d(.s, .{ .start = trunc }); } - // Convert to weight dtype (bf16) before transformer layers. - // Conv stem operates in f32; transformer weights are stored in bf16. - h = h.convert(self.layers[0].attention.wq.weight.dtype()); - // Transformer layers for (self.layers) |layer| { h = layer.forward(h, self.config); @@ -120,7 +118,9 @@ pub const CausalConv1d = struct { const padding_right = target_length - input_len; const dtype = input.dtype(); - var h = input.conv1d(self.weight.convert(dtype), .{ + std.debug.assert(dtype == self.weight.dtype()); + + var h = input.conv1d(self.weight, .{ .window_strides = stride, .padding = &.{ padding_left, padding_right }, }); @@ -178,10 +178,10 @@ pub const SelfAttention = struct { pub fn init(store: zml.io.TensorStore.View) SelfAttention { return .{ - .wq = linear(store.withPrefix("wq")), - .wk = linear(store.withPrefix("wk")), - .wv = linear(store.withPrefix("wv")), - .wo = linear(store.withPrefix("wo")), + .wq = initLinear(store.withPrefix("wq")), + .wk = initLinear(store.withPrefix("wk")), + .wv = initLinear(store.withPrefix("wv")), + .wo = initLinear(store.withPrefix("wo")), }; } @@ -196,7 +196,6 @@ pub const SelfAttention = struct { pub fn forward(self: SelfAttention, x: Tensor, config: Config) Tensor { const enc = config.encoder(); const dtype = x.dtype(); - // const converted_x = x.convert(self.wq.weight.dtype()); // Q, K, V projections (encoder: Q and V have biases, K does not) var q = self.wq.forward(x); @@ -240,9 +239,9 @@ pub const SwiGluFfn = struct { pub fn init(store: zml.io.TensorStore.View) SwiGluFfn { return .{ - .w1 = linear(store.withPrefix("w1")), - .w2 = linear(store.withPrefix("w2")), - .w3 = linear(store.withPrefix("w3")), + .w1 = initLinear(store.withPrefix("w1")), + .w2 = initLinear(store.withPrefix("w2")), + .w3 = initLinear(store.withPrefix("w3")), }; } @@ -262,7 +261,7 @@ pub const SwiGluFfn = struct { // -- Helpers -fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { +fn initLinear(store: zml.io.TensorStore.View) zml.nn.Linear { return .init( store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 3289f3ea..e6b5ac62 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -131,11 +131,12 @@ const ConvStemTestHarness = struct { } pub fn forward(self: ConvStemTestHarness, mel_input: Tensor) Tensor { - var h = mel_input.withTags(.{ .channels, .time }).insertAxes(.channels, .{.batch}); + const dtype = self.conv0.weight.dtype(); + var h = mel_input.convert(dtype).withTags(.{ .channels, .time }).insertAxes(.channels, .{.batch}); h = self.conv0.forward(h).gelu(); h = self.conv1.forward(h).gelu(); h = h.squeeze(.batch); - return h.transpose(.{ .time, .channels }); + return h.transpose(.{ .time, .channels }).convert(.f32); } }; @@ -148,7 +149,8 @@ const TransformerLayerTestHarness = struct { } pub fn forward(self: TransformerLayerTestHarness, h: Tensor) Tensor { - return self.inner.forward(h.withTags(.{ .s, .d }), self.config); + const dtype = self.inner.attention.wq.weight.dtype(); + return self.inner.forward(h.convert(dtype).withTags(.{ .s, .d }), self.config).convert(.f32); } }; From e8c7afe77b4693cf9000e2ff621dddbdb89fb8e4 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 12:13:25 +0100 Subject: [PATCH 45/98] new testLayer and remove convert --- examples/voxtral/encoder.zig | 6 +++--- examples/voxtral/test.zig | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 524b4eab..17d92886 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -81,7 +81,7 @@ pub const Encoder = struct { h = layer.forward(h, self.config); } - return rmsNorm(h, self.norm, self.norm_eps).convert(.f32); + return rmsNorm(h, self.norm, self.norm_eps); } }; @@ -126,7 +126,7 @@ pub const CausalConv1d = struct { }); // Bias: [channels] broadcasts to [batch, channels, time] - h = h.add(self.bias.convert(dtype).broad(h.shape())); + h = h.add(self.bias.broad(h.shape())); return h; } }; @@ -275,5 +275,5 @@ fn deinitLinear(l: *zml.Bufferized(zml.nn.Linear)) void { } pub fn rmsNorm(x: Tensor, weight: Tensor, eps: f32) Tensor { - return zml.nn.rmsNorm(x, .d, eps).mul(weight.convert(x.dtype()).broad(x.shape())); + return zml.nn.rmsNorm(x, .d, eps).mul(weight.broad(x.shape())); } diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index e6b5ac62..8613fdb8 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -81,7 +81,7 @@ pub fn main() !void { var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); defer LogMelSpectrogram.unload(&mel_spectrum_buffers); - try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, ref_store.view(), "mel", .{ .inner = mel_spectrum_buffers }, 1e-3); + try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, ref_store.view(), "mel", .{ .inner = mel_spectrum_buffers }, .{}); } // -- Conv stem test @@ -94,7 +94,7 @@ pub fn main() !void { var conv_buffers = try zml.io.load(ConvStemTestHarness, &conv_stem, allocator, io, platform, load_opts); defer ConvStemTestHarness.unloadBuffers(&conv_buffers); - try zml.testing.testLayer(allocator, io, platform, conv_stem, .forward, ref_store.view(), "conv_stem", conv_buffers, 1e-3); + try zml.testing.testLayer(allocator, io, platform, conv_stem, .forward, ref_store.view(), "conv_stem", conv_buffers, .{}); } // -- Encoder test @@ -105,7 +105,7 @@ pub fn main() !void { defer Encoder.unloadBuffers(&encoder_buffers, allocator); const harness = EncoderTestHarness{ .inner = encoder_model }; - try zml.testing.testLayer(allocator, io, platform, harness, .forward, ref_store.view(), "encoder", .{ .inner = encoder_buffers }, 1e-3); + try zml.testing.testLayer(allocator, io, platform, harness, .forward, ref_store.view(), "encoder", .{ .inner = encoder_buffers }, .{}); } } @@ -133,9 +133,11 @@ const ConvStemTestHarness = struct { pub fn forward(self: ConvStemTestHarness, mel_input: Tensor) Tensor { const dtype = self.conv0.weight.dtype(); var h = mel_input.convert(dtype).withTags(.{ .channels, .time }).insertAxes(.channels, .{.batch}); + h = self.conv0.forward(h).gelu(); h = self.conv1.forward(h).gelu(); h = h.squeeze(.batch); + return h.transpose(.{ .time, .channels }).convert(.f32); } }; @@ -158,6 +160,6 @@ const EncoderTestHarness = struct { inner: Encoder, pub fn forward(self: EncoderTestHarness, mel_input: Tensor) Tensor { - return self.inner.forward(mel_input.withTags(.{ .channels, .time })); + return self.inner.forward(mel_input.withTags(.{ .channels, .time })).convert(.f32); } }; From 8c83dcc7c867049656374113820d65bfda24e3ec Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 14:32:29 +0100 Subject: [PATCH 46/98] add layer by layer encoder transformer test --- examples/voxtral/test.zig | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 8613fdb8..77a3c2b5 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -97,16 +97,36 @@ pub fn main() !void { try zml.testing.testLayer(allocator, io, platform, conv_stem, .forward, ref_store.view(), "conv_stem", conv_buffers, .{}); } - // -- Encoder test + // -- Transformer Layer tests { - const encoder_model = Encoder.init(allocator, model_store.view().withPrefix(encoder_prefix), config); - defer encoder_model.deinit(allocator); - var encoder_buffers = try zml.io.load(Encoder, &encoder_model, allocator, io, platform, load_opts); - defer Encoder.unloadBuffers(&encoder_buffers, allocator); + const transformer_store = model_store.view().withPrefix(encoder_prefix).withPrefix("transformer").withPrefix("layers"); + const enc_cfg = config.encoder(); - const harness = EncoderTestHarness{ .inner = encoder_model }; - try zml.testing.testLayer(allocator, io, platform, harness, .forward, ref_store.view(), "encoder", .{ .inner = encoder_buffers }, .{}); + inline for (0..32) |i| { + if (i < enc_cfg.n_layers) { + const layer = TransformerLayerTestHarness{ + .inner = enc.TransformerLayer.init(transformer_store.withLayer(i), config), + .config = config, + }; + + var layer_buffers = try zml.io.load(TransformerLayerTestHarness, &layer, allocator, io, platform, load_opts); + defer TransformerLayerTestHarness.unloadBuffers(&layer_buffers); + + try zml.testing.testLayer(allocator, io, platform, layer, .forward, ref_store.view(), std.fmt.comptimePrint("encoder.layers.{d}", .{i}), layer_buffers, .{.minimum_close_fraction = 0.9}); + } + } } + + // -- Encoder test + // { + // const encoder_model = Encoder.init(allocator, model_store.view().withPrefix(encoder_prefix), config); + // defer encoder_model.deinit(allocator); + // var encoder_buffers = try zml.io.load(Encoder, &encoder_model, allocator, io, platform, load_opts); + // defer Encoder.unloadBuffers(&encoder_buffers, allocator); + + // const harness = EncoderTestHarness{ .inner = encoder_model }; + // try zml.testing.testLayer(allocator, io, platform, harness, .forward, ref_store.view(), "encoder", .{ .inner = encoder_buffers }, .{}); + // } } // ================================================================ From f1c48500419cf17e3bfdfa8638fc08c61dae745c Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 15:04:19 +0100 Subject: [PATCH 47/98] Decoder prefill tests --- examples/voxtral/decoder.zig | 125 ++++++++++++++++++----------------- examples/voxtral/test.zig | 121 ++++++++++++++++++++++++++++----- 2 files changed, 169 insertions(+), 77 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 6f9a40e9..4b4adcbd 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -43,6 +43,69 @@ pub const Adapter = struct { } }; + +/// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. +pub const Decoder = struct { + tok_embeddings: Tensor, + layers: []DecoderLayer, + norm: Tensor, + norm_eps: f32, + config: Config, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Decoder { + const layers = allocator.alloc(DecoderLayer, config.n_layers) catch unreachable; + for (layers, 0..) |*layer, i| { + layer.* = DecoderLayer.init(store.withPrefix("layers").withLayer(i), config); + } + + return .{ + .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .vocab, .d }), + .layers = layers, + .norm = store.withPrefix("norm").createTensorWithTags("weight", .{.d}), + .norm_eps = config.norm_eps, + .config = config, + }; + } + + pub fn deinit(self: Decoder, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { + self.tok_embeddings.deinit(); + for (self.layers) |*layer| { + DecoderLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + self.norm.deinit(); + } + + /// Run all decoder layers on input embeddings. + /// input_embeds: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] + /// Returns: ([s, d], KvCache) + pub fn forward(self: Decoder, input_embeds: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor) struct { Tensor, KvCache } { + var h = input_embeds; + var cache = kv_cache; + + for (self.layers, 0..) |layer, i| { + const layer_cache = cache.atLayer(i); + const result = layer.forward(h, token_index, layer_cache, t_cond, self.config); + h = result[0]; + const updated_layer_cache = result[1]; + cache = updated_layer_cache.reuseBuffer(layer_cache); + } + + h = rmsNorm(h, self.norm, self.norm_eps); + return .{ h, cache }; + } + + /// Compute output logits via tied embeddings: hidden.dot(tok_embeddings, .d) + /// hidden: [s, d] → [s, vocab] + pub fn logits(self: Decoder, hidden: Tensor) Tensor { + return hidden.dot(self.tok_embeddings.convert(hidden.dtype()), .d); + } +}; + /// AdaRmsNorm: per-layer time conditioning. /// Computes ada_scale = up(gelu(down(t_cond))), then returns h * (1 + ada_scale). pub const AdaRmsNorm = struct { @@ -288,68 +351,6 @@ pub const DecoderLayer = struct { } }; -/// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. -pub const Decoder = struct { - tok_embeddings: Tensor, - layers: []DecoderLayer, - norm: Tensor, - norm_eps: f32, - config: Config, - - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Decoder { - const layers = allocator.alloc(DecoderLayer, config.n_layers) catch unreachable; - for (layers, 0..) |*layer, i| { - layer.* = DecoderLayer.init(store.withPrefix("layers").withLayer(i), config); - } - - return .{ - .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .vocab, .d }), - .layers = layers, - .norm = store.withPrefix("norm").createTensorWithTags("weight", .{.d}), - .norm_eps = config.norm_eps, - .config = config, - }; - } - - pub fn deinit(self: Decoder, allocator: std.mem.Allocator) void { - allocator.free(self.layers); - } - - pub fn unloadBuffers(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { - self.tok_embeddings.deinit(); - for (self.layers) |*layer| { - DecoderLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - self.norm.deinit(); - } - - /// Run all decoder layers on input embeddings. - /// input_embeds: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] - /// Returns: ([s, d], KvCache) - pub fn forward(self: Decoder, input_embeds: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor) struct { Tensor, KvCache } { - var h = input_embeds; - var cache = kv_cache; - - for (self.layers, 0..) |layer, i| { - const layer_cache = cache.atLayer(i); - const result = layer.forward(h, token_index, layer_cache, t_cond, self.config); - h = result[0]; - const updated_layer_cache = result[1]; - cache = updated_layer_cache.reuseBuffer(layer_cache); - } - - h = rmsNorm(h, self.norm, self.norm_eps); - return .{ h, cache }; - } - - /// Compute output logits via tied embeddings: hidden.dot(tok_embeddings, .d) - /// hidden: [s, d] → [s, vocab] - pub fn logits(self: Decoder, hidden: Tensor) Tensor { - return hidden.dot(self.tok_embeddings.convert(hidden.dtype()), .d); - } -}; - // -- Helpers fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 77a3c2b5..4867d60d 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -14,6 +14,8 @@ const LogMelSpectrogram = mel.LogMelSpectrogram; const enc = @import("encoder.zig"); const Encoder = enc.Encoder; +const dec = @import("decoder.zig"); + const CliArgs = struct { reference: []const u8, model: []const u8, @@ -98,24 +100,24 @@ pub fn main() !void { } // -- Transformer Layer tests - { - const transformer_store = model_store.view().withPrefix(encoder_prefix).withPrefix("transformer").withPrefix("layers"); - const enc_cfg = config.encoder(); + // { + // const transformer_store = model_store.view().withPrefix(encoder_prefix).withPrefix("transformer").withPrefix("layers"); + // const enc_cfg = config.encoder(); - inline for (0..32) |i| { - if (i < enc_cfg.n_layers) { - const layer = TransformerLayerTestHarness{ - .inner = enc.TransformerLayer.init(transformer_store.withLayer(i), config), - .config = config, - }; + // inline for (0..32) |i| { + // if (i < enc_cfg.n_layers) { + // const layer = TransformerLayerTestHarness{ + // .inner = enc.TransformerLayer.init(transformer_store.withLayer(i), config), + // .config = config, + // }; - var layer_buffers = try zml.io.load(TransformerLayerTestHarness, &layer, allocator, io, platform, load_opts); - defer TransformerLayerTestHarness.unloadBuffers(&layer_buffers); + // var layer_buffers = try zml.io.load(TransformerLayerTestHarness, &layer, allocator, io, platform, load_opts); + // defer TransformerLayerTestHarness.unloadBuffers(&layer_buffers); - try zml.testing.testLayer(allocator, io, platform, layer, .forward, ref_store.view(), std.fmt.comptimePrint("encoder.layers.{d}", .{i}), layer_buffers, .{.minimum_close_fraction = 0.9}); - } - } - } + // try zml.testing.testLayer(allocator, io, platform, layer, .forward, ref_store.view(), std.fmt.comptimePrint("encoder.layers.{d}", .{i}), layer_buffers, .{.minimum_close_fraction = 0.9}); + // } + // } + // } // -- Encoder test // { @@ -127,6 +129,55 @@ pub fn main() !void { // const harness = EncoderTestHarness{ .inner = encoder_model }; // try zml.testing.testLayer(allocator, io, platform, harness, .forward, ref_store.view(), "encoder", .{ .inner = encoder_buffers }, .{}); // } + + // -- Adapter test + { + const adapter = AdapterTestHarness{ + .inner = dec.Adapter.init(model_store.view()), + }; + var adapter_buffers = try zml.io.load(AdapterTestHarness, &adapter, allocator, io, platform, load_opts); + defer AdapterTestHarness.unloadBuffers(&adapter_buffers); + + try zml.testing.testLayer(allocator, io, platform, adapter, .forward, ref_store.view(), "adapter", adapter_buffers, .{.minimum_close_fraction = 0.9}); + } + + // -- Decoder Layer tests + { + const dec_layer_store = model_store.view().withPrefix("layers"); + + // Load t_cond from reference activations (shared across all layers) + const t_cond_tensor = ref_store.view().withPrefix("t_cond.out").createTensorWithTags("0", .{.d}); + const TCond = struct { t_cond: Tensor }; + const t_cond_spec = TCond{ .t_cond = t_cond_tensor }; + const ref_load_opts: zml.io.LoadOpts = .{ + .store = &ref_store, + .parallelism = 1, + .dma_chunks = 1, + .dma_chunk_size = 4096, + }; + var t_cond_buf = try zml.io.load(TCond, &t_cond_spec, allocator, io, platform, ref_load_opts); + defer t_cond_buf.t_cond.deinit(); + + inline for (0..32) |i| { + if (i < config.n_layers) { + const layer = DecoderLayerTestHarness{ + .inner = dec.DecoderLayer.init(dec_layer_store.withLayer(i), config), + .t_cond = t_cond_tensor, + .config = config, + }; + + var inner_buffers = try zml.io.load(dec.DecoderLayer, &layer.inner, allocator, io, platform, load_opts); + defer dec.DecoderLayer.unloadBuffers(&inner_buffers); + + const layer_buffers: zml.Bufferized(DecoderLayerTestHarness) = .{ + .inner = inner_buffers, + .t_cond = t_cond_buf.t_cond, + }; + + try zml.testing.testLayer(allocator, io, platform, layer, .forward, ref_store.view(), std.fmt.comptimePrint("decoder.prefill.layers.{d}", .{i}), layer_buffers, .{ .minimum_close_fraction = 0.85 }); + } + } + } } // ================================================================ @@ -183,3 +234,43 @@ const EncoderTestHarness = struct { return self.inner.forward(mel_input.withTags(.{ .channels, .time })).convert(.f32); } }; + +const AdapterTestHarness = struct { + inner: dec.Adapter, + + pub fn unloadBuffers(self: *zml.Bufferized(AdapterTestHarness)) void { + dec.Adapter.unloadBuffers(&self.inner); + } + + pub fn forward(self: AdapterTestHarness, encoder_out: Tensor) Tensor { + return self.inner.forward(encoder_out.convert(.bf16).withTags(.{ .s, .d })).convert(.f32); + } +}; + +const DecoderLayerTestHarness = struct { + inner: dec.DecoderLayer, + t_cond: Tensor, + config: Config, + + pub fn forward(self: DecoderLayerTestHarness, h: Tensor) Tensor { + const dtype = self.inner.attention.wq.weight.dtype(); + const h_typed = h.convert(dtype).withTags(.{ .s, .d }); + + // Create zero KV cache for prefill (empty cache for this layer) + const kv_shape = zml.Shape.init(.{ + .layer = 1, + .k = h_typed.dim(.s), + .h = @as(i64, @intCast(self.config.n_kv_heads)), + .hd = @as(i64, @intCast(self.config.head_dim)), + }, dtype); + const kv_cache = dec.KvCache{ + .k = Tensor.zeroes(kv_shape), + .v = Tensor.zeroes(kv_shape), + .layer_index = Tensor.scalar(0, .u32), + }; + + const token_index = Tensor.scalar(0, .u32); + const result = self.inner.forward(h_typed, token_index, kv_cache, self.t_cond.convert(dtype), self.config); + return result[0].convert(.f32); + } +}; From 234f814404d7a02c6e3ca22c27cd2d08470dee46 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 16:45:58 +0100 Subject: [PATCH 48/98] hardcode tokens and compile encoder --- examples/voxtral/voxtral.zig | 121 +++++++++++++++++++++++++++++++++-- 1 file changed, 117 insertions(+), 4 deletions(-) diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index c018199a..7d0236b2 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -12,11 +12,24 @@ const Tensor = zml.Tensor; const mel = @import("mel_spectrogram.zig"); const LogMelSpectrogram = mel.LogMelSpectrogram; +const enc = @import("encoder.zig"); +const Encoder = enc.Encoder; + +const dec = @import("decoder.zig"); +const Decoder = dec.Decoder; + const CliArgs = struct { input: []const u8, model: []const u8, }; +const prompt = ""; +const tokens: []u32 = .{1, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32}; +const audio_len = 293699; + +const max_seq_len = 1024; +const seq_len = 1024; + pub fn main() !void { log.info("Start of Voxtral", .{}); @@ -54,7 +67,16 @@ pub fn main() !void { const wav_file = try loadWav(allocator, &reader.interface); defer allocator.free(wav_file); + const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); + + var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); + defer model_registry.deinit(); + + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); + defer model_store.deinit(); + + var parsed_config = try cfg.parseConfig(allocator, io, model_dir); defer parsed_config.deinit(); const config = parsed_config.value; @@ -64,9 +86,35 @@ pub fn main() !void { log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); var melspectro_model: LogMelSpectrogram = .init(config); + const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; + var encoder_model: Encoder = .init(allocator, model_store.view().withPrefix(encoder_prefix), config); + defer encoder_model.deinit(allocator); + + var model: Decoder = .init(allocator, model_store.view(), config); + const dtype = model.tok_embeddings.dtype(); + const model_params: VoxtralParameters = .{ + .prefill_tokens = .init(.{.s = max_seq_len}, .u32), + .decode_tokens = .init(.{.s = 1}, .u32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(.init(.{ + .layer = model.layers.len, + + }), dtype), + }; + + var tokenizer_future = try io.concurrent(loadTokenizer, .{allocator, io, model_dir}); var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model}); + var compiled_encoder_future = try io.concurrent(compileEncoder, .{allocator, io, platform, encoder_model}); + var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); + var tokenizer = try tokenizer_future.await(io); + tokenizer.deinit(); + + var compiled_encoder = try compiled_encoder_future.await(io); + defer compiled_encoder.deinit(); + _ = &compiled_encoder; + var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); defer compiled_mel_spectrum.deinit(); var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); @@ -84,16 +132,18 @@ pub fn main() !void { const num_frames = wav_file.len / config.audio().hop_length; const output_shape = zml.Shape.init(.{ 128, num_frames }, .f32); + const output_slice: zml.Slice = try zml.Slice.alloc(allocator, output_shape); defer output_slice.free(allocator); + var output_buffer: zml.Buffer = try .fromSlice(io, platform, output_slice); defer output_buffer.deinit(); exec_args.set(.{mel_spectrum_buffers, input_buffer}); compiled_mel_spectrum.call(exec_args, &results); - results.fill(.{&output_buffer}); - - try output_buffer.toSlice(io, output_slice); + results.fill(.{&output_buffer}); + + // try output_buffer.toSlice(io, output_slice); // const outfile = try std.Io.Dir.createFile(.cwd(), io, args.output, .{}); // defer outfile.close(io); @@ -135,5 +185,68 @@ fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { } pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram) !zml.Exe { - return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{293699}, .f32).withTags(.{.samples})}); + return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{audio_len}, .f32).withTags(.{.samples})}); +} + +pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder) !zml.Exe { + const num_frames = audio_len / model.config.audio().hop_length; + + return try platform.compile(allocator, io, model, .forward, .{ + Tensor.init(.{ .channels = 128, .time = num_frames }, .f32), + }); +} + +const CompileModelResult = struct { + prefill: zml.Exe, + decode: zml.Exe, +}; + +const VoxtralParameters = struct { + prefill_tokens: Tensor, + decode_tokens: Tensor, + token_index: Tensor, + kv_cache: dec.KvCache, + t_cond: Tensor, +}; + +pub fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, params: VoxtralParameters) !CompileModelResult { + var prefill_future = try io.concurrent(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Encoder, params_: VoxtralParameters) !zml.Exe { + platform_.compile(allocator_, io_, model_, .forward, .{ + params_.prefill_tokens, + params_.token_index, + params_.kv_cache, + params_.t_cond, + }); + } + }.call, .{allocator, io, platform, model, params}); + + var decode_future = try io.concurrent(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Encoder, params_: VoxtralParameters) !zml.Exe { + platform_.compile(allocator_, io_, model_, .forward, .{ + params_.decode_tokens, + params_.token_index, + params_.kv_cache, + params_.t_cond, + }); + } + }.call, .{allocator, io, platform, model, params}); + + + const prefill_exe = try prefill_future.await(io); + const decode_exe = try decode_future.await(io); + + return .{ .prefill = prefill_exe, .decode = decode_exe }; +} + +pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + + return try .fromBytes(allocator, io, bytes); } From ff0e3ba39a021403cf04f126718d0cece86a1d50 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 18:10:41 +0100 Subject: [PATCH 49/98] Add embedder, compile every model and load weights --- examples/voxtral/decoder.zig | 65 +++++++++++++++ examples/voxtral/encoder.zig | 23 ++++++ examples/voxtral/voxtral.zig | 148 ++++++++++++++++++++++++++--------- zml/module.zig | 2 +- 4 files changed, 201 insertions(+), 37 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 4b4adcbd..4d3329e1 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -27,6 +27,29 @@ pub const Adapter = struct { }; } + pub fn load(self: *const Adapter, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Adapter){ + progress.increaseEstimatedTotalItems(store.view().count()); + var timer: std.time.Timer = try .start(); + var total_bytes: usize = 0; + defer { + const took = timer.read(); + log.info("Loaded adapter weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + total_bytes, + took, + total_bytes / took * std.time.ns_per_s, + }); + } + + return zml.io.load(Adapter, self, allocator, io, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + pub fn unloadBuffers(self: *zml.Bufferized(Adapter)) void { deinitLinear(&self.proj0); deinitLinear(&self.proj1); @@ -43,6 +66,48 @@ pub const Adapter = struct { } }; +pub const Embedder = struct { + tok_embeddings: Tensor, + + pub fn init(store: zml.io.TensorStore.View) Embedder { + return .{ + .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .vocab, .d }), + }; + } + + pub fn forward(self: Embedder, audio_embeds: Tensor, text_tokens: Tensor) Tensor { + const text_embeds = self.tok_embeddings.gather(.{ .vocab = text_tokens }, .{}); + return text_embeds.add(audio_embeds); + } + + pub fn load(self: *const Embedder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Embedder) { + progress.increaseEstimatedTotalItems(store.view().count()); + var timer: std.time.Timer = try .start(); + var total_bytes: usize = 0; + defer { + const took = timer.read(); + log.info("Loaded embedder weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + total_bytes, + took, + total_bytes / took * std.time.ns_per_s, + }); + } + + return zml.io.load(Embedder, self, allocator, io, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Embedder)) !void { + self.tok_embeddings.deinit(); + } +}; + /// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. pub const Decoder = struct { diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 17d92886..e49b0f07 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -41,6 +41,29 @@ pub const Encoder = struct { allocator.free(self.layers); } + pub fn load(self: *const Encoder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Encoder){ + progress.increaseEstimatedTotalItems(store.view().count()); + var timer: std.time.Timer = try .start(); + var total_bytes: usize = 0; + defer { + const took = timer.read(); + log.info("Loaded encoder weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + total_bytes, + took, + total_bytes / took * std.time.ns_per_s, + }); + } + + return zml.io.load(Encoder, self, allocator, io, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + pub fn unloadBuffers(self: *zml.Bufferized(Encoder), allocator: std.mem.Allocator) void { CausalConv1d.unloadBuffers(&self.conv0); CausalConv1d.unloadBuffers(&self.conv1); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 7d0236b2..4725291e 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -17,6 +17,8 @@ const Encoder = enc.Encoder; const dec = @import("decoder.zig"); const Decoder = dec.Decoder; +const Adapter = dec.Adapter; +const Embedder = dec.Embedder; const CliArgs = struct { input: []const u8, @@ -27,6 +29,7 @@ const prompt = ""; const tokens: []u32 = .{1, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32}; const audio_len = 293699; +const n_delay_tokens = 6; const max_seq_len = 1024; const seq_len = 1024; @@ -54,6 +57,8 @@ pub fn main() !void { try vfs.register("file", vfs_file.io()); const io = vfs.io(); + + var progress = std.Progress.start(io, .{ .root_name = "Voxtral" }); const arena: std.heap.ArenaAllocator = .init(allocator); defer arena.deinit(); @@ -89,59 +94,75 @@ pub fn main() !void { const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; var encoder_model: Encoder = .init(allocator, model_store.view().withPrefix(encoder_prefix), config); defer encoder_model.deinit(allocator); + + const adapter: Adapter = .init(model_store.view()); + const embedder: Embedder = .init(model_store.view()); var model: Decoder = .init(allocator, model_store.view(), config); + defer model.deinit(allocator); const dtype = model.tok_embeddings.dtype(); const model_params: VoxtralParameters = .{ - .prefill_tokens = .init(.{.s = max_seq_len}, .u32), - .decode_tokens = .init(.{.s = 1}, .u32), + .prefill_embeds = .init(.{.s = max_seq_len, .d = config.dim}, .bf16), + .decode_embeds = .init(.{.s = 1, .d = config.dim}, .bf16), .token_index = .init(.{}, .u32), .kv_cache = .init(.init(.{ .layer = model.layers.len, - - }), dtype), + .k = max_seq_len, + .h = config.n_kv_heads, + .hd = config.head_dim, + }, dtype)), + .t_cond = Tensor.init(.{.d = config.dim}, dtype), }; var tokenizer_future = try io.concurrent(loadTokenizer, .{allocator, io, model_dir}); var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model}); var compiled_encoder_future = try io.concurrent(compileEncoder, .{allocator, io, platform, encoder_model}); + const num_frames = audio_len / config.audio().hop_length; + const encoder_seq_len: u32 = @intCast((num_frames / 2) - (num_frames / 2) % config.downsample_factor()); + var compiled_adapter_future = try io.concurrent(compileAdapter, .{allocator, io, platform, adapter, encoder_seq_len}); + const adapter_seq_len = encoder_seq_len / config.downsample_factor(); + var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len}); + var compiled_decoder_future = try io.concurrent(compileDecoder, .{allocator, io, platform, model, model_params}); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); + var encoder_buffers_future = try io.concurrent(Encoder.load, .{&encoder_model,allocator,io, platform, &model_store, &progress}); + var adapter_buffers_future = try io.concurrent(Adapter.load, .{&adapter, allocator, io, platform, &model_store, &progress}); var tokenizer = try tokenizer_future.await(io); tokenizer.deinit(); + + // -- Compiled models + + var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); + defer compiled_mel_spectrum.deinit(); var compiled_encoder = try compiled_encoder_future.await(io); defer compiled_encoder.deinit(); _ = &compiled_encoder; + + var compiled_adapter = try compiled_adapter_future.await(io); + defer compiled_adapter.deinit(); + _ = &compiled_adapter; + + var compiled_embedder = try compiled_embedder_future.await(io); + defer compiled_embedder.deinit(); + _ = &compiled_embedder; + + var compiled_model = try compiled_decoder_future.await(io); + defer compiled_model.prefill.deinit(); + defer compiled_model.decode.deinit(); + + // -- Buffers - var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); - defer compiled_mel_spectrum.deinit(); var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); defer LogMelSpectrogram.unload(&mel_spectrum_buffers); - // Execute - var exec_args = try compiled_mel_spectrum.args(allocator); - defer exec_args.deinit(allocator); - var results = try compiled_mel_spectrum.results(allocator); - defer results.deinit(allocator); + var encoder_buffers = try encoder_buffers_future.await(io); + defer Encoder.unloadBuffers(&encoder_buffers, allocator); - const input_slice = zml.Slice.init(.init(.{ .freq = wav_file.len }, .f32), std.mem.sliceAsBytes(wav_file)); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); - defer input_buffer.deinit(); - - const num_frames = wav_file.len / config.audio().hop_length; - const output_shape = zml.Shape.init(.{ 128, num_frames }, .f32); - - const output_slice: zml.Slice = try zml.Slice.alloc(allocator, output_shape); - defer output_slice.free(allocator); - - var output_buffer: zml.Buffer = try .fromSlice(io, platform, output_slice); - defer output_buffer.deinit(); - - exec_args.set(.{mel_spectrum_buffers, input_buffer}); - compiled_mel_spectrum.call(exec_args, &results); - results.fill(.{&output_buffer}); + var adapter_buffers = try adapter_buffers_future.await(io); + defer Adapter.unloadBuffers(&adapter_buffers); + progress.end(); // try output_buffer.toSlice(io, output_slice); @@ -155,6 +176,32 @@ pub fn main() !void { // try writer_interface.writeAll(output_slice.data()); } +fn generateTranscript(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, config: Config, wav_file: []f32, mel_spectogram: zml.Exe) !void { + // Execute + var mel_args = try mel_spectogram.args(allocator); + defer mel_args.deinit(allocator); + var mel_results = try mel_spectogram.results(allocator); + defer mel_results.deinit(allocator); + + const num_frames = audio_len / config.audio().hop_length; + const input_slice = zml.Slice.init(.init(.{ .freq = wav_file.len }, .f32), std.mem.sliceAsBytes(wav_file)); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); + defer input_buffer.deinit(); + + const output_shape = zml.Shape.init(.{ 128, num_frames }, .f32); + + const output_slice: zml.Slice = try zml.Slice.alloc(allocator, output_shape); + defer output_slice.free(allocator); + + var output_buffer: zml.Buffer = try .fromSlice(io, platform, output_slice); + defer output_buffer.deinit(); + + // mel_args.set(.{mel_spectrum_buffers, input_buffer}); + // compiled_mel_spectrum.call(exec_args, &results); + // results.fill(.{&output_buffer}); + +} + fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { var arena_state: std.heap.ArenaAllocator = .init(allocator); defer arena_state.deinit(); @@ -196,24 +243,37 @@ pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const }); } +pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, encoder_seq_len: u32) !zml.Exe { + return try platform.compile(allocator, io, model, .forward, .{ + Tensor.init(.{ .s = encoder_seq_len, .d = 1280 }, .bf16), + }); +} + +pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, adapter_seq_len: u32) !zml.Exe { + return try platform.compile(allocator, io, model, .forward, .{ + Tensor.init(.{ .s = adapter_seq_len, .d = 3072 }, .bf16), + Tensor.init(.{ .s = adapter_seq_len }, .u32), + }); +} + const CompileModelResult = struct { prefill: zml.Exe, decode: zml.Exe, }; const VoxtralParameters = struct { - prefill_tokens: Tensor, - decode_tokens: Tensor, + prefill_embeds: Tensor, + decode_embeds: Tensor, token_index: Tensor, kv_cache: dec.KvCache, t_cond: Tensor, }; -pub fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, params: VoxtralParameters) !CompileModelResult { +pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, params: VoxtralParameters) !CompileModelResult { var prefill_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Encoder, params_: VoxtralParameters) !zml.Exe { - platform_.compile(allocator_, io_, model_, .forward, .{ - params_.prefill_tokens, + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, params_: VoxtralParameters) !zml.Exe { + return try platform_.compile(allocator_, io_, model_, .forward, .{ + params_.prefill_embeds, params_.token_index, params_.kv_cache, params_.t_cond, @@ -222,9 +282,9 @@ pub fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: *const z }.call, .{allocator, io, platform, model, params}); var decode_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Encoder, params_: VoxtralParameters) !zml.Exe { - platform_.compile(allocator_, io_, model_, .forward, .{ - params_.decode_tokens, + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, params_: VoxtralParameters) !zml.Exe { + return try platform_.compile(allocator_, io_, model_, .forward, .{ + params_.decode_embeds, params_.token_index, params_.kv_cache, params_.t_cond, @@ -239,6 +299,22 @@ pub fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: *const z return .{ .prefill = prefill_exe, .decode = decode_exe }; } +/// Sinusoidal time embedding: encodes a scalar t into a [dim]-dimensional vector. +/// out[i] = cos(t * inv_freq[i]) for i < half_dim +/// out[i] = sin(t * inv_freq[i-half_dim]) for i >= half_dim +/// where inv_freq[i] = exp(-log(theta) * i / half_dim) +// fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32) []f32 { +// const half_dim = dim / 2; +// const result = allocator.alloc(f32, dim) catch unreachable; +// for (0..half_dim) |i| { +// const freq = @exp(-@log(@as(f32, 10000.0)) * @as(f32, @floatFromInt(i)) / @as(f32, @floatFromInt(half_dim))); +// const angle = t_value * freq; +// result[i] = @cos(angle); +// result[i + half_dim] = @sin(angle); +// } +// return result; +// } + pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { const bytes = b: { const file = try dir.openFile(io, "tokenizer.json", .{}); diff --git a/zml/module.zig b/zml/module.zig index 31637852..a87866d9 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -195,7 +195,7 @@ pub fn compile( const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; - log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); + // log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); const num_devices = sharding.num_partitions * sharding.num_replicas; const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); From e4d80948deb91e6be4494fb40c784ebcb2951f03 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 12 Feb 2026 18:19:07 +0100 Subject: [PATCH 50/98] Remove hardcoded values and read from config --- examples/voxtral/voxtral.zig | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 4725291e..f0e7698d 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -30,8 +30,6 @@ const tokens: []u32 = .{1, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 3 const audio_len = 293699; const n_delay_tokens = 6; -const max_seq_len = 1024; -const seq_len = 1024; pub fn main() !void { log.info("Start of Voxtral", .{}); @@ -102,12 +100,12 @@ pub fn main() !void { defer model.deinit(allocator); const dtype = model.tok_embeddings.dtype(); const model_params: VoxtralParameters = .{ - .prefill_embeds = .init(.{.s = max_seq_len, .d = config.dim}, .bf16), + .prefill_embeds = .init(.{.s = config.sliding_window, .d = config.dim}, .bf16), .decode_embeds = .init(.{.s = 1, .d = config.dim}, .bf16), .token_index = .init(.{}, .u32), .kv_cache = .init(.init(.{ .layer = model.layers.len, - .k = max_seq_len, + .k = config.sliding_window, .h = config.n_kv_heads, .hd = config.head_dim, }, dtype)), @@ -119,9 +117,9 @@ pub fn main() !void { var compiled_encoder_future = try io.concurrent(compileEncoder, .{allocator, io, platform, encoder_model}); const num_frames = audio_len / config.audio().hop_length; const encoder_seq_len: u32 = @intCast((num_frames / 2) - (num_frames / 2) % config.downsample_factor()); - var compiled_adapter_future = try io.concurrent(compileAdapter, .{allocator, io, platform, adapter, encoder_seq_len}); + var compiled_adapter_future = try io.concurrent(compileAdapter, .{allocator, io, platform, adapter, encoder_seq_len, config}); const adapter_seq_len = encoder_seq_len / config.downsample_factor(); - var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len}); + var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len, config}); var compiled_decoder_future = try io.concurrent(compileDecoder, .{allocator, io, platform, model, model_params}); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); @@ -243,15 +241,15 @@ pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const }); } -pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, encoder_seq_len: u32) !zml.Exe { +pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, encoder_seq_len: u32, config: Config) !zml.Exe { return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .s = encoder_seq_len, .d = 1280 }, .bf16), + Tensor.init(.{ .s = encoder_seq_len, .d = config.encoder().dim }, .bf16), }); } -pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, adapter_seq_len: u32) !zml.Exe { +pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, adapter_seq_len: u32, config: Config) !zml.Exe { return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .s = adapter_seq_len, .d = 3072 }, .bf16), + Tensor.init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16), Tensor.init(.{ .s = adapter_seq_len }, .u32), }); } From 3e32eb20635a05a9d38e9d78f3b00e59f21823c3 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 13 Feb 2026 10:36:45 +0100 Subject: [PATCH 51/98] Load embedder and decoder weights --- examples/voxtral/decoder.zig | 25 ++++++++++++++++++++++++- examples/voxtral/mel_spectrogram.zig | 2 +- examples/voxtral/voxtral.zig | 19 ++++++++++++++++--- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 4d3329e1..cc75022c 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -103,7 +103,7 @@ pub const Embedder = struct { }); } - pub fn unloadBuffers(self: *zml.Bufferized(Embedder)) !void { + pub fn unloadBuffers(self: *zml.Bufferized(Embedder)) void { self.tok_embeddings.deinit(); } }; @@ -136,6 +136,29 @@ pub const Decoder = struct { allocator.free(self.layers); } + pub fn load(self: *const Decoder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Decoder) { + progress.increaseEstimatedTotalItems(store.view().count()); + var timer: std.time.Timer = try .start(); + var total_bytes: usize = 0; + defer { + const took = timer.read(); + log.info("Loaded decoder weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + total_bytes, + took, + total_bytes / took * std.time.ns_per_s, + }); + } + + return zml.io.load(Decoder, self, allocator, io, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + pub fn unloadBuffers(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { self.tok_embeddings.deinit(); for (self.layers) |*layer| { diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig index 844956df..21c9febf 100644 --- a/examples/voxtral/mel_spectrogram.zig +++ b/examples/voxtral/mel_spectrogram.zig @@ -37,7 +37,7 @@ pub const LogMelSpectrogram = struct { const window_weight = self.window.getWeights(self.n_fft, dtype); const fft_len = window_weight.dim(.samples); const num_frames: u63 = @intCast(@divFloor(waveform.dim(.samples), self.hop_len)); - + // var wav = waveform; // const force_num_frames = self.force_num_frames; // const force_num_samples = force_num_frames * self.hop_len; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index f0e7698d..9ff1d153 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -88,6 +88,10 @@ pub fn main() !void { defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); + const num_frames = audio_len / config.audio().hop_length; + const encoder_seq_len: u32 = @intCast((num_frames / 2) - (num_frames / 2) % config.downsample_factor()); + const adapter_seq_len = encoder_seq_len / config.downsample_factor(); + var melspectro_model: LogMelSpectrogram = .init(config); const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; var encoder_model: Encoder = .init(allocator, model_store.view().withPrefix(encoder_prefix), config); @@ -98,6 +102,7 @@ pub fn main() !void { var model: Decoder = .init(allocator, model_store.view(), config); defer model.deinit(allocator); + const dtype = model.tok_embeddings.dtype(); const model_params: VoxtralParameters = .{ .prefill_embeds = .init(.{.s = config.sliding_window, .d = config.dim}, .bf16), @@ -115,16 +120,15 @@ pub fn main() !void { var tokenizer_future = try io.concurrent(loadTokenizer, .{allocator, io, model_dir}); var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model}); var compiled_encoder_future = try io.concurrent(compileEncoder, .{allocator, io, platform, encoder_model}); - const num_frames = audio_len / config.audio().hop_length; - const encoder_seq_len: u32 = @intCast((num_frames / 2) - (num_frames / 2) % config.downsample_factor()); var compiled_adapter_future = try io.concurrent(compileAdapter, .{allocator, io, platform, adapter, encoder_seq_len, config}); - const adapter_seq_len = encoder_seq_len / config.downsample_factor(); var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len, config}); var compiled_decoder_future = try io.concurrent(compileDecoder, .{allocator, io, platform, model, model_params}); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); var encoder_buffers_future = try io.concurrent(Encoder.load, .{&encoder_model,allocator,io, platform, &model_store, &progress}); var adapter_buffers_future = try io.concurrent(Adapter.load, .{&adapter, allocator, io, platform, &model_store, &progress}); + var embedder_buffers_future = try io.concurrent(Embedder.load, .{&embedder, allocator, io, platform, &model_store, &progress}); + var decoder_buffers_future = try io.concurrent(Decoder.load, .{&model, allocator, io, platform, &model_store, &progress}); var tokenizer = try tokenizer_future.await(io); tokenizer.deinit(); @@ -160,7 +164,16 @@ pub fn main() !void { var adapter_buffers = try adapter_buffers_future.await(io); defer Adapter.unloadBuffers(&adapter_buffers); + + var embedder_buffers = try embedder_buffers_future.await(io); + defer Embedder.unloadBuffers(&embedder_buffers); + + var decoder_buffers = try decoder_buffers_future.await(io); + defer Decoder.unloadBuffers(&decoder_buffers, allocator); + progress.end(); + + // try output_buffer.toSlice(io, output_slice); From cf5d01a5ea0c9624edede0bcb6d54556fae8075d Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 13 Feb 2026 11:32:31 +0100 Subject: [PATCH 52/98] Change acceptable error --- examples/voxtral/test.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 4867d60d..509562ee 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -138,7 +138,7 @@ pub fn main() !void { var adapter_buffers = try zml.io.load(AdapterTestHarness, &adapter, allocator, io, platform, load_opts); defer AdapterTestHarness.unloadBuffers(&adapter_buffers); - try zml.testing.testLayer(allocator, io, platform, adapter, .forward, ref_store.view(), "adapter", adapter_buffers, .{.minimum_close_fraction = 0.9}); + try zml.testing.testLayer(allocator, io, platform, adapter, .forward, ref_store.view(), "adapter", adapter_buffers, .{.minimum_close_fraction = 0.8}); } // -- Decoder Layer tests From 1a9688e662a4ad66db8a87a4a98c3077d68dc556 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 13 Feb 2026 11:32:48 +0100 Subject: [PATCH 53/98] First decode try --- examples/voxtral/decoder.zig | 14 +- examples/voxtral/voxtral.zig | 313 +++++++++++++++++++++++++++++------ 2 files changed, 264 insertions(+), 63 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index cc75022c..efdb5d95 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -168,9 +168,9 @@ pub const Decoder = struct { self.norm.deinit(); } - /// Run all decoder layers on input embeddings. + /// Run all decoder layers on input embeddings, compute logits and return argmax tokens. /// input_embeds: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] - /// Returns: ([s, d], KvCache) + /// Returns: (tokens [s] u32, KvCache) pub fn forward(self: Decoder, input_embeds: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor) struct { Tensor, KvCache } { var h = input_embeds; var cache = kv_cache; @@ -184,13 +184,11 @@ pub const Decoder = struct { } h = rmsNorm(h, self.norm, self.norm_eps); - return .{ h, cache }; - } - /// Compute output logits via tied embeddings: hidden.dot(tok_embeddings, .d) - /// hidden: [s, d] → [s, vocab] - pub fn logits(self: Decoder, hidden: Tensor) Tensor { - return hidden.dot(self.tok_embeddings.convert(hidden.dtype()), .d); + // Compute logits via tied embeddings and take argmax + const output_logits = h.dot(self.tok_embeddings.convert(h.dtype()), .d); + const output_tokens = output_logits.argMax(.vocab).indices.convert(.u32); + return .{ output_tokens, cache }; } }; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 9ff1d153..dbe788dd 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -26,7 +26,8 @@ const CliArgs = struct { }; const prompt = ""; -const tokens: []u32 = .{1, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32}; +const tokens_array = [_]u32{1, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32}; +const tokens: []const u32 = &tokens_array; const audio_len = 293699; const n_delay_tokens = 6; @@ -105,7 +106,7 @@ pub fn main() !void { const dtype = model.tok_embeddings.dtype(); const model_params: VoxtralParameters = .{ - .prefill_embeds = .init(.{.s = config.sliding_window, .d = config.dim}, .bf16), + .prefill_embeds = .init(.{.s = adapter_seq_len, .d = config.dim}, .bf16), .decode_embeds = .init(.{.s = 1, .d = config.dim}, .bf16), .token_index = .init(.{}, .u32), .kv_cache = .init(.init(.{ @@ -122,6 +123,7 @@ pub fn main() !void { var compiled_encoder_future = try io.concurrent(compileEncoder, .{allocator, io, platform, encoder_model}); var compiled_adapter_future = try io.concurrent(compileAdapter, .{allocator, io, platform, adapter, encoder_seq_len, config}); var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len, config}); + var compiled_decode_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, @as(u32, 1), config}); var compiled_decoder_future = try io.concurrent(compileDecoder, .{allocator, io, platform, model, model_params}); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); @@ -131,7 +133,7 @@ pub fn main() !void { var decoder_buffers_future = try io.concurrent(Decoder.load, .{&model, allocator, io, platform, &model_store, &progress}); var tokenizer = try tokenizer_future.await(io); - tokenizer.deinit(); + defer tokenizer.deinit(); // -- Compiled models @@ -140,15 +142,15 @@ pub fn main() !void { var compiled_encoder = try compiled_encoder_future.await(io); defer compiled_encoder.deinit(); - _ = &compiled_encoder; var compiled_adapter = try compiled_adapter_future.await(io); defer compiled_adapter.deinit(); - _ = &compiled_adapter; var compiled_embedder = try compiled_embedder_future.await(io); defer compiled_embedder.deinit(); - _ = &compiled_embedder; + + var compiled_decode_embedder = try compiled_decode_embedder_future.await(io); + defer compiled_decode_embedder.deinit(); var compiled_model = try compiled_decoder_future.await(io); defer compiled_model.prefill.deinit(); @@ -173,45 +175,246 @@ pub fn main() !void { progress.end(); + // --- Execution pipeline --- + log.info("Running inference pipeline...", .{}); + + // 1. Mel spectrogram + log.info("Running mel spectrogram...", .{}); + var mel_output: zml.Buffer = undefined; + { + var mel_args = try compiled_mel_spectrum.args(allocator); + defer mel_args.deinit(allocator); + var mel_results = try compiled_mel_spectrum.results(allocator); + defer mel_results.deinit(allocator); + + const audio_slice: zml.Slice = .init( + .init(.{audio_len}, .f32), + std.mem.sliceAsBytes(wav_file), + ); + var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); + defer audio_buffer.deinit(); + + mel_output = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = num_frames }, .f32), .{}); + + mel_args.set(.{ &mel_spectrum_buffers, audio_buffer }); + compiled_mel_spectrum.call(mel_args, &mel_results); + mel_results.fill(.{&mel_output}); + } + defer mel_output.deinit(); + log.info("Mel spectrogram done.", .{}); + + // 2. Encoder + log.info("Running encoder...", .{}); + var encoder_output: zml.Buffer = undefined; + { + var enc_args = try compiled_encoder.args(allocator); + defer enc_args.deinit(allocator); + var enc_results = try compiled_encoder.results(allocator); + defer enc_results.deinit(allocator); + + encoder_output = try .uninitialized(io, platform, .init(.{ .s = encoder_seq_len, .d = config.encoder().dim }, .bf16), .{}); + + enc_args.set(.{ &encoder_buffers, mel_output }); + compiled_encoder.call(enc_args, &enc_results); + enc_results.fill(.{&encoder_output}); + } + defer encoder_output.deinit(); + log.info("Encoder done.", .{}); + + // 3. Adapter + log.info("Running adapter...", .{}); + var adapter_output: zml.Buffer = undefined; + { + var adp_args = try compiled_adapter.args(allocator); + defer adp_args.deinit(allocator); + var adp_results = try compiled_adapter.results(allocator); + defer adp_results.deinit(allocator); + + adapter_output = try .uninitialized(io, platform, .init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16), .{}); + + adp_args.set(.{ &adapter_buffers, encoder_output }); + compiled_adapter.call(adp_args, &adp_results); + adp_results.fill(.{&adapter_output}); + } + defer adapter_output.deinit(); + log.info("Adapter done.", .{}); + + // Download adapter_out to CPU for per-position slicing in the decode loop + const adapter_out_slice: zml.Slice = try .alloc(allocator, .init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16)); + defer adapter_out_slice.free(allocator); + try adapter_output.toSlice(io, adapter_out_slice); + const adapter_out_cpu = adapter_out_slice.constItems(u16); // bf16 stored as u16 + + // 4. Embedder: combine audio + token embeddings for the prefix (39 positions) + log.info("Running prefix embedder...", .{}); + var combined_embeds: zml.Buffer = undefined; + { + var emb_args = try compiled_embedder.args(allocator); + defer emb_args.deinit(allocator); + var emb_results = try compiled_embedder.results(allocator); + defer emb_results.deinit(allocator); + + const padded_tokens = try allocator.alloc(u32, adapter_seq_len); + defer allocator.free(padded_tokens); + @memcpy(padded_tokens[0..tokens.len], tokens); + @memset(padded_tokens[tokens.len..], 32); // STREAMING_PAD + + const tokens_slice: zml.Slice = .init( + .init(.{ .s = adapter_seq_len }, .u32), + std.mem.sliceAsBytes(padded_tokens), + ); + var tokens_buffer: zml.Buffer = try .fromSlice(io, platform, tokens_slice); + defer tokens_buffer.deinit(); + + combined_embeds = try .uninitialized(io, platform, .init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16), .{}); + + emb_args.set(.{ &embedder_buffers, adapter_output, tokens_buffer }); + compiled_embedder.call(emb_args, &emb_results); + emb_results.fill(.{&combined_embeds}); + } + defer combined_embeds.deinit(); + log.info("Prefix embedder done.", .{}); + + // 5. Compute time embedding on CPU and upload + const t_cond_data = computeTimeEmbedding(allocator, @floatFromInt(n_delay_tokens), config.dim); + defer allocator.free(t_cond_data); + const t_cond_bf16 = try allocator.alloc(u16, config.dim); + defer allocator.free(t_cond_bf16); + for (0..config.dim) |i| { + t_cond_bf16[i] = @bitCast(zml.floats.BFloat16.fromF32(t_cond_data[i])); + } + const t_cond_slice: zml.Slice = .init( + .init(.{ .d = config.dim }, .bf16), + std.mem.sliceAsBytes(t_cond_bf16), + ); + var t_cond_buffer: zml.Buffer = try .fromSlice(io, platform, t_cond_slice); + defer t_cond_buffer.deinit(); + log.info("Time embedding computed (t={d}).", .{n_delay_tokens}); + + // 6. Init KV cache + var kv_cache_buffers = try model_params.kv_cache.initBuffer(io, platform); + defer dec.KvCache.deinitBuffer(&kv_cache_buffers); + + // 7. Decoder prefill: process all adapter_seq_len positions at once + // (fills KV cache; we take the token at position tokens.len-1 as the first generated token) + log.info("Running decoder prefill ({d} positions)...", .{adapter_seq_len}); + var prefill_tokens_buffer: zml.Buffer = undefined; + { + var dec_args = try compiled_model.prefill.args(allocator); + defer dec_args.deinit(allocator); + var dec_results = try compiled_model.prefill.results(allocator); + defer dec_results.deinit(allocator); + + var token_index_buffer = try zml.Buffer.scalar(io, platform, @as(u32, 0), .u32); + defer token_index_buffer.deinit(); + + prefill_tokens_buffer = try .uninitialized(io, platform, .init(.{ .s = adapter_seq_len }, .u32), .{}); + + dec_args.set(.{ &decoder_buffers, combined_embeds, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); + compiled_model.prefill.call(dec_args, &dec_results); + dec_results.fill(.{ &prefill_tokens_buffer, &kv_cache_buffers }); + } + defer prefill_tokens_buffer.deinit(); + log.info("Decoder prefill done.", .{}); + + // Extract the first generated token from prefill (at position tokens.len - 1) + const prefill_out_slice: zml.Slice = try .alloc(allocator, .init(.{ .s = adapter_seq_len }, .u32)); + defer prefill_out_slice.free(allocator); + try prefill_tokens_buffer.toSlice(io, prefill_out_slice); + var prev_token: u32 = prefill_out_slice.items(u32)[tokens.len - 1]; + log.info("First generated token (from prefill pos {d}): {d}", .{ tokens.len - 1, prev_token }); + + // 8. Autoregressive decode loop + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + var stdout = std.Io.File.stdout().writer(io, &.{}); + const d: usize = @intCast(config.dim); + + // Prepare reusable args/results for decode embedder and decoder + var demb_args = try compiled_decode_embedder.args(allocator); + defer demb_args.deinit(allocator); + var demb_results = try compiled_decode_embedder.results(allocator); + defer demb_results.deinit(allocator); + + var decode_args = try compiled_model.decode.args(allocator); + defer decode_args.deinit(allocator); + var decode_results = try compiled_model.decode.results(allocator); + defer decode_results.deinit(allocator); + + // Reusable token output slice + var gen_token_slice: zml.Slice = try .alloc(allocator, .init(.{ .s = 1 }, .u32)); + defer gen_token_slice.free(allocator); + gen_token_slice.items(u32)[0] = prev_token; + var gen_token_buffer: zml.Buffer = try .fromSlice(io, platform, gen_token_slice); + defer gen_token_buffer.deinit(); + + // Reusable combined embed buffer + var combined_embed_buffer: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = 1, .d = config.dim }, .bf16), .{}); + defer combined_embed_buffer.deinit(); + + const n_audio: usize = @intCast(adapter_seq_len); + log.info("Running decoder decode ({d} steps)...", .{n_audio - tokens.len}); + + for (tokens.len..n_audio) |pos| { + // Check EOS (token_id=2) + if (prev_token == 2) break; + + // Print decoded text for the current token + if (try tokenizer_decoder.next(prev_token)) |chunk| { + try stdout.interface.print("{d}: {s}\n", .{pos, chunk}); + try stdout.interface.flush(); + } + + // Create audio embed [1, d] from adapter_out_cpu[pos] + const pos_offset = pos * d; + const audio_embed_slice: zml.Slice = .init( + .init(.{ .s = 1, .d = config.dim }, .bf16), + std.mem.sliceAsBytes(adapter_out_cpu[pos_offset .. pos_offset + d]), + ); + var audio_embed_buf: zml.Buffer = try .fromSlice(io, platform, audio_embed_slice); + defer audio_embed_buf.deinit(); + + // Create token [1] buffer + const token_data = [_]u32{prev_token}; + const tok_slice: zml.Slice = .init( + .init(.{ .s = 1 }, .u32), + std.mem.sliceAsBytes(&token_data), + ); + var tok_buf: zml.Buffer = try .fromSlice(io, platform, tok_slice); + defer tok_buf.deinit(); + + // Run decode embedder: adapter_out[pos] + tok_embed(prev_token) → [1, d] + demb_args.set(.{ &embedder_buffers, audio_embed_buf, tok_buf }); + compiled_decode_embedder.call(demb_args, &demb_results); + demb_results.fill(.{&combined_embed_buffer}); + + // Run decoder: combined_embed, token_index=pos, kv_cache, t_cond → next token + const pos_data = [_]u32{@intCast(pos)}; + const pos_slice: zml.Slice = .init(.init(.{}, .u32), std.mem.sliceAsBytes(&pos_data)); + var pos_buf: zml.Buffer = try .fromSlice(io, platform, pos_slice); + defer pos_buf.deinit(); + + decode_args.set(.{ &decoder_buffers, combined_embed_buffer, pos_buf, &kv_cache_buffers, t_cond_buffer }); + compiled_model.decode.call(decode_args, &decode_results); + decode_results.fill(.{ &gen_token_buffer, &kv_cache_buffers }); + + // Download generated token + try gen_token_buffer.toSlice(io, gen_token_slice); + prev_token = gen_token_slice.items(u32)[0]; + } + + // Print final token + if (try tokenizer_decoder.next(prev_token)) |chunk| { + try stdout.interface.print("{s}", .{chunk}); + } - - // try output_buffer.toSlice(io, output_slice); + try stdout.interface.writeAll("\n"); + try stdout.interface.flush(); - // const outfile = try std.Io.Dir.createFile(.cwd(), io, args.output, .{}); - // defer outfile.close(io); - - // var outbuff: [4096]u8 = undefined; - // var writer = outfile.writer(io, &outbuff); - // const writer_interface = &writer.interface; - - // try writer_interface.writeAll(output_slice.data()); + log.info("Decode complete.", .{}); } -fn generateTranscript(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, config: Config, wav_file: []f32, mel_spectogram: zml.Exe) !void { - // Execute - var mel_args = try mel_spectogram.args(allocator); - defer mel_args.deinit(allocator); - var mel_results = try mel_spectogram.results(allocator); - defer mel_results.deinit(allocator); - - const num_frames = audio_len / config.audio().hop_length; - const input_slice = zml.Slice.init(.init(.{ .freq = wav_file.len }, .f32), std.mem.sliceAsBytes(wav_file)); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); - defer input_buffer.deinit(); - - const output_shape = zml.Shape.init(.{ 128, num_frames }, .f32); - - const output_slice: zml.Slice = try zml.Slice.alloc(allocator, output_shape); - defer output_slice.free(allocator); - - var output_buffer: zml.Buffer = try .fromSlice(io, platform, output_slice); - defer output_buffer.deinit(); - - // mel_args.set(.{mel_spectrum_buffers, input_buffer}); - // compiled_mel_spectrum.call(exec_args, &results); - // results.fill(.{&output_buffer}); - -} fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { var arena_state: std.heap.ArenaAllocator = .init(allocator); @@ -260,10 +463,10 @@ pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const }); } -pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, adapter_seq_len: u32, config: Config) !zml.Exe { +pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, seq_len: u32, config: Config) !zml.Exe { return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16), - Tensor.init(.{ .s = adapter_seq_len }, .u32), + Tensor.init(.{ .s = seq_len, .d = config.dim }, .bf16), + Tensor.init(.{ .s = seq_len }, .u32), }); } @@ -314,17 +517,17 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const /// out[i] = cos(t * inv_freq[i]) for i < half_dim /// out[i] = sin(t * inv_freq[i-half_dim]) for i >= half_dim /// where inv_freq[i] = exp(-log(theta) * i / half_dim) -// fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32) []f32 { -// const half_dim = dim / 2; -// const result = allocator.alloc(f32, dim) catch unreachable; -// for (0..half_dim) |i| { -// const freq = @exp(-@log(@as(f32, 10000.0)) * @as(f32, @floatFromInt(i)) / @as(f32, @floatFromInt(half_dim))); -// const angle = t_value * freq; -// result[i] = @cos(angle); -// result[i + half_dim] = @sin(angle); -// } -// return result; -// } +fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32) []f32 { + const half_dim = dim / 2; + const result = allocator.alloc(f32, dim) catch unreachable; + for (0..half_dim) |i| { + const freq = @exp(-@log(@as(f32, 10000.0)) * @as(f32, @floatFromInt(i)) / @as(f32, @floatFromInt(half_dim))); + const angle = t_value * freq; + result[i] = @cos(angle); + result[i + half_dim] = @sin(angle); + } + return result; +} pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { const bytes = b: { From 3f5e00564fabb62060d76de57b942adaa9af7aa3 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 13 Feb 2026 11:59:47 +0100 Subject: [PATCH 54/98] Pad audio --- examples/voxtral/decoder.zig | 4 +- examples/voxtral/voxtral.zig | 216 +++++------------------------------ 2 files changed, 31 insertions(+), 189 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index efdb5d95..483fa8d7 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -368,7 +368,7 @@ pub const SelfAttention = struct { v = v.rename(.{ .s = .k }); // Update KV cache and retrieve cached K/V - const new_kv_cache = kv_cache.update(k, v, token_index); + const new_kv_cache = kv_cache.update(k, v, pos_index.rename(.{ .s = .k })); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); @@ -377,7 +377,7 @@ pub const SelfAttention = struct { var attn_mask = zml.nn.causalAttnMask(.{ .q = full_seq_len, .k = full_seq_len }, dtype, config.sliding_window); attn_mask = attn_mask.gatherSlices( Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), - token_index.reshape(.{ .coord = 1 }), + pos_index.rename(.{ .s = .coord }), .{}, ); const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index dbe788dd..09bca30b 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -28,9 +28,10 @@ const CliArgs = struct { const prompt = ""; const tokens_array = [_]u32{1, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32}; const tokens: []const u32 = &tokens_array; -const audio_len = 293699; - const n_delay_tokens = 6; +const n_left_pad_tokens = 32; +const n_right_pad_tokens = (n_delay_tokens + 1) + 10; // 17 +const raw_audio_length_per_tok = 1280; // SAMPLE_RATE / FRAME_RATE = 16000 / 12.5 pub fn main() !void { log.info("Start of Voxtral", .{}); @@ -71,7 +72,18 @@ pub fn main() !void { const wav_file = try loadWav(allocator, &reader.interface); defer allocator.free(wav_file); - + // Pad audio for offline streaming mode (matching mistral_common AudioEncoder.pad) + const left_pad = n_left_pad_tokens * raw_audio_length_per_tok; // 32 * 1280 = 40960 + const align_pad = (raw_audio_length_per_tok - (wav_file.len % raw_audio_length_per_tok)) % raw_audio_length_per_tok; + const right_pad = align_pad + n_right_pad_tokens * raw_audio_length_per_tok; // align + 17 * 1280 + const audio_len = left_pad + wav_file.len + right_pad; + + const padded_audio = try allocator.alloc(f32, audio_len); + defer allocator.free(padded_audio); + + @memset(padded_audio, 0); + @memcpy(padded_audio[left_pad .. left_pad + wav_file.len], wav_file); + const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); @@ -106,7 +118,7 @@ pub fn main() !void { const dtype = model.tok_embeddings.dtype(); const model_params: VoxtralParameters = .{ - .prefill_embeds = .init(.{.s = adapter_seq_len, .d = config.dim}, .bf16), + .prefill_embeds = .init(.{.s = @as(u32, @intCast(tokens.len)), .d = config.dim}, .bf16), .decode_embeds = .init(.{.s = 1, .d = config.dim}, .bf16), .token_index = .init(.{}, .u32), .kv_cache = .init(.init(.{ @@ -119,10 +131,10 @@ pub fn main() !void { }; var tokenizer_future = try io.concurrent(loadTokenizer, .{allocator, io, model_dir}); - var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model}); - var compiled_encoder_future = try io.concurrent(compileEncoder, .{allocator, io, platform, encoder_model}); + var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model, audio_len}); + var compiled_encoder_future = try io.concurrent(compileEncoder, .{allocator, io, platform, encoder_model, audio_len}); var compiled_adapter_future = try io.concurrent(compileAdapter, .{allocator, io, platform, adapter, encoder_seq_len, config}); - var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len, config}); + var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, @as(u32, @intCast(tokens.len)), config}); var compiled_decode_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, @as(u32, 1), config}); var compiled_decoder_future = try io.concurrent(compileDecoder, .{allocator, io, platform, model, model_params}); @@ -178,6 +190,9 @@ pub fn main() !void { // --- Execution pipeline --- log.info("Running inference pipeline...", .{}); + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + // 1. Mel spectrogram log.info("Running mel spectrogram...", .{}); var mel_output: zml.Buffer = undefined; @@ -189,7 +204,7 @@ pub fn main() !void { const audio_slice: zml.Slice = .init( .init(.{audio_len}, .f32), - std.mem.sliceAsBytes(wav_file), + std.mem.sliceAsBytes(padded_audio), ); var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); defer audio_buffer.deinit(); @@ -235,184 +250,11 @@ pub fn main() !void { adp_args.set(.{ &adapter_buffers, encoder_output }); compiled_adapter.call(adp_args, &adp_results); adp_results.fill(.{&adapter_output}); + + std.debug.print("{}\n", .{adapter_output.shape()}); } defer adapter_output.deinit(); log.info("Adapter done.", .{}); - - // Download adapter_out to CPU for per-position slicing in the decode loop - const adapter_out_slice: zml.Slice = try .alloc(allocator, .init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16)); - defer adapter_out_slice.free(allocator); - try adapter_output.toSlice(io, adapter_out_slice); - const adapter_out_cpu = adapter_out_slice.constItems(u16); // bf16 stored as u16 - - // 4. Embedder: combine audio + token embeddings for the prefix (39 positions) - log.info("Running prefix embedder...", .{}); - var combined_embeds: zml.Buffer = undefined; - { - var emb_args = try compiled_embedder.args(allocator); - defer emb_args.deinit(allocator); - var emb_results = try compiled_embedder.results(allocator); - defer emb_results.deinit(allocator); - - const padded_tokens = try allocator.alloc(u32, adapter_seq_len); - defer allocator.free(padded_tokens); - @memcpy(padded_tokens[0..tokens.len], tokens); - @memset(padded_tokens[tokens.len..], 32); // STREAMING_PAD - - const tokens_slice: zml.Slice = .init( - .init(.{ .s = adapter_seq_len }, .u32), - std.mem.sliceAsBytes(padded_tokens), - ); - var tokens_buffer: zml.Buffer = try .fromSlice(io, platform, tokens_slice); - defer tokens_buffer.deinit(); - - combined_embeds = try .uninitialized(io, platform, .init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16), .{}); - - emb_args.set(.{ &embedder_buffers, adapter_output, tokens_buffer }); - compiled_embedder.call(emb_args, &emb_results); - emb_results.fill(.{&combined_embeds}); - } - defer combined_embeds.deinit(); - log.info("Prefix embedder done.", .{}); - - // 5. Compute time embedding on CPU and upload - const t_cond_data = computeTimeEmbedding(allocator, @floatFromInt(n_delay_tokens), config.dim); - defer allocator.free(t_cond_data); - const t_cond_bf16 = try allocator.alloc(u16, config.dim); - defer allocator.free(t_cond_bf16); - for (0..config.dim) |i| { - t_cond_bf16[i] = @bitCast(zml.floats.BFloat16.fromF32(t_cond_data[i])); - } - const t_cond_slice: zml.Slice = .init( - .init(.{ .d = config.dim }, .bf16), - std.mem.sliceAsBytes(t_cond_bf16), - ); - var t_cond_buffer: zml.Buffer = try .fromSlice(io, platform, t_cond_slice); - defer t_cond_buffer.deinit(); - log.info("Time embedding computed (t={d}).", .{n_delay_tokens}); - - // 6. Init KV cache - var kv_cache_buffers = try model_params.kv_cache.initBuffer(io, platform); - defer dec.KvCache.deinitBuffer(&kv_cache_buffers); - - // 7. Decoder prefill: process all adapter_seq_len positions at once - // (fills KV cache; we take the token at position tokens.len-1 as the first generated token) - log.info("Running decoder prefill ({d} positions)...", .{adapter_seq_len}); - var prefill_tokens_buffer: zml.Buffer = undefined; - { - var dec_args = try compiled_model.prefill.args(allocator); - defer dec_args.deinit(allocator); - var dec_results = try compiled_model.prefill.results(allocator); - defer dec_results.deinit(allocator); - - var token_index_buffer = try zml.Buffer.scalar(io, platform, @as(u32, 0), .u32); - defer token_index_buffer.deinit(); - - prefill_tokens_buffer = try .uninitialized(io, platform, .init(.{ .s = adapter_seq_len }, .u32), .{}); - - dec_args.set(.{ &decoder_buffers, combined_embeds, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); - compiled_model.prefill.call(dec_args, &dec_results); - dec_results.fill(.{ &prefill_tokens_buffer, &kv_cache_buffers }); - } - defer prefill_tokens_buffer.deinit(); - log.info("Decoder prefill done.", .{}); - - // Extract the first generated token from prefill (at position tokens.len - 1) - const prefill_out_slice: zml.Slice = try .alloc(allocator, .init(.{ .s = adapter_seq_len }, .u32)); - defer prefill_out_slice.free(allocator); - try prefill_tokens_buffer.toSlice(io, prefill_out_slice); - var prev_token: u32 = prefill_out_slice.items(u32)[tokens.len - 1]; - log.info("First generated token (from prefill pos {d}): {d}", .{ tokens.len - 1, prev_token }); - - // 8. Autoregressive decode loop - var tokenizer_decoder = try tokenizer.decoder(); - defer tokenizer_decoder.deinit(); - - var stdout = std.Io.File.stdout().writer(io, &.{}); - const d: usize = @intCast(config.dim); - - // Prepare reusable args/results for decode embedder and decoder - var demb_args = try compiled_decode_embedder.args(allocator); - defer demb_args.deinit(allocator); - var demb_results = try compiled_decode_embedder.results(allocator); - defer demb_results.deinit(allocator); - - var decode_args = try compiled_model.decode.args(allocator); - defer decode_args.deinit(allocator); - var decode_results = try compiled_model.decode.results(allocator); - defer decode_results.deinit(allocator); - - // Reusable token output slice - var gen_token_slice: zml.Slice = try .alloc(allocator, .init(.{ .s = 1 }, .u32)); - defer gen_token_slice.free(allocator); - gen_token_slice.items(u32)[0] = prev_token; - var gen_token_buffer: zml.Buffer = try .fromSlice(io, platform, gen_token_slice); - defer gen_token_buffer.deinit(); - - // Reusable combined embed buffer - var combined_embed_buffer: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = 1, .d = config.dim }, .bf16), .{}); - defer combined_embed_buffer.deinit(); - - const n_audio: usize = @intCast(adapter_seq_len); - log.info("Running decoder decode ({d} steps)...", .{n_audio - tokens.len}); - - for (tokens.len..n_audio) |pos| { - // Check EOS (token_id=2) - if (prev_token == 2) break; - - // Print decoded text for the current token - if (try tokenizer_decoder.next(prev_token)) |chunk| { - try stdout.interface.print("{d}: {s}\n", .{pos, chunk}); - try stdout.interface.flush(); - } - - // Create audio embed [1, d] from adapter_out_cpu[pos] - const pos_offset = pos * d; - const audio_embed_slice: zml.Slice = .init( - .init(.{ .s = 1, .d = config.dim }, .bf16), - std.mem.sliceAsBytes(adapter_out_cpu[pos_offset .. pos_offset + d]), - ); - var audio_embed_buf: zml.Buffer = try .fromSlice(io, platform, audio_embed_slice); - defer audio_embed_buf.deinit(); - - // Create token [1] buffer - const token_data = [_]u32{prev_token}; - const tok_slice: zml.Slice = .init( - .init(.{ .s = 1 }, .u32), - std.mem.sliceAsBytes(&token_data), - ); - var tok_buf: zml.Buffer = try .fromSlice(io, platform, tok_slice); - defer tok_buf.deinit(); - - // Run decode embedder: adapter_out[pos] + tok_embed(prev_token) → [1, d] - demb_args.set(.{ &embedder_buffers, audio_embed_buf, tok_buf }); - compiled_decode_embedder.call(demb_args, &demb_results); - demb_results.fill(.{&combined_embed_buffer}); - - // Run decoder: combined_embed, token_index=pos, kv_cache, t_cond → next token - const pos_data = [_]u32{@intCast(pos)}; - const pos_slice: zml.Slice = .init(.init(.{}, .u32), std.mem.sliceAsBytes(&pos_data)); - var pos_buf: zml.Buffer = try .fromSlice(io, platform, pos_slice); - defer pos_buf.deinit(); - - decode_args.set(.{ &decoder_buffers, combined_embed_buffer, pos_buf, &kv_cache_buffers, t_cond_buffer }); - compiled_model.decode.call(decode_args, &decode_results); - decode_results.fill(.{ &gen_token_buffer, &kv_cache_buffers }); - - // Download generated token - try gen_token_buffer.toSlice(io, gen_token_slice); - prev_token = gen_token_slice.items(u32)[0]; - } - - // Print final token - if (try tokenizer_decoder.next(prev_token)) |chunk| { - try stdout.interface.print("{s}", .{chunk}); - } - - try stdout.interface.writeAll("\n"); - try stdout.interface.flush(); - - log.info("Decode complete.", .{}); } @@ -445,12 +287,12 @@ fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { return samples; } -pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram) !zml.Exe { - return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{audio_len}, .f32).withTags(.{.samples})}); +pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, padded_audio_len: usize) !zml.Exe { + return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{padded_audio_len}, .f32).withTags(.{.samples})}); } -pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder) !zml.Exe { - const num_frames = audio_len / model.config.audio().hop_length; +pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, padded_audio_len: usize) !zml.Exe { + const num_frames = padded_audio_len / model.config.audio().hop_length; return try platform.compile(allocator, io, model, .forward, .{ Tensor.init(.{ .channels = 128, .time = num_frames }, .f32), From f5b9372bbef35dc8d68ef65d796fc6c8763d4381 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 13 Feb 2026 14:05:27 +0100 Subject: [PATCH 55/98] test embedder --- examples/voxtral/decoder.zig | 7 ++++-- examples/voxtral/voxtral.zig | 44 ++++++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 483fa8d7..4e1ab43e 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -75,7 +75,10 @@ pub const Embedder = struct { }; } - pub fn forward(self: Embedder, audio_embeds: Tensor, text_tokens: Tensor) Tensor { + pub fn forward(self: Embedder, full_adapter_out: Tensor, text_tokens: Tensor, pos: Tensor) Tensor { + const audio_embeds = full_adapter_out.dynamicSlice(.{ + .s = Tensor.DynSlice{ .start = pos, .len = @intCast(text_tokens.dim(.s)) }, + }); const text_embeds = self.tok_embeddings.gather(.{ .vocab = text_tokens }, .{}); return text_embeds.add(audio_embeds); } @@ -377,7 +380,7 @@ pub const SelfAttention = struct { var attn_mask = zml.nn.causalAttnMask(.{ .q = full_seq_len, .k = full_seq_len }, dtype, config.sliding_window); attn_mask = attn_mask.gatherSlices( Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), - pos_index.rename(.{ .s = .coord }), + token_index.reshape(.{ .coord = 1 }), .{}, ); const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 09bca30b..ff055e78 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -134,8 +134,8 @@ pub fn main() !void { var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model, audio_len}); var compiled_encoder_future = try io.concurrent(compileEncoder, .{allocator, io, platform, encoder_model, audio_len}); var compiled_adapter_future = try io.concurrent(compileAdapter, .{allocator, io, platform, adapter, encoder_seq_len, config}); - var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, @as(u32, @intCast(tokens.len)), config}); - var compiled_decode_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, @as(u32, 1), config}); + var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len, @as(u32, @intCast(tokens.len)), config}); + var compiled_decode_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len, @as(u32, 1), config}); var compiled_decoder_future = try io.concurrent(compileDecoder, .{allocator, io, platform, model, model_params}); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); @@ -232,6 +232,12 @@ pub fn main() !void { enc_args.set(.{ &encoder_buffers, mel_output }); compiled_encoder.call(enc_args, &enc_results); enc_results.fill(.{&encoder_output}); + + + const output = try encoder_output.toSliceAlloc(allocator, io); + defer output.free(allocator); + + std.debug.print("{any}\n", .{output.items(u32)[0..100]}); } defer encoder_output.deinit(); log.info("Encoder done.", .{}); @@ -255,6 +261,35 @@ pub fn main() !void { } defer adapter_output.deinit(); log.info("Adapter done.", .{}); + + // 4. Embedder - combine audio embeddings with token embeddings + log.info("Running embedder...", .{}); + var embedder_output: zml.Buffer = undefined; + { + var emb_args = try compiled_embedder.args(allocator); + defer emb_args.deinit(allocator); + var emb_results = try compiled_embedder.results(allocator); + defer emb_results.deinit(allocator); + + const token_slice: zml.Slice = .init( + .init(.{@as(u32, @intCast(tokens.len))}, .u32), + std.mem.sliceAsBytes(tokens), + ); + var token_buffer: zml.Buffer = try .fromSlice(io, platform, token_slice); + defer token_buffer.deinit(); + + var pos_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); + defer pos_buffer.deinit(); + + embedder_output = try .uninitialized(io, platform, + .init(.{ .s = @as(u32, @intCast(tokens.len)), .d = config.dim }, .bf16), .{}); + + emb_args.set(.{ &embedder_buffers, adapter_output, token_buffer, pos_buffer }); + compiled_embedder.call(emb_args, &emb_results); + emb_results.fill(.{&embedder_output}); + } + defer embedder_output.deinit(); + log.info("Embedder done.", .{}); } @@ -305,10 +340,11 @@ pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const }); } -pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, seq_len: u32, config: Config) !zml.Exe { +pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, adapter_seq_len_: u32, seq_len: u32, config: Config) !zml.Exe { return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .s = seq_len, .d = config.dim }, .bf16), + Tensor.init(.{ .s = adapter_seq_len_, .d = config.dim }, .bf16), Tensor.init(.{ .s = seq_len }, .u32), + Tensor.init(.{}, .u32), }); } From f2d79dfcb901d0ee8ba795b2b27a9e30d5a82938 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 13 Feb 2026 15:34:35 +0100 Subject: [PATCH 56/98] decode loop test --- examples/voxtral/decoder.zig | 7 ++- examples/voxtral/voxtral.zig | 118 ++++++++++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 4e1ab43e..3539eb26 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -175,12 +175,13 @@ pub const Decoder = struct { /// input_embeds: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] /// Returns: (tokens [s] u32, KvCache) pub fn forward(self: Decoder, input_embeds: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor) struct { Tensor, KvCache } { + const t = t_cond.convert(input_embeds.dtype()); var h = input_embeds; var cache = kv_cache; for (self.layers, 0..) |layer, i| { const layer_cache = cache.atLayer(i); - const result = layer.forward(h, token_index, layer_cache, t_cond, self.config); + const result = layer.forward(h, token_index, layer_cache, t, self.config); h = result[0]; const updated_layer_cache = result[1]; cache = updated_layer_cache.reuseBuffer(layer_cache); @@ -188,8 +189,8 @@ pub const Decoder = struct { h = rmsNorm(h, self.norm, self.norm_eps); - // Compute logits via tied embeddings and take argmax - const output_logits = h.dot(self.tok_embeddings.convert(h.dtype()), .d); + // Compute logits in f32 for precision (matching Python reference) + const output_logits = h.convert(.f32).dot(self.tok_embeddings.convert(.f32), .d); const output_tokens = output_logits.argMax(.vocab).indices.convert(.u32); return .{ output_tokens, cache }; } diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index ff055e78..3c0a59d6 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -72,7 +72,6 @@ pub fn main() !void { const wav_file = try loadWav(allocator, &reader.interface); defer allocator.free(wav_file); - // Pad audio for offline streaming mode (matching mistral_common AudioEncoder.pad) const left_pad = n_left_pad_tokens * raw_audio_length_per_tok; // 32 * 1280 = 40960 const align_pad = (raw_audio_length_per_tok - (wav_file.len % raw_audio_length_per_tok)) % raw_audio_length_per_tok; const right_pad = align_pad + n_right_pad_tokens * raw_audio_length_per_tok; // align + 17 * 1280 @@ -127,7 +126,7 @@ pub fn main() !void { .h = config.n_kv_heads, .hd = config.head_dim, }, dtype)), - .t_cond = Tensor.init(.{.d = config.dim}, dtype), + .t_cond = Tensor.init(.{.d = config.dim}, .f32), }; var tokenizer_future = try io.concurrent(loadTokenizer, .{allocator, io, model_dir}); @@ -290,6 +289,121 @@ pub fn main() !void { } defer embedder_output.deinit(); log.info("Embedder done.", .{}); + + // 5. Init KV cache + t_cond buffer + var kv_cache_buffers = try model_params.kv_cache.initBuffer(io, platform); + defer dec.KvCache.deinitBuffer(&kv_cache_buffers); + + const t_cond_data = computeTimeEmbedding(allocator, @as(f32, @floatFromInt(n_delay_tokens)), config.dim); + defer allocator.free(t_cond_data); + const t_cond_slice: zml.Slice = .init( + .init(.{config.dim}, .f32), + std.mem.sliceAsBytes(t_cond_data), + ); + var t_cond_buffer: zml.Buffer = try .fromSlice(io, platform, t_cond_slice); + defer t_cond_buffer.deinit(); + std.debug.print("t_cond first 10 values: ", .{}); + for (0..@min(10, t_cond_data.len)) |j| { + std.debug.print("{d:.4} ", .{t_cond_data[j]}); + } + std.debug.print("\n", .{}); + + // 6. Decoder prefill + log.info("Running decoder prefill...", .{}); + var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); + defer generated_token_slice.free(allocator); + { + var prefill_args = try compiled_model.prefill.args(allocator); + defer prefill_args.deinit(allocator); + var prefill_results = try compiled_model.prefill.results(allocator); + defer prefill_results.deinit(allocator); + + var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); + defer token_index_buffer.deinit(); + + var prefill_tokens_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, @intCast(tokens.len))}, .u32)); + defer prefill_tokens_slice.free(allocator); + var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); + defer prefill_output.deinit(); + + prefill_args.set(.{ &decoder_buffers, embedder_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); + compiled_model.prefill.call(prefill_args, &prefill_results); + prefill_results.fill(.{ &prefill_output, &kv_cache_buffers }); + + // Extract last token from prefill output (first generated token) + try prefill_output.toSlice(io, prefill_tokens_slice); + log.info("Prefill output tokens: {any}", .{prefill_tokens_slice.items(u32)[0..tokens.len]}); + generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[tokens.len - 1]; + log.info("First generated token (from prefill): {}", .{generated_token_slice.items(u32)[0]}); + } + log.info("Prefill done.", .{}); + + // 7. Autoregressive decode loop + log.info("Running decoder autoregressive...", .{}); + const eos_token: u32 = 2; + { + var decode_args = try compiled_model.decode.args(allocator); + defer decode_args.deinit(allocator); + var decode_results = try compiled_model.decode.results(allocator); + defer decode_results.deinit(allocator); + + var emb_args = try compiled_decode_embedder.args(allocator); + defer emb_args.deinit(allocator); + var emb_results = try compiled_decode_embedder.results(allocator); + defer emb_results.deinit(allocator); + + var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); + defer current_token_buffer.deinit(); + + var decode_embed: zml.Buffer = try .uninitialized(io, platform, + .init(.{ .s = @as(u32, 1), .d = config.dim }, .bf16), .{}); + defer decode_embed.deinit(); + + const max_pos = adapter_seq_len; + log.info("Decode loop: tokens.len={}, max_pos={}", .{tokens.len, max_pos}); + var num_generated: usize = 0; + for (tokens.len..max_pos) |i| { + const generated_token = generated_token_slice.items(u32)[0]; + num_generated += 1; + + if (generated_token != 32) { + if (try tokenizer_decoder.next(generated_token)) |chunk| { + std.debug.print("{s}", .{chunk}); + } + } + + + if (generated_token == eos_token) { + log.info(" EOS reached at pos={}", .{i}); + break; + } + + // 1. Embedder: combine adapter_output[pos] + tok_embed(token) + const token_slice: zml.Slice = .init(.init(.{@as(u32, 1)}, .u32), + std.mem.sliceAsBytes(generated_token_slice.items(u32)[0..1])); + var token_buf: zml.Buffer = try .fromSlice(io, platform, token_slice); + defer token_buf.deinit(); + + var pos_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(i)), .u32); + defer pos_buffer.deinit(); + + emb_args.set(.{ &embedder_buffers, adapter_output, token_buf, pos_buffer }); + compiled_decode_embedder.call(emb_args, &emb_results); + emb_results.fill(.{&decode_embed}); + + // 2. Decoder: get next token + var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(i)), .u32); + defer token_index_buffer.deinit(); + + decode_args.set(.{ &decoder_buffers, decode_embed, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); + compiled_model.decode.call(decode_args, &decode_results); + decode_results.fill(.{ ¤t_token_buffer, &kv_cache_buffers }); + + try current_token_buffer.toSlice(io, generated_token_slice); + } + std.debug.print("\n", .{}); + log.info("Decode done. Generated {} tokens total.", .{num_generated}); + } } From c3f9e1ecc2a9f0c08eba0245bf7fa7ee57baeaeb Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 13 Feb 2026 16:43:41 +0100 Subject: [PATCH 57/98] Some cleanup --- examples/voxtral/BUILD.bazel | 2 + examples/voxtral/common.zig | 54 +++++++++ examples/voxtral/decoder.zig | 157 +++++++-------------------- examples/voxtral/encoder.zig | 75 ++++--------- examples/voxtral/mel_spectrogram.zig | 26 +---- examples/voxtral/test.zig | 31 ------ examples/voxtral/voxtral.zig | 125 ++++++--------------- examples/voxtral/wav.zig | 38 +++++-- 8 files changed, 178 insertions(+), 330 deletions(-) create mode 100644 examples/voxtral/common.zig diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel index 049e2f8a..d7c1da2a 100644 --- a/examples/voxtral/BUILD.bazel +++ b/examples/voxtral/BUILD.bazel @@ -10,6 +10,7 @@ zig_binary( "config.zig", "encoder.zig", "decoder.zig", + "common.zig", ], extra_srcs = [ "//examples/voxtral/assets:voxtral_mel_filter", @@ -26,6 +27,7 @@ zig_binary( "config.zig", "encoder.zig", "decoder.zig", + "common.zig", ], extra_srcs = [ "//examples/voxtral/assets:voxtral_mel_filter", diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig new file mode 100644 index 00000000..c445c373 --- /dev/null +++ b/examples/voxtral/common.zig @@ -0,0 +1,54 @@ +const std = @import("std"); +const log = std.log; + +const zml = @import("zml"); +const Tensor = zml.Tensor; + +pub fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init( + store.createTensorWithTags("weight", .{ .dout, .d }), + store.maybeCreateTensorWithTags("bias", .{.dout}), + .d, + ); +} + +pub fn deinitLinear(l: *zml.Bufferized(zml.nn.Linear)) void { + l.weight.deinit(); + if (l.bias) |*b| b.deinit(); +} + +pub fn rmsNorm(x: Tensor, weight: Tensor, eps: f32) Tensor { + return zml.nn.rmsNorm(x, .d, eps).mul(weight.broad(x.shape())); +} + +pub fn loadModel( + comptime T: type, + self: *const T, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + comptime label: []const u8, +) !zml.Bufferized(T) { + progress.increaseEstimatedTotalItems(store.view().count()); + var timer: std.time.Timer = try .start(); + var total_bytes: usize = 0; + defer { + const took = timer.read(); + log.info("Loaded " ++ label ++ " weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + total_bytes, + took, + total_bytes / took * std.time.ns_per_s, + }); + } + + return zml.io.load(T, self, allocator, io, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .parallelism = 16, + .total_bytes = &total_bytes, + }); +} diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 3539eb26..42361eb1 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const log = std.log; const zml = @import("zml"); const Tensor = zml.Tensor; @@ -10,7 +9,9 @@ const Config = cfg.Config; const enc = @import("encoder.zig"); const SwiGluFfn = enc.SwiGluFfn; -const rmsNorm = enc.rmsNorm; + +const common = @import("common.zig"); +const rmsNorm = common.rmsNorm; /// Adapter: encoder→decoder projection. /// Reshapes encoder output [s, d=1280] → [s/4, d=5120] (concatenate 4 timesteps), @@ -22,37 +23,18 @@ pub const Adapter = struct { pub fn init(store: zml.io.TensorStore.View) Adapter { const proj_store = store.withPrefix("mm_streams_embeddings.embedding_module.audio_language_projection"); return .{ - .proj0 = linear(proj_store.withLayer(0)), - .proj1 = linear(proj_store.withLayer(2)), + .proj0 = common.linear(proj_store.withLayer(0)), + .proj1 = common.linear(proj_store.withLayer(2)), }; } - pub fn load(self: *const Adapter, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Adapter){ - progress.increaseEstimatedTotalItems(store.view().count()); - var timer: std.time.Timer = try .start(); - var total_bytes: usize = 0; - defer { - const took = timer.read(); - log.info("Loaded adapter weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ - total_bytes, - took, - total_bytes / took * std.time.ns_per_s, - }); - } - - return zml.io.load(Adapter, self, allocator, io, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, - .progress = progress, - .store = store, - .parallelism = 16, - .total_bytes = &total_bytes, - }); + pub fn load(self: *const Adapter, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Adapter) { + return common.loadModel(Adapter, self, allocator, io, platform, store, progress, "adapter"); } pub fn unloadBuffers(self: *zml.Bufferized(Adapter)) void { - deinitLinear(&self.proj0); - deinitLinear(&self.proj1); + common.deinitLinear(&self.proj0); + common.deinitLinear(&self.proj1); } /// encoder_out: [s, d=1280] → [s/4, d=3072] @@ -84,26 +66,7 @@ pub const Embedder = struct { } pub fn load(self: *const Embedder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Embedder) { - progress.increaseEstimatedTotalItems(store.view().count()); - var timer: std.time.Timer = try .start(); - var total_bytes: usize = 0; - defer { - const took = timer.read(); - log.info("Loaded embedder weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ - total_bytes, - took, - total_bytes / took * std.time.ns_per_s, - }); - } - - return zml.io.load(Embedder, self, allocator, io, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, - .progress = progress, - .store = store, - .parallelism = 16, - .total_bytes = &total_bytes, - }); + return common.loadModel(Embedder, self, allocator, io, platform, store, progress, "embedder"); } pub fn unloadBuffers(self: *zml.Bufferized(Embedder)) void { @@ -114,6 +77,7 @@ pub const Embedder = struct { /// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. pub const Decoder = struct { + /// Same weights as Embedder.tok_embeddings; duplicated here for logit computation. tok_embeddings: Tensor, layers: []DecoderLayer, norm: Tensor, @@ -140,26 +104,7 @@ pub const Decoder = struct { } pub fn load(self: *const Decoder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Decoder) { - progress.increaseEstimatedTotalItems(store.view().count()); - var timer: std.time.Timer = try .start(); - var total_bytes: usize = 0; - defer { - const took = timer.read(); - log.info("Loaded decoder weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ - total_bytes, - took, - total_bytes / took * std.time.ns_per_s, - }); - } - - return zml.io.load(Decoder, self, allocator, io, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, - .progress = progress, - .store = store, - .parallelism = 16, - .total_bytes = &total_bytes, - }); + return common.loadModel(Decoder, self, allocator, io, platform, store, progress, "decoder"); } pub fn unloadBuffers(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { @@ -205,14 +150,14 @@ pub const AdaRmsNorm = struct { pub fn init(store: zml.io.TensorStore.View) AdaRmsNorm { const ada_store = store.withPrefix("ada_rms_norm_t_cond"); return .{ - .down = linear(ada_store.withLayer(0)), - .up = linear(ada_store.withLayer(2)), + .down = common.linear(ada_store.withLayer(0)), + .up = common.linear(ada_store.withLayer(2)), }; } pub fn unloadBuffers(self: *zml.Bufferized(AdaRmsNorm)) void { - deinitLinear(&self.down); - deinitLinear(&self.up); + common.deinitLinear(&self.down); + common.deinitLinear(&self.up); } /// h: [s, d], t_cond: [d] → [s, d] @@ -269,37 +214,23 @@ pub const KvCache = struct { } pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { - const k_shape = self.k.shape().drop(.layer); - var layer = self.layer_index; - layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; - - return if (token_index) |idx| .{ - .k = self.k.scatterSlices( - .{ .layer = layer, .k = idx }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer, .k = idx }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - } else .{ - .k = self.k.scatterSlices( - .{ .layer = layer }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), + return .{ + .k = scatterCache(self.k, new_k, self.layer_index, token_index), + .v = scatterCache(self.v, new_v, self.layer_index, token_index), .layer_index = self.layer_index, }; } + fn scatterCache(cache: Tensor, new: Tensor, layer_index: Tensor, token_index: ?Tensor) Tensor { + const k_shape = cache.shape().drop(.layer); + const converted = new.convert(cache.dtype()).transpose(k_shape); + const scatter_opts: Tensor.ScatterOpts = .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }; + return if (token_index) |idx| + cache.scatterSlices(.{ .layer = layer_index.broad(idx.shape()), .k = idx }, converted, scatter_opts).reuseBuffer(cache) + else + cache.scatterSlices(.{ .layer = layer_index }, converted, scatter_opts).reuseBuffer(cache); + } + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { return .{ .k = self.k, @@ -326,18 +257,18 @@ pub const SelfAttention = struct { pub fn init(store: zml.io.TensorStore.View) SelfAttention { return .{ - .wq = linear(store.withPrefix("wq")), - .wk = linear(store.withPrefix("wk")), - .wv = linear(store.withPrefix("wv")), - .wo = linear(store.withPrefix("wo")), + .wq = common.linear(store.withPrefix("wq")), + .wk = common.linear(store.withPrefix("wk")), + .wv = common.linear(store.withPrefix("wv")), + .wo = common.linear(store.withPrefix("wo")), }; } pub fn unloadBuffers(self: *zml.Bufferized(SelfAttention)) void { - deinitLinear(&self.wq); - deinitLinear(&self.wk); - deinitLinear(&self.wv); - deinitLinear(&self.wo); + common.deinitLinear(&self.wq); + common.deinitLinear(&self.wk); + common.deinitLinear(&self.wv); + common.deinitLinear(&self.wo); } /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) @@ -441,17 +372,3 @@ pub const DecoderLayer = struct { } }; -// -- Helpers - -fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { - return .init( - store.createTensorWithTags("weight", .{ .dout, .d }), - store.maybeCreateTensorWithTags("bias", .{.dout}), - .d, - ); -} - -fn deinitLinear(l: *zml.Bufferized(zml.nn.Linear)) void { - l.weight.deinit(); - if (l.bias) |*b| b.deinit(); -} diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index e49b0f07..0503b164 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -1,14 +1,14 @@ const std = @import("std"); -const builtin = @import("builtin"); -const log = std.log; const zml = @import("zml"); const Tensor = zml.Tensor; -const Shape = zml.Shape; const cfg = @import("config.zig"); const Config = cfg.Config; +const common = @import("common.zig"); +const rmsNorm = common.rmsNorm; + pub const Encoder = struct { conv0: CausalConv1d, conv1: CausalConv1d, @@ -41,27 +41,8 @@ pub const Encoder = struct { allocator.free(self.layers); } - pub fn load(self: *const Encoder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Encoder){ - progress.increaseEstimatedTotalItems(store.view().count()); - var timer: std.time.Timer = try .start(); - var total_bytes: usize = 0; - defer { - const took = timer.read(); - log.info("Loaded encoder weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ - total_bytes, - took, - total_bytes / took * std.time.ns_per_s, - }); - } - - return zml.io.load(Encoder, self, allocator, io, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, - .progress = progress, - .store = store, - .parallelism = 16, - .total_bytes = &total_bytes, - }); + pub fn load(self: *const Encoder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Encoder) { + return common.loadModel(Encoder, self, allocator, io, platform, store, progress, "encoder"); } pub fn unloadBuffers(self: *zml.Bufferized(Encoder), allocator: std.mem.Allocator) void { @@ -201,18 +182,18 @@ pub const SelfAttention = struct { pub fn init(store: zml.io.TensorStore.View) SelfAttention { return .{ - .wq = initLinear(store.withPrefix("wq")), - .wk = initLinear(store.withPrefix("wk")), - .wv = initLinear(store.withPrefix("wv")), - .wo = initLinear(store.withPrefix("wo")), + .wq = common.linear(store.withPrefix("wq")), + .wk = common.linear(store.withPrefix("wk")), + .wv = common.linear(store.withPrefix("wv")), + .wo = common.linear(store.withPrefix("wo")), }; } pub fn unloadBuffers(self: *zml.Bufferized(SelfAttention)) void { - deinitLinear(&self.wq); - deinitLinear(&self.wk); - deinitLinear(&self.wv); - deinitLinear(&self.wo); + common.deinitLinear(&self.wq); + common.deinitLinear(&self.wk); + common.deinitLinear(&self.wv); + common.deinitLinear(&self.wo); } /// x: [s, d] -> [s, d] @@ -262,16 +243,16 @@ pub const SwiGluFfn = struct { pub fn init(store: zml.io.TensorStore.View) SwiGluFfn { return .{ - .w1 = initLinear(store.withPrefix("w1")), - .w2 = initLinear(store.withPrefix("w2")), - .w3 = initLinear(store.withPrefix("w3")), + .w1 = common.linear(store.withPrefix("w1")), + .w2 = common.linear(store.withPrefix("w2")), + .w3 = common.linear(store.withPrefix("w3")), }; } pub fn unloadBuffers(self: *zml.Bufferized(SwiGluFfn)) void { - deinitLinear(&self.w1); - deinitLinear(&self.w2); - deinitLinear(&self.w3); + common.deinitLinear(&self.w1); + common.deinitLinear(&self.w2); + common.deinitLinear(&self.w3); } /// x: [s, d] -> [s, d] @@ -282,21 +263,3 @@ pub const SwiGluFfn = struct { } }; -// -- Helpers - -fn initLinear(store: zml.io.TensorStore.View) zml.nn.Linear { - return .init( - store.createTensorWithTags("weight", .{ .dout, .d }), - store.maybeCreateTensorWithTags("bias", .{.dout}), - .d, - ); -} - -fn deinitLinear(l: *zml.Bufferized(zml.nn.Linear)) void { - l.weight.deinit(); - if (l.bias) |*b| b.deinit(); -} - -pub fn rmsNorm(x: Tensor, weight: Tensor, eps: f32) Tensor { - return zml.nn.rmsNorm(x, .d, eps).mul(weight.broad(x.shape())); -} diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig index 21c9febf..06640219 100644 --- a/examples/voxtral/mel_spectrogram.zig +++ b/examples/voxtral/mel_spectrogram.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const log = std.log; const cfg = @import("config.zig"); const Config = cfg.Config; @@ -16,8 +15,6 @@ pub const LogMelSpectrogram = struct { hop_len: u63, global_log_mel_max: f32, - // force_num_frames: u32 = 3000, - precision: zml.DataType = .f32, pub fn init(config: Config) LogMelSpectrogram { @@ -37,20 +34,6 @@ pub const LogMelSpectrogram = struct { const window_weight = self.window.getWeights(self.n_fft, dtype); const fft_len = window_weight.dim(.samples); const num_frames: u63 = @intCast(@divFloor(waveform.dim(.samples), self.hop_len)); - - // var wav = waveform; - // const force_num_frames = self.force_num_frames; - // const force_num_samples = force_num_frames * self.hop_len; - // if (num_frames > force_num_frames) { - // wav = wav.slice1d(-1, .{ .end = force_num_samples }); - // num_frames = force_num_frames; - // } else if (num_frames < force_num_frames) { - // const tagged = wav.withTags(.{.t}); - // wav = tagged.pad(0.0, .{ .t = Tensor.Pad{ .high = force_num_samples - tagged.dim(.t) } }); - // } - - // num_frames = @intCast(@divFloor(wav.dim(0), self.hop_len)); - // std.debug.assert(num_frames == force_num_frames); // Reflect padding const padded_wav = blk: { @@ -95,11 +78,7 @@ pub const LogMelSpectrogram = struct { pub fn stft(waveform: Tensor, weight: Tensor, num_frames: usize, stride: u63, precision: zml.DataType) Tensor { const fft_len = weight.dim(0); - const num_samples = waveform.dim(0); - - // const num_frames = 3000; const indices = Tensor.arange(.{ .end = @intCast(num_frames * stride), .step = stride }, .i32); - std.log.warn("num_samples: {d}, num_frames: {d}, end: {d}, stride: {d}, indices: {}", .{ num_samples, num_frames, num_frames * stride, stride, indices }); var windows = waveform.gatherSlices(.{fft_len}, indices.appendAxes(.{.coord }), .{ .indices_are_sorted = true }); windows = windows.mul(weight.broadcastLeft(windows.shape())); @@ -107,10 +86,7 @@ pub fn stft(waveform: Tensor, weight: Tensor, num_frames: usize, stride: u63, pr var fft = windows.convert(precision).fft(.{ .kind = .RFFT, .length = &.{fft_len} }); const spectrogram = fft.abs(); - const ret = spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{.frames, .freq_bins}); - log.info("{f}", .{ret.shape()}); - - return ret; + return spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{.frames, .freq_bins}); } pub const AudioWindow = enum { diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 509562ee..27e44bb9 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -98,37 +98,6 @@ pub fn main() !void { try zml.testing.testLayer(allocator, io, platform, conv_stem, .forward, ref_store.view(), "conv_stem", conv_buffers, .{}); } - - // -- Transformer Layer tests - // { - // const transformer_store = model_store.view().withPrefix(encoder_prefix).withPrefix("transformer").withPrefix("layers"); - // const enc_cfg = config.encoder(); - - // inline for (0..32) |i| { - // if (i < enc_cfg.n_layers) { - // const layer = TransformerLayerTestHarness{ - // .inner = enc.TransformerLayer.init(transformer_store.withLayer(i), config), - // .config = config, - // }; - - // var layer_buffers = try zml.io.load(TransformerLayerTestHarness, &layer, allocator, io, platform, load_opts); - // defer TransformerLayerTestHarness.unloadBuffers(&layer_buffers); - - // try zml.testing.testLayer(allocator, io, platform, layer, .forward, ref_store.view(), std.fmt.comptimePrint("encoder.layers.{d}", .{i}), layer_buffers, .{.minimum_close_fraction = 0.9}); - // } - // } - // } - - // -- Encoder test - // { - // const encoder_model = Encoder.init(allocator, model_store.view().withPrefix(encoder_prefix), config); - // defer encoder_model.deinit(allocator); - // var encoder_buffers = try zml.io.load(Encoder, &encoder_model, allocator, io, platform, load_opts); - // defer Encoder.unloadBuffers(&encoder_buffers, allocator); - - // const harness = EncoderTestHarness{ .inner = encoder_model }; - // try zml.testing.testLayer(allocator, io, platform, harness, .forward, ref_store.view(), "encoder", .{ .inner = encoder_buffers }, .{}); - // } // -- Adapter test { diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 3c0a59d6..4892ea41 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -69,7 +69,7 @@ pub fn main() !void { var wav_buffer: [4096]u8 = undefined; var reader = file.reader(io, &wav_buffer); - const wav_file = try loadWav(allocator, &reader.interface); + const wav_file = try wav_utils.loadWav(allocator, &reader.interface); defer allocator.free(wav_file); const left_pad = n_left_pad_tokens * raw_audio_length_per_tok; // 32 * 1280 = 40960 @@ -163,9 +163,9 @@ pub fn main() !void { var compiled_decode_embedder = try compiled_decode_embedder_future.await(io); defer compiled_decode_embedder.deinit(); - var compiled_model = try compiled_decoder_future.await(io); - defer compiled_model.prefill.deinit(); - defer compiled_model.decode.deinit(); + const compiled_prefill, const compiled_decoder = try compiled_decoder_future.await(io); + errdefer compiled_prefill.deinit(); + errdefer compiled_decoder.deinit(); // -- Buffers @@ -231,12 +231,6 @@ pub fn main() !void { enc_args.set(.{ &encoder_buffers, mel_output }); compiled_encoder.call(enc_args, &enc_results); enc_results.fill(.{&encoder_output}); - - - const output = try encoder_output.toSliceAlloc(allocator, io); - defer output.free(allocator); - - std.debug.print("{any}\n", .{output.items(u32)[0..100]}); } defer encoder_output.deinit(); log.info("Encoder done.", .{}); @@ -255,8 +249,6 @@ pub fn main() !void { adp_args.set(.{ &adapter_buffers, encoder_output }); compiled_adapter.call(adp_args, &adp_results); adp_results.fill(.{&adapter_output}); - - std.debug.print("{}\n", .{adapter_output.shape()}); } defer adapter_output.deinit(); log.info("Adapter done.", .{}); @@ -302,20 +294,17 @@ pub fn main() !void { ); var t_cond_buffer: zml.Buffer = try .fromSlice(io, platform, t_cond_slice); defer t_cond_buffer.deinit(); - std.debug.print("t_cond first 10 values: ", .{}); - for (0..@min(10, t_cond_data.len)) |j| { - std.debug.print("{d:.4} ", .{t_cond_data[j]}); - } - std.debug.print("\n", .{}); // 6. Decoder prefill log.info("Running decoder prefill...", .{}); var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); defer generated_token_slice.free(allocator); { - var prefill_args = try compiled_model.prefill.args(allocator); + defer compiled_prefill.deinit(); + + var prefill_args = try compiled_prefill.args(allocator); defer prefill_args.deinit(allocator); - var prefill_results = try compiled_model.prefill.results(allocator); + var prefill_results = try compiled_prefill.results(allocator); defer prefill_results.deinit(allocator); var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); @@ -327,7 +316,7 @@ pub fn main() !void { defer prefill_output.deinit(); prefill_args.set(.{ &decoder_buffers, embedder_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); - compiled_model.prefill.call(prefill_args, &prefill_results); + compiled_prefill.call(prefill_args, &prefill_results); prefill_results.fill(.{ &prefill_output, &kv_cache_buffers }); // Extract last token from prefill output (first generated token) @@ -342,9 +331,11 @@ pub fn main() !void { log.info("Running decoder autoregressive...", .{}); const eos_token: u32 = 2; { - var decode_args = try compiled_model.decode.args(allocator); + defer compiled_decoder.deinit(); + + var decode_args = try compiled_decoder.args(allocator); defer decode_args.deinit(allocator); - var decode_results = try compiled_model.decode.results(allocator); + var decode_results = try compiled_decoder.results(allocator); defer decode_results.deinit(allocator); var emb_args = try compiled_decode_embedder.args(allocator); @@ -396,7 +387,7 @@ pub fn main() !void { defer token_index_buffer.deinit(); decode_args.set(.{ &decoder_buffers, decode_embed, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); - compiled_model.decode.call(decode_args, &decode_results); + compiled_decoder.call(decode_args, &decode_results); decode_results.fill(.{ ¤t_token_buffer, &kv_cache_buffers }); try current_token_buffer.toSlice(io, generated_token_slice); @@ -406,36 +397,6 @@ pub fn main() !void { } } - -fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - - const arena = arena_state.allocator(); - var sample_list: std.ArrayList(u8) = .empty; - - const wav_fmt = try wav_utils.readPcmWav(arena, reader, &sample_list); - const byte_per_sample = wav_fmt.bits_per_sample / 8; - const sample_count = sample_list.items.len / (byte_per_sample * wav_fmt.num_channels); - - const samples = try allocator.alloc(f32, sample_count); - for (0..sample_count) |i| { - const offset = i * byte_per_sample * wav_fmt.num_channels; - - const sample = switch(byte_per_sample) { - 1 => (@as(f32, @floatFromInt(std.mem.bytesToValue(u8, sample_list.items[offset .. offset + 1]))) - 128.0) / 128.0, - 2 => @as(f32, @floatFromInt(std.mem.bytesToValue(i16, sample_list.items[offset .. offset + 2]))) / 32768.0, - 3 => @as(f32, @floatFromInt(std.mem.bytesToValue(i24, sample_list.items[offset .. offset + 3]))) / 8388608.0, - 4 => @as(f32, @floatFromInt(std.mem.bytesToValue(i32, sample_list.items[offset .. offset + 4]))) / 2147483648.0, - else => unreachable, - }; - - samples[i] = sample; - } - - return samples; -} - pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, padded_audio_len: usize) !zml.Exe { return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{padded_audio_len}, .f32).withTags(.{.samples})}); } @@ -462,11 +423,6 @@ pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *cons }); } -const CompileModelResult = struct { - prefill: zml.Exe, - decode: zml.Exe, -}; - const VoxtralParameters = struct { prefill_embeds: Tensor, decode_embeds: Tensor, @@ -475,34 +431,37 @@ const VoxtralParameters = struct { t_cond: Tensor, }; -pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, params: VoxtralParameters) !CompileModelResult { - var prefill_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, params_: VoxtralParameters) !zml.Exe { +pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, params: VoxtralParameters) !struct { zml.Exe, zml.Exe} { + const compile = struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, embeds_: Tensor, params_: VoxtralParameters) !zml.Exe { return try platform_.compile(allocator_, io_, model_, .forward, .{ - params_.prefill_embeds, + embeds_, params_.token_index, params_.kv_cache, params_.t_cond, }); } - }.call, .{allocator, io, platform, model, params}); - - var decode_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, params_: VoxtralParameters) !zml.Exe { - return try platform_.compile(allocator_, io_, model_, .forward, .{ - params_.decode_embeds, - params_.token_index, - params_.kv_cache, - params_.t_cond, - }); - } - }.call, .{allocator, io, platform, model, params}); + }.call; + var prefill_future = try io.concurrent(compile, .{ allocator, io, platform, model, params.prefill_embeds, params }); + var decode_future = try io.concurrent(compile, .{ allocator, io, platform, model, params.decode_embeds, params }); - const prefill_exe = try prefill_future.await(io); - const decode_exe = try decode_future.await(io); + return .{ + try prefill_future.await(io), + try decode_future.await(io), + }; +} - return .{ .prefill = prefill_exe, .decode = decode_exe }; +pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + + return try .fromBytes(allocator, io, bytes); } /// Sinusoidal time embedding: encodes a scalar t into a [dim]-dimensional vector. @@ -520,15 +479,3 @@ fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32) [] } return result; } - -pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { - const bytes = b: { - const file = try dir.openFile(io, "tokenizer.json", .{}); - defer file.close(io); - var reader = file.reader(io, &.{}); - break :b try reader.interface.readAlloc(allocator, try file.length(io)); - }; - defer allocator.free(bytes); - - return try .fromBytes(allocator, io, bytes); -} diff --git a/examples/voxtral/wav.zig b/examples/voxtral/wav.zig index a4683939..89f4cd89 100644 --- a/examples/voxtral/wav.zig +++ b/examples/voxtral/wav.zig @@ -37,7 +37,7 @@ pub const Chunk = struct { }; }; -pub fn readPcmWav(allocator: std.mem.Allocator, reader: *std.Io.Reader, out: *std.ArrayList(u8)) !Chunk.Fmt { +fn readPcmWav(allocator: std.mem.Allocator, reader: *std.Io.Reader, out: *std.ArrayList(u8)) !Chunk.Fmt { std.debug.assert((try reader.takeStruct(Chunk.Header, .little)).id == .riff); std.debug.assert((try reader.takeStruct(Chunk.Riff, .little)).format == .wave); var fmt: ?Chunk.Fmt = null; @@ -62,11 +62,31 @@ pub fn readPcmWav(allocator: std.mem.Allocator, reader: *std.Io.Reader, out: *st return fmt.?; } -// pub fn writePcmWav(writer: anytype, fmt: Chunk.Fmt, data: []const u8) !void { -// try writer.writeStruct(Chunk.Header{ .id = .riff, .size = @sizeOf(Chunk.Riff) }); -// try writer.writeStruct(Chunk.Riff{ .format = .wave }); -// try writer.writeStruct(Chunk.Header{ .id = .fmt, .size = @sizeOf(Chunk.Fmt) }); -// try writer.writeStruct(fmt); -// try writer.writeStruct(Chunk.Header{ .id = .data, .size = @intCast(data.len) }); -// try writer.writeAll(data); -// } +pub fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + + const arena = arena_state.allocator(); + var sample_list: std.ArrayList(u8) = .empty; + + const wav_fmt = try readPcmWav(arena, reader, &sample_list); + const byte_per_sample = wav_fmt.bits_per_sample / 8; + const sample_count = sample_list.items.len / (byte_per_sample * wav_fmt.num_channels); + + const samples = try allocator.alloc(f32, sample_count); + for (0..sample_count) |i| { + const offset = i * byte_per_sample * wav_fmt.num_channels; + + const sample = switch(byte_per_sample) { + 1 => (@as(f32, @floatFromInt(std.mem.bytesToValue(u8, sample_list.items[offset .. offset + 1]))) - 128.0) / 128.0, + 2 => @as(f32, @floatFromInt(std.mem.bytesToValue(i16, sample_list.items[offset .. offset + 2]))) / 32768.0, + 3 => @as(f32, @floatFromInt(std.mem.bytesToValue(i24, sample_list.items[offset .. offset + 3]))) / 8388608.0, + 4 => @as(f32, @floatFromInt(std.mem.bytesToValue(i32, sample_list.items[offset .. offset + 4]))) / 2147483648.0, + else => unreachable, + }; + + samples[i] = sample; + } + + return samples; +} From eb6d66e0a834e913040bee79e0024d73d82337d8 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 13 Feb 2026 17:25:40 +0100 Subject: [PATCH 58/98] better naming and formatting --- examples/voxtral/BUILD.bazel | 4 +- examples/voxtral/common.zig | 1 + examples/voxtral/config.zig | 2 + examples/voxtral/decoder.zig | 36 ++- examples/voxtral/encoder.zig | 22 +- examples/voxtral/main.zig | 201 +++++++++++++ examples/voxtral/mel_spectrogram.zig | 4 + examples/voxtral/test.zig | 22 +- examples/voxtral/voxtral.zig | 406 ++++++++++----------------- examples/voxtral/wav.zig | 21 ++ 10 files changed, 435 insertions(+), 284 deletions(-) create mode 100644 examples/voxtral/main.zig diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel index d7c1da2a..a7bcf6f6 100644 --- a/examples/voxtral/BUILD.bazel +++ b/examples/voxtral/BUILD.bazel @@ -3,8 +3,9 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( name = "voxtral", - main = "voxtral.zig", + main = "main.zig", srcs = [ + "voxtral.zig", "mel_spectrogram.zig", "wav.zig", "config.zig", @@ -22,6 +23,7 @@ zig_binary( name = "voxtral_test", main = "test.zig", srcs = [ + "voxtral.zig", "mel_spectrogram.zig", "wav.zig", "config.zig", diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index c445c373..ef3cd466 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -32,6 +32,7 @@ pub fn loadModel( comptime label: []const u8, ) !zml.Bufferized(T) { progress.increaseEstimatedTotalItems(store.view().count()); + var timer: std.time.Timer = try .start(); var total_bytes: usize = 0; defer { diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index b852342a..9c7798e4 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -72,9 +72,11 @@ pub const Config = struct { pub fn parseConfig(allocator: std.mem.Allocator, io: std.Io, model_dir: std.Io.Dir) !std.json.Parsed(Config) { const config_file = try model_dir.openFile(io, "params.json", .{}); // it is named params not config here defer config_file.close(io); + var buffer: [4096]u8 = undefined; var file_reader = config_file.reader(io, &buffer); var reader: std.json.Reader = .init(allocator, &file_reader.interface); defer reader.deinit(); + return try std.json.parseFromTokenSource(Config, allocator, &reader, .{ .ignore_unknown_fields = true }); } diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 42361eb1..8053836c 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -22,6 +22,7 @@ pub const Adapter = struct { pub fn init(store: zml.io.TensorStore.View) Adapter { const proj_store = store.withPrefix("mm_streams_embeddings.embedding_module.audio_language_projection"); + return .{ .proj0 = common.linear(proj_store.withLayer(0)), .proj1 = common.linear(proj_store.withLayer(2)), @@ -32,7 +33,7 @@ pub const Adapter = struct { return common.loadModel(Adapter, self, allocator, io, platform, store, progress, "adapter"); } - pub fn unloadBuffers(self: *zml.Bufferized(Adapter)) void { + pub fn unload(self: *zml.Bufferized(Adapter)) void { common.deinitLinear(&self.proj0); common.deinitLinear(&self.proj1); } @@ -61,7 +62,9 @@ pub const Embedder = struct { const audio_embeds = full_adapter_out.dynamicSlice(.{ .s = Tensor.DynSlice{ .start = pos, .len = @intCast(text_tokens.dim(.s)) }, }); + const text_embeds = self.tok_embeddings.gather(.{ .vocab = text_tokens }, .{}); + return text_embeds.add(audio_embeds); } @@ -69,7 +72,7 @@ pub const Embedder = struct { return common.loadModel(Embedder, self, allocator, io, platform, store, progress, "embedder"); } - pub fn unloadBuffers(self: *zml.Bufferized(Embedder)) void { + pub fn unload(self: *zml.Bufferized(Embedder)) void { self.tok_embeddings.deinit(); } }; @@ -86,6 +89,7 @@ pub const Decoder = struct { pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Decoder { const layers = allocator.alloc(DecoderLayer, config.n_layers) catch unreachable; + for (layers, 0..) |*layer, i| { layer.* = DecoderLayer.init(store.withPrefix("layers").withLayer(i), config); } @@ -107,11 +111,13 @@ pub const Decoder = struct { return common.loadModel(Decoder, self, allocator, io, platform, store, progress, "decoder"); } - pub fn unloadBuffers(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { + pub fn unload(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { self.tok_embeddings.deinit(); + for (self.layers) |*layer| { - DecoderLayer.unloadBuffers(layer); + DecoderLayer.unload(layer); } + allocator.free(self.layers); self.norm.deinit(); } @@ -129,6 +135,7 @@ pub const Decoder = struct { const result = layer.forward(h, token_index, layer_cache, t, self.config); h = result[0]; const updated_layer_cache = result[1]; + cache = updated_layer_cache.reuseBuffer(layer_cache); } @@ -137,6 +144,7 @@ pub const Decoder = struct { // Compute logits in f32 for precision (matching Python reference) const output_logits = h.convert(.f32).dot(self.tok_embeddings.convert(.f32), .d); const output_tokens = output_logits.argMax(.vocab).indices.convert(.u32); + return .{ output_tokens, cache }; } }; @@ -155,7 +163,7 @@ pub const AdaRmsNorm = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(AdaRmsNorm)) void { + pub fn unload(self: *zml.Bufferized(AdaRmsNorm)) void { common.deinitLinear(&self.down); common.deinitLinear(&self.up); } @@ -164,6 +172,7 @@ pub const AdaRmsNorm = struct { pub fn forward(self: AdaRmsNorm, h: Tensor, t_cond: Tensor) Tensor { const ada_scale = self.up.forward(self.down.forward(t_cond).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); const one_plus_scale = ada_scale.broad(h.shape()).add(Tensor.scalar(1.0, h.dtype()).broad(h.shape())); + return h.mul(one_plus_scale); } }; @@ -225,6 +234,7 @@ pub const KvCache = struct { const k_shape = cache.shape().drop(.layer); const converted = new.convert(cache.dtype()).transpose(k_shape); const scatter_opts: Tensor.ScatterOpts = .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }; + return if (token_index) |idx| cache.scatterSlices(.{ .layer = layer_index.broad(idx.shape()), .k = idx }, converted, scatter_opts).reuseBuffer(cache) else @@ -264,7 +274,7 @@ pub const SelfAttention = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttention)) void { + pub fn unload(self: *zml.Bufferized(SelfAttention)) void { common.deinitLinear(&self.wq); common.deinitLinear(&self.wk); common.deinitLinear(&self.wv); @@ -295,6 +305,7 @@ pub const SelfAttention = struct { .layout = .interleaved, .freq_base = config.rope_theta, }; + q = zml.nn.rope(q, pos_index, rope_opts); k = zml.nn.rope(k, pos_index, rope_opts); @@ -315,6 +326,7 @@ pub const SelfAttention = struct { token_index.reshape(.{ .coord = 1 }), .{}, ); + const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); // Merge heads and output projection: [q, h, hd] → [s, d] @@ -343,12 +355,14 @@ pub const DecoderLayer = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { + pub fn unload(self: *zml.Bufferized(DecoderLayer)) void { self.attention_norm.deinit(); - SelfAttention.unloadBuffers(&self.attention); + SelfAttention.unload(&self.attention); + self.ffn_norm.deinit(); - SwiGluFfn.unloadBuffers(&self.feed_forward); - AdaRmsNorm.unloadBuffers(&self.ada_norm); + SwiGluFfn.unload(&self.feed_forward); + + AdaRmsNorm.unload(&self.ada_norm); } /// h: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] → ([s, d], KvCache) @@ -360,12 +374,14 @@ pub const DecoderLayer = struct { kv_cache, config, ); + var out = h.add(attn_out); // Pre-FFN norm → ada conditioning → FFN → residual const normed = rmsNorm(out, self.ffn_norm, self.norm_eps); const conditioned = self.ada_norm.forward(normed, t_cond); const ffn_out = self.feed_forward.forward(conditioned); + out = out.add(ffn_out); return .{ out, updated_kv_cache }; diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 0503b164..7f457319 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -45,12 +45,12 @@ pub const Encoder = struct { return common.loadModel(Encoder, self, allocator, io, platform, store, progress, "encoder"); } - pub fn unloadBuffers(self: *zml.Bufferized(Encoder), allocator: std.mem.Allocator) void { - CausalConv1d.unloadBuffers(&self.conv0); - CausalConv1d.unloadBuffers(&self.conv1); + pub fn unload(self: *zml.Bufferized(Encoder), allocator: std.mem.Allocator) void { + CausalConv1d.unload(&self.conv0); + CausalConv1d.unload(&self.conv1); for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); + TransformerLayer.unload(layer); } allocator.free(self.layers); @@ -103,7 +103,7 @@ pub const CausalConv1d = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(CausalConv1d)) void { + pub fn unload(self: *zml.Bufferized(CausalConv1d)) void { self.weight.deinit(); self.bias.deinit(); } @@ -153,11 +153,12 @@ pub const TransformerLayer = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + pub fn unload(self: *zml.Bufferized(TransformerLayer)) void { self.attention_norm.deinit(); - SelfAttention.unloadBuffers(&self.attention); + SelfAttention.unload(&self.attention); + self.ffn_norm.deinit(); - SwiGluFfn.unloadBuffers(&self.feed_forward); + SwiGluFfn.unload(&self.feed_forward); } /// h: [s, d] -> [s, d] @@ -189,7 +190,7 @@ pub const SelfAttention = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttention)) void { + pub fn unload(self: *zml.Bufferized(SelfAttention)) void { common.deinitLinear(&self.wq); common.deinitLinear(&self.wk); common.deinitLinear(&self.wv); @@ -249,7 +250,7 @@ pub const SwiGluFfn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SwiGluFfn)) void { + pub fn unload(self: *zml.Bufferized(SwiGluFfn)) void { common.deinitLinear(&self.w1); common.deinitLinear(&self.w2); common.deinitLinear(&self.w3); @@ -259,6 +260,7 @@ pub const SwiGluFfn = struct { pub fn forward(self: SwiGluFfn, x: Tensor) Tensor { const gate = self.w1.forward(x).silu(); const up = self.w3.forward(x); + return self.w2.forward(gate.mul(up).rename(.{ .dout = .d })).rename(.{ .dout = .d }); } }; diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig new file mode 100644 index 00000000..828db49f --- /dev/null +++ b/examples/voxtral/main.zig @@ -0,0 +1,201 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const log = std.log; +const wav_utils = @import("wav.zig"); +const cfg = @import("config.zig"); +const Config = cfg.Config; + +const zml = @import("zml"); +const stdx = zml.stdx; +const Tensor = zml.Tensor; + +const mel = @import("mel_spectrogram.zig"); +const LogMelSpectrogram = mel.LogMelSpectrogram; + +const enc = @import("encoder.zig"); +const Encoder = enc.Encoder; + +const dec = @import("decoder.zig"); +const Decoder = dec.Decoder; +const Adapter = dec.Adapter; +const Embedder = dec.Embedder; + +const voxtral = @import("voxtral.zig"); + +const CliArgs = struct { + input: []const u8, + model: []const u8, +}; + +const tokens_array = [_]u32{ 1, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 }; +const tokens: []const u32 = &tokens_array; +const n_delay_tokens = 6; +const n_left_pad_tokens = 32; +const n_right_pad_tokens = (n_delay_tokens + 1) + 10; // 17 +const raw_audio_length_per_tok = 1280; // SAMPLE_RATE / FRAME_RATE = 16000 / 12.5 + +pub fn main() !void { + log.info("Start of Voxtral", .{}); + + var dbg = std.heap.DebugAllocator(.{}).init; + defer if (builtin.mode == .Debug) std.debug.assert(dbg.deinit() == .ok); + + const allocator = switch (builtin.mode) { + .Debug => dbg.allocator(), + else => std.heap.c_allocator, + }; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const args = stdx.flags.parseProcessArgs(CliArgs); + + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + defer vfs.deinit(); + + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); + defer vfs_file.deinit(); + try vfs.register("file", vfs_file.io()); + + const io = vfs.io(); + + var progress = std.Progress.start(io, .{ .root_name = "Voxtral" }); + + const file = try std.Io.Dir.openFile(.cwd(), io, args.input, .{}); + defer file.close(io); + + var wav_buffer: [4096]u8 = undefined; + var reader = file.reader(io, &wav_buffer); + + const left_pad = n_left_pad_tokens * raw_audio_length_per_tok; + const padded_wav = try wav_utils.loadAndPadWav(allocator, &reader.interface, left_pad, n_right_pad_tokens, raw_audio_length_per_tok); + defer allocator.free(padded_wav); + const audio_len = padded_wav.len; + + const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); + + var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); + defer model_registry.deinit(); + + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); + defer model_store.deinit(); + + var parsed_config = try cfg.parseConfig(allocator, io, model_dir); + defer parsed_config.deinit(); + const config = parsed_config.value; + + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); + + const num_frames = audio_len / config.audio().hop_length; + const encoder_seq_len: u32 = @intCast((num_frames / 2) - (num_frames / 2) % config.downsample_factor()); + const adapter_seq_len = encoder_seq_len / config.downsample_factor(); + + var melspectro_model: LogMelSpectrogram = .init(config); + const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; + var encoder_model: Encoder = .init(allocator, model_store.view().withPrefix(encoder_prefix), config); + defer encoder_model.deinit(allocator); + + const adapter: Adapter = .init(model_store.view()); + const embedder: Embedder = .init(model_store.view()); + + var model: Decoder = .init(allocator, model_store.view(), config); + defer model.deinit(allocator); + + const dtype = model.tok_embeddings.dtype(); + const model_params: voxtral.VoxtralParameters = .{ + .prefill_embeds = .init(.{ .s = @as(u32, @intCast(tokens.len)), .d = config.dim }, .bf16), + .decode_embeds = .init(.{ .s = 1, .d = config.dim }, .bf16), + .token_index = .init(.{}, .u32), + .kv_cache = .init(.init(.{ + .layer = model.layers.len, + .k = config.sliding_window, + .h = config.n_kv_heads, + .hd = config.head_dim, + }, dtype)), + .t_cond = Tensor.init(.{ .d = config.dim }, .f32), + }; + + // Launch concurrent compilation and buffer loading + var tokenizer_future = try io.concurrent(voxtral.loadTokenizer, .{ allocator, io, model_dir, &progress }); + var compiled_mel_spectrum_future = try io.concurrent(voxtral.compileMelSpectrum, .{ allocator, io, platform, melspectro_model, audio_len, &progress }); + var compiled_encoder_future = try io.concurrent(voxtral.compileEncoder, .{ allocator, io, platform, encoder_model, audio_len, &progress }); + var compiled_adapter_future = try io.concurrent(voxtral.compileAdapter, .{ allocator, io, platform, adapter, encoder_seq_len, config, &progress }); + var compiled_embedder_future = try io.concurrent(voxtral.compileEmbedder, .{ allocator, io, platform, embedder, adapter_seq_len, @as(u32, @intCast(tokens.len)), config, &progress }); + var compiled_decode_embedder_future = try io.concurrent(voxtral.compileEmbedder, .{ allocator, io, platform, embedder, adapter_seq_len, @as(u32, 1), config, &progress }); + var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, model, model_params, &progress }); + + var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{ &melspectro_model, io, platform }); + var encoder_buffers_future = try io.concurrent(Encoder.load, .{ &encoder_model, allocator, io, platform, &model_store, &progress }); + var adapter_buffers_future = try io.concurrent(Adapter.load, .{ &adapter, allocator, io, platform, &model_store, &progress }); + var embedder_buffers_future = try io.concurrent(Embedder.load, .{ &embedder, allocator, io, platform, &model_store, &progress }); + var decoder_buffers_future = try io.concurrent(Decoder.load, .{ &model, allocator, io, platform, &model_store, &progress }); + + // Await tokenizer + var tokenizer = try tokenizer_future.await(io); + defer tokenizer.deinit(); + + // -- Compiled models + var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); + defer compiled_mel_spectrum.deinit(); + + var compiled_encoder = try compiled_encoder_future.await(io); + defer compiled_encoder.deinit(); + + var compiled_adapter = try compiled_adapter_future.await(io); + defer compiled_adapter.deinit(); + + var compiled_embedder = try compiled_embedder_future.await(io); + defer compiled_embedder.deinit(); + + var compiled_decode_embedder = try compiled_decode_embedder_future.await(io); + defer compiled_decode_embedder.deinit(); + + var compiled_prefill, var compiled_decoder = try compiled_decoder_future.await(io); + defer compiled_prefill.deinit(); + defer compiled_decoder.deinit(); + + // -- Buffers + var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); + defer LogMelSpectrogram.unload(&mel_spectrum_buffers); + + var encoder_buffers = try encoder_buffers_future.await(io); + defer Encoder.unload(&encoder_buffers, allocator); + + var adapter_buffers = try adapter_buffers_future.await(io); + defer Adapter.unload(&adapter_buffers); + + var embedder_buffers = try embedder_buffers_future.await(io); + defer Embedder.unload(&embedder_buffers); + + var decoder_buffers = try decoder_buffers_future.await(io); + defer Decoder.unload(&decoder_buffers, allocator); + + progress.end(); + + try voxtral.runPipeline( + allocator, + io, + platform, + config, + &tokenizer, + padded_wav, + audio_len, + tokens, + n_delay_tokens, + &compiled_mel_spectrum, + &compiled_encoder, + &compiled_adapter, + &compiled_embedder, + &compiled_decode_embedder, + &compiled_prefill, + &compiled_decoder, + &mel_spectrum_buffers, + &encoder_buffers, + &adapter_buffers, + &embedder_buffers, + &decoder_buffers, + model_params, + ); +} diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig index 06640219..e492c9d5 100644 --- a/examples/voxtral/mel_spectrogram.zig +++ b/examples/voxtral/mel_spectrogram.zig @@ -19,6 +19,7 @@ pub const LogMelSpectrogram = struct { pub fn init(config: Config) LogMelSpectrogram { const audio = config.audio(); + return .{ .window = .hann, .mel_filters = Tensor.init(.{201, 128}, .f32).withTags(.{.freq_bins, .mel}), @@ -48,6 +49,7 @@ pub const LogMelSpectrogram = struct { var spectrogram = stft(padded_wav, window_weight, num_frames + 1, self.hop_len, self.precision); spectrogram = spectrogram.slice1d(.frames, .{ .end = -1 }); spectrogram = spectrogram.convert(dtype); + // Re-weight frequencies for speech spectrogram = spectrogram.dot(self.mel_filters, .freq_bins); @@ -100,9 +102,11 @@ pub const AudioWindow = enum { .boxcar => Tensor.constant(dtype.one()).withTags(.{.samples}), .hann => { if (len <= 1) return Tensor.constant(dtype.one()); + const flen: f64 = @floatFromInt(len); const freq = Tensor.constant(dtype.constant(std.math.pi / flen)); const steps = Tensor.arange(.{ .start = -len, .end = len, .step = 2 }, dtype); + return steps.mul(freq).cos().scale(0.5).addConstant(0.5).convert(dtype).withTags(.{.samples}); }, }; diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 27e44bb9..79bd2037 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -94,7 +94,7 @@ pub fn main() !void { .conv1 = enc.CausalConv1d.init(conv_store.withLayer(1).withPrefix("conv"), 2), }; var conv_buffers = try zml.io.load(ConvStemTestHarness, &conv_stem, allocator, io, platform, load_opts); - defer ConvStemTestHarness.unloadBuffers(&conv_buffers); + defer ConvStemTestHarness.unload(&conv_buffers); try zml.testing.testLayer(allocator, io, platform, conv_stem, .forward, ref_store.view(), "conv_stem", conv_buffers, .{}); } @@ -105,7 +105,7 @@ pub fn main() !void { .inner = dec.Adapter.init(model_store.view()), }; var adapter_buffers = try zml.io.load(AdapterTestHarness, &adapter, allocator, io, platform, load_opts); - defer AdapterTestHarness.unloadBuffers(&adapter_buffers); + defer AdapterTestHarness.unload(&adapter_buffers); try zml.testing.testLayer(allocator, io, platform, adapter, .forward, ref_store.view(), "adapter", adapter_buffers, .{.minimum_close_fraction = 0.8}); } @@ -136,7 +136,7 @@ pub fn main() !void { }; var inner_buffers = try zml.io.load(dec.DecoderLayer, &layer.inner, allocator, io, platform, load_opts); - defer dec.DecoderLayer.unloadBuffers(&inner_buffers); + defer dec.DecoderLayer.unload(&inner_buffers); const layer_buffers: zml.Bufferized(DecoderLayerTestHarness) = .{ .inner = inner_buffers, @@ -165,9 +165,9 @@ const ConvStemTestHarness = struct { conv0: enc.CausalConv1d, conv1: enc.CausalConv1d, - pub fn unloadBuffers(self: *zml.Bufferized(ConvStemTestHarness)) void { - enc.CausalConv1d.unloadBuffers(&self.conv0); - enc.CausalConv1d.unloadBuffers(&self.conv1); + pub fn unload(self: *zml.Bufferized(ConvStemTestHarness)) void { + enc.CausalConv1d.unload(&self.conv0); + enc.CausalConv1d.unload(&self.conv1); } pub fn forward(self: ConvStemTestHarness, mel_input: Tensor) Tensor { @@ -186,8 +186,8 @@ const TransformerLayerTestHarness = struct { inner: enc.TransformerLayer, config: Config, - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayerTestHarness)) void { - enc.TransformerLayer.unloadBuffers(&self.inner); + pub fn unload(self: *zml.Bufferized(TransformerLayerTestHarness)) void { + enc.TransformerLayer.unload(&self.inner); } pub fn forward(self: TransformerLayerTestHarness, h: Tensor) Tensor { @@ -207,8 +207,8 @@ const EncoderTestHarness = struct { const AdapterTestHarness = struct { inner: dec.Adapter, - pub fn unloadBuffers(self: *zml.Bufferized(AdapterTestHarness)) void { - dec.Adapter.unloadBuffers(&self.inner); + pub fn unload(self: *zml.Bufferized(AdapterTestHarness)) void { + dec.Adapter.unload(&self.inner); } pub fn forward(self: AdapterTestHarness, encoder_out: Tensor) Tensor { @@ -232,6 +232,7 @@ const DecoderLayerTestHarness = struct { .h = @as(i64, @intCast(self.config.n_kv_heads)), .hd = @as(i64, @intCast(self.config.head_dim)), }, dtype); + const kv_cache = dec.KvCache{ .k = Tensor.zeroes(kv_shape), .v = Tensor.zeroes(kv_shape), @@ -240,6 +241,7 @@ const DecoderLayerTestHarness = struct { const token_index = Tensor.scalar(0, .u32); const result = self.inner.forward(h_typed, token_index, kv_cache, self.t_cond.convert(dtype), self.config); + return result[0].convert(.f32); } }; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 4892ea41..01059d38 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -1,12 +1,9 @@ const std = @import("std"); -const builtin = @import("builtin"); const log = std.log; -const wav_utils = @import("wav.zig"); const cfg = @import("config.zig"); const Config = cfg.Config; const zml = @import("zml"); -const stdx = zml.stdx; const Tensor = zml.Tensor; const mel = @import("mel_spectrogram.zig"); @@ -20,178 +17,169 @@ const Decoder = dec.Decoder; const Adapter = dec.Adapter; const Embedder = dec.Embedder; -const CliArgs = struct { - input: []const u8, - model: []const u8, +pub const VoxtralParameters = struct { + prefill_embeds: Tensor, + decode_embeds: Tensor, + token_index: Tensor, + kv_cache: dec.KvCache, + t_cond: Tensor, }; -const prompt = ""; -const tokens_array = [_]u32{1, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32}; -const tokens: []const u32 = &tokens_array; -const n_delay_tokens = 6; -const n_left_pad_tokens = 32; -const n_right_pad_tokens = (n_delay_tokens + 1) + 10; // 17 -const raw_audio_length_per_tok = 1280; // SAMPLE_RATE / FRAME_RATE = 16000 / 12.5 +pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, padded_audio_len: usize, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling mel spectrogram...", 1); + defer node.end(); + + return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{padded_audio_len}, .f32).withTags(.{.samples})}); +} -pub fn main() !void { - log.info("Start of Voxtral", .{}); +pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, padded_audio_len: usize, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling encoder...", 1); + defer node.end(); + + const num_frames = padded_audio_len / model.config.audio().hop_length; + + return try platform.compile(allocator, io, model, .forward, .{ + Tensor.init(.{ .channels = 128, .time = num_frames }, .f32), + }); +} - var dbg = std.heap.DebugAllocator(.{}).init; - defer if (builtin.mode == .Debug) std.debug.assert(dbg.deinit() == .ok); +pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, encoder_seq_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + + var node = progress.start("Compiling adapter...", 1); + defer node.end(); + + return try platform.compile(allocator, io, model, .forward, .{ + Tensor.init(.{ .s = encoder_seq_len, .d = config.encoder().dim }, .bf16), + }); +} - const allocator = switch (builtin.mode) { - .Debug => dbg.allocator(), - else => std.heap.c_allocator, +pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, adapter_seq_len_: u32, seq_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + + var node = progress.start("Compiling embedder...", 1); + defer node.end(); + + return try platform.compile(allocator, io, model, .forward, .{ + Tensor.init(.{ .s = adapter_seq_len_, .d = config.dim }, .bf16), + Tensor.init(.{ .s = seq_len }, .u32), + Tensor.init(.{}, .u32), + }); +} + +pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, params: VoxtralParameters, progress: *std.Progress.Node) !struct { zml.Exe, zml.Exe } { + const compilePrefill = struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, embeds_: Tensor, params_: VoxtralParameters, progress_: *std.Progress.Node) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + + var node_ = progress_.start("Compiling decoder prefill...", 1); + defer node_.end(); + + return try platform_.compile(allocator_, io_, model_, .forward, .{ + embeds_, + params_.token_index, + params_.kv_cache, + params_.t_cond, + }); + } + }.call; + + const compileDecode = struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, embeds_: Tensor, params_: VoxtralParameters, progress_: *std.Progress.Node) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + + var node_ = progress_.start("Compiling decoder decode...", 1); + defer node_.end(); + + return try platform_.compile(allocator_, io_, model_, .forward, .{ + embeds_, + params_.token_index, + params_.kv_cache, + params_.t_cond, + }); + } + }.call; + + var prefill_future = try io.concurrent(compilePrefill, .{ allocator, io, platform, model, params.prefill_embeds, params, progress }); + var decode_future = try io.concurrent(compileDecode, .{ allocator, io, platform, model, params.decode_embeds, params, progress }); + + return .{ + try prefill_future.await(io), + try decode_future.await(io), }; +} - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - const args = stdx.flags.parseProcessArgs(CliArgs); - - var vfs: zml.io.VFS = try .init(allocator, threaded.io()); - defer vfs.deinit(); - - var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); - defer vfs_file.deinit(); - try vfs.register("file", vfs_file.io()); - - const io = vfs.io(); +pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { + progress.increaseEstimatedTotalItems(1); - var progress = std.Progress.start(io, .{ .root_name = "Voxtral" }); - - const arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - const file = try std.Io.Dir.openFile(.cwd(), io, args.input, .{}); - defer file.close(io); - - var wav_buffer: [4096]u8 = undefined; - var reader = file.reader(io, &wav_buffer); - - const wav_file = try wav_utils.loadWav(allocator, &reader.interface); - defer allocator.free(wav_file); - - const left_pad = n_left_pad_tokens * raw_audio_length_per_tok; // 32 * 1280 = 40960 - const align_pad = (raw_audio_length_per_tok - (wav_file.len % raw_audio_length_per_tok)) % raw_audio_length_per_tok; - const right_pad = align_pad + n_right_pad_tokens * raw_audio_length_per_tok; // align + 17 * 1280 - const audio_len = left_pad + wav_file.len + right_pad; - - const padded_audio = try allocator.alloc(f32, audio_len); - defer allocator.free(padded_audio); - - @memset(padded_audio, 0); - @memcpy(padded_audio[left_pad .. left_pad + wav_file.len], wav_file); - - const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); - - var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); - defer model_registry.deinit(); + var node = progress.start("Loading tokenizer...", 1); + defer node.end(); - var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); - defer model_store.deinit(); + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + return try .fromBytes(allocator, io, bytes); +} + +/// Sinusoidal time embedding: encodes a scalar t into a [dim]-dimensional vector. +/// out[i] = cos(t * inv_freq[i]) for i < half_dim +/// out[i] = sin(t * inv_freq[i-half_dim]) for i >= half_dim +/// where inv_freq[i] = exp(-log(theta) * i / half_dim) +pub fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32) []f32 { + const half_dim = dim / 2; + const result = allocator.alloc(f32, dim) catch unreachable; - var parsed_config = try cfg.parseConfig(allocator, io, model_dir); - defer parsed_config.deinit(); - const config = parsed_config.value; + for (0..half_dim) |i| { + const freq = @exp(-@log(@as(f32, 10000.0)) * @as(f32, @floatFromInt(i)) / @as(f32, @floatFromInt(half_dim))); + const angle = t_value * freq; + result[i] = @cos(angle); + result[i + half_dim] = @sin(angle); + } + + return result; +} - var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); - log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); +pub fn runPipeline( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + config: Config, + tokenizer: *zml.tokenizer.Tokenizer, + padded_audio: []const f32, + audio_len: usize, + tokens: []const u32, + n_delay_tokens: u32, + compiled_mel_spectrum: *zml.Exe, + compiled_encoder: *zml.Exe, + compiled_adapter: *zml.Exe, + compiled_embedder: *zml.Exe, + compiled_decode_embedder: *zml.Exe, + compiled_prefill: *zml.Exe, + compiled_decoder: *zml.Exe, + mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), + encoder_buffers: *zml.Bufferized(Encoder), + adapter_buffers: *zml.Bufferized(Adapter), + embedder_buffers: *zml.Bufferized(Embedder), + decoder_buffers: *zml.Bufferized(Decoder), + model_params: VoxtralParameters, +) !void { + log.info("Running inference pipeline...", .{}); const num_frames = audio_len / config.audio().hop_length; const encoder_seq_len: u32 = @intCast((num_frames / 2) - (num_frames / 2) % config.downsample_factor()); const adapter_seq_len = encoder_seq_len / config.downsample_factor(); - - var melspectro_model: LogMelSpectrogram = .init(config); - const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; - var encoder_model: Encoder = .init(allocator, model_store.view().withPrefix(encoder_prefix), config); - defer encoder_model.deinit(allocator); - - const adapter: Adapter = .init(model_store.view()); - const embedder: Embedder = .init(model_store.view()); - - var model: Decoder = .init(allocator, model_store.view(), config); - defer model.deinit(allocator); - - const dtype = model.tok_embeddings.dtype(); - const model_params: VoxtralParameters = .{ - .prefill_embeds = .init(.{.s = @as(u32, @intCast(tokens.len)), .d = config.dim}, .bf16), - .decode_embeds = .init(.{.s = 1, .d = config.dim}, .bf16), - .token_index = .init(.{}, .u32), - .kv_cache = .init(.init(.{ - .layer = model.layers.len, - .k = config.sliding_window, - .h = config.n_kv_heads, - .hd = config.head_dim, - }, dtype)), - .t_cond = Tensor.init(.{.d = config.dim}, .f32), - }; - - var tokenizer_future = try io.concurrent(loadTokenizer, .{allocator, io, model_dir}); - var compiled_mel_spectrum_future = try io.concurrent(compileMelSpectrum, .{allocator, io, platform, melspectro_model, audio_len}); - var compiled_encoder_future = try io.concurrent(compileEncoder, .{allocator, io, platform, encoder_model, audio_len}); - var compiled_adapter_future = try io.concurrent(compileAdapter, .{allocator, io, platform, adapter, encoder_seq_len, config}); - var compiled_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len, @as(u32, @intCast(tokens.len)), config}); - var compiled_decode_embedder_future = try io.concurrent(compileEmbedder, .{allocator, io, platform, embedder, adapter_seq_len, @as(u32, 1), config}); - var compiled_decoder_future = try io.concurrent(compileDecoder, .{allocator, io, platform, model, model_params}); - - var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); - var encoder_buffers_future = try io.concurrent(Encoder.load, .{&encoder_model,allocator,io, platform, &model_store, &progress}); - var adapter_buffers_future = try io.concurrent(Adapter.load, .{&adapter, allocator, io, platform, &model_store, &progress}); - var embedder_buffers_future = try io.concurrent(Embedder.load, .{&embedder, allocator, io, platform, &model_store, &progress}); - var decoder_buffers_future = try io.concurrent(Decoder.load, .{&model, allocator, io, platform, &model_store, &progress}); - - var tokenizer = try tokenizer_future.await(io); - defer tokenizer.deinit(); - - // -- Compiled models - - var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); - defer compiled_mel_spectrum.deinit(); - - var compiled_encoder = try compiled_encoder_future.await(io); - defer compiled_encoder.deinit(); - - var compiled_adapter = try compiled_adapter_future.await(io); - defer compiled_adapter.deinit(); - - var compiled_embedder = try compiled_embedder_future.await(io); - defer compiled_embedder.deinit(); - - var compiled_decode_embedder = try compiled_decode_embedder_future.await(io); - defer compiled_decode_embedder.deinit(); - - const compiled_prefill, const compiled_decoder = try compiled_decoder_future.await(io); - errdefer compiled_prefill.deinit(); - errdefer compiled_decoder.deinit(); - - // -- Buffers - - var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); - defer LogMelSpectrogram.unload(&mel_spectrum_buffers); - - var encoder_buffers = try encoder_buffers_future.await(io); - defer Encoder.unloadBuffers(&encoder_buffers, allocator); - - var adapter_buffers = try adapter_buffers_future.await(io); - defer Adapter.unloadBuffers(&adapter_buffers); - - var embedder_buffers = try embedder_buffers_future.await(io); - defer Embedder.unloadBuffers(&embedder_buffers); - - var decoder_buffers = try decoder_buffers_future.await(io); - defer Decoder.unloadBuffers(&decoder_buffers, allocator); - - progress.end(); - - // --- Execution pipeline --- - log.info("Running inference pipeline...", .{}); var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); - + // 1. Mel spectrogram log.info("Running mel spectrogram...", .{}); var mel_output: zml.Buffer = undefined; @@ -210,7 +198,7 @@ pub fn main() !void { mel_output = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = num_frames }, .f32), .{}); - mel_args.set(.{ &mel_spectrum_buffers, audio_buffer }); + mel_args.set(.{ mel_spectrum_buffers, audio_buffer }); compiled_mel_spectrum.call(mel_args, &mel_results); mel_results.fill(.{&mel_output}); } @@ -228,7 +216,7 @@ pub fn main() !void { encoder_output = try .uninitialized(io, platform, .init(.{ .s = encoder_seq_len, .d = config.encoder().dim }, .bf16), .{}); - enc_args.set(.{ &encoder_buffers, mel_output }); + enc_args.set(.{ encoder_buffers, mel_output }); compiled_encoder.call(enc_args, &enc_results); enc_results.fill(.{&encoder_output}); } @@ -246,7 +234,7 @@ pub fn main() !void { adapter_output = try .uninitialized(io, platform, .init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16), .{}); - adp_args.set(.{ &adapter_buffers, encoder_output }); + adp_args.set(.{ adapter_buffers, encoder_output }); compiled_adapter.call(adp_args, &adp_results); adp_results.fill(.{&adapter_output}); } @@ -275,7 +263,7 @@ pub fn main() !void { embedder_output = try .uninitialized(io, platform, .init(.{ .s = @as(u32, @intCast(tokens.len)), .d = config.dim }, .bf16), .{}); - emb_args.set(.{ &embedder_buffers, adapter_output, token_buffer, pos_buffer }); + emb_args.set(.{ embedder_buffers, adapter_output, token_buffer, pos_buffer }); compiled_embedder.call(emb_args, &emb_results); emb_results.fill(.{&embedder_output}); } @@ -300,8 +288,6 @@ pub fn main() !void { var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); defer generated_token_slice.free(allocator); { - defer compiled_prefill.deinit(); - var prefill_args = try compiled_prefill.args(allocator); defer prefill_args.deinit(allocator); var prefill_results = try compiled_prefill.results(allocator); @@ -315,7 +301,7 @@ pub fn main() !void { var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); defer prefill_output.deinit(); - prefill_args.set(.{ &decoder_buffers, embedder_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); + prefill_args.set(.{ decoder_buffers, embedder_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); compiled_prefill.call(prefill_args, &prefill_results); prefill_results.fill(.{ &prefill_output, &kv_cache_buffers }); @@ -331,8 +317,6 @@ pub fn main() !void { log.info("Running decoder autoregressive...", .{}); const eos_token: u32 = 2; { - defer compiled_decoder.deinit(); - var decode_args = try compiled_decoder.args(allocator); defer decode_args.deinit(allocator); var decode_results = try compiled_decoder.results(allocator); @@ -351,7 +335,7 @@ pub fn main() !void { defer decode_embed.deinit(); const max_pos = adapter_seq_len; - log.info("Decode loop: tokens.len={}, max_pos={}", .{tokens.len, max_pos}); + log.info("Decode loop: tokens.len={}, max_pos={}", .{ tokens.len, max_pos }); var num_generated: usize = 0; for (tokens.len..max_pos) |i| { const generated_token = generated_token_slice.items(u32)[0]; @@ -360,10 +344,9 @@ pub fn main() !void { if (generated_token != 32) { if (try tokenizer_decoder.next(generated_token)) |chunk| { std.debug.print("{s}", .{chunk}); - } + } } - if (generated_token == eos_token) { log.info(" EOS reached at pos={}", .{i}); break; @@ -378,7 +361,7 @@ pub fn main() !void { var pos_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(i)), .u32); defer pos_buffer.deinit(); - emb_args.set(.{ &embedder_buffers, adapter_output, token_buf, pos_buffer }); + emb_args.set(.{ embedder_buffers, adapter_output, token_buf, pos_buffer }); compiled_decode_embedder.call(emb_args, &emb_results); emb_results.fill(.{&decode_embed}); @@ -386,7 +369,7 @@ pub fn main() !void { var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(i)), .u32); defer token_index_buffer.deinit(); - decode_args.set(.{ &decoder_buffers, decode_embed, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); + decode_args.set(.{ decoder_buffers, decode_embed, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); compiled_decoder.call(decode_args, &decode_results); decode_results.fill(.{ ¤t_token_buffer, &kv_cache_buffers }); @@ -396,86 +379,3 @@ pub fn main() !void { log.info("Decode done. Generated {} tokens total.", .{num_generated}); } } - -pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, padded_audio_len: usize) !zml.Exe { - return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{padded_audio_len}, .f32).withTags(.{.samples})}); -} - -pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, padded_audio_len: usize) !zml.Exe { - const num_frames = padded_audio_len / model.config.audio().hop_length; - - return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .channels = 128, .time = num_frames }, .f32), - }); -} - -pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, encoder_seq_len: u32, config: Config) !zml.Exe { - return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .s = encoder_seq_len, .d = config.encoder().dim }, .bf16), - }); -} - -pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, adapter_seq_len_: u32, seq_len: u32, config: Config) !zml.Exe { - return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .s = adapter_seq_len_, .d = config.dim }, .bf16), - Tensor.init(.{ .s = seq_len }, .u32), - Tensor.init(.{}, .u32), - }); -} - -const VoxtralParameters = struct { - prefill_embeds: Tensor, - decode_embeds: Tensor, - token_index: Tensor, - kv_cache: dec.KvCache, - t_cond: Tensor, -}; - -pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, params: VoxtralParameters) !struct { zml.Exe, zml.Exe} { - const compile = struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, embeds_: Tensor, params_: VoxtralParameters) !zml.Exe { - return try platform_.compile(allocator_, io_, model_, .forward, .{ - embeds_, - params_.token_index, - params_.kv_cache, - params_.t_cond, - }); - } - }.call; - - var prefill_future = try io.concurrent(compile, .{ allocator, io, platform, model, params.prefill_embeds, params }); - var decode_future = try io.concurrent(compile, .{ allocator, io, platform, model, params.decode_embeds, params }); - - return .{ - try prefill_future.await(io), - try decode_future.await(io), - }; -} - -pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { - const bytes = b: { - const file = try dir.openFile(io, "tokenizer.json", .{}); - defer file.close(io); - var reader = file.reader(io, &.{}); - break :b try reader.interface.readAlloc(allocator, try file.length(io)); - }; - defer allocator.free(bytes); - - return try .fromBytes(allocator, io, bytes); -} - -/// Sinusoidal time embedding: encodes a scalar t into a [dim]-dimensional vector. -/// out[i] = cos(t * inv_freq[i]) for i < half_dim -/// out[i] = sin(t * inv_freq[i-half_dim]) for i >= half_dim -/// where inv_freq[i] = exp(-log(theta) * i / half_dim) -fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32) []f32 { - const half_dim = dim / 2; - const result = allocator.alloc(f32, dim) catch unreachable; - for (0..half_dim) |i| { - const freq = @exp(-@log(@as(f32, 10000.0)) * @as(f32, @floatFromInt(i)) / @as(f32, @floatFromInt(half_dim))); - const angle = t_value * freq; - result[i] = @cos(angle); - result[i + half_dim] = @sin(angle); - } - return result; -} diff --git a/examples/voxtral/wav.zig b/examples/voxtral/wav.zig index 89f4cd89..d0ffdd78 100644 --- a/examples/voxtral/wav.zig +++ b/examples/voxtral/wav.zig @@ -90,3 +90,24 @@ pub fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f3 return samples; } + +pub fn loadAndPadWav( + allocator: std.mem.Allocator, + reader: *std.Io.Reader, + left_pad: usize, + right_pad_tokens: usize, + raw_audio_length_per_tok: usize, +) ![]const f32 { + const wav_file = try loadWav(allocator, reader); + defer allocator.free(wav_file); + + const align_pad = (raw_audio_length_per_tok - (wav_file.len % raw_audio_length_per_tok)) % raw_audio_length_per_tok; + const right_pad = align_pad + right_pad_tokens * raw_audio_length_per_tok; + const audio_len = left_pad + wav_file.len + right_pad; + + const padded_audio = try allocator.alloc(f32, audio_len); + @memset(padded_audio, 0); + @memcpy(padded_audio[left_pad .. left_pad + wav_file.len], wav_file); + + return padded_audio; +} From 93f0f6713375d8c489a1e0f1cf9b0bb97e1c6880 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Sat, 14 Feb 2026 19:26:40 +0100 Subject: [PATCH 59/98] Remove hardcoded values --- examples/voxtral/main.zig | 46 ++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 828db49f..9d11f2c4 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -25,19 +25,18 @@ const voxtral = @import("voxtral.zig"); const CliArgs = struct { input: []const u8, model: []const u8, + transcription_delay_ms: f32 = 480.0, }; -const tokens_array = [_]u32{ 1, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 }; -const tokens: []const u32 = &tokens_array; -const n_delay_tokens = 6; -const n_left_pad_tokens = 32; -const n_right_pad_tokens = (n_delay_tokens + 1) + 10; // 17 -const raw_audio_length_per_tok = 1280; // SAMPLE_RATE / FRAME_RATE = 16000 / 12.5 +// Streaming protocol constants +const token_bos: u32 = 1; +const token_streaming_pad: u32 = 32; +const n_left_pad_tokens: u32 = 32; pub fn main() !void { log.info("Start of Voxtral", .{}); - var dbg = std.heap.DebugAllocator(.{}).init; + var dbg = std.heap.DebugAllocator(.{ .thread_safe = true }).init; defer if (builtin.mode == .Debug) std.debug.assert(dbg.deinit() == .ok); const allocator = switch (builtin.mode) { @@ -64,14 +63,6 @@ pub fn main() !void { const file = try std.Io.Dir.openFile(.cwd(), io, args.input, .{}); defer file.close(io); - var wav_buffer: [4096]u8 = undefined; - var reader = file.reader(io, &wav_buffer); - - const left_pad = n_left_pad_tokens * raw_audio_length_per_tok; - const padded_wav = try wav_utils.loadAndPadWav(allocator, &reader.interface, left_pad, n_right_pad_tokens, raw_audio_length_per_tok); - defer allocator.free(padded_wav); - const audio_len = padded_wav.len; - const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); @@ -84,6 +75,31 @@ pub fn main() !void { defer parsed_config.deinit(); const config = parsed_config.value; + const sample_rate: f32 = @floatFromInt(config.audio().sampling_rate); + const frame_rate = config.audio().frame_rate; + const raw_audio_length_per_tok: u32 = @intFromFloat(sample_rate / frame_rate); + const hop_length = config.audio().hop_length; + + const delay_samples: u32 = @intFromFloat(args.transcription_delay_ms / 1000.0 * sample_rate); + const audio_length_per_tok = raw_audio_length_per_tok / hop_length; + const n_delay_tokens = std.math.divCeil(u32, delay_samples / hop_length, audio_length_per_tok) catch unreachable; + const n_right_pad_tokens = (n_delay_tokens + 1) + 10; + + // Build tokens: [BOS] ++ [STREAMING_PAD] * (n_left_pad_tokens + n_delay_tokens) + const n_prompt_tokens = 1 + n_left_pad_tokens + n_delay_tokens; + const tokens = try allocator.alloc(u32, n_prompt_tokens); + defer allocator.free(tokens); + tokens[0] = token_bos; + @memset(tokens[1..], token_streaming_pad); + + var wav_buffer: [4096]u8 = undefined; + var reader = file.reader(io, &wav_buffer); + + const left_pad = n_left_pad_tokens * raw_audio_length_per_tok; + const padded_wav = try wav_utils.loadAndPadWav(allocator, &reader.interface, left_pad, n_right_pad_tokens, raw_audio_length_per_tok); + defer allocator.free(padded_wav); + const audio_len = padded_wav.len; + var platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); From bccd580635aad860b16316bd25ed01faae6ffbd4 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Sun, 15 Feb 2026 11:56:31 +0100 Subject: [PATCH 60/98] Remove hardcoded dsf and pad encoder output instead of truncating after --- examples/voxtral/decoder.zig | 81 ++++++++++++++++++++---------------- examples/voxtral/encoder.zig | 9 ---- examples/voxtral/main.zig | 20 +++++---- examples/voxtral/test.zig | 2 +- examples/voxtral/voxtral.zig | 11 +++-- 5 files changed, 64 insertions(+), 59 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 8053836c..d620df57 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -14,18 +14,20 @@ const common = @import("common.zig"); const rmsNorm = common.rmsNorm; /// Adapter: encoder→decoder projection. -/// Reshapes encoder output [s, d=1280] → [s/4, d=5120] (concatenate 4 timesteps), -/// then projects through a 2-layer MLP: Linear(5120→3072) → GELU → Linear(3072→3072). +/// Reshapes encoder output [s, d] → [s/dsf, d*dsf] (concatenate `downsample_factor` consecutive timesteps), +/// then projects through a 2-layer MLP: Linear(d*dsf→dim) → GELU → Linear(dim→dim). pub const Adapter = struct { proj0: zml.nn.Linear, proj1: zml.nn.Linear, + downsample_factor: u32, - pub fn init(store: zml.io.TensorStore.View) Adapter { + pub fn init(store: zml.io.TensorStore.View, config: Config) Adapter { const proj_store = store.withPrefix("mm_streams_embeddings.embedding_module.audio_language_projection"); - + return .{ .proj0 = common.linear(proj_store.withLayer(0)), .proj1 = common.linear(proj_store.withLayer(2)), + .downsample_factor = config.downsample_factor(), }; } @@ -38,11 +40,20 @@ pub const Adapter = struct { common.deinitLinear(&self.proj1); } - /// encoder_out: [s, d=1280] → [s/4, d=3072] + /// encoder_out: [s, d] → [s/dsf, dim] pub fn forward(self: Adapter, encoder_out: Tensor) Tensor { - // Reshape [s, d=1280] → [s/4, d=5120] by concatenating 4 consecutive timesteps. - const h = encoder_out - .splitAxis(.s, .{ .s = .auto, .group = 4 }) + var h = encoder_out; + const dsf = self.downsample_factor; + + // Pad encoder output to next multiple of downsample_factor + const seq_len: usize = @intCast(h.dim(.s)); + const remainder = seq_len % dsf; + if (remainder > 0) { + h = h.pad(0, .{ .s = Tensor.Pad{ .high = @as(i64, @intCast(dsf - remainder)) } }); + } + + // Reshape [s, d] → [s/dsf, d*dsf] by concatenating consecutive timesteps. + h = h.splitAxis(.s, .{ .s = .auto, .group = dsf }) .merge(.{ .d = .{ .group, .d } }); return self.proj1.forward(self.proj0.forward(h).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); @@ -53,31 +64,30 @@ pub const Embedder = struct { tok_embeddings: Tensor, pub fn init(store: zml.io.TensorStore.View) Embedder { - return .{ - .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .vocab, .d }), - }; + return .{ + .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .vocab, .d }), + }; } pub fn forward(self: Embedder, full_adapter_out: Tensor, text_tokens: Tensor, pos: Tensor) Tensor { - const audio_embeds = full_adapter_out.dynamicSlice(.{ - .s = Tensor.DynSlice{ .start = pos, .len = @intCast(text_tokens.dim(.s)) }, - }); - - const text_embeds = self.tok_embeddings.gather(.{ .vocab = text_tokens }, .{}); - - return text_embeds.add(audio_embeds); + const audio_embeds = full_adapter_out.dynamicSlice(.{ + .s = Tensor.DynSlice{ .start = pos, .len = @intCast(text_tokens.dim(.s)) }, + }); + + const text_embeds = self.tok_embeddings.gather(.{ .vocab = text_tokens }, .{}); + + return text_embeds.add(audio_embeds); } pub fn load(self: *const Embedder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Embedder) { return common.loadModel(Embedder, self, allocator, io, platform, store, progress, "embedder"); } - + pub fn unload(self: *zml.Bufferized(Embedder)) void { - self.tok_embeddings.deinit(); + self.tok_embeddings.deinit(); } }; - /// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. pub const Decoder = struct { /// Same weights as Embedder.tok_embeddings; duplicated here for logit computation. @@ -89,7 +99,7 @@ pub const Decoder = struct { pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Decoder { const layers = allocator.alloc(DecoderLayer, config.n_layers) catch unreachable; - + for (layers, 0..) |*layer, i| { layer.* = DecoderLayer.init(store.withPrefix("layers").withLayer(i), config); } @@ -110,14 +120,14 @@ pub const Decoder = struct { pub fn load(self: *const Decoder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Decoder) { return common.loadModel(Decoder, self, allocator, io, platform, store, progress, "decoder"); } - + pub fn unload(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { self.tok_embeddings.deinit(); - + for (self.layers) |*layer| { DecoderLayer.unload(layer); } - + allocator.free(self.layers); self.norm.deinit(); } @@ -135,7 +145,7 @@ pub const Decoder = struct { const result = layer.forward(h, token_index, layer_cache, t, self.config); h = result[0]; const updated_layer_cache = result[1]; - + cache = updated_layer_cache.reuseBuffer(layer_cache); } @@ -144,7 +154,7 @@ pub const Decoder = struct { // Compute logits in f32 for precision (matching Python reference) const output_logits = h.convert(.f32).dot(self.tok_embeddings.convert(.f32), .d); const output_tokens = output_logits.argMax(.vocab).indices.convert(.u32); - + return .{ output_tokens, cache }; } }; @@ -172,7 +182,7 @@ pub const AdaRmsNorm = struct { pub fn forward(self: AdaRmsNorm, h: Tensor, t_cond: Tensor) Tensor { const ada_scale = self.up.forward(self.down.forward(t_cond).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); const one_plus_scale = ada_scale.broad(h.shape()).add(Tensor.scalar(1.0, h.dtype()).broad(h.shape())); - + return h.mul(one_plus_scale); } }; @@ -234,7 +244,7 @@ pub const KvCache = struct { const k_shape = cache.shape().drop(.layer); const converted = new.convert(cache.dtype()).transpose(k_shape); const scatter_opts: Tensor.ScatterOpts = .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }; - + return if (token_index) |idx| cache.scatterSlices(.{ .layer = layer_index.broad(idx.shape()), .k = idx }, converted, scatter_opts).reuseBuffer(cache) else @@ -305,7 +315,7 @@ pub const SelfAttention = struct { .layout = .interleaved, .freq_base = config.rope_theta, }; - + q = zml.nn.rope(q, pos_index, rope_opts); k = zml.nn.rope(k, pos_index, rope_opts); @@ -326,7 +336,7 @@ pub const SelfAttention = struct { token_index.reshape(.{ .coord = 1 }), .{}, ); - + const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); // Merge heads and output projection: [q, h, hd] → [s, d] @@ -358,10 +368,10 @@ pub const DecoderLayer = struct { pub fn unload(self: *zml.Bufferized(DecoderLayer)) void { self.attention_norm.deinit(); SelfAttention.unload(&self.attention); - + self.ffn_norm.deinit(); SwiGluFfn.unload(&self.feed_forward); - + AdaRmsNorm.unload(&self.ada_norm); } @@ -374,17 +384,16 @@ pub const DecoderLayer = struct { kv_cache, config, ); - + var out = h.add(attn_out); // Pre-FFN norm → ada conditioning → FFN → residual const normed = rmsNorm(out, self.ffn_norm, self.norm_eps); const conditioned = self.ada_norm.forward(normed, t_cond); const ffn_out = self.feed_forward.forward(conditioned); - + out = out.add(ffn_out); return .{ out, updated_kv_cache }; } }; - diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 7f457319..ac2d6363 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -67,19 +67,10 @@ pub const Encoder = struct { h = self.conv0.forward(h).gelu(); h = self.conv1.forward(h).gelu(); - // [batch=1, channels=1280, time'] -> [s, d=1280] h = h.squeeze(.batch); h = h.transpose(.{ .time, .channels }).rename(.{ .time = .s, .channels = .d }); - // Left-truncate to multiple of downsample_factor - const seq_len = h.dim(.s); - const trunc: i64 = @intCast(@as(usize, @intCast(seq_len)) % self.config.downsample_factor()); - - if (trunc > 0) { - h = h.slice1d(.s, .{ .start = trunc }); - } - // Transformer layers for (self.layers) |layer| { h = layer.forward(h, self.config); diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 9d11f2c4..38bc4eb0 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -28,9 +28,6 @@ const CliArgs = struct { transcription_delay_ms: f32 = 480.0, }; -// Streaming protocol constants -const token_bos: u32 = 1; -const token_streaming_pad: u32 = 32; const n_left_pad_tokens: u32 = 32; pub fn main() !void { @@ -89,8 +86,6 @@ pub fn main() !void { const n_prompt_tokens = 1 + n_left_pad_tokens + n_delay_tokens; const tokens = try allocator.alloc(u32, n_prompt_tokens); defer allocator.free(tokens); - tokens[0] = token_bos; - @memset(tokens[1..], token_streaming_pad); var wav_buffer: [4096]u8 = undefined; var reader = file.reader(io, &wav_buffer); @@ -105,15 +100,17 @@ pub fn main() !void { log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); const num_frames = audio_len / config.audio().hop_length; - const encoder_seq_len: u32 = @intCast((num_frames / 2) - (num_frames / 2) % config.downsample_factor()); - const adapter_seq_len = encoder_seq_len / config.downsample_factor(); + const dsf: u32 = config.downsample_factor(); + // Natural encoder output length after stride-2 causal conv (adapter pads to multiple of dsf) + const encoder_seq_len: u32 = @intCast((num_frames + 1) / 2); + const adapter_seq_len = (encoder_seq_len + dsf - 1) / dsf; var melspectro_model: LogMelSpectrogram = .init(config); const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; var encoder_model: Encoder = .init(allocator, model_store.view().withPrefix(encoder_prefix), config); defer encoder_model.deinit(allocator); - const adapter: Adapter = .init(model_store.view()); + const adapter: Adapter = .init(model_store.view(), config); const embedder: Embedder = .init(model_store.view()); var model: Decoder = .init(allocator, model_store.view(), config); @@ -148,10 +145,15 @@ pub fn main() !void { var embedder_buffers_future = try io.concurrent(Embedder.load, .{ &embedder, allocator, io, platform, &model_store, &progress }); var decoder_buffers_future = try io.concurrent(Decoder.load, .{ &model, allocator, io, platform, &model_store, &progress }); - // Await tokenizer + // Await tokenizer and look up special token IDs var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); + const token_bos = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); + const token_streaming_pad = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); + tokens[0] = token_bos; + @memset(tokens[1..], token_streaming_pad); + // -- Compiled models var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); defer compiled_mel_spectrum.deinit(); diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index 79bd2037..f78aa22e 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -102,7 +102,7 @@ pub fn main() !void { // -- Adapter test { const adapter = AdapterTestHarness{ - .inner = dec.Adapter.init(model_store.view()), + .inner = dec.Adapter.init(model_store.view(), config), }; var adapter_buffers = try zml.io.load(AdapterTestHarness, &adapter, allocator, io, platform, load_opts); defer AdapterTestHarness.unload(&adapter_buffers); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 01059d38..549d6b53 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -174,8 +174,10 @@ pub fn runPipeline( log.info("Running inference pipeline...", .{}); const num_frames = audio_len / config.audio().hop_length; - const encoder_seq_len: u32 = @intCast((num_frames / 2) - (num_frames / 2) % config.downsample_factor()); - const adapter_seq_len = encoder_seq_len / config.downsample_factor(); + const dsf: u32 = config.downsample_factor(); + // Natural encoder output length after stride-2 causal conv (adapter pads to multiple of dsf) + const encoder_seq_len: u32 = @intCast((num_frames + 1) / 2); + const adapter_seq_len = (encoder_seq_len + dsf - 1) / dsf; var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); @@ -315,7 +317,8 @@ pub fn runPipeline( // 7. Autoregressive decode loop log.info("Running decoder autoregressive...", .{}); - const eos_token: u32 = 2; + const eos_token = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); + const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); { var decode_args = try compiled_decoder.args(allocator); defer decode_args.deinit(allocator); @@ -341,7 +344,7 @@ pub fn runPipeline( const generated_token = generated_token_slice.items(u32)[0]; num_generated += 1; - if (generated_token != 32) { + if (generated_token != streaming_pad_token) { if (try tokenizer_decoder.next(generated_token)) |chunk| { std.debug.print("{s}", .{chunk}); } From ea56f50ee820ce8336f0f014c31a1df1071e72c7 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Sun, 15 Feb 2026 18:19:23 +0100 Subject: [PATCH 61/98] inline embedder into decoder --- examples/voxtral/decoder.zig | 42 +++------- examples/voxtral/main.zig | 22 +---- examples/voxtral/voxtral.zig | 157 +++++++++++------------------------ 3 files changed, 63 insertions(+), 158 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index d620df57..676d3501 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -60,37 +60,9 @@ pub const Adapter = struct { } }; -pub const Embedder = struct { - tok_embeddings: Tensor, - - pub fn init(store: zml.io.TensorStore.View) Embedder { - return .{ - .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .vocab, .d }), - }; - } - - pub fn forward(self: Embedder, full_adapter_out: Tensor, text_tokens: Tensor, pos: Tensor) Tensor { - const audio_embeds = full_adapter_out.dynamicSlice(.{ - .s = Tensor.DynSlice{ .start = pos, .len = @intCast(text_tokens.dim(.s)) }, - }); - - const text_embeds = self.tok_embeddings.gather(.{ .vocab = text_tokens }, .{}); - - return text_embeds.add(audio_embeds); - } - - pub fn load(self: *const Embedder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Embedder) { - return common.loadModel(Embedder, self, allocator, io, platform, store, progress, "embedder"); - } - - pub fn unload(self: *zml.Bufferized(Embedder)) void { - self.tok_embeddings.deinit(); - } -}; - /// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. pub const Decoder = struct { - /// Same weights as Embedder.tok_embeddings; duplicated here for logit computation. + /// Token embeddings: used for both input embedding and output logit computation. tok_embeddings: Tensor, layers: []DecoderLayer, norm: Tensor, @@ -135,8 +107,16 @@ pub const Decoder = struct { /// Run all decoder layers on input embeddings, compute logits and return argmax tokens. /// input_embeds: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] /// Returns: (tokens [s] u32, KvCache) - pub fn forward(self: Decoder, input_embeds: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor) struct { Tensor, KvCache } { + pub fn forward(self: Decoder, text_tokens: Tensor, adapter_out: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor) struct { Tensor, KvCache } { + const audio_embeds = adapter_out.dynamicSlice(.{ + .s = Tensor.DynSlice{ .start = token_index, .len = @intCast(text_tokens.dim(.s)) }, + }); + + const text_embeds = self.tok_embeddings.gather(.{ .vocab = text_tokens }, .{}); + const input_embeds = text_embeds.add(audio_embeds); + const t = t_cond.convert(input_embeds.dtype()); + var h = input_embeds; var cache = kv_cache; @@ -153,7 +133,7 @@ pub const Decoder = struct { // Compute logits in f32 for precision (matching Python reference) const output_logits = h.convert(.f32).dot(self.tok_embeddings.convert(.f32), .d); - const output_tokens = output_logits.argMax(.vocab).indices.convert(.u32); + const output_tokens = output_logits.argMax(.vocab).indices.squeeze(.vocab).convert(.u32); return .{ output_tokens, cache }; } diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 38bc4eb0..c97dff3e 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -18,7 +18,6 @@ const Encoder = enc.Encoder; const dec = @import("decoder.zig"); const Decoder = dec.Decoder; const Adapter = dec.Adapter; -const Embedder = dec.Embedder; const voxtral = @import("voxtral.zig"); @@ -111,15 +110,15 @@ pub fn main() !void { defer encoder_model.deinit(allocator); const adapter: Adapter = .init(model_store.view(), config); - const embedder: Embedder = .init(model_store.view()); var model: Decoder = .init(allocator, model_store.view(), config); defer model.deinit(allocator); const dtype = model.tok_embeddings.dtype(); const model_params: voxtral.VoxtralParameters = .{ - .prefill_embeds = .init(.{ .s = @as(u32, @intCast(tokens.len)), .d = config.dim }, .bf16), - .decode_embeds = .init(.{ .s = 1, .d = config.dim }, .bf16), + .prefill_tokens = .init(.{ .s = @as(u32, @intCast(tokens.len)) }, .u32), + .decode_tokens = .init(.{ .s = 1 }, .u32), + .adapter_out = .init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16), .token_index = .init(.{}, .u32), .kv_cache = .init(.init(.{ .layer = model.layers.len, @@ -135,14 +134,11 @@ pub fn main() !void { var compiled_mel_spectrum_future = try io.concurrent(voxtral.compileMelSpectrum, .{ allocator, io, platform, melspectro_model, audio_len, &progress }); var compiled_encoder_future = try io.concurrent(voxtral.compileEncoder, .{ allocator, io, platform, encoder_model, audio_len, &progress }); var compiled_adapter_future = try io.concurrent(voxtral.compileAdapter, .{ allocator, io, platform, adapter, encoder_seq_len, config, &progress }); - var compiled_embedder_future = try io.concurrent(voxtral.compileEmbedder, .{ allocator, io, platform, embedder, adapter_seq_len, @as(u32, @intCast(tokens.len)), config, &progress }); - var compiled_decode_embedder_future = try io.concurrent(voxtral.compileEmbedder, .{ allocator, io, platform, embedder, adapter_seq_len, @as(u32, 1), config, &progress }); var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, model, model_params, &progress }); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{ &melspectro_model, io, platform }); var encoder_buffers_future = try io.concurrent(Encoder.load, .{ &encoder_model, allocator, io, platform, &model_store, &progress }); var adapter_buffers_future = try io.concurrent(Adapter.load, .{ &adapter, allocator, io, platform, &model_store, &progress }); - var embedder_buffers_future = try io.concurrent(Embedder.load, .{ &embedder, allocator, io, platform, &model_store, &progress }); var decoder_buffers_future = try io.concurrent(Decoder.load, .{ &model, allocator, io, platform, &model_store, &progress }); // Await tokenizer and look up special token IDs @@ -164,12 +160,6 @@ pub fn main() !void { var compiled_adapter = try compiled_adapter_future.await(io); defer compiled_adapter.deinit(); - var compiled_embedder = try compiled_embedder_future.await(io); - defer compiled_embedder.deinit(); - - var compiled_decode_embedder = try compiled_decode_embedder_future.await(io); - defer compiled_decode_embedder.deinit(); - var compiled_prefill, var compiled_decoder = try compiled_decoder_future.await(io); defer compiled_prefill.deinit(); defer compiled_decoder.deinit(); @@ -184,9 +174,6 @@ pub fn main() !void { var adapter_buffers = try adapter_buffers_future.await(io); defer Adapter.unload(&adapter_buffers); - var embedder_buffers = try embedder_buffers_future.await(io); - defer Embedder.unload(&embedder_buffers); - var decoder_buffers = try decoder_buffers_future.await(io); defer Decoder.unload(&decoder_buffers, allocator); @@ -205,14 +192,11 @@ pub fn main() !void { &compiled_mel_spectrum, &compiled_encoder, &compiled_adapter, - &compiled_embedder, - &compiled_decode_embedder, &compiled_prefill, &compiled_decoder, &mel_spectrum_buffers, &encoder_buffers, &adapter_buffers, - &embedder_buffers, &decoder_buffers, model_params, ); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 549d6b53..b19168e6 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -15,11 +15,11 @@ const Encoder = enc.Encoder; const dec = @import("decoder.zig"); const Decoder = dec.Decoder; const Adapter = dec.Adapter; -const Embedder = dec.Embedder; pub const VoxtralParameters = struct { - prefill_embeds: Tensor, - decode_embeds: Tensor, + prefill_tokens: Tensor, + decode_tokens: Tensor, + adapter_out: Tensor, token_index: Tensor, kv_cache: dec.KvCache, t_cond: Tensor, @@ -29,7 +29,7 @@ pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *c progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling mel spectrogram...", 1); defer node.end(); - + return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{padded_audio_len}, .f32).withTags(.{.samples})}); } @@ -37,9 +37,9 @@ pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling encoder...", 1); defer node.end(); - + const num_frames = padded_audio_len / model.config.audio().hop_length; - + return try platform.compile(allocator, io, model, .forward, .{ Tensor.init(.{ .channels = 128, .time = num_frames }, .f32), }); @@ -47,54 +47,26 @@ pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, encoder_seq_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); - + var node = progress.start("Compiling adapter...", 1); defer node.end(); - + return try platform.compile(allocator, io, model, .forward, .{ Tensor.init(.{ .s = encoder_seq_len, .d = config.encoder().dim }, .bf16), }); } -pub fn compileEmbedder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Embedder, adapter_seq_len_: u32, seq_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - - var node = progress.start("Compiling embedder...", 1); - defer node.end(); - - return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .s = adapter_seq_len_, .d = config.dim }, .bf16), - Tensor.init(.{ .s = seq_len }, .u32), - Tensor.init(.{}, .u32), - }); -} - pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, params: VoxtralParameters, progress: *std.Progress.Node) !struct { zml.Exe, zml.Exe } { const compilePrefill = struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, embeds_: Tensor, params_: VoxtralParameters, progress_: *std.Progress.Node) !zml.Exe { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, tokens_: Tensor, params_: VoxtralParameters, progress_: *std.Progress.Node) !zml.Exe { progress_.increaseEstimatedTotalItems(1); - + var node_ = progress_.start("Compiling decoder prefill...", 1); defer node_.end(); - + return try platform_.compile(allocator_, io_, model_, .forward, .{ - embeds_, - params_.token_index, - params_.kv_cache, - params_.t_cond, - }); - } - }.call; - - const compileDecode = struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, embeds_: Tensor, params_: VoxtralParameters, progress_: *std.Progress.Node) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - - var node_ = progress_.start("Compiling decoder decode...", 1); - defer node_.end(); - - return try platform_.compile(allocator_, io_, model_, .forward, .{ - embeds_, + tokens_, + params_.adapter_out, params_.token_index, params_.kv_cache, params_.t_cond, @@ -102,8 +74,25 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const } }.call; - var prefill_future = try io.concurrent(compilePrefill, .{ allocator, io, platform, model, params.prefill_embeds, params, progress }); - var decode_future = try io.concurrent(compileDecode, .{ allocator, io, platform, model, params.decode_embeds, params, progress }); + const compileDecode = struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, tokens_: Tensor, params_: VoxtralParameters, progress_: *std.Progress.Node) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + + var node_ = progress_.start("Compiling decoder decode...", 1); + defer node_.end(); + + return try platform_.compile(allocator_, io_, model_, .forward, .{ + tokens_, + params_.adapter_out, + params_.token_index, + params_.kv_cache, + params_.t_cond, + }); + } + }.call; + + var prefill_future = try io.concurrent(compilePrefill, .{ allocator, io, platform, model, params.prefill_tokens, params, progress }); + var decode_future = try io.concurrent(compileDecode, .{ allocator, io, platform, model, params.decode_tokens, params, progress }); return .{ try prefill_future.await(io), @@ -113,14 +102,14 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { progress.increaseEstimatedTotalItems(1); - + var node = progress.start("Loading tokenizer...", 1); defer node.end(); - + const bytes = b: { const file = try dir.openFile(io, "tokenizer.json", .{}); defer file.close(io); - + var reader = file.reader(io, &.{}); break :b try reader.interface.readAlloc(allocator, try file.length(io)); }; @@ -136,14 +125,14 @@ pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, pub fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32) []f32 { const half_dim = dim / 2; const result = allocator.alloc(f32, dim) catch unreachable; - + for (0..half_dim) |i| { const freq = @exp(-@log(@as(f32, 10000.0)) * @as(f32, @floatFromInt(i)) / @as(f32, @floatFromInt(half_dim))); const angle = t_value * freq; result[i] = @cos(angle); result[i + half_dim] = @sin(angle); } - + return result; } @@ -160,14 +149,11 @@ pub fn runPipeline( compiled_mel_spectrum: *zml.Exe, compiled_encoder: *zml.Exe, compiled_adapter: *zml.Exe, - compiled_embedder: *zml.Exe, - compiled_decode_embedder: *zml.Exe, compiled_prefill: *zml.Exe, compiled_decoder: *zml.Exe, mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), encoder_buffers: *zml.Bufferized(Encoder), adapter_buffers: *zml.Bufferized(Adapter), - embedder_buffers: *zml.Bufferized(Embedder), decoder_buffers: *zml.Bufferized(Decoder), model_params: VoxtralParameters, ) !void { @@ -243,36 +229,7 @@ pub fn runPipeline( defer adapter_output.deinit(); log.info("Adapter done.", .{}); - // 4. Embedder - combine audio embeddings with token embeddings - log.info("Running embedder...", .{}); - var embedder_output: zml.Buffer = undefined; - { - var emb_args = try compiled_embedder.args(allocator); - defer emb_args.deinit(allocator); - var emb_results = try compiled_embedder.results(allocator); - defer emb_results.deinit(allocator); - - const token_slice: zml.Slice = .init( - .init(.{@as(u32, @intCast(tokens.len))}, .u32), - std.mem.sliceAsBytes(tokens), - ); - var token_buffer: zml.Buffer = try .fromSlice(io, platform, token_slice); - defer token_buffer.deinit(); - - var pos_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); - defer pos_buffer.deinit(); - - embedder_output = try .uninitialized(io, platform, - .init(.{ .s = @as(u32, @intCast(tokens.len)), .d = config.dim }, .bf16), .{}); - - emb_args.set(.{ embedder_buffers, adapter_output, token_buffer, pos_buffer }); - compiled_embedder.call(emb_args, &emb_results); - emb_results.fill(.{&embedder_output}); - } - defer embedder_output.deinit(); - log.info("Embedder done.", .{}); - - // 5. Init KV cache + t_cond buffer + // 4. Init KV cache + t_cond buffer var kv_cache_buffers = try model_params.kv_cache.initBuffer(io, platform); defer dec.KvCache.deinitBuffer(&kv_cache_buffers); @@ -285,7 +242,7 @@ pub fn runPipeline( var t_cond_buffer: zml.Buffer = try .fromSlice(io, platform, t_cond_slice); defer t_cond_buffer.deinit(); - // 6. Decoder prefill + // 5. Decoder prefill log.info("Running decoder prefill...", .{}); var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); defer generated_token_slice.free(allocator); @@ -295,6 +252,13 @@ pub fn runPipeline( var prefill_results = try compiled_prefill.results(allocator); defer prefill_results.deinit(allocator); + const token_slice: zml.Slice = .init( + .init(.{@as(u32, @intCast(tokens.len))}, .u32), + std.mem.sliceAsBytes(tokens), + ); + var token_buffer: zml.Buffer = try .fromSlice(io, platform, token_slice); + defer token_buffer.deinit(); + var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); defer token_index_buffer.deinit(); @@ -303,7 +267,7 @@ pub fn runPipeline( var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); defer prefill_output.deinit(); - prefill_args.set(.{ decoder_buffers, embedder_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); + prefill_args.set(.{ decoder_buffers, token_buffer, adapter_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); compiled_prefill.call(prefill_args, &prefill_results); prefill_results.fill(.{ &prefill_output, &kv_cache_buffers }); @@ -315,7 +279,7 @@ pub fn runPipeline( } log.info("Prefill done.", .{}); - // 7. Autoregressive decode loop + // 6. Autoregressive decode loop log.info("Running decoder autoregressive...", .{}); const eos_token = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); @@ -325,18 +289,9 @@ pub fn runPipeline( var decode_results = try compiled_decoder.results(allocator); defer decode_results.deinit(allocator); - var emb_args = try compiled_decode_embedder.args(allocator); - defer emb_args.deinit(allocator); - var emb_results = try compiled_decode_embedder.results(allocator); - defer emb_results.deinit(allocator); - var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); defer current_token_buffer.deinit(); - var decode_embed: zml.Buffer = try .uninitialized(io, platform, - .init(.{ .s = @as(u32, 1), .d = config.dim }, .bf16), .{}); - defer decode_embed.deinit(); - const max_pos = adapter_seq_len; log.info("Decode loop: tokens.len={}, max_pos={}", .{ tokens.len, max_pos }); var num_generated: usize = 0; @@ -355,24 +310,10 @@ pub fn runPipeline( break; } - // 1. Embedder: combine adapter_output[pos] + tok_embed(token) - const token_slice: zml.Slice = .init(.init(.{@as(u32, 1)}, .u32), - std.mem.sliceAsBytes(generated_token_slice.items(u32)[0..1])); - var token_buf: zml.Buffer = try .fromSlice(io, platform, token_slice); - defer token_buf.deinit(); - - var pos_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(i)), .u32); - defer pos_buffer.deinit(); - - emb_args.set(.{ embedder_buffers, adapter_output, token_buf, pos_buffer }); - compiled_decode_embedder.call(emb_args, &emb_results); - emb_results.fill(.{&decode_embed}); - - // 2. Decoder: get next token var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(i)), .u32); defer token_index_buffer.deinit(); - decode_args.set(.{ decoder_buffers, decode_embed, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); + decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); compiled_decoder.call(decode_args, &decode_results); decode_results.fill(.{ ¤t_token_buffer, &kv_cache_buffers }); From b14e898a808c8295503471df94cbfce2be8534dd Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Sun, 15 Feb 2026 18:25:55 +0100 Subject: [PATCH 62/98] use sampleTokens --- examples/voxtral/decoder.zig | 18 +++++++++--------- examples/voxtral/main.zig | 1 + examples/voxtral/voxtral.zig | 14 ++++++++++---- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 676d3501..85242e1a 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -77,7 +77,7 @@ pub const Decoder = struct { } return .{ - .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .vocab, .d }), + .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .voc, .d }), .layers = layers, .norm = store.withPrefix("norm").createTensorWithTags("weight", .{.d}), .norm_eps = config.norm_eps, @@ -104,15 +104,15 @@ pub const Decoder = struct { self.norm.deinit(); } - /// Run all decoder layers on input embeddings, compute logits and return argmax tokens. - /// input_embeds: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] - /// Returns: (tokens [s] u32, KvCache) - pub fn forward(self: Decoder, text_tokens: Tensor, adapter_out: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor) struct { Tensor, KvCache } { + /// Run all decoder layers on input embeddings, sample next tokens and return updated KV cache + RNG. + /// text_tokens: [s] u32, adapter_out: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d], rng: Rng + /// Returns: (tokens [s] u32, KvCache, Rng) + pub fn forward(self: Decoder, text_tokens: Tensor, adapter_out: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng) struct { Tensor, KvCache, Tensor.Rng } { const audio_embeds = adapter_out.dynamicSlice(.{ .s = Tensor.DynSlice{ .start = token_index, .len = @intCast(text_tokens.dim(.s)) }, }); - const text_embeds = self.tok_embeddings.gather(.{ .vocab = text_tokens }, .{}); + const text_embeds = self.tok_embeddings.gather(.{ .voc = text_tokens }, .{}); const input_embeds = text_embeds.add(audio_embeds); const t = t_cond.convert(input_embeds.dtype()); @@ -132,10 +132,10 @@ pub const Decoder = struct { h = rmsNorm(h, self.norm, self.norm_eps); // Compute logits in f32 for precision (matching Python reference) - const output_logits = h.convert(.f32).dot(self.tok_embeddings.convert(.f32), .d); - const output_tokens = output_logits.argMax(.vocab).indices.squeeze(.vocab).convert(.u32); + const logits = h.convert(.f32).dot(self.tok_embeddings.convert(.f32), .d); + const output_tokens, const new_rng = zml.nn.sampleTokens(logits, .{}, rng); - return .{ output_tokens, cache }; + return .{ output_tokens.convert(.u32), cache, new_rng }; } }; diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index c97dff3e..175298fa 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -127,6 +127,7 @@ pub fn main() !void { .hd = config.head_dim, }, dtype)), .t_cond = Tensor.init(.{ .d = config.dim }, .f32), + .rng = .init(), }; // Launch concurrent compilation and buffer loading diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index b19168e6..6d2d5e61 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -23,6 +23,7 @@ pub const VoxtralParameters = struct { token_index: Tensor, kv_cache: dec.KvCache, t_cond: Tensor, + rng: Tensor.Rng, }; pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, padded_audio_len: usize, progress: *std.Progress.Node) !zml.Exe { @@ -70,6 +71,7 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const params_.token_index, params_.kv_cache, params_.t_cond, + params_.rng, }); } }.call; @@ -87,6 +89,7 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const params_.token_index, params_.kv_cache, params_.t_cond, + params_.rng, }); } }.call; @@ -242,6 +245,9 @@ pub fn runPipeline( var t_cond_buffer: zml.Buffer = try .fromSlice(io, platform, t_cond_slice); defer t_cond_buffer.deinit(); + var rng_buffers = try Tensor.Rng.initBuffer(platform, 0, io); + defer Tensor.Rng.deinitBuffer(&rng_buffers); + // 5. Decoder prefill log.info("Running decoder prefill...", .{}); var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); @@ -267,9 +273,9 @@ pub fn runPipeline( var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); defer prefill_output.deinit(); - prefill_args.set(.{ decoder_buffers, token_buffer, adapter_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); + prefill_args.set(.{ decoder_buffers, token_buffer, adapter_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer, rng_buffers }); compiled_prefill.call(prefill_args, &prefill_results); - prefill_results.fill(.{ &prefill_output, &kv_cache_buffers }); + prefill_results.fill(.{ &prefill_output, &kv_cache_buffers, &rng_buffers }); // Extract last token from prefill output (first generated token) try prefill_output.toSlice(io, prefill_tokens_slice); @@ -313,9 +319,9 @@ pub fn runPipeline( var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(i)), .u32); defer token_index_buffer.deinit(); - decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer }); + decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer, rng_buffers }); compiled_decoder.call(decode_args, &decode_results); - decode_results.fill(.{ ¤t_token_buffer, &kv_cache_buffers }); + decode_results.fill(.{ ¤t_token_buffer, &kv_cache_buffers, &rng_buffers }); try current_token_buffer.toSlice(io, generated_token_slice); } From 8338a2fe154505071a79a70d2c3af38495e2bb53 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Sun, 15 Feb 2026 23:11:32 +0100 Subject: [PATCH 63/98] start of paper match --- examples/voxtral/common.zig | 84 +++++++++- examples/voxtral/decoder.zig | 104 ++---------- examples/voxtral/encoder.zig | 104 +++++++++++- examples/voxtral/main.zig | 77 ++++----- examples/voxtral/voxtral.zig | 314 ++++++++++++++++++----------------- 5 files changed, 396 insertions(+), 287 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index ef3cd466..e0d40aa9 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -3,6 +3,7 @@ const log = std.log; const zml = @import("zml"); const Tensor = zml.Tensor; +const Shape = zml.Shape; pub fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { return .init( @@ -32,7 +33,7 @@ pub fn loadModel( comptime label: []const u8, ) !zml.Bufferized(T) { progress.increaseEstimatedTotalItems(store.view().count()); - + var timer: std.time.Timer = try .start(); var total_bytes: usize = 0; defer { @@ -53,3 +54,84 @@ pub fn loadModel( .total_bytes = &total_bytes, }); } + +/// KV cache for all layers of a transformer. +/// Stores K/V tensors with shape {layer, k=max_seq_len, h, hd}. +pub const KvCache = struct { + k: Tensor, + v: Tensor, + layer_index: Tensor, + + pub fn init(kv_shape: Shape) KvCache { + return .{ + .k = .fromShape(kv_shape), + .v = .fromShape(kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initShape(kv_shape: Shape) zml.ShapeOf(KvCache) { + return .{ + .k = kv_shape, + .v = kv_shape, + .layer_index = Shape.init(.{}, .u32), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + self.k.deinit(); + self.v.deinit(); + self.layer_index.deinit(); + } + + pub fn keys(self: KvCache) Tensor { + return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache) Tensor { + return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + return .{ + .k = scatterCache(self.k, new_k, self.layer_index, token_index), + .v = scatterCache(self.v, new_v, self.layer_index, token_index), + .layer_index = self.layer_index, + }; + } + + fn scatterCache(cache: Tensor, new: Tensor, layer_index: Tensor, token_index: ?Tensor) Tensor { + const k_shape = cache.shape().drop(.layer); + const converted = new.convert(cache.dtype()).transpose(k_shape); + const scatter_opts: Tensor.ScatterOpts = .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }; + + return if (token_index) |idx| + cache.scatterSlices(.{ .layer = layer_index.broad(idx.shape()), .k = idx }, converted, scatter_opts).reuseBuffer(cache) + else + cache.scatterSlices(.{ .layer = layer_index }, converted, scatter_opts).reuseBuffer(cache); + } + + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } +}; diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 85242e1a..4b953794 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -58,6 +58,19 @@ pub const Adapter = struct { return self.proj1.forward(self.proj0.forward(h).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); } + + /// encoder_chunk: [s=dsf, d] → [s=1, dim] + /// Like forward but input is always exactly dsf frames, so no padding needed. + pub fn forwardStep(self: Adapter, encoder_chunk: Tensor) Tensor { + const dsf = self.downsample_factor; + + // Reshape [s=dsf, d] → [s=1, d*dsf] + const h = encoder_chunk + .splitAxis(.s, .{ .s = .auto, .group = dsf }) + .merge(.{ .d = .{ .group, .d } }); + + return self.proj1.forward(self.proj0.forward(h).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); + } }; /// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. @@ -105,15 +118,11 @@ pub const Decoder = struct { } /// Run all decoder layers on input embeddings, sample next tokens and return updated KV cache + RNG. - /// text_tokens: [s] u32, adapter_out: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d], rng: Rng + /// text_tokens: [s] u32, audio_embed: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d], rng: Rng /// Returns: (tokens [s] u32, KvCache, Rng) - pub fn forward(self: Decoder, text_tokens: Tensor, adapter_out: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng) struct { Tensor, KvCache, Tensor.Rng } { - const audio_embeds = adapter_out.dynamicSlice(.{ - .s = Tensor.DynSlice{ .start = token_index, .len = @intCast(text_tokens.dim(.s)) }, - }); - + pub fn forward(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng) struct { Tensor, KvCache, Tensor.Rng } { const text_embeds = self.tok_embeddings.gather(.{ .voc = text_tokens }, .{}); - const input_embeds = text_embeds.add(audio_embeds); + const input_embeds = text_embeds.add(audio_embed); const t = t_cond.convert(input_embeds.dtype()); @@ -167,86 +176,7 @@ pub const AdaRmsNorm = struct { } }; -/// KV cache for all decoder layers. -/// Stores K/V tensors with shape {layer, k=max_seq_len, h, hd}. -pub const KvCache = struct { - k: Tensor, - v: Tensor, - layer_index: Tensor, - - pub fn init(kv_shape: Shape) KvCache { - return .{ - .k = .fromShape(kv_shape), - .v = .fromShape(kv_shape), - .layer_index = .init(.{}, .u32), - }; - } - - pub fn initShape(kv_shape: Shape) zml.ShapeOf(KvCache) { - return .{ - .k = kv_shape, - .v = kv_shape, - .layer_index = Shape.init(.{}, .u32), - }; - } - - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { - return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { - self.k.deinit(); - self.v.deinit(); - self.layer_index.deinit(); - } - - pub fn keys(self: KvCache) Tensor { - return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); - } - - pub fn values(self: KvCache) Tensor { - return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); - } - - pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { - return .{ - .k = scatterCache(self.k, new_k, self.layer_index, token_index), - .v = scatterCache(self.v, new_v, self.layer_index, token_index), - .layer_index = self.layer_index, - }; - } - - fn scatterCache(cache: Tensor, new: Tensor, layer_index: Tensor, token_index: ?Tensor) Tensor { - const k_shape = cache.shape().drop(.layer); - const converted = new.convert(cache.dtype()).transpose(k_shape); - const scatter_opts: Tensor.ScatterOpts = .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }; - - return if (token_index) |idx| - cache.scatterSlices(.{ .layer = layer_index.broad(idx.shape()), .k = idx }, converted, scatter_opts).reuseBuffer(cache) - else - cache.scatterSlices(.{ .layer = layer_index }, converted, scatter_opts).reuseBuffer(cache); - } - - pub fn atLayer(self: KvCache, layer_index: usize) KvCache { - return .{ - .k = self.k, - .v = self.v, - .layer_index = Tensor.scalar(layer_index, .u32), - }; - } - - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), - }; - } -}; +pub const KvCache = common.KvCache; /// Decoder self-attention with KV cache. pub const SelfAttention = struct { diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index ac2d6363..b8eb9db2 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -8,6 +8,7 @@ const Config = cfg.Config; const common = @import("common.zig"); const rmsNorm = common.rmsNorm; +const KvCache = common.KvCache; pub const Encoder = struct { conv0: CausalConv1d, @@ -60,7 +61,19 @@ pub const Encoder = struct { /// mel: [channels=128, time=frames] /// Returns: [s, d=1280] pub fn forward(self: Encoder, mel: Tensor) Tensor { - // Standardize precision to match model weights (usually bf16) + var h = self.convStem(mel); + + // Transformer layers + for (self.layers) |layer| { + h = layer.forward(h, self.config); + } + + return rmsNorm(h, self.norm, self.norm_eps); + } + + /// Run the convolutional stem: conv0 + gelu + conv1 + gelu + reshape. + /// mel: [channels=128, time=frames] → [s=(frames+1)/2, d=1280] + pub fn convStem(self: Encoder, mel: Tensor) Tensor { const dtype = self.conv0.weight.dtype(); var h = mel.convert(dtype).insertAxes(.channels, .{.batch}); @@ -69,14 +82,25 @@ pub const Encoder = struct { // [batch=1, channels=1280, time'] -> [s, d=1280] h = h.squeeze(.batch); - h = h.transpose(.{ .time, .channels }).rename(.{ .time = .s, .channels = .d }); + return h.transpose(.{ .time, .channels }).rename(.{ .time = .s, .channels = .d }); + } - // Transformer layers - for (self.layers) |layer| { - h = layer.forward(h, self.config); + /// Process a chunk of conv stem output through all transformer layers with KV cache. + /// x: [s=dsf, d=1280], token_index: scalar, kv_cache: KvCache + /// Returns: ([s=dsf, d=1280], KvCache) + pub fn transformerStep(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { + var h = x; + var cache = kv_cache; + + for (self.layers, 0..) |layer, i| { + const layer_cache = cache.atLayer(i); + const result = layer.forwardStep(h, token_index, layer_cache, self.config); + h = result[0]; + const updated_layer_cache = result[1]; + cache = updated_layer_cache.reuseBuffer(layer_cache); } - return rmsNorm(h, self.norm, self.norm_eps); + return .{ rmsNorm(h, self.norm, self.norm_eps), cache }; } }; @@ -164,6 +188,22 @@ pub const TransformerLayer = struct { return out; } + + /// h: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) + pub fn forwardStep(self: TransformerLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config) struct { Tensor, KvCache } { + const attn_out, const updated_kv_cache = self.attention.forwardStep( + rmsNorm(h, self.attention_norm, self.norm_eps), + token_index, + kv_cache, + config, + ); + var out = h.add(attn_out); + + const ffn_out = self.feed_forward.forward(rmsNorm(out, self.ffn_norm, self.norm_eps)); + out = out.add(ffn_out); + + return .{ out, updated_kv_cache }; + } }; pub const SelfAttention = struct { @@ -226,6 +266,58 @@ pub const SelfAttention = struct { const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return self.wo.forward(merged).rename(.{ .dout = .d }); } + + /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) + pub fn forwardStep(self: SelfAttention, x: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config) struct { Tensor, KvCache } { + const enc = config.encoder(); + const dtype = x.dtype(); + + var q = self.wq.forward(x); + var k = self.wk.forward(x); + var v = self.wv.forward(x); + + q = q.splitAxis(.dout, .{ .h = enc.n_heads, .hd = enc.head_dim }); + k = k.splitAxis(.dout, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); + v = v.splitAxis(.dout, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); + + // Position indices for RoPE: token_index + arange(s) + const pos_index = b: { + const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) + .withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + const rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = enc.rope_theta, + }; + + q = zml.nn.rope(q, pos_index, rope_opts); + k = zml.nn.rope(k, pos_index, rope_opts); + + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + // Update KV cache and retrieve cached K/V + const new_kv_cache = kv_cache.update(k, v, pos_index.rename(.{ .s = .k })); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + // Causal attention with sliding window + const full_seq_len = k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = full_seq_len, .k = full_seq_len }, dtype, enc.sliding_window); + attn_mask = attn_mask.gatherSlices( + zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), + token_index.reshape(.{ .coord = 1 }), + .{}, + ); + + const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); + + const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; + } }; pub const SwiGluFfn = struct { diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 175298fa..0f823b39 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -19,6 +19,9 @@ const dec = @import("decoder.zig"); const Decoder = dec.Decoder; const Adapter = dec.Adapter; +const common = @import("common.zig"); +const KvCache = common.KvCache; + const voxtral = @import("voxtral.zig"); const CliArgs = struct { @@ -98,12 +101,6 @@ pub fn main() !void { defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); - const num_frames = audio_len / config.audio().hop_length; - const dsf: u32 = config.downsample_factor(); - // Natural encoder output length after stride-2 causal conv (adapter pads to multiple of dsf) - const encoder_seq_len: u32 = @intCast((num_frames + 1) / 2); - const adapter_seq_len = (encoder_seq_len + dsf - 1) / dsf; - var melspectro_model: LogMelSpectrogram = .init(config); const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; var encoder_model: Encoder = .init(allocator, model_store.view().withPrefix(encoder_prefix), config); @@ -111,36 +108,39 @@ pub fn main() !void { const adapter: Adapter = .init(model_store.view(), config); - var model: Decoder = .init(allocator, model_store.view(), config); - defer model.deinit(allocator); + var decoder_model: Decoder = .init(allocator, model_store.view(), config); + defer decoder_model.deinit(allocator); - const dtype = model.tok_embeddings.dtype(); - const model_params: voxtral.VoxtralParameters = .{ - .prefill_tokens = .init(.{ .s = @as(u32, @intCast(tokens.len)) }, .u32), - .decode_tokens = .init(.{ .s = 1 }, .u32), - .adapter_out = .init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16), - .token_index = .init(.{}, .u32), - .kv_cache = .init(.init(.{ - .layer = model.layers.len, - .k = config.sliding_window, - .h = config.n_kv_heads, - .hd = config.head_dim, - }, dtype)), - .t_cond = Tensor.init(.{ .d = config.dim }, .f32), - .rng = .init(), - }; + // KV cache shapes for encoder and decoder + const enc_cfg = config.encoder(); + const enc_dtype = encoder_model.norm.dtype(); + const enc_kv_cache: KvCache = .init(.init(.{ + .layer = enc_cfg.n_layers, + .k = enc_cfg.sliding_window, + .h = enc_cfg.n_kv_heads, + .hd = enc_cfg.head_dim, + }, enc_dtype)); + + const dec_dtype = decoder_model.tok_embeddings.dtype(); + const dec_kv_cache: KvCache = .init(.init(.{ + .layer = decoder_model.layers.len, + .k = config.sliding_window, + .h = config.n_kv_heads, + .hd = config.head_dim, + }, dec_dtype)); // Launch concurrent compilation and buffer loading var tokenizer_future = try io.concurrent(voxtral.loadTokenizer, .{ allocator, io, model_dir, &progress }); var compiled_mel_spectrum_future = try io.concurrent(voxtral.compileMelSpectrum, .{ allocator, io, platform, melspectro_model, audio_len, &progress }); - var compiled_encoder_future = try io.concurrent(voxtral.compileEncoder, .{ allocator, io, platform, encoder_model, audio_len, &progress }); - var compiled_adapter_future = try io.concurrent(voxtral.compileAdapter, .{ allocator, io, platform, adapter, encoder_seq_len, config, &progress }); - var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, model, model_params, &progress }); + var compiled_conv_stem_future = try io.concurrent(voxtral.compileConvStem, .{ allocator, io, platform, encoder_model, audio_len, &progress }); + var compiled_encoder_step_future = try io.concurrent(voxtral.compileEncoderStep, .{ allocator, io, platform, encoder_model, enc_kv_cache, &progress }); + var compiled_adapter_step_future = try io.concurrent(voxtral.compileAdapterStep, .{ allocator, io, platform, adapter, config, &progress }); + var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, decoder_model, dec_kv_cache, &progress }); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{ &melspectro_model, io, platform }); var encoder_buffers_future = try io.concurrent(Encoder.load, .{ &encoder_model, allocator, io, platform, &model_store, &progress }); var adapter_buffers_future = try io.concurrent(Adapter.load, .{ &adapter, allocator, io, platform, &model_store, &progress }); - var decoder_buffers_future = try io.concurrent(Decoder.load, .{ &model, allocator, io, platform, &model_store, &progress }); + var decoder_buffers_future = try io.concurrent(Decoder.load, .{ &decoder_model, allocator, io, platform, &model_store, &progress }); // Await tokenizer and look up special token IDs var tokenizer = try tokenizer_future.await(io); @@ -155,14 +155,16 @@ pub fn main() !void { var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); defer compiled_mel_spectrum.deinit(); - var compiled_encoder = try compiled_encoder_future.await(io); - defer compiled_encoder.deinit(); + var compiled_conv_stem = try compiled_conv_stem_future.await(io); + defer compiled_conv_stem.deinit(); - var compiled_adapter = try compiled_adapter_future.await(io); - defer compiled_adapter.deinit(); + var compiled_encoder_step = try compiled_encoder_step_future.await(io); + defer compiled_encoder_step.deinit(); - var compiled_prefill, var compiled_decoder = try compiled_decoder_future.await(io); - defer compiled_prefill.deinit(); + var compiled_adapter_step = try compiled_adapter_step_future.await(io); + defer compiled_adapter_step.deinit(); + + var compiled_decoder = try compiled_decoder_future.await(io); defer compiled_decoder.deinit(); // -- Buffers @@ -191,14 +193,15 @@ pub fn main() !void { tokens, n_delay_tokens, &compiled_mel_spectrum, - &compiled_encoder, - &compiled_adapter, - &compiled_prefill, + &compiled_conv_stem, + &compiled_encoder_step, + &compiled_adapter_step, &compiled_decoder, &mel_spectrum_buffers, &encoder_buffers, &adapter_buffers, &decoder_buffers, - model_params, + enc_kv_cache, + dec_kv_cache, ); } diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 6d2d5e61..edc6a25e 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -16,15 +16,8 @@ const dec = @import("decoder.zig"); const Decoder = dec.Decoder; const Adapter = dec.Adapter; -pub const VoxtralParameters = struct { - prefill_tokens: Tensor, - decode_tokens: Tensor, - adapter_out: Tensor, - token_index: Tensor, - kv_cache: dec.KvCache, - t_cond: Tensor, - rng: Tensor.Rng, -}; +const common = @import("common.zig"); +const KvCache = common.KvCache; pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, padded_audio_len: usize, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); @@ -34,73 +27,60 @@ pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *c return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{padded_audio_len}, .f32).withTags(.{.samples})}); } -pub fn compileEncoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, padded_audio_len: usize, progress: *std.Progress.Node) !zml.Exe { +pub fn compileConvStem(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, padded_audio_len: usize, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling encoder...", 1); + var node = progress.start("Compiling conv stem...", 1); defer node.end(); const num_frames = padded_audio_len / model.config.audio().hop_length; - return try platform.compile(allocator, io, model, .forward, .{ + return try platform.compile(allocator, io, model, .convStem, .{ Tensor.init(.{ .channels = 128, .time = num_frames }, .f32), }); } -pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, encoder_seq_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { +pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, enc_kv_cache: KvCache, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); - - var node = progress.start("Compiling adapter...", 1); + var node = progress.start("Compiling encoder step...", 1); defer node.end(); - return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .s = encoder_seq_len, .d = config.encoder().dim }, .bf16), + const enc_cfg = model.config.encoder(); + const dsf = model.config.downsample_factor(); + + return try platform.compile(allocator, io, model, .transformerStep, .{ + Tensor.init(.{ .s = dsf, .d = enc_cfg.dim }, .bf16), + Tensor.init(.{}, .u32), + enc_kv_cache, }); } -pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, params: VoxtralParameters, progress: *std.Progress.Node) !struct { zml.Exe, zml.Exe } { - const compilePrefill = struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, tokens_: Tensor, params_: VoxtralParameters, progress_: *std.Progress.Node) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); +pub fn compileAdapterStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, config: Config, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling adapter step...", 1); + defer node.end(); - var node_ = progress_.start("Compiling decoder prefill...", 1); - defer node_.end(); + const dsf = config.downsample_factor(); - return try platform_.compile(allocator_, io_, model_, .forward, .{ - tokens_, - params_.adapter_out, - params_.token_index, - params_.kv_cache, - params_.t_cond, - params_.rng, - }); - } - }.call; + return try platform.compile(allocator, io, model, .forwardStep, .{ + Tensor.init(.{ .s = dsf, .d = config.encoder().dim }, .bf16), + }); +} - const compileDecode = struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, tokens_: Tensor, params_: VoxtralParameters, progress_: *std.Progress.Node) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); +pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, dec_kv_cache: KvCache, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling decoder...", 1); + defer node.end(); - var node_ = progress_.start("Compiling decoder decode...", 1); - defer node_.end(); + const dim = model.config.dim; - return try platform_.compile(allocator_, io_, model_, .forward, .{ - tokens_, - params_.adapter_out, - params_.token_index, - params_.kv_cache, - params_.t_cond, - params_.rng, - }); - } - }.call; - - var prefill_future = try io.concurrent(compilePrefill, .{ allocator, io, platform, model, params.prefill_tokens, params, progress }); - var decode_future = try io.concurrent(compileDecode, .{ allocator, io, platform, model, params.decode_tokens, params, progress }); - - return .{ - try prefill_future.await(io), - try decode_future.await(io), - }; + return try platform.compile(allocator, io, model, .forward, .{ + Tensor.init(.{ .s = 1 }, .u32), + Tensor.init(.{ .s = 1, .d = dim }, .bf16), + Tensor.init(.{}, .u32), + dec_kv_cache, + Tensor.init(.{ .d = dim }, .f32), + Tensor.Rng.init(), + }); } pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { @@ -150,23 +130,24 @@ pub fn runPipeline( tokens: []const u32, n_delay_tokens: u32, compiled_mel_spectrum: *zml.Exe, - compiled_encoder: *zml.Exe, - compiled_adapter: *zml.Exe, - compiled_prefill: *zml.Exe, + compiled_conv_stem: *zml.Exe, + compiled_encoder_step: *zml.Exe, + compiled_adapter_step: *zml.Exe, compiled_decoder: *zml.Exe, mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), encoder_buffers: *zml.Bufferized(Encoder), adapter_buffers: *zml.Bufferized(Adapter), decoder_buffers: *zml.Bufferized(Decoder), - model_params: VoxtralParameters, + enc_kv_cache: KvCache, + dec_kv_cache: KvCache, ) !void { log.info("Running inference pipeline...", .{}); const num_frames = audio_len / config.audio().hop_length; const dsf: u32 = config.downsample_factor(); - // Natural encoder output length after stride-2 causal conv (adapter pads to multiple of dsf) const encoder_seq_len: u32 = @intCast((num_frames + 1) / 2); - const adapter_seq_len = (encoder_seq_len + dsf - 1) / dsf; + const total_steps = (encoder_seq_len + dsf - 1) / dsf; + const enc_dim: u32 = config.encoder().dim; var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); @@ -196,45 +177,48 @@ pub fn runPipeline( defer mel_output.deinit(); log.info("Mel spectrogram done.", .{}); - // 2. Encoder - log.info("Running encoder...", .{}); - var encoder_output: zml.Buffer = undefined; + // 2. Conv stem + log.info("Running conv stem...", .{}); + var conv_output: zml.Buffer = undefined; { - var enc_args = try compiled_encoder.args(allocator); - defer enc_args.deinit(allocator); - var enc_results = try compiled_encoder.results(allocator); - defer enc_results.deinit(allocator); + var args = try compiled_conv_stem.args(allocator); + defer args.deinit(allocator); + var results = try compiled_conv_stem.results(allocator); + defer results.deinit(allocator); - encoder_output = try .uninitialized(io, platform, .init(.{ .s = encoder_seq_len, .d = config.encoder().dim }, .bf16), .{}); + conv_output = try .uninitialized(io, platform, .init(.{ .s = encoder_seq_len, .d = enc_dim }, .bf16), .{}); - enc_args.set(.{ encoder_buffers, mel_output }); - compiled_encoder.call(enc_args, &enc_results); - enc_results.fill(.{&encoder_output}); + args.set(.{ encoder_buffers, mel_output }); + compiled_conv_stem.call(args, &results); + results.fill(.{&conv_output}); } - defer encoder_output.deinit(); - log.info("Encoder done.", .{}); + defer conv_output.deinit(); + log.info("Conv stem done.", .{}); + + // 3. Download conv output to host (for per-step slicing) + const bf16_bytes: usize = 2; + const frame_bytes: usize = @as(usize, enc_dim) * bf16_bytes; + const chunk_bytes: usize = @as(usize, dsf) * frame_bytes; + const padded_enc_len: u32 = total_steps * dsf; + + const conv_host = try allocator.alloc(u8, @as(usize, padded_enc_len) * frame_bytes); + defer allocator.free(conv_host); + @memset(conv_host, 0); // zero-pad for last chunk if encoder_seq_len % dsf != 0 - // 3. Adapter - log.info("Running adapter...", .{}); - var adapter_output: zml.Buffer = undefined; { - var adp_args = try compiled_adapter.args(allocator); - defer adp_args.deinit(allocator); - var adp_results = try compiled_adapter.results(allocator); - defer adp_results.deinit(allocator); - - adapter_output = try .uninitialized(io, platform, .init(.{ .s = adapter_seq_len, .d = config.dim }, .bf16), .{}); - - adp_args.set(.{ adapter_buffers, encoder_output }); - compiled_adapter.call(adp_args, &adp_results); - adp_results.fill(.{&adapter_output}); + var download = try conv_output.toSliceAlloc(allocator, io); + defer download.free(allocator); + const src = download.constData(); + @memcpy(conv_host[0..src.len], src); } - defer adapter_output.deinit(); - log.info("Adapter done.", .{}); + log.info("Conv output downloaded to host ({} frames, {} steps).", .{ encoder_seq_len, total_steps }); - // 4. Init KV cache + t_cond buffer - var kv_cache_buffers = try model_params.kv_cache.initBuffer(io, platform); - defer dec.KvCache.deinitBuffer(&kv_cache_buffers); + // 4. Init KV caches + t_cond + RNG + var enc_kv_buffers = try enc_kv_cache.initBuffer(io, platform); + defer KvCache.deinitBuffer(&enc_kv_buffers); + + var dec_kv_buffers = try dec_kv_cache.initBuffer(io, platform); + defer KvCache.deinitBuffer(&dec_kv_buffers); const t_cond_data = computeTimeEmbedding(allocator, @as(f32, @floatFromInt(n_delay_tokens)), config.dim); defer allocator.free(t_cond_data); @@ -248,61 +232,87 @@ pub fn runPipeline( var rng_buffers = try Tensor.Rng.initBuffer(platform, 0, io); defer Tensor.Rng.deinitBuffer(&rng_buffers); - // 5. Decoder prefill - log.info("Running decoder prefill...", .{}); - var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); - defer generated_token_slice.free(allocator); - { - var prefill_args = try compiled_prefill.args(allocator); - defer prefill_args.deinit(allocator); - var prefill_results = try compiled_prefill.results(allocator); - defer prefill_results.deinit(allocator); - - const token_slice: zml.Slice = .init( - .init(.{@as(u32, @intCast(tokens.len))}, .u32), - std.mem.sliceAsBytes(tokens), - ); - var token_buffer: zml.Buffer = try .fromSlice(io, platform, token_slice); - defer token_buffer.deinit(); - - var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); - defer token_index_buffer.deinit(); - - var prefill_tokens_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, @intCast(tokens.len))}, .u32)); - defer prefill_tokens_slice.free(allocator); - var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); - defer prefill_output.deinit(); - - prefill_args.set(.{ decoder_buffers, token_buffer, adapter_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer, rng_buffers }); - compiled_prefill.call(prefill_args, &prefill_results); - prefill_results.fill(.{ &prefill_output, &kv_cache_buffers, &rng_buffers }); - - // Extract last token from prefill output (first generated token) - try prefill_output.toSlice(io, prefill_tokens_slice); - log.info("Prefill output tokens: {any}", .{prefill_tokens_slice.items(u32)[0..tokens.len]}); - generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[tokens.len - 1]; - log.info("First generated token (from prefill): {}", .{generated_token_slice.items(u32)[0]}); - } - log.info("Prefill done.", .{}); - - // 6. Autoregressive decode loop - log.info("Running decoder autoregressive...", .{}); + // 5. Step-by-step loop + log.info("Starting step-by-step loop ({} steps, {} prompt tokens)...", .{ total_steps, tokens.len }); const eos_token = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); - { - var decode_args = try compiled_decoder.args(allocator); - defer decode_args.deinit(allocator); - var decode_results = try compiled_decoder.results(allocator); - defer decode_results.deinit(allocator); - var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); - defer current_token_buffer.deinit(); + // Pre-allocate exe args/results (reused across iterations) + var enc_step_args = try compiled_encoder_step.args(allocator); + defer enc_step_args.deinit(allocator); + var enc_step_results = try compiled_encoder_step.results(allocator); + defer enc_step_results.deinit(allocator); - const max_pos = adapter_seq_len; - log.info("Decode loop: tokens.len={}, max_pos={}", .{ tokens.len, max_pos }); - var num_generated: usize = 0; - for (tokens.len..max_pos) |i| { - const generated_token = generated_token_slice.items(u32)[0]; + var adp_step_args = try compiled_adapter_step.args(allocator); + defer adp_step_args.deinit(allocator); + var adp_step_results = try compiled_adapter_step.results(allocator); + defer adp_step_results.deinit(allocator); + + var dec_args = try compiled_decoder.args(allocator); + defer dec_args.deinit(allocator); + var dec_results = try compiled_decoder.results(allocator); + defer dec_results.deinit(allocator); + + // Pre-allocate step output buffers (reused across iterations) + var enc_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = dsf, .d = enc_dim }, .bf16), .{}); + defer enc_step_output.deinit(); + + var adapter_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = 1, .d = config.dim }, .bf16), .{}); + defer adapter_output.deinit(); + + var token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); + defer token_slice.free(allocator); + var token_output: zml.Buffer = try .uninitialized(io, platform, .init(.{@as(u32, 1)}, .u32), .{}); + defer token_output.deinit(); + + var num_generated: usize = 0; + + for (0..total_steps) |t| { + const t_u32: u32 = @intCast(t); + + // --- Encoder step: process dsf frames through transformer with KV cache --- + const byte_offset = t * chunk_bytes; + const chunk_slice: zml.Slice = .init( + .init(.{ .s = dsf, .d = enc_dim }, .bf16), + conv_host[byte_offset .. byte_offset + chunk_bytes], + ); + var chunk_buffer: zml.Buffer = try .fromSlice(io, platform, chunk_slice); + defer chunk_buffer.deinit(); + + var enc_token_index: zml.Buffer = try .scalar(io, platform, t_u32 * dsf, .u32); + defer enc_token_index.deinit(); + + enc_step_args.set(.{ encoder_buffers, chunk_buffer, enc_token_index, &enc_kv_buffers }); + compiled_encoder_step.call(enc_step_args, &enc_step_results); + enc_step_results.fill(.{ &enc_step_output, &enc_kv_buffers }); + + // --- Adapter step: dsf encoded frames → 1 audio embedding --- + adp_step_args.set(.{ adapter_buffers, enc_step_output }); + compiled_adapter_step.call(adp_step_args, &adp_step_results); + adp_step_results.fill(.{&adapter_output}); + + // --- Decoder step: 1 token + 1 audio embedding → 1 output token --- + // During prompt: use prompt token. After prompt: use previous output. + if (t < tokens.len) { + token_slice.items(u32)[0] = tokens[t]; + } + + var text_token_buffer: zml.Buffer = try .fromSlice(io, platform, token_slice); + defer text_token_buffer.deinit(); + + var dec_token_index: zml.Buffer = try .scalar(io, platform, t_u32, .u32); + defer dec_token_index.deinit(); + + dec_args.set(.{ decoder_buffers, text_token_buffer, adapter_output, dec_token_index, &dec_kv_buffers, t_cond_buffer, rng_buffers }); + compiled_decoder.call(dec_args, &dec_results); + dec_results.fill(.{ &token_output, &dec_kv_buffers, &rng_buffers }); + + // Download output token + try token_output.toSlice(io, token_slice); + const generated_token = token_slice.items(u32)[0]; + + // Print tokens after prompt phase + if (t >= tokens.len) { num_generated += 1; if (generated_token != streaming_pad_token) { @@ -312,20 +322,12 @@ pub fn runPipeline( } if (generated_token == eos_token) { - log.info(" EOS reached at pos={}", .{i}); + log.info("EOS reached at step={}", .{t}); break; } - - var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(i)), .u32); - defer token_index_buffer.deinit(); - - decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_output, token_index_buffer, &kv_cache_buffers, t_cond_buffer, rng_buffers }); - compiled_decoder.call(decode_args, &decode_results); - decode_results.fill(.{ ¤t_token_buffer, &kv_cache_buffers, &rng_buffers }); - - try current_token_buffer.toSlice(io, generated_token_slice); } - std.debug.print("\n", .{}); - log.info("Decode done. Generated {} tokens total.", .{num_generated}); } + + std.debug.print("\n", .{}); + log.info("Done. Generated {} tokens.", .{num_generated}); } From 81effa4e408afbd446bb21078915ec0492cb5df8 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 16 Feb 2026 09:31:35 +0100 Subject: [PATCH 64/98] support diff len input --- examples/voxtral/main.zig | 22 ++- examples/voxtral/voxtral.zig | 268 ++++++++++++++++++++++++++--------- 2 files changed, 218 insertions(+), 72 deletions(-) diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 0f823b39..3d1c40f4 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -129,13 +129,17 @@ pub fn main() !void { .hd = config.head_dim, }, dec_dtype)); + const prompt_len: u32 = @intCast(tokens.len); + // Launch concurrent compilation and buffer loading var tokenizer_future = try io.concurrent(voxtral.loadTokenizer, .{ allocator, io, model_dir, &progress }); var compiled_mel_spectrum_future = try io.concurrent(voxtral.compileMelSpectrum, .{ allocator, io, platform, melspectro_model, audio_len, &progress }); var compiled_conv_stem_future = try io.concurrent(voxtral.compileConvStem, .{ allocator, io, platform, encoder_model, audio_len, &progress }); + var compiled_encoder_prefill_future = try io.concurrent(voxtral.compileEncoderPrefill, .{ allocator, io, platform, encoder_model, prompt_len, enc_kv_cache, &progress }); var compiled_encoder_step_future = try io.concurrent(voxtral.compileEncoderStep, .{ allocator, io, platform, encoder_model, enc_kv_cache, &progress }); + var compiled_adapter_future = try io.concurrent(voxtral.compileAdapter, .{ allocator, io, platform, adapter, prompt_len, config, &progress }); var compiled_adapter_step_future = try io.concurrent(voxtral.compileAdapterStep, .{ allocator, io, platform, adapter, config, &progress }); - var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, decoder_model, dec_kv_cache, &progress }); + var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, decoder_model, prompt_len, dec_kv_cache, &progress }); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{ &melspectro_model, io, platform }); var encoder_buffers_future = try io.concurrent(Encoder.load, .{ &encoder_model, allocator, io, platform, &model_store, &progress }); @@ -158,14 +162,21 @@ pub fn main() !void { var compiled_conv_stem = try compiled_conv_stem_future.await(io); defer compiled_conv_stem.deinit(); + var compiled_encoder_prefill = try compiled_encoder_prefill_future.await(io); + defer compiled_encoder_prefill.deinit(); + var compiled_encoder_step = try compiled_encoder_step_future.await(io); defer compiled_encoder_step.deinit(); + var compiled_adapter = try compiled_adapter_future.await(io); + defer compiled_adapter.deinit(); + var compiled_adapter_step = try compiled_adapter_step_future.await(io); defer compiled_adapter_step.deinit(); - var compiled_decoder = try compiled_decoder_future.await(io); - defer compiled_decoder.deinit(); + var compiled_decoder_prefill, var compiled_decoder_decode = try compiled_decoder_future.await(io); + defer compiled_decoder_prefill.deinit(); + defer compiled_decoder_decode.deinit(); // -- Buffers var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); @@ -194,9 +205,12 @@ pub fn main() !void { n_delay_tokens, &compiled_mel_spectrum, &compiled_conv_stem, + &compiled_encoder_prefill, &compiled_encoder_step, + &compiled_adapter, &compiled_adapter_step, - &compiled_decoder, + &compiled_decoder_prefill, + &compiled_decoder_decode, &mel_spectrum_buffers, &encoder_buffers, &adapter_buffers, diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index edc6a25e..a67dcff2 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -39,6 +39,23 @@ pub fn compileConvStem(allocator: std.mem.Allocator, io: std.Io, platform: *cons }); } +/// Encoder prefill: processes prompt_len * dsf frames at once, populating the KV cache. +pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, prompt_len: u32, enc_kv_cache: KvCache, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling encoder prefill...", 1); + defer node.end(); + + const enc_cfg = model.config.encoder(); + const dsf = model.config.downsample_factor(); + + return try platform.compile(allocator, io, model, .transformerStep, .{ + Tensor.init(.{ .s = prompt_len * dsf, .d = enc_cfg.dim }, .bf16), + Tensor.init(.{}, .u32), + enc_kv_cache, + }); +} + +/// Encoder step: processes dsf frames with KV cache (for streaming decode). pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, enc_kv_cache: KvCache, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling encoder step...", 1); @@ -54,6 +71,20 @@ pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *c }); } +/// Adapter batch: processes prompt_len * dsf encoder frames → prompt_len embeddings. +pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, prompt_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling adapter...", 1); + defer node.end(); + + const dsf = config.downsample_factor(); + + return try platform.compile(allocator, io, model, .forward, .{ + Tensor.init(.{ .s = prompt_len * dsf, .d = config.encoder().dim }, .bf16), + }); +} + +/// Adapter step: processes dsf encoder frames → 1 embedding (for streaming decode). pub fn compileAdapterStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, config: Config, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling adapter step...", 1); @@ -66,21 +97,51 @@ pub fn compileAdapterStep(allocator: std.mem.Allocator, io: std.Io, platform: *c }); } -pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, dec_kv_cache: KvCache, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling decoder...", 1); - defer node.end(); - +/// Decoder prefill (s=prompt_len) + decode (s=1), compiled concurrently. +pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, prompt_len: u32, dec_kv_cache: KvCache, progress: *std.Progress.Node) !struct { zml.Exe, zml.Exe } { const dim = model.config.dim; - return try platform.compile(allocator, io, model, .forward, .{ - Tensor.init(.{ .s = 1 }, .u32), - Tensor.init(.{ .s = 1, .d = dim }, .bf16), - Tensor.init(.{}, .u32), - dec_kv_cache, - Tensor.init(.{ .d = dim }, .f32), - Tensor.Rng.init(), - }); + const compilePrefill = struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, s: u32, kv_cache_: KvCache, dim_: u32, progress_: *std.Progress.Node) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling decoder prefill...", 1); + defer node_.end(); + + return try platform_.compile(allocator_, io_, model_, .forward, .{ + Tensor.init(.{ .s = s }, .u32), + Tensor.init(.{ .s = s, .d = dim_ }, .bf16), + Tensor.init(.{}, .u32), + kv_cache_, + Tensor.init(.{ .d = dim_ }, .f32), + Tensor.Rng.init(), + }); + } + }.call; + + const compileDecode = struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, kv_cache_: KvCache, dim_: u32, progress_: *std.Progress.Node) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling decoder decode...", 1); + defer node_.end(); + + return try platform_.compile(allocator_, io_, model_, .forward, .{ + Tensor.init(.{ .s = 1 }, .u32), + Tensor.init(.{ .s = 1, .d = dim_ }, .bf16), + Tensor.init(.{}, .u32), + kv_cache_, + Tensor.init(.{ .d = dim_ }, .f32), + Tensor.Rng.init(), + }); + } + }.call; + + var prefill_future = try io.concurrent(compilePrefill, .{ allocator, io, platform, model, prompt_len, dec_kv_cache, dim, progress }); + var decode_future = try io.concurrent(compileDecode, .{ allocator, io, platform, model, dec_kv_cache, dim, progress }); + + return .{ + try prefill_future.await(io), + try decode_future.await(io), + }; } pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { @@ -131,9 +192,12 @@ pub fn runPipeline( n_delay_tokens: u32, compiled_mel_spectrum: *zml.Exe, compiled_conv_stem: *zml.Exe, + compiled_encoder_prefill: *zml.Exe, compiled_encoder_step: *zml.Exe, + compiled_adapter: *zml.Exe, compiled_adapter_step: *zml.Exe, - compiled_decoder: *zml.Exe, + compiled_decoder_prefill: *zml.Exe, + compiled_decoder_decode: *zml.Exe, mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), encoder_buffers: *zml.Bufferized(Encoder), adapter_buffers: *zml.Bufferized(Adapter), @@ -148,6 +212,7 @@ pub fn runPipeline( const encoder_seq_len: u32 = @intCast((num_frames + 1) / 2); const total_steps = (encoder_seq_len + dsf - 1) / dsf; const enc_dim: u32 = config.encoder().dim; + const prompt_len: u32 = @intCast(tokens.len); var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); @@ -232,12 +297,98 @@ pub fn runPipeline( var rng_buffers = try Tensor.Rng.initBuffer(platform, 0, io); defer Tensor.Rng.deinitBuffer(&rng_buffers); - // 5. Step-by-step loop - log.info("Starting step-by-step loop ({} steps, {} prompt tokens)...", .{ total_steps, tokens.len }); const eos_token = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); - // Pre-allocate exe args/results (reused across iterations) + // ===== Prefill phase ===== + // 5. Encoder prefill: first prompt_len*dsf frames through transformer with KV cache + log.info("Running encoder prefill ({} frames)...", .{prompt_len * dsf}); + var encoder_prefill_output: zml.Buffer = undefined; + { + const prefill_frames = prompt_len * dsf; + const prefill_conv_bytes = @as(usize, prefill_frames) * frame_bytes; + const prefill_conv_slice: zml.Slice = .init( + .init(.{ .s = prefill_frames, .d = enc_dim }, .bf16), + conv_host[0..prefill_conv_bytes], + ); + var prefill_conv_buffer: zml.Buffer = try .fromSlice(io, platform, prefill_conv_slice); + defer prefill_conv_buffer.deinit(); + + var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); + defer enc_token_index.deinit(); + + encoder_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_frames, .d = enc_dim }, .bf16), .{}); + + var args = try compiled_encoder_prefill.args(allocator); + defer args.deinit(allocator); + var results = try compiled_encoder_prefill.results(allocator); + defer results.deinit(allocator); + + args.set(.{ encoder_buffers, prefill_conv_buffer, enc_token_index, &enc_kv_buffers }); + compiled_encoder_prefill.call(args, &results); + results.fill(.{ &encoder_prefill_output, &enc_kv_buffers }); + } + defer encoder_prefill_output.deinit(); + log.info("Encoder prefill done.", .{}); + + // 6. Adapter batch: encoder prefill output → prompt_len audio embeddings + log.info("Running adapter...", .{}); + var adapter_prefill_output: zml.Buffer = undefined; + { + adapter_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prompt_len, .d = config.dim }, .bf16), .{}); + + var args = try compiled_adapter.args(allocator); + defer args.deinit(allocator); + var results = try compiled_adapter.results(allocator); + defer results.deinit(allocator); + + args.set(.{ adapter_buffers, encoder_prefill_output }); + compiled_adapter.call(args, &results); + results.fill(.{&adapter_prefill_output}); + } + defer adapter_prefill_output.deinit(); + log.info("Adapter done.", .{}); + + // 7. Decoder prefill: all prompt tokens + audio embeddings → first generated token + log.info("Running decoder prefill...", .{}); + var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); + defer generated_token_slice.free(allocator); + { + var args = try compiled_decoder_prefill.args(allocator); + defer args.deinit(allocator); + var results = try compiled_decoder_prefill.results(allocator); + defer results.deinit(allocator); + + const token_data_slice: zml.Slice = .init( + .init(.{prompt_len}, .u32), + std.mem.sliceAsBytes(tokens), + ); + var token_buffer: zml.Buffer = try .fromSlice(io, platform, token_data_slice); + defer token_buffer.deinit(); + + var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); + defer token_index_buffer.deinit(); + + var prefill_output_slice: zml.Slice = try .alloc(allocator, .init(.{prompt_len}, .u32)); + defer prefill_output_slice.free(allocator); + var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_output_slice); + defer prefill_output.deinit(); + + args.set(.{ decoder_buffers, token_buffer, adapter_prefill_output, token_index_buffer, &dec_kv_buffers, t_cond_buffer, rng_buffers }); + compiled_decoder_prefill.call(args, &results); + results.fill(.{ &prefill_output, &dec_kv_buffers, &rng_buffers }); + + // Extract last token from prefill output (first generated token) + try prefill_output.toSlice(io, prefill_output_slice); + log.info("Prefill output tokens: {any}", .{prefill_output_slice.items(u32)[0..prompt_len]}); + generated_token_slice.items(u32)[0] = prefill_output_slice.items(u32)[prompt_len - 1]; + log.info("First generated token (from prefill): {}", .{generated_token_slice.items(u32)[0]}); + } + log.info("Decoder prefill done.", .{}); + + // ===== Decode phase: step-by-step encoder → adapter → decoder ===== + log.info("Running decode loop (steps {}..{})...", .{ prompt_len, total_steps }); + var enc_step_args = try compiled_encoder_step.args(allocator); defer enc_step_args.deinit(allocator); var enc_step_results = try compiled_encoder_step.results(allocator); @@ -248,29 +399,37 @@ pub fn runPipeline( var adp_step_results = try compiled_adapter_step.results(allocator); defer adp_step_results.deinit(allocator); - var dec_args = try compiled_decoder.args(allocator); - defer dec_args.deinit(allocator); - var dec_results = try compiled_decoder.results(allocator); - defer dec_results.deinit(allocator); + var decode_args = try compiled_decoder_decode.args(allocator); + defer decode_args.deinit(allocator); + var decode_results = try compiled_decoder_decode.results(allocator); + defer decode_results.deinit(allocator); - // Pre-allocate step output buffers (reused across iterations) var enc_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = dsf, .d = enc_dim }, .bf16), .{}); defer enc_step_output.deinit(); - var adapter_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = 1, .d = config.dim }, .bf16), .{}); - defer adapter_output.deinit(); + var adapter_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = 1, .d = config.dim }, .bf16), .{}); + defer adapter_step_output.deinit(); - var token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); - defer token_slice.free(allocator); - var token_output: zml.Buffer = try .uninitialized(io, platform, .init(.{@as(u32, 1)}, .u32), .{}); - defer token_output.deinit(); + var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); + defer current_token_buffer.deinit(); var num_generated: usize = 0; + for (prompt_len..total_steps) |t| { + const generated_token = generated_token_slice.items(u32)[0]; + num_generated += 1; - for (0..total_steps) |t| { - const t_u32: u32 = @intCast(t); + if (generated_token != streaming_pad_token) { + if (try tokenizer_decoder.next(generated_token)) |chunk| { + std.debug.print("{s}", .{chunk}); + } + } - // --- Encoder step: process dsf frames through transformer with KV cache --- + if (generated_token == eos_token) { + log.info(" EOS reached at step={}", .{t}); + break; + } + + // Encoder step: dsf frames → encoded chunk const byte_offset = t * chunk_bytes; const chunk_slice: zml.Slice = .init( .init(.{ .s = dsf, .d = enc_dim }, .bf16), @@ -279,55 +438,28 @@ pub fn runPipeline( var chunk_buffer: zml.Buffer = try .fromSlice(io, platform, chunk_slice); defer chunk_buffer.deinit(); - var enc_token_index: zml.Buffer = try .scalar(io, platform, t_u32 * dsf, .u32); + var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)) * dsf, .u32); defer enc_token_index.deinit(); enc_step_args.set(.{ encoder_buffers, chunk_buffer, enc_token_index, &enc_kv_buffers }); compiled_encoder_step.call(enc_step_args, &enc_step_results); enc_step_results.fill(.{ &enc_step_output, &enc_kv_buffers }); - // --- Adapter step: dsf encoded frames → 1 audio embedding --- + // Adapter step: encoded chunk → 1 audio embedding adp_step_args.set(.{ adapter_buffers, enc_step_output }); compiled_adapter_step.call(adp_step_args, &adp_step_results); - adp_step_results.fill(.{&adapter_output}); + adp_step_results.fill(.{&adapter_step_output}); - // --- Decoder step: 1 token + 1 audio embedding → 1 output token --- - // During prompt: use prompt token. After prompt: use previous output. - if (t < tokens.len) { - token_slice.items(u32)[0] = tokens[t]; - } - - var text_token_buffer: zml.Buffer = try .fromSlice(io, platform, token_slice); - defer text_token_buffer.deinit(); - - var dec_token_index: zml.Buffer = try .scalar(io, platform, t_u32, .u32); + // Decoder decode: 1 token + 1 audio embedding → next token + var dec_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)), .u32); defer dec_token_index.deinit(); - dec_args.set(.{ decoder_buffers, text_token_buffer, adapter_output, dec_token_index, &dec_kv_buffers, t_cond_buffer, rng_buffers }); - compiled_decoder.call(dec_args, &dec_results); - dec_results.fill(.{ &token_output, &dec_kv_buffers, &rng_buffers }); + decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_step_output, dec_token_index, &dec_kv_buffers, t_cond_buffer, rng_buffers }); + compiled_decoder_decode.call(decode_args, &decode_results); + decode_results.fill(.{ ¤t_token_buffer, &dec_kv_buffers, &rng_buffers }); - // Download output token - try token_output.toSlice(io, token_slice); - const generated_token = token_slice.items(u32)[0]; - - // Print tokens after prompt phase - if (t >= tokens.len) { - num_generated += 1; - - if (generated_token != streaming_pad_token) { - if (try tokenizer_decoder.next(generated_token)) |chunk| { - std.debug.print("{s}", .{chunk}); - } - } - - if (generated_token == eos_token) { - log.info("EOS reached at step={}", .{t}); - break; - } - } + try current_token_buffer.toSlice(io, generated_token_slice); } - std.debug.print("\n", .{}); - log.info("Done. Generated {} tokens.", .{num_generated}); + log.info("Decode done. Generated {} tokens total.", .{num_generated}); } From cc06ee95b893815ee02e6a74eac25c389b8a5bff Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 16 Feb 2026 11:35:43 +0100 Subject: [PATCH 65/98] encoder uses attention directly as well --- examples/voxtral/decoder.zig | 21 +++----- examples/voxtral/encoder.zig | 93 ++++-------------------------------- examples/voxtral/main.zig | 33 +++++++++++-- examples/voxtral/voxtral.zig | 38 +++++++++------ 4 files changed, 70 insertions(+), 115 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 4b953794..4334e47b 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -120,7 +120,7 @@ pub const Decoder = struct { /// Run all decoder layers on input embeddings, sample next tokens and return updated KV cache + RNG. /// text_tokens: [s] u32, audio_embed: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d], rng: Rng /// Returns: (tokens [s] u32, KvCache, Rng) - pub fn forward(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng) struct { Tensor, KvCache, Tensor.Rng } { + pub fn forward(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor.Rng } { const text_embeds = self.tok_embeddings.gather(.{ .voc = text_tokens }, .{}); const input_embeds = text_embeds.add(audio_embed); @@ -131,7 +131,7 @@ pub const Decoder = struct { for (self.layers, 0..) |layer, i| { const layer_cache = cache.atLayer(i); - const result = layer.forward(h, token_index, layer_cache, t, self.config); + const result = layer.forward(h, token_index, layer_cache, t, self.config, attention_metadata, attention_parameters); h = result[0]; const updated_layer_cache = result[1]; @@ -202,7 +202,7 @@ pub const SelfAttention = struct { } /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) - pub fn forward(self: SelfAttention, x: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config) struct { Tensor, KvCache } { + pub fn forward(self: SelfAttention, x: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { const dtype = x.dtype(); var q = self.wq.forward(x); @@ -238,16 +238,7 @@ pub const SelfAttention = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - // Causal attention with sliding window - const full_seq_len = k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = full_seq_len, .k = full_seq_len }, dtype, config.sliding_window); - attn_mask = attn_mask.gatherSlices( - Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), - token_index.reshape(.{ .coord = 1 }), - .{}, - ); - - const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); + const attn_out = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); // Merge heads and output projection: [q, h, hd] → [s, d] const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); @@ -286,13 +277,15 @@ pub const DecoderLayer = struct { } /// h: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] → ([s, d], KvCache) - pub fn forward(self: DecoderLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, config: Config) struct { Tensor, KvCache } { + pub fn forward(self: DecoderLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, config: Config, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { // Pre-attention norm → attention (with KV cache) → residual const attn_out, const updated_kv_cache = self.attention.forward( rmsNorm(h, self.attention_norm, self.norm_eps), token_index, kv_cache, config, + attention_metadata, + attention_parameters, ); var out = h.add(attn_out); diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index b8eb9db2..017850d0 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -58,19 +58,6 @@ pub const Encoder = struct { self.norm.deinit(); } - /// mel: [channels=128, time=frames] - /// Returns: [s, d=1280] - pub fn forward(self: Encoder, mel: Tensor) Tensor { - var h = self.convStem(mel); - - // Transformer layers - for (self.layers) |layer| { - h = layer.forward(h, self.config); - } - - return rmsNorm(h, self.norm, self.norm_eps); - } - /// Run the convolutional stem: conv0 + gelu + conv1 + gelu + reshape. /// mel: [channels=128, time=frames] → [s=(frames+1)/2, d=1280] pub fn convStem(self: Encoder, mel: Tensor) Tensor { @@ -88,13 +75,13 @@ pub const Encoder = struct { /// Process a chunk of conv stem output through all transformer layers with KV cache. /// x: [s=dsf, d=1280], token_index: scalar, kv_cache: KvCache /// Returns: ([s=dsf, d=1280], KvCache) - pub fn transformerStep(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { + pub fn transformer(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { var h = x; var cache = kv_cache; for (self.layers, 0..) |layer, i| { const layer_cache = cache.atLayer(i); - const result = layer.forwardStep(h, token_index, layer_cache, self.config); + const result = layer.forward(h, token_index, layer_cache, self.config, attention_metadata, attention_parameters); h = result[0]; const updated_layer_cache = result[1]; cache = updated_layer_cache.reuseBuffer(layer_cache); @@ -171,31 +158,20 @@ pub const TransformerLayer = struct { pub fn unload(self: *zml.Bufferized(TransformerLayer)) void { self.attention_norm.deinit(); SelfAttention.unload(&self.attention); - + self.ffn_norm.deinit(); SwiGluFfn.unload(&self.feed_forward); } - /// h: [s, d] -> [s, d] - pub fn forward(self: TransformerLayer, h: Tensor, config: Config) Tensor { - // Pre-attention norm + attention + residual - const attn_out = self.attention.forward(rmsNorm(h, self.attention_norm, self.norm_eps), config); - var out = h.add(attn_out); - - // Pre-FFN norm + FFN + residual - const ffn_out = self.feed_forward.forward(rmsNorm(out, self.ffn_norm, self.norm_eps)); - out = out.add(ffn_out); - - return out; - } - /// h: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) - pub fn forwardStep(self: TransformerLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config) struct { Tensor, KvCache } { - const attn_out, const updated_kv_cache = self.attention.forwardStep( + pub fn forward(self: TransformerLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { + const attn_out, const updated_kv_cache = self.attention.forward( rmsNorm(h, self.attention_norm, self.norm_eps), token_index, kv_cache, config, + attention_metadata, + attention_parameters, ); var out = h.add(attn_out); @@ -228,47 +204,8 @@ pub const SelfAttention = struct { common.deinitLinear(&self.wo); } - /// x: [s, d] -> [s, d] - pub fn forward(self: SelfAttention, x: Tensor, config: Config) Tensor { - const enc = config.encoder(); - const dtype = x.dtype(); - - // Q, K, V projections (encoder: Q and V have biases, K does not) - var q = self.wq.forward(x); - var k = self.wk.forward(x); - var v = self.wv.forward(x); - - // Split into heads: [s, dout] -> [s, h, hd] - q = q.splitAxis(.dout, .{ .h = enc.n_heads, .hd = enc.head_dim }); - k = k.splitAxis(.dout, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); - v = v.splitAxis(.dout, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); - - // RoPE (interleaved, matching is_neox_style=False in Python) - const rope_opts: zml.nn.RopeOpts = .{ - .layout = .interleaved, - .freq_base = enc.rope_theta, - }; - - q = zml.nn.rope(q, null, rope_opts); - k = zml.nn.rope(k, null, rope_opts); - - // Rename seq dim for sdpa: .s -> .q/.k - q = q.rename(.{ .s = .q }); - k = k.rename(.{ .s = .k }); - v = v.rename(.{ .s = .k }); - - // Causal attention with sliding window - const seq_len = q.dim(.q); - const attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, enc.sliding_window); - const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); - - // Merge heads and output projection: [q, h, hd] -> [s, d] - const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return self.wo.forward(merged).rename(.{ .dout = .d }); - } - /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) - pub fn forwardStep(self: SelfAttention, x: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config) struct { Tensor, KvCache } { + pub fn forward(self: SelfAttention, x: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { const enc = config.encoder(); const dtype = x.dtype(); @@ -304,16 +241,7 @@ pub const SelfAttention = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - // Causal attention with sliding window - const full_seq_len = k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = full_seq_len, .k = full_seq_len }, dtype, enc.sliding_window); - attn_mask = attn_mask.gatherSlices( - zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), - token_index.reshape(.{ .coord = 1 }), - .{}, - ); - - const attn_out = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); + const attn_out = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; @@ -343,8 +271,7 @@ pub const SwiGluFfn = struct { pub fn forward(self: SwiGluFfn, x: Tensor) Tensor { const gate = self.w1.forward(x).silu(); const up = self.w3.forward(x); - + return self.w2.forward(gate.mul(up).rename(.{ .dout = .d })).rename(.{ .dout = .d }); } }; - diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 3d1c40f4..ef3a7655 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -28,6 +28,7 @@ const CliArgs = struct { input: []const u8, model: []const u8, transcription_delay_ms: f32 = 480.0, + backend: ?zml.attention.Backend = null, }; const n_left_pad_tokens: u32 = 32; @@ -97,10 +98,18 @@ pub fn main() !void { defer allocator.free(padded_wav); const audio_len = padded_wav.len; - var platform: *zml.Platform = try .auto(allocator, io, .{}); + var platform: *zml.Platform = try .auto(allocator, io, .{ + .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.75 } } }, + }); defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); + const backend = args.backend orelse b: { + const selected = zml.attention.Backend.auto(platform); + log.info("Selected backend: {}", .{selected}); + break :b selected; + }; + var melspectro_model: LogMelSpectrogram = .init(config); const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; var encoder_model: Encoder = .init(allocator, model_store.view().withPrefix(encoder_prefix), config); @@ -131,15 +140,23 @@ pub fn main() !void { const prompt_len: u32 = @intCast(tokens.len); + const enc_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(enc_cfg.sliding_window))); + const dec_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(config.sliding_window))); + const attention_parameters: zml.attention.Parameters = .init(.fromBackend(backend)); + // Launch concurrent compilation and buffer loading var tokenizer_future = try io.concurrent(voxtral.loadTokenizer, .{ allocator, io, model_dir, &progress }); + + // -- Fixed len for now var compiled_mel_spectrum_future = try io.concurrent(voxtral.compileMelSpectrum, .{ allocator, io, platform, melspectro_model, audio_len, &progress }); var compiled_conv_stem_future = try io.concurrent(voxtral.compileConvStem, .{ allocator, io, platform, encoder_model, audio_len, &progress }); - var compiled_encoder_prefill_future = try io.concurrent(voxtral.compileEncoderPrefill, .{ allocator, io, platform, encoder_model, prompt_len, enc_kv_cache, &progress }); - var compiled_encoder_step_future = try io.concurrent(voxtral.compileEncoderStep, .{ allocator, io, platform, encoder_model, enc_kv_cache, &progress }); + // -- + + var compiled_encoder_prefill_future = try io.concurrent(voxtral.compileEncoderPrefill, .{ allocator, io, platform, encoder_model, prompt_len, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); + var compiled_encoder_step_future = try io.concurrent(voxtral.compileEncoderStep, .{ allocator, io, platform, encoder_model, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); var compiled_adapter_future = try io.concurrent(voxtral.compileAdapter, .{ allocator, io, platform, adapter, prompt_len, config, &progress }); var compiled_adapter_step_future = try io.concurrent(voxtral.compileAdapterStep, .{ allocator, io, platform, adapter, config, &progress }); - var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, decoder_model, prompt_len, dec_kv_cache, &progress }); + var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, decoder_model, prompt_len, dec_kv_cache, dec_attention_metadata, attention_parameters, &progress }); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{ &melspectro_model, io, platform }); var encoder_buffers_future = try io.concurrent(Encoder.load, .{ &encoder_model, allocator, io, platform, &model_store, &progress }); @@ -191,6 +208,12 @@ pub fn main() !void { var decoder_buffers = try decoder_buffers_future.await(io); defer Decoder.unload(&decoder_buffers, allocator); + var enc_attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try enc_attention_metadata.initBuffer(io, platform); + defer zml.attention.Metadata.deinitBuffer(&enc_attention_metadata_buffers); + + var dec_attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try dec_attention_metadata.initBuffer(io, platform); + defer zml.attention.Metadata.deinitBuffer(&dec_attention_metadata_buffers); + progress.end(); try voxtral.runPipeline( @@ -217,5 +240,7 @@ pub fn main() !void { &decoder_buffers, enc_kv_cache, dec_kv_cache, + &enc_attention_metadata_buffers, + &dec_attention_metadata_buffers, ); } diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index a67dcff2..973c045c 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -40,7 +40,7 @@ pub fn compileConvStem(allocator: std.mem.Allocator, io: std.Io, platform: *cons } /// Encoder prefill: processes prompt_len * dsf frames at once, populating the KV cache. -pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, prompt_len: u32, enc_kv_cache: KvCache, progress: *std.Progress.Node) !zml.Exe { +pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, prompt_len: u32, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling encoder prefill...", 1); defer node.end(); @@ -48,15 +48,17 @@ pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: const enc_cfg = model.config.encoder(); const dsf = model.config.downsample_factor(); - return try platform.compile(allocator, io, model, .transformerStep, .{ + return try platform.compile(allocator, io, model, .transformer, .{ Tensor.init(.{ .s = prompt_len * dsf, .d = enc_cfg.dim }, .bf16), Tensor.init(.{}, .u32), enc_kv_cache, + attention_metadata, + attention_parameters, }); } /// Encoder step: processes dsf frames with KV cache (for streaming decode). -pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, enc_kv_cache: KvCache, progress: *std.Progress.Node) !zml.Exe { +pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling encoder step...", 1); defer node.end(); @@ -64,10 +66,12 @@ pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *c const enc_cfg = model.config.encoder(); const dsf = model.config.downsample_factor(); - return try platform.compile(allocator, io, model, .transformerStep, .{ + return try platform.compile(allocator, io, model, .transformer, .{ Tensor.init(.{ .s = dsf, .d = enc_cfg.dim }, .bf16), Tensor.init(.{}, .u32), enc_kv_cache, + attention_metadata, + attention_parameters, }); } @@ -92,17 +96,17 @@ pub fn compileAdapterStep(allocator: std.mem.Allocator, io: std.Io, platform: *c const dsf = config.downsample_factor(); - return try platform.compile(allocator, io, model, .forwardStep, .{ + return try platform.compile(allocator, io, model, .forward, .{ Tensor.init(.{ .s = dsf, .d = config.encoder().dim }, .bf16), }); } /// Decoder prefill (s=prompt_len) + decode (s=1), compiled concurrently. -pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, prompt_len: u32, dec_kv_cache: KvCache, progress: *std.Progress.Node) !struct { zml.Exe, zml.Exe } { +pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, prompt_len: u32, dec_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !struct { zml.Exe, zml.Exe } { const dim = model.config.dim; const compilePrefill = struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, s: u32, kv_cache_: KvCache, dim_: u32, progress_: *std.Progress.Node) !zml.Exe { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, s: u32, kv_cache_: KvCache, dim_: u32, attention_metadata_: zml.attention.Metadata, attention_parameters_: zml.attention.Parameters, progress_: *std.Progress.Node) !zml.Exe { progress_.increaseEstimatedTotalItems(1); var node_ = progress_.start("Compiling decoder prefill...", 1); defer node_.end(); @@ -114,12 +118,14 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const kv_cache_, Tensor.init(.{ .d = dim_ }, .f32), Tensor.Rng.init(), + attention_metadata_, + attention_parameters_, }); } }.call; const compileDecode = struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, kv_cache_: KvCache, dim_: u32, progress_: *std.Progress.Node) !zml.Exe { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, kv_cache_: KvCache, dim_: u32, attention_metadata_: zml.attention.Metadata, attention_parameters_: zml.attention.Parameters, progress_: *std.Progress.Node) !zml.Exe { progress_.increaseEstimatedTotalItems(1); var node_ = progress_.start("Compiling decoder decode...", 1); defer node_.end(); @@ -131,12 +137,14 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const kv_cache_, Tensor.init(.{ .d = dim_ }, .f32), Tensor.Rng.init(), + attention_metadata_, + attention_parameters_, }); } }.call; - var prefill_future = try io.concurrent(compilePrefill, .{ allocator, io, platform, model, prompt_len, dec_kv_cache, dim, progress }); - var decode_future = try io.concurrent(compileDecode, .{ allocator, io, platform, model, dec_kv_cache, dim, progress }); + var prefill_future = try io.concurrent(compilePrefill, .{ allocator, io, platform, model, prompt_len, dec_kv_cache, dim, attention_metadata, attention_parameters, progress }); + var decode_future = try io.concurrent(compileDecode, .{ allocator, io, platform, model, dec_kv_cache, dim, attention_metadata, attention_parameters, progress }); return .{ try prefill_future.await(io), @@ -204,6 +212,8 @@ pub fn runPipeline( decoder_buffers: *zml.Bufferized(Decoder), enc_kv_cache: KvCache, dec_kv_cache: KvCache, + enc_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), ) !void { log.info("Running inference pipeline...", .{}); @@ -324,7 +334,7 @@ pub fn runPipeline( var results = try compiled_encoder_prefill.results(allocator); defer results.deinit(allocator); - args.set(.{ encoder_buffers, prefill_conv_buffer, enc_token_index, &enc_kv_buffers }); + args.set(.{ encoder_buffers, prefill_conv_buffer, enc_token_index, &enc_kv_buffers, enc_attention_metadata_buffers }); compiled_encoder_prefill.call(args, &results); results.fill(.{ &encoder_prefill_output, &enc_kv_buffers }); } @@ -374,7 +384,7 @@ pub fn runPipeline( var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_output_slice); defer prefill_output.deinit(); - args.set(.{ decoder_buffers, token_buffer, adapter_prefill_output, token_index_buffer, &dec_kv_buffers, t_cond_buffer, rng_buffers }); + args.set(.{ decoder_buffers, token_buffer, adapter_prefill_output, token_index_buffer, &dec_kv_buffers, t_cond_buffer, rng_buffers, dec_attention_metadata_buffers }); compiled_decoder_prefill.call(args, &results); results.fill(.{ &prefill_output, &dec_kv_buffers, &rng_buffers }); @@ -441,7 +451,7 @@ pub fn runPipeline( var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)) * dsf, .u32); defer enc_token_index.deinit(); - enc_step_args.set(.{ encoder_buffers, chunk_buffer, enc_token_index, &enc_kv_buffers }); + enc_step_args.set(.{ encoder_buffers, chunk_buffer, enc_token_index, &enc_kv_buffers, enc_attention_metadata_buffers }); compiled_encoder_step.call(enc_step_args, &enc_step_results); enc_step_results.fill(.{ &enc_step_output, &enc_kv_buffers }); @@ -454,7 +464,7 @@ pub fn runPipeline( var dec_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)), .u32); defer dec_token_index.deinit(); - decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_step_output, dec_token_index, &dec_kv_buffers, t_cond_buffer, rng_buffers }); + decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_step_output, dec_token_index, &dec_kv_buffers, t_cond_buffer, rng_buffers, dec_attention_metadata_buffers }); compiled_decoder_decode.call(decode_args, &decode_results); decode_results.fill(.{ ¤t_token_buffer, &dec_kv_buffers, &rng_buffers }); From 3002cc985cecaefb5252d657849144e08796fb06 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 16 Feb 2026 12:49:52 +0100 Subject: [PATCH 66/98] step conv stem + prefill --- examples/voxtral/encoder.zig | 12 +++ examples/voxtral/main.zig | 8 +- examples/voxtral/voxtral.zig | 166 +++++++++++++++++++++++------------ 3 files changed, 130 insertions(+), 56 deletions(-) diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 017850d0..e136de9a 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -58,6 +58,18 @@ pub const Encoder = struct { self.norm.deinit(); } + /// Streaming conv stem: processes a mel chunk with history prepended. + /// Input: mel_chunk [channels=128, time=mel_history+mel_per_step] + /// Output: [s=dsf, d=1280] — only the new frames (history-derived outputs discarded) + pub fn convStemStep(self: Encoder, mel_chunk: Tensor) Tensor { + const full_output = self.convStem(mel_chunk); + // full_output: [s=(12+1)/2=6, d=1280] when chunk is 12 frames + // Skip first 2 (history-derived), keep last dsf=4 + const dsf: u32 = self.config.downsample_factor(); + const total: u32 = @intCast(full_output.dim(.s)); + return full_output.slice1d(.s, .{ .start = total - dsf }); + } + /// Run the convolutional stem: conv0 + gelu + conv1 + gelu + reshape. /// mel: [channels=128, time=frames] → [s=(frames+1)/2, d=1280] pub fn convStem(self: Encoder, mel: Tensor) Tensor { diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index ef3a7655..74a1f6a4 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -149,7 +149,9 @@ pub fn main() !void { // -- Fixed len for now var compiled_mel_spectrum_future = try io.concurrent(voxtral.compileMelSpectrum, .{ allocator, io, platform, melspectro_model, audio_len, &progress }); - var compiled_conv_stem_future = try io.concurrent(voxtral.compileConvStem, .{ allocator, io, platform, encoder_model, audio_len, &progress }); + const mel_per_step: u32 = config.downsample_factor() * 2; + var compiled_conv_stem_future = try io.concurrent(voxtral.compileConvStem, .{ allocator, io, platform, encoder_model, prompt_len * mel_per_step, &progress }); + var compiled_conv_stem_step_future = try io.concurrent(voxtral.compileConvStemStep, .{ allocator, io, platform, encoder_model, &progress }); // -- var compiled_encoder_prefill_future = try io.concurrent(voxtral.compileEncoderPrefill, .{ allocator, io, platform, encoder_model, prompt_len, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); @@ -179,6 +181,9 @@ pub fn main() !void { var compiled_conv_stem = try compiled_conv_stem_future.await(io); defer compiled_conv_stem.deinit(); + var compiled_conv_stem_step = try compiled_conv_stem_step_future.await(io); + defer compiled_conv_stem_step.deinit(); + var compiled_encoder_prefill = try compiled_encoder_prefill_future.await(io); defer compiled_encoder_prefill.deinit(); @@ -228,6 +233,7 @@ pub fn main() !void { n_delay_tokens, &compiled_mel_spectrum, &compiled_conv_stem, + &compiled_conv_stem_step, &compiled_encoder_prefill, &compiled_encoder_step, &compiled_adapter, diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 973c045c..41113558 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -27,15 +27,27 @@ pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *c return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{padded_audio_len}, .f32).withTags(.{.samples})}); } -pub fn compileConvStem(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, padded_audio_len: usize, progress: *std.Progress.Node) !zml.Exe { +pub fn compileConvStem(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, num_mel_frames: u32, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling conv stem...", 1); defer node.end(); - const num_frames = padded_audio_len / model.config.audio().hop_length; - return try platform.compile(allocator, io, model, .convStem, .{ - Tensor.init(.{ .channels = 128, .time = num_frames }, .f32), + Tensor.init(.{ .channels = 128, .time = num_mel_frames }, .f32), + }); +} + +pub fn compileConvStemStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling conv stem step...", 1); + defer node.end(); + + const mel_history = 4; + const mel_per_step = model.config.downsample_factor() * 2; + const chunk_mel = mel_history + mel_per_step; + + return try platform.compile(allocator, io, model, .convStemStep, .{ + Tensor.init(.{ .channels = 128, .time = chunk_mel }, .f32), }); } @@ -200,6 +212,7 @@ pub fn runPipeline( n_delay_tokens: u32, compiled_mel_spectrum: *zml.Exe, compiled_conv_stem: *zml.Exe, + compiled_conv_stem_step: *zml.Exe, compiled_encoder_prefill: *zml.Exe, compiled_encoder_step: *zml.Exe, compiled_adapter: *zml.Exe, @@ -217,8 +230,11 @@ pub fn runPipeline( ) !void { log.info("Running inference pipeline...", .{}); - const num_frames = audio_len / config.audio().hop_length; + const num_frames: u32 = @intCast(audio_len / config.audio().hop_length); const dsf: u32 = config.downsample_factor(); + const mel_per_step: u32 = dsf * 2; + const mel_history: u32 = 4; + const chunk_mel: u32 = mel_history + mel_per_step; const encoder_seq_len: u32 = @intCast((num_frames + 1) / 2); const total_steps = (encoder_seq_len + dsf - 1) / dsf; const enc_dim: u32 = config.encoder().dim; @@ -252,43 +268,28 @@ pub fn runPipeline( defer mel_output.deinit(); log.info("Mel spectrogram done.", .{}); - // 2. Conv stem - log.info("Running conv stem...", .{}); - var conv_output: zml.Buffer = undefined; - { - var args = try compiled_conv_stem.args(allocator); - defer args.deinit(allocator); - var results = try compiled_conv_stem.results(allocator); - defer results.deinit(allocator); + // 2. Download mel to host (channels-major: [128][padded_mel_frames] in f32) + const f32_bytes: usize = @sizeOf(f32); + const padded_mel_frames: u32 = total_steps * mel_per_step; - conv_output = try .uninitialized(io, platform, .init(.{ .s = encoder_seq_len, .d = enc_dim }, .bf16), .{}); - - args.set(.{ encoder_buffers, mel_output }); - compiled_conv_stem.call(args, &results); - results.fill(.{&conv_output}); - } - defer conv_output.deinit(); - log.info("Conv stem done.", .{}); - - // 3. Download conv output to host (for per-step slicing) - const bf16_bytes: usize = 2; - const frame_bytes: usize = @as(usize, enc_dim) * bf16_bytes; - const chunk_bytes: usize = @as(usize, dsf) * frame_bytes; - const padded_enc_len: u32 = total_steps * dsf; - - const conv_host = try allocator.alloc(u8, @as(usize, padded_enc_len) * frame_bytes); - defer allocator.free(conv_host); - @memset(conv_host, 0); // zero-pad for last chunk if encoder_seq_len % dsf != 0 + const mel_host = try allocator.alloc(u8, @as(usize, 128) * padded_mel_frames * f32_bytes); + defer allocator.free(mel_host); + @memset(mel_host, 0); // zero-pad beyond num_frames { - var download = try conv_output.toSliceAlloc(allocator, io); + var download = try mel_output.toSliceAlloc(allocator, io); defer download.free(allocator); const src = download.constData(); - @memcpy(conv_host[0..src.len], src); + const src_stride = @as(usize, num_frames) * f32_bytes; + const dst_stride = @as(usize, padded_mel_frames) * f32_bytes; + const copy_len = @as(usize, num_frames) * f32_bytes; + for (0..128) |c| { + @memcpy(mel_host[c * dst_stride ..][0..copy_len], src[c * src_stride ..][0..copy_len]); + } } - log.info("Conv output downloaded to host ({} frames, {} steps).", .{ encoder_seq_len, total_steps }); + log.info("Mel downloaded to host ({} frames, padded to {}, {} steps).", .{ num_frames, padded_mel_frames, total_steps }); - // 4. Init KV caches + t_cond + RNG + // 3. Init KV caches + t_cond + RNG var enc_kv_buffers = try enc_kv_cache.initBuffer(io, platform); defer KvCache.deinitBuffer(&enc_kv_buffers); @@ -311,22 +312,52 @@ pub fn runPipeline( const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); // ===== Prefill phase ===== + // 4. Conv stem prefill: process first prompt_len*mel_per_step mel frames + log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * mel_per_step}); + var conv_prefill_output: zml.Buffer = undefined; + { + const prefill_mel_time: u32 = prompt_len * mel_per_step; + const prefill_staging_bytes: usize = @as(usize, 128) * prefill_mel_time * f32_bytes; + const prefill_mel_staging = try allocator.alloc(u8, prefill_staging_bytes); + defer allocator.free(prefill_mel_staging); + + const src_stride = @as(usize, padded_mel_frames) * f32_bytes; + const dst_stride = @as(usize, prefill_mel_time) * f32_bytes; + const copy_len = @as(usize, prefill_mel_time) * f32_bytes; + for (0..128) |c| { + @memcpy(prefill_mel_staging[c * dst_stride ..][0..copy_len], mel_host[c * src_stride ..][0..copy_len]); + } + + const mel_slice: zml.Slice = .init( + .init(.{ .channels = 128, .time = prefill_mel_time }, .f32), + prefill_mel_staging[0..prefill_staging_bytes], + ); + var mel_buffer: zml.Buffer = try .fromSlice(io, platform, mel_slice); + defer mel_buffer.deinit(); + + const prefill_enc_frames = prompt_len * dsf; + conv_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_enc_frames, .d = enc_dim }, .bf16), .{}); + + var args = try compiled_conv_stem.args(allocator); + defer args.deinit(allocator); + var results = try compiled_conv_stem.results(allocator); + defer results.deinit(allocator); + + args.set(.{ encoder_buffers, mel_buffer }); + compiled_conv_stem.call(args, &results); + results.fill(.{&conv_prefill_output}); + } + defer conv_prefill_output.deinit(); + log.info("Conv stem prefill done.", .{}); + // 5. Encoder prefill: first prompt_len*dsf frames through transformer with KV cache log.info("Running encoder prefill ({} frames)...", .{prompt_len * dsf}); var encoder_prefill_output: zml.Buffer = undefined; { - const prefill_frames = prompt_len * dsf; - const prefill_conv_bytes = @as(usize, prefill_frames) * frame_bytes; - const prefill_conv_slice: zml.Slice = .init( - .init(.{ .s = prefill_frames, .d = enc_dim }, .bf16), - conv_host[0..prefill_conv_bytes], - ); - var prefill_conv_buffer: zml.Buffer = try .fromSlice(io, platform, prefill_conv_slice); - defer prefill_conv_buffer.deinit(); - var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); defer enc_token_index.deinit(); + const prefill_frames = prompt_len * dsf; encoder_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_frames, .d = enc_dim }, .bf16), .{}); var args = try compiled_encoder_prefill.args(allocator); @@ -334,7 +365,7 @@ pub fn runPipeline( var results = try compiled_encoder_prefill.results(allocator); defer results.deinit(allocator); - args.set(.{ encoder_buffers, prefill_conv_buffer, enc_token_index, &enc_kv_buffers, enc_attention_metadata_buffers }); + args.set(.{ encoder_buffers, conv_prefill_output, enc_token_index, &enc_kv_buffers, enc_attention_metadata_buffers }); compiled_encoder_prefill.call(args, &results); results.fill(.{ &encoder_prefill_output, &enc_kv_buffers }); } @@ -396,9 +427,14 @@ pub fn runPipeline( } log.info("Decoder prefill done.", .{}); - // ===== Decode phase: step-by-step encoder → adapter → decoder ===== + // ===== Decode phase: step-by-step conv_stem → encoder → adapter → decoder ===== log.info("Running decode loop (steps {}..{})...", .{ prompt_len, total_steps }); + var conv_step_args = try compiled_conv_stem_step.args(allocator); + defer conv_step_args.deinit(allocator); + var conv_step_results = try compiled_conv_stem_step.results(allocator); + defer conv_step_results.deinit(allocator); + var enc_step_args = try compiled_encoder_step.args(allocator); defer enc_step_args.deinit(allocator); var enc_step_results = try compiled_encoder_step.results(allocator); @@ -414,6 +450,9 @@ pub fn runPipeline( var decode_results = try compiled_decoder_decode.results(allocator); defer decode_results.deinit(allocator); + var conv_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = dsf, .d = enc_dim }, .bf16), .{}); + defer conv_step_output.deinit(); + var enc_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = dsf, .d = enc_dim }, .bf16), .{}); defer enc_step_output.deinit(); @@ -423,6 +462,14 @@ pub fn runPipeline( var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); defer current_token_buffer.deinit(); + // Mel staging buffer for conv stem step (reused each iteration) + const mel_staging_bytes: usize = @as(usize, 128) * chunk_mel * f32_bytes; + const mel_staging = try allocator.alloc(u8, mel_staging_bytes); + defer allocator.free(mel_staging); + + const mel_src_stride = @as(usize, padded_mel_frames) * f32_bytes; + const mel_dst_stride = @as(usize, chunk_mel) * f32_bytes; + var num_generated: usize = 0; for (prompt_len..total_steps) |t| { const generated_token = generated_token_slice.items(u32)[0]; @@ -439,19 +486,28 @@ pub fn runPipeline( break; } - // Encoder step: dsf frames → encoded chunk - const byte_offset = t * chunk_bytes; - const chunk_slice: zml.Slice = .init( - .init(.{ .s = dsf, .d = enc_dim }, .bf16), - conv_host[byte_offset .. byte_offset + chunk_bytes], - ); - var chunk_buffer: zml.Buffer = try .fromSlice(io, platform, chunk_slice); - defer chunk_buffer.deinit(); + // Conv stem step: prepare mel chunk with history, run conv stem + const mel_t_start: usize = t * mel_per_step - mel_history; + for (0..128) |c| { + @memcpy(mel_staging[c * mel_dst_stride ..][0..mel_dst_stride], mel_host[c * mel_src_stride + mel_t_start * f32_bytes ..][0..mel_dst_stride]); + } + const mel_chunk_slice: zml.Slice = .init( + .init(.{ .channels = 128, .time = chunk_mel }, .f32), + mel_staging[0..mel_staging_bytes], + ); + var mel_chunk_buffer: zml.Buffer = try .fromSlice(io, platform, mel_chunk_slice); + defer mel_chunk_buffer.deinit(); + + conv_step_args.set(.{ encoder_buffers, mel_chunk_buffer }); + compiled_conv_stem_step.call(conv_step_args, &conv_step_results); + conv_step_results.fill(.{&conv_step_output}); + + // Encoder step: conv stem output → encoded chunk var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)) * dsf, .u32); defer enc_token_index.deinit(); - enc_step_args.set(.{ encoder_buffers, chunk_buffer, enc_token_index, &enc_kv_buffers, enc_attention_metadata_buffers }); + enc_step_args.set(.{ encoder_buffers, conv_step_output, enc_token_index, &enc_kv_buffers, enc_attention_metadata_buffers }); compiled_encoder_step.call(enc_step_args, &enc_step_results); enc_step_results.fill(.{ &enc_step_output, &enc_kv_buffers }); From ee0644c6a999a919816ec80c72dc074887d5379e Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 16 Feb 2026 13:05:16 +0100 Subject: [PATCH 67/98] store params in a struct --- examples/voxtral/config.zig | 62 +++++++++++++++++++++++++++++++++--- examples/voxtral/main.zig | 32 ++++++------------- examples/voxtral/voxtral.zig | 50 ++++++++++++----------------- 3 files changed, 89 insertions(+), 55 deletions(-) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index 9c7798e4..f79aca82 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -53,9 +53,9 @@ pub const Config = struct { window_size: u32 = 400, global_log_mel_max: f32 = 1.5, }; - + // -- Shortcuts - + pub fn encoder(self: Config) EncoderArgs { return self.multimodal.whisper_model_args.encoder_args; } @@ -72,11 +72,65 @@ pub const Config = struct { pub fn parseConfig(allocator: std.mem.Allocator, io: std.Io, model_dir: std.Io.Dir) !std.json.Parsed(Config) { const config_file = try model_dir.openFile(io, "params.json", .{}); // it is named params not config here defer config_file.close(io); - + var buffer: [4096]u8 = undefined; var file_reader = config_file.reader(io, &buffer); var reader: std.json.Reader = .init(allocator, &file_reader.interface); defer reader.deinit(); - + return try std.json.parseFromTokenSource(Config, allocator, &reader, .{ .ignore_unknown_fields = true }); } + +pub const StreamParams = struct { + dsf: u32, + mel_per_step: u32, + mel_history: u32, + chunk_mel: u32, + raw_audio_length_per_tok: u32, + _hop_length: u32, + n_delay_tokens: u32, + n_right_pad_tokens: u32, + left_pad: u32, + prompt_len: u32, + + pub fn init(config: Config, transcription_delay_ms: f32, n_left_pad_tokens: u32) StreamParams { + const sample_rate: f32 = @floatFromInt(config.audio().sampling_rate); + const frame_rate = config.audio().frame_rate; + const raw_audio_length_per_tok: u32 = @intFromFloat(sample_rate / frame_rate); + const hop_length = config.audio().hop_length; + + const delay_samples: u32 = @intFromFloat(transcription_delay_ms / 1000.0 * sample_rate); + const audio_length_per_tok = raw_audio_length_per_tok / hop_length; + const n_delay_tokens = std.math.divCeil(u32, delay_samples / hop_length, audio_length_per_tok) catch unreachable; + + const dsf = config.downsample_factor(); + const mel_history: u32 = 4; + + return .{ + .dsf = dsf, + .mel_per_step = dsf * 2, + .mel_history = mel_history, + .chunk_mel = mel_history + dsf * 2, + .raw_audio_length_per_tok = raw_audio_length_per_tok, + ._hop_length = hop_length, + .n_delay_tokens = n_delay_tokens, + .n_right_pad_tokens = (n_delay_tokens + 1) + 10, + .left_pad = n_left_pad_tokens * raw_audio_length_per_tok, + .prompt_len = 1 + n_left_pad_tokens + n_delay_tokens, + }; + } + + pub fn numFrames(self: StreamParams, audio_len: usize) u32 { + return @intCast(audio_len / self._hop_length); + } + + pub fn totalSteps(self: StreamParams, audio_len: usize) u32 { + const nf = self.numFrames(audio_len); + const encoder_seq_len = (nf + 1) / 2; + return (encoder_seq_len + self.dsf - 1) / self.dsf; + } + + pub fn paddedMelFrames(self: StreamParams, audio_len: usize) u32 { + return self.totalSteps(audio_len) * self.mel_per_step; + } +}; diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 74a1f6a4..17231a81 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -4,6 +4,7 @@ const log = std.log; const wav_utils = @import("wav.zig"); const cfg = @import("config.zig"); const Config = cfg.Config; +const StreamParams = cfg.StreamParams; const zml = @import("zml"); const stdx = zml.stdx; @@ -75,26 +76,16 @@ pub fn main() !void { defer parsed_config.deinit(); const config = parsed_config.value; - const sample_rate: f32 = @floatFromInt(config.audio().sampling_rate); - const frame_rate = config.audio().frame_rate; - const raw_audio_length_per_tok: u32 = @intFromFloat(sample_rate / frame_rate); - const hop_length = config.audio().hop_length; - - const delay_samples: u32 = @intFromFloat(args.transcription_delay_ms / 1000.0 * sample_rate); - const audio_length_per_tok = raw_audio_length_per_tok / hop_length; - const n_delay_tokens = std.math.divCeil(u32, delay_samples / hop_length, audio_length_per_tok) catch unreachable; - const n_right_pad_tokens = (n_delay_tokens + 1) + 10; + const sp: StreamParams = .init(config, args.transcription_delay_ms, n_left_pad_tokens); // Build tokens: [BOS] ++ [STREAMING_PAD] * (n_left_pad_tokens + n_delay_tokens) - const n_prompt_tokens = 1 + n_left_pad_tokens + n_delay_tokens; - const tokens = try allocator.alloc(u32, n_prompt_tokens); + const tokens = try allocator.alloc(u32, sp.prompt_len); defer allocator.free(tokens); var wav_buffer: [4096]u8 = undefined; var reader = file.reader(io, &wav_buffer); - const left_pad = n_left_pad_tokens * raw_audio_length_per_tok; - const padded_wav = try wav_utils.loadAndPadWav(allocator, &reader.interface, left_pad, n_right_pad_tokens, raw_audio_length_per_tok); + const padded_wav = try wav_utils.loadAndPadWav(allocator, &reader.interface, sp.left_pad, sp.n_right_pad_tokens, sp.raw_audio_length_per_tok); defer allocator.free(padded_wav); const audio_len = padded_wav.len; @@ -138,8 +129,6 @@ pub fn main() !void { .hd = config.head_dim, }, dec_dtype)); - const prompt_len: u32 = @intCast(tokens.len); - const enc_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(enc_cfg.sliding_window))); const dec_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(config.sliding_window))); const attention_parameters: zml.attention.Parameters = .init(.fromBackend(backend)); @@ -149,16 +138,15 @@ pub fn main() !void { // -- Fixed len for now var compiled_mel_spectrum_future = try io.concurrent(voxtral.compileMelSpectrum, .{ allocator, io, platform, melspectro_model, audio_len, &progress }); - const mel_per_step: u32 = config.downsample_factor() * 2; - var compiled_conv_stem_future = try io.concurrent(voxtral.compileConvStem, .{ allocator, io, platform, encoder_model, prompt_len * mel_per_step, &progress }); - var compiled_conv_stem_step_future = try io.concurrent(voxtral.compileConvStemStep, .{ allocator, io, platform, encoder_model, &progress }); // -- - var compiled_encoder_prefill_future = try io.concurrent(voxtral.compileEncoderPrefill, .{ allocator, io, platform, encoder_model, prompt_len, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); + var compiled_conv_stem_future = try io.concurrent(voxtral.compileConvStem, .{ allocator, io, platform, encoder_model, sp.prompt_len * sp.mel_per_step, &progress }); + var compiled_conv_stem_step_future = try io.concurrent(voxtral.compileConvStemStep, .{ allocator, io, platform, encoder_model, sp, &progress }); + var compiled_encoder_prefill_future = try io.concurrent(voxtral.compileEncoderPrefill, .{ allocator, io, platform, encoder_model, sp.prompt_len, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); var compiled_encoder_step_future = try io.concurrent(voxtral.compileEncoderStep, .{ allocator, io, platform, encoder_model, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); - var compiled_adapter_future = try io.concurrent(voxtral.compileAdapter, .{ allocator, io, platform, adapter, prompt_len, config, &progress }); + var compiled_adapter_future = try io.concurrent(voxtral.compileAdapter, .{ allocator, io, platform, adapter, sp.prompt_len, config, &progress }); var compiled_adapter_step_future = try io.concurrent(voxtral.compileAdapterStep, .{ allocator, io, platform, adapter, config, &progress }); - var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, decoder_model, prompt_len, dec_kv_cache, dec_attention_metadata, attention_parameters, &progress }); + var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, decoder_model, sp.prompt_len, dec_kv_cache, dec_attention_metadata, attention_parameters, &progress }); var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{ &melspectro_model, io, platform }); var encoder_buffers_future = try io.concurrent(Encoder.load, .{ &encoder_model, allocator, io, platform, &model_store, &progress }); @@ -230,7 +218,7 @@ pub fn main() !void { padded_wav, audio_len, tokens, - n_delay_tokens, + sp, &compiled_mel_spectrum, &compiled_conv_stem, &compiled_conv_stem_step, diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 41113558..dc1beea4 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -2,6 +2,7 @@ const std = @import("std"); const log = std.log; const cfg = @import("config.zig"); const Config = cfg.Config; +const StreamParams = cfg.StreamParams; const zml = @import("zml"); const Tensor = zml.Tensor; @@ -37,17 +38,13 @@ pub fn compileConvStem(allocator: std.mem.Allocator, io: std.Io, platform: *cons }); } -pub fn compileConvStemStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, progress: *std.Progress.Node) !zml.Exe { +pub fn compileConvStemStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling conv stem step...", 1); defer node.end(); - const mel_history = 4; - const mel_per_step = model.config.downsample_factor() * 2; - const chunk_mel = mel_history + mel_per_step; - return try platform.compile(allocator, io, model, .convStemStep, .{ - Tensor.init(.{ .channels = 128, .time = chunk_mel }, .f32), + Tensor.init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), }); } @@ -209,7 +206,7 @@ pub fn runPipeline( padded_audio: []const f32, audio_len: usize, tokens: []const u32, - n_delay_tokens: u32, + sp: StreamParams, compiled_mel_spectrum: *zml.Exe, compiled_conv_stem: *zml.Exe, compiled_conv_stem_step: *zml.Exe, @@ -230,15 +227,11 @@ pub fn runPipeline( ) !void { log.info("Running inference pipeline...", .{}); - const num_frames: u32 = @intCast(audio_len / config.audio().hop_length); - const dsf: u32 = config.downsample_factor(); - const mel_per_step: u32 = dsf * 2; - const mel_history: u32 = 4; - const chunk_mel: u32 = mel_history + mel_per_step; - const encoder_seq_len: u32 = @intCast((num_frames + 1) / 2); - const total_steps = (encoder_seq_len + dsf - 1) / dsf; + const num_frames = sp.numFrames(audio_len); + const total_steps = sp.totalSteps(audio_len); + const padded_mel_frames = sp.paddedMelFrames(audio_len); const enc_dim: u32 = config.encoder().dim; - const prompt_len: u32 = @intCast(tokens.len); + const prompt_len = sp.prompt_len; var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); @@ -270,7 +263,6 @@ pub fn runPipeline( // 2. Download mel to host (channels-major: [128][padded_mel_frames] in f32) const f32_bytes: usize = @sizeOf(f32); - const padded_mel_frames: u32 = total_steps * mel_per_step; const mel_host = try allocator.alloc(u8, @as(usize, 128) * padded_mel_frames * f32_bytes); defer allocator.free(mel_host); @@ -296,7 +288,7 @@ pub fn runPipeline( var dec_kv_buffers = try dec_kv_cache.initBuffer(io, platform); defer KvCache.deinitBuffer(&dec_kv_buffers); - const t_cond_data = computeTimeEmbedding(allocator, @as(f32, @floatFromInt(n_delay_tokens)), config.dim); + const t_cond_data = computeTimeEmbedding(allocator, @as(f32, @floatFromInt(sp.n_delay_tokens)), config.dim); defer allocator.free(t_cond_data); const t_cond_slice: zml.Slice = .init( .init(.{config.dim}, .f32), @@ -313,10 +305,10 @@ pub fn runPipeline( // ===== Prefill phase ===== // 4. Conv stem prefill: process first prompt_len*mel_per_step mel frames - log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * mel_per_step}); + log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * sp.mel_per_step}); var conv_prefill_output: zml.Buffer = undefined; { - const prefill_mel_time: u32 = prompt_len * mel_per_step; + const prefill_mel_time: u32 = prompt_len * sp.mel_per_step; const prefill_staging_bytes: usize = @as(usize, 128) * prefill_mel_time * f32_bytes; const prefill_mel_staging = try allocator.alloc(u8, prefill_staging_bytes); defer allocator.free(prefill_mel_staging); @@ -335,7 +327,7 @@ pub fn runPipeline( var mel_buffer: zml.Buffer = try .fromSlice(io, platform, mel_slice); defer mel_buffer.deinit(); - const prefill_enc_frames = prompt_len * dsf; + const prefill_enc_frames = prompt_len * sp.dsf; conv_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_enc_frames, .d = enc_dim }, .bf16), .{}); var args = try compiled_conv_stem.args(allocator); @@ -351,13 +343,13 @@ pub fn runPipeline( log.info("Conv stem prefill done.", .{}); // 5. Encoder prefill: first prompt_len*dsf frames through transformer with KV cache - log.info("Running encoder prefill ({} frames)...", .{prompt_len * dsf}); + log.info("Running encoder prefill ({} frames)...", .{prompt_len * sp.dsf}); var encoder_prefill_output: zml.Buffer = undefined; { var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); defer enc_token_index.deinit(); - const prefill_frames = prompt_len * dsf; + const prefill_frames = prompt_len * sp.dsf; encoder_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_frames, .d = enc_dim }, .bf16), .{}); var args = try compiled_encoder_prefill.args(allocator); @@ -450,10 +442,10 @@ pub fn runPipeline( var decode_results = try compiled_decoder_decode.results(allocator); defer decode_results.deinit(allocator); - var conv_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = dsf, .d = enc_dim }, .bf16), .{}); + var conv_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = sp.dsf, .d = enc_dim }, .bf16), .{}); defer conv_step_output.deinit(); - var enc_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = dsf, .d = enc_dim }, .bf16), .{}); + var enc_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = sp.dsf, .d = enc_dim }, .bf16), .{}); defer enc_step_output.deinit(); var adapter_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = 1, .d = config.dim }, .bf16), .{}); @@ -463,12 +455,12 @@ pub fn runPipeline( defer current_token_buffer.deinit(); // Mel staging buffer for conv stem step (reused each iteration) - const mel_staging_bytes: usize = @as(usize, 128) * chunk_mel * f32_bytes; + const mel_staging_bytes: usize = @as(usize, 128) * sp.chunk_mel * f32_bytes; const mel_staging = try allocator.alloc(u8, mel_staging_bytes); defer allocator.free(mel_staging); const mel_src_stride = @as(usize, padded_mel_frames) * f32_bytes; - const mel_dst_stride = @as(usize, chunk_mel) * f32_bytes; + const mel_dst_stride = @as(usize, sp.chunk_mel) * f32_bytes; var num_generated: usize = 0; for (prompt_len..total_steps) |t| { @@ -487,13 +479,13 @@ pub fn runPipeline( } // Conv stem step: prepare mel chunk with history, run conv stem - const mel_t_start: usize = t * mel_per_step - mel_history; + const mel_t_start: usize = t * sp.mel_per_step - sp.mel_history; for (0..128) |c| { @memcpy(mel_staging[c * mel_dst_stride ..][0..mel_dst_stride], mel_host[c * mel_src_stride + mel_t_start * f32_bytes ..][0..mel_dst_stride]); } const mel_chunk_slice: zml.Slice = .init( - .init(.{ .channels = 128, .time = chunk_mel }, .f32), + .init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), mel_staging[0..mel_staging_bytes], ); var mel_chunk_buffer: zml.Buffer = try .fromSlice(io, platform, mel_chunk_slice); @@ -504,7 +496,7 @@ pub fn runPipeline( conv_step_results.fill(.{&conv_step_output}); // Encoder step: conv stem output → encoded chunk - var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)) * dsf, .u32); + var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)) * sp.dsf, .u32); defer enc_token_index.deinit(); enc_step_args.set(.{ encoder_buffers, conv_step_output, enc_token_index, &enc_kv_buffers, enc_attention_metadata_buffers }); From 394a5e59e84a0c4e6fa129539a2af6a33ddda87f Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 16 Feb 2026 14:34:48 +0100 Subject: [PATCH 68/98] divide runPipeline into 3 parts --- examples/voxtral/decoder.zig | 2 +- examples/voxtral/encoder.zig | 5 +- examples/voxtral/main.zig | 3 +- examples/voxtral/voxtral.zig | 279 +++++++++++++++++++++++++---------- 4 files changed, 209 insertions(+), 80 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 4334e47b..3b900b27 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -140,7 +140,7 @@ pub const Decoder = struct { h = rmsNorm(h, self.norm, self.norm_eps); - // Compute logits in f32 for precision (matching Python reference) + // Compute logits in f32 for precision const logits = h.convert(.f32).dot(self.tok_embeddings.convert(.f32), .d); const output_tokens, const new_rng = zml.nn.sampleTokens(logits, .{}, rng); diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index e136de9a..a45b5b06 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -19,8 +19,9 @@ pub const Encoder = struct { config: Config, pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Encoder { - const conv_store = store.withPrefix("conv_layers"); - const transformer_store = store.withPrefix("transformer"); + const encoder_store = store.withPrefix("mm_streams_embeddings.embedding_module.whisper_encoder"); + const conv_store = encoder_store.withPrefix("conv_layers"); + const transformer_store = encoder_store.withPrefix("transformer"); const enc = config.encoder(); const layers = allocator.alloc(TransformerLayer, enc.n_layers) catch unreachable; diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 17231a81..31c783a5 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -102,8 +102,7 @@ pub fn main() !void { }; var melspectro_model: LogMelSpectrogram = .init(config); - const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; - var encoder_model: Encoder = .init(allocator, model_store.view().withPrefix(encoder_prefix), config); + var encoder_model: Encoder = .init(allocator, model_store.view(), config); defer encoder_model.deinit(allocator); const adapter: Adapter = .init(model_store.view(), config); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index dc1beea4..c28ed74b 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -84,7 +84,7 @@ pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *c }); } -/// Adapter batch: processes prompt_len * dsf encoder frames → prompt_len embeddings. +/// Adapter batch: processes prompt_len * dsf encoder frames -> prompt_len embeddings. pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, prompt_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling adapter...", 1); @@ -97,7 +97,7 @@ pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const }); } -/// Adapter step: processes dsf encoder frames → 1 embedding (for streaming decode). +/// Adapter step: processes dsf encoder frames -> 1 embedding (for streaming decode). pub fn compileAdapterStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, config: Config, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling adapter step...", 1); @@ -197,46 +197,23 @@ pub fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32 return result; } -pub fn runPipeline( +/// Run mel spectrogram and download result to host memory. +/// Returns zero-padded mel data as bytes (channels-major: [128][padded_mel_frames]). Caller owns the memory. +pub fn runMelSpectrogram( allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, - config: Config, - tokenizer: *zml.tokenizer.Tokenizer, + sp: StreamParams, padded_audio: []const f32, audio_len: usize, - tokens: []const u32, - sp: StreamParams, compiled_mel_spectrum: *zml.Exe, - compiled_conv_stem: *zml.Exe, - compiled_conv_stem_step: *zml.Exe, - compiled_encoder_prefill: *zml.Exe, - compiled_encoder_step: *zml.Exe, - compiled_adapter: *zml.Exe, - compiled_adapter_step: *zml.Exe, - compiled_decoder_prefill: *zml.Exe, - compiled_decoder_decode: *zml.Exe, mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), - encoder_buffers: *zml.Bufferized(Encoder), - adapter_buffers: *zml.Bufferized(Adapter), - decoder_buffers: *zml.Bufferized(Decoder), - enc_kv_cache: KvCache, - dec_kv_cache: KvCache, - enc_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), - dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), -) !void { - log.info("Running inference pipeline...", .{}); - +) ![]u8 { const num_frames = sp.numFrames(audio_len); - const total_steps = sp.totalSteps(audio_len); const padded_mel_frames = sp.paddedMelFrames(audio_len); - const enc_dim: u32 = config.encoder().dim; - const prompt_len = sp.prompt_len; + const total_steps = sp.totalSteps(audio_len); + const f32_bytes: usize = @sizeOf(f32); - var tokenizer_decoder = try tokenizer.decoder(); - defer tokenizer_decoder.deinit(); - - // 1. Mel spectrogram log.info("Running mel spectrogram...", .{}); var mel_output: zml.Buffer = undefined; { @@ -261,11 +238,8 @@ pub fn runPipeline( defer mel_output.deinit(); log.info("Mel spectrogram done.", .{}); - // 2. Download mel to host (channels-major: [128][padded_mel_frames] in f32) - const f32_bytes: usize = @sizeOf(f32); - + // Download mel to host (channels-major: [128][padded_mel_frames] in f32) const mel_host = try allocator.alloc(u8, @as(usize, 128) * padded_mel_frames * f32_bytes); - defer allocator.free(mel_host); @memset(mel_host, 0); // zero-pad beyond num_frames { @@ -281,30 +255,41 @@ pub fn runPipeline( } log.info("Mel downloaded to host ({} frames, padded to {}, {} steps).", .{ num_frames, padded_mel_frames, total_steps }); - // 3. Init KV caches + t_cond + RNG - var enc_kv_buffers = try enc_kv_cache.initBuffer(io, platform); - defer KvCache.deinitBuffer(&enc_kv_buffers); + return mel_host; +} - var dec_kv_buffers = try dec_kv_cache.initBuffer(io, platform); - defer KvCache.deinitBuffer(&dec_kv_buffers); +/// Run the prefill phase: conv stem -> encoder -> adapter -> decoder prefill. +/// Populates KV caches and produces the first generated token into generated_token_slice. +pub fn runPrefill( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + config: Config, + sp: StreamParams, + mel_host: []const u8, + audio_len: usize, + tokens: []const u32, + compiled_conv_stem: *zml.Exe, + compiled_encoder_prefill: *zml.Exe, + compiled_adapter: *zml.Exe, + compiled_decoder_prefill: *zml.Exe, + encoder_buffers: *zml.Bufferized(Encoder), + adapter_buffers: *zml.Bufferized(Adapter), + decoder_buffers: *zml.Bufferized(Decoder), + enc_kv_buffers: *zml.Bufferized(KvCache), + dec_kv_buffers: *zml.Bufferized(KvCache), + t_cond_buffer: zml.Buffer, + rng_buffers: *zml.Bufferized(Tensor.Rng), + enc_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + generated_token_slice: zml.Slice, +) !void { + const padded_mel_frames = sp.paddedMelFrames(audio_len); + const enc_dim: u32 = config.encoder().dim; + const prompt_len = sp.prompt_len; + const f32_bytes: usize = @sizeOf(f32); - const t_cond_data = computeTimeEmbedding(allocator, @as(f32, @floatFromInt(sp.n_delay_tokens)), config.dim); - defer allocator.free(t_cond_data); - const t_cond_slice: zml.Slice = .init( - .init(.{config.dim}, .f32), - std.mem.sliceAsBytes(t_cond_data), - ); - var t_cond_buffer: zml.Buffer = try .fromSlice(io, platform, t_cond_slice); - defer t_cond_buffer.deinit(); - - var rng_buffers = try Tensor.Rng.initBuffer(platform, 0, io); - defer Tensor.Rng.deinitBuffer(&rng_buffers); - - const eos_token = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); - const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); - - // ===== Prefill phase ===== - // 4. Conv stem prefill: process first prompt_len*mel_per_step mel frames + // Conv stem prefill: process first prompt_len*mel_per_step mel frames log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * sp.mel_per_step}); var conv_prefill_output: zml.Buffer = undefined; { @@ -342,7 +327,7 @@ pub fn runPipeline( defer conv_prefill_output.deinit(); log.info("Conv stem prefill done.", .{}); - // 5. Encoder prefill: first prompt_len*dsf frames through transformer with KV cache + // Encoder prefill: first prompt_len*dsf frames through transformer with KV cache log.info("Running encoder prefill ({} frames)...", .{prompt_len * sp.dsf}); var encoder_prefill_output: zml.Buffer = undefined; { @@ -357,14 +342,14 @@ pub fn runPipeline( var results = try compiled_encoder_prefill.results(allocator); defer results.deinit(allocator); - args.set(.{ encoder_buffers, conv_prefill_output, enc_token_index, &enc_kv_buffers, enc_attention_metadata_buffers }); + args.set(.{ encoder_buffers, conv_prefill_output, enc_token_index, enc_kv_buffers, enc_attention_metadata_buffers }); compiled_encoder_prefill.call(args, &results); - results.fill(.{ &encoder_prefill_output, &enc_kv_buffers }); + results.fill(.{ &encoder_prefill_output, enc_kv_buffers }); } defer encoder_prefill_output.deinit(); log.info("Encoder prefill done.", .{}); - // 6. Adapter batch: encoder prefill output → prompt_len audio embeddings + // Adapter batch: encoder prefill output -> prompt_len audio embeddings log.info("Running adapter...", .{}); var adapter_prefill_output: zml.Buffer = undefined; { @@ -382,10 +367,8 @@ pub fn runPipeline( defer adapter_prefill_output.deinit(); log.info("Adapter done.", .{}); - // 7. Decoder prefill: all prompt tokens + audio embeddings → first generated token + // Decoder prefill: all prompt tokens + audio embeddings -> first generated token log.info("Running decoder prefill...", .{}); - var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); - defer generated_token_slice.free(allocator); { var args = try compiled_decoder_prefill.args(allocator); defer args.deinit(allocator); @@ -407,9 +390,9 @@ pub fn runPipeline( var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_output_slice); defer prefill_output.deinit(); - args.set(.{ decoder_buffers, token_buffer, adapter_prefill_output, token_index_buffer, &dec_kv_buffers, t_cond_buffer, rng_buffers, dec_attention_metadata_buffers }); + args.set(.{ decoder_buffers, token_buffer, adapter_prefill_output, token_index_buffer, dec_kv_buffers, t_cond_buffer, rng_buffers.*, dec_attention_metadata_buffers }); compiled_decoder_prefill.call(args, &results); - results.fill(.{ &prefill_output, &dec_kv_buffers, &rng_buffers }); + results.fill(.{ &prefill_output, dec_kv_buffers, rng_buffers }); // Extract last token from prefill output (first generated token) try prefill_output.toSlice(io, prefill_output_slice); @@ -418,8 +401,45 @@ pub fn runPipeline( log.info("First generated token (from prefill): {}", .{generated_token_slice.items(u32)[0]}); } log.info("Decoder prefill done.", .{}); +} + +/// Run the streaming decode loop: step-by-step conv_stem -> encoder -> adapter -> decoder. +pub fn runGenerationLoop( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + config: Config, + sp: StreamParams, + tokenizer: *zml.tokenizer.Tokenizer, + mel_host: []const u8, + audio_len: usize, + compiled_conv_stem_step: *zml.Exe, + compiled_encoder_step: *zml.Exe, + compiled_adapter_step: *zml.Exe, + compiled_decoder_decode: *zml.Exe, + encoder_buffers: *zml.Bufferized(Encoder), + adapter_buffers: *zml.Bufferized(Adapter), + decoder_buffers: *zml.Bufferized(Decoder), + enc_kv_buffers: *zml.Bufferized(KvCache), + dec_kv_buffers: *zml.Bufferized(KvCache), + t_cond_buffer: zml.Buffer, + rng_buffers: *zml.Bufferized(Tensor.Rng), + enc_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + generated_token_slice: zml.Slice, +) !void { + const padded_mel_frames = sp.paddedMelFrames(audio_len); + const total_steps = sp.totalSteps(audio_len); + const enc_dim: u32 = config.encoder().dim; + const prompt_len = sp.prompt_len; + const f32_bytes: usize = @sizeOf(f32); + + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + const eos_token = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); + const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); - // ===== Decode phase: step-by-step conv_stem → encoder → adapter → decoder ===== log.info("Running decode loop (steps {}..{})...", .{ prompt_len, total_steps }); var conv_step_args = try compiled_conv_stem_step.args(allocator); @@ -495,29 +515,138 @@ pub fn runPipeline( compiled_conv_stem_step.call(conv_step_args, &conv_step_results); conv_step_results.fill(.{&conv_step_output}); - // Encoder step: conv stem output → encoded chunk + // Encoder step: conv stem output -> encoded chunk var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)) * sp.dsf, .u32); defer enc_token_index.deinit(); - enc_step_args.set(.{ encoder_buffers, conv_step_output, enc_token_index, &enc_kv_buffers, enc_attention_metadata_buffers }); + enc_step_args.set(.{ encoder_buffers, conv_step_output, enc_token_index, enc_kv_buffers, enc_attention_metadata_buffers }); compiled_encoder_step.call(enc_step_args, &enc_step_results); - enc_step_results.fill(.{ &enc_step_output, &enc_kv_buffers }); + enc_step_results.fill(.{ &enc_step_output, enc_kv_buffers }); - // Adapter step: encoded chunk → 1 audio embedding + // Adapter step: encoded chunk -> 1 audio embedding adp_step_args.set(.{ adapter_buffers, enc_step_output }); compiled_adapter_step.call(adp_step_args, &adp_step_results); adp_step_results.fill(.{&adapter_step_output}); - // Decoder decode: 1 token + 1 audio embedding → next token + // Decoder decode: 1 token + 1 audio embedding -> next token var dec_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)), .u32); defer dec_token_index.deinit(); - decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_step_output, dec_token_index, &dec_kv_buffers, t_cond_buffer, rng_buffers, dec_attention_metadata_buffers }); + decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_step_output, dec_token_index, dec_kv_buffers, t_cond_buffer, rng_buffers.*, dec_attention_metadata_buffers }); compiled_decoder_decode.call(decode_args, &decode_results); - decode_results.fill(.{ ¤t_token_buffer, &dec_kv_buffers, &rng_buffers }); + decode_results.fill(.{ ¤t_token_buffer, dec_kv_buffers, rng_buffers }); try current_token_buffer.toSlice(io, generated_token_slice); } std.debug.print("\n", .{}); log.info("Decode done. Generated {} tokens total.", .{num_generated}); } + +pub fn runPipeline( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + config: Config, + tokenizer: *zml.tokenizer.Tokenizer, + padded_audio: []const f32, + audio_len: usize, + tokens: []const u32, + sp: StreamParams, + compiled_mel_spectrum: *zml.Exe, + compiled_conv_stem: *zml.Exe, + compiled_conv_stem_step: *zml.Exe, + compiled_encoder_prefill: *zml.Exe, + compiled_encoder_step: *zml.Exe, + compiled_adapter: *zml.Exe, + compiled_adapter_step: *zml.Exe, + compiled_decoder_prefill: *zml.Exe, + compiled_decoder_decode: *zml.Exe, + mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), + encoder_buffers: *zml.Bufferized(Encoder), + adapter_buffers: *zml.Bufferized(Adapter), + decoder_buffers: *zml.Bufferized(Decoder), + enc_kv_cache: KvCache, + dec_kv_cache: KvCache, + enc_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), +) !void { + log.info("Running inference pipeline...", .{}); + + // 1. Mel spectrogram + const mel_host = try runMelSpectrogram(allocator, io, platform, sp, padded_audio, audio_len, compiled_mel_spectrum, mel_spectrum_buffers); + defer allocator.free(mel_host); + + // 2. Init KV caches + t_cond + RNG + var enc_kv_buffers = try enc_kv_cache.initBuffer(io, platform); + defer KvCache.deinitBuffer(&enc_kv_buffers); + + var dec_kv_buffers = try dec_kv_cache.initBuffer(io, platform); + defer KvCache.deinitBuffer(&dec_kv_buffers); + + const t_cond_data = computeTimeEmbedding(allocator, @as(f32, @floatFromInt(sp.n_delay_tokens)), config.dim); + defer allocator.free(t_cond_data); + const t_cond_slice: zml.Slice = .init( + .init(.{config.dim}, .f32), + std.mem.sliceAsBytes(t_cond_data), + ); + var t_cond_buffer: zml.Buffer = try .fromSlice(io, platform, t_cond_slice); + defer t_cond_buffer.deinit(); + + var rng_buffers = try Tensor.Rng.initBuffer(platform, 0, io); + defer Tensor.Rng.deinitBuffer(&rng_buffers); + + var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); + defer generated_token_slice.free(allocator); + + // 3. Prefill + try runPrefill( + allocator, + io, + platform, + config, + sp, + mel_host, + audio_len, + tokens, + compiled_conv_stem, + compiled_encoder_prefill, + compiled_adapter, + compiled_decoder_prefill, + encoder_buffers, + adapter_buffers, + decoder_buffers, + &enc_kv_buffers, + &dec_kv_buffers, + t_cond_buffer, + &rng_buffers, + enc_attention_metadata_buffers, + dec_attention_metadata_buffers, + generated_token_slice, + ); + + // 4. Generation loop + try runGenerationLoop( + allocator, + io, + platform, + config, + sp, + tokenizer, + mel_host, + audio_len, + compiled_conv_stem_step, + compiled_encoder_step, + compiled_adapter_step, + compiled_decoder_decode, + encoder_buffers, + adapter_buffers, + decoder_buffers, + &enc_kv_buffers, + &dec_kv_buffers, + t_cond_buffer, + &rng_buffers, + enc_attention_metadata_buffers, + dec_attention_metadata_buffers, + generated_token_slice, + ); +} From 946421c6d83b4265f1aa791a0b7f09d3cd623e85 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 16 Feb 2026 15:21:59 +0100 Subject: [PATCH 69/98] fix new zig and disable vfs for now --- examples/voxtral/common.zig | 9 +++++---- examples/voxtral/main.zig | 20 +++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index e0d40aa9..82fd495e 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -2,6 +2,7 @@ const std = @import("std"); const log = std.log; const zml = @import("zml"); +const stdx = zml.stdx; const Tensor = zml.Tensor; const Shape = zml.Shape; @@ -34,14 +35,14 @@ pub fn loadModel( ) !zml.Bufferized(T) { progress.increaseEstimatedTotalItems(store.view().count()); - var timer: std.time.Timer = try .start(); + const now: std.Io.Timestamp = .now(io, .awake); var total_bytes: usize = 0; defer { - const took = timer.read(); + const took = now.untilNow(io, .awake); log.info("Loaded " ++ label ++ " weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ total_bytes, - took, - total_bytes / took * std.time.ns_per_s, + stdx.fmt.fmtDuration(took), + total_bytes / @as(usize, @intCast(took.toNanoseconds())) * std.time.ns_per_s, }); } diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 31c783a5..501bb1c5 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -34,7 +34,7 @@ const CliArgs = struct { const n_left_pad_tokens: u32 = 32; -pub fn main() !void { +pub fn main(init: std.process.Init) !void { log.info("Start of Voxtral", .{}); var dbg = std.heap.DebugAllocator(.{ .thread_safe = true }).init; @@ -45,19 +45,17 @@ pub fn main() !void { else => std.heap.c_allocator, }; - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); + const args = stdx.flags.parse(init.minimal.args, CliArgs); - const args = stdx.flags.parseProcessArgs(CliArgs); + // var vfs: zml.io.VFS = try .init(allocator, init.io); + // defer vfs.deinit(); - var vfs: zml.io.VFS = try .init(allocator, threaded.io()); - defer vfs.deinit(); + // var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + // defer vfs_file.deinit(); + // try vfs.register("file", vfs_file.io()); - var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); - defer vfs_file.deinit(); - try vfs.register("file", vfs_file.io()); - - const io = vfs.io(); + // const io = vfs.io(); + const io = init.io; var progress = std.Progress.start(io, .{ .root_name = "Voxtral" }); From f43fff4f88d3d7309574cdec9bdfac165ffbe10a Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 16 Feb 2026 15:40:54 +0100 Subject: [PATCH 70/98] remove forwardstep --- examples/voxtral/decoder.zig | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 3b900b27..112d5c61 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -58,19 +58,6 @@ pub const Adapter = struct { return self.proj1.forward(self.proj0.forward(h).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); } - - /// encoder_chunk: [s=dsf, d] → [s=1, dim] - /// Like forward but input is always exactly dsf frames, so no padding needed. - pub fn forwardStep(self: Adapter, encoder_chunk: Tensor) Tensor { - const dsf = self.downsample_factor; - - // Reshape [s=dsf, d] → [s=1, d*dsf] - const h = encoder_chunk - .splitAxis(.s, .{ .s = .auto, .group = dsf }) - .merge(.{ .d = .{ .group, .d } }); - - return self.proj1.forward(self.proj0.forward(h).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); - } }; /// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. From 0035ce2ed192356ce05f54f1e04f9b292c732b59 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 16 Feb 2026 18:01:42 +0100 Subject: [PATCH 71/98] args derived kvcache size + metrics --- examples/voxtral/common.zig | 4 ++-- examples/voxtral/main.zig | 11 ++++++----- examples/voxtral/voxtral.zig | 12 ++++++++++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index 82fd495e..684c7ca2 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -81,8 +81,8 @@ pub const KvCache = struct { pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), + .k = try .uninitialized(io, platform, self.k.shape(), .{}), + .v = try .uninitialized(io, platform, self.v.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 501bb1c5..b3c32328 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -30,6 +30,7 @@ const CliArgs = struct { model: []const u8, transcription_delay_ms: f32 = 480.0, backend: ?zml.attention.Backend = null, + max_seqlen: u32 = 1500, }; const n_left_pad_tokens: u32 = 32; @@ -88,7 +89,7 @@ pub fn main(init: std.process.Init) !void { const audio_len = padded_wav.len; var platform: *zml.Platform = try .auto(allocator, io, .{ - .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.75 } } }, + .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.50 } } }, }); defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); @@ -113,7 +114,7 @@ pub fn main(init: std.process.Init) !void { const enc_dtype = encoder_model.norm.dtype(); const enc_kv_cache: KvCache = .init(.init(.{ .layer = enc_cfg.n_layers, - .k = enc_cfg.sliding_window, + .k = args.max_seqlen, .h = enc_cfg.n_kv_heads, .hd = enc_cfg.head_dim, }, enc_dtype)); @@ -121,13 +122,13 @@ pub fn main(init: std.process.Init) !void { const dec_dtype = decoder_model.tok_embeddings.dtype(); const dec_kv_cache: KvCache = .init(.init(.{ .layer = decoder_model.layers.len, - .k = config.sliding_window, + .k = args.max_seqlen, .h = config.n_kv_heads, .hd = config.head_dim, }, dec_dtype)); - const enc_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(enc_cfg.sliding_window))); - const dec_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(config.sliding_window))); + const enc_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(args.max_seqlen))); + const dec_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(args.max_seqlen))); const attention_parameters: zml.attention.Parameters = .init(.fromBackend(backend)); // Launch concurrent compilation and buffer loading diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index c28ed74b..e09937bf 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -5,6 +5,7 @@ const Config = cfg.Config; const StreamParams = cfg.StreamParams; const zml = @import("zml"); +const stdx = zml.stdx; const Tensor = zml.Tensor; const mel = @import("mel_spectrogram.zig"); @@ -439,8 +440,10 @@ pub fn runGenerationLoop( const eos_token = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); + const streaming_word_token = tokenizer.tokenToId("[STREAMING_WORD]"); log.info("Running decode loop (steps {}..{})...", .{ prompt_len, total_steps }); + const decode_start: std.Io.Timestamp = .now(io, .awake); var conv_step_args = try compiled_conv_stem_step.args(allocator); defer conv_step_args.deinit(allocator); @@ -487,7 +490,7 @@ pub fn runGenerationLoop( const generated_token = generated_token_slice.items(u32)[0]; num_generated += 1; - if (generated_token != streaming_pad_token) { + if (generated_token != streaming_pad_token and (streaming_word_token == null or generated_token != streaming_word_token.?)) { if (try tokenizer_decoder.next(generated_token)) |chunk| { std.debug.print("{s}", .{chunk}); } @@ -538,8 +541,13 @@ pub fn runGenerationLoop( try current_token_buffer.toSlice(io, generated_token_slice); } + const decode_duration = decode_start.untilNow(io, .awake); std.debug.print("\n", .{}); - log.info("Decode done. Generated {} tokens total.", .{num_generated}); + log.info("Decode done. Generated {} tokens in {D}: {:.3}tok/s", .{ + num_generated, + stdx.fmt.fmtDuration(decode_duration), + stdx.Io.Duration.hzFloat(stdx.Io.Duration.div(decode_duration, num_generated)), + }); } pub fn runPipeline( From d018298cb1f744413b1a12fe9d7b2e7d4ed630b4 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 16 Feb 2026 21:51:22 +0100 Subject: [PATCH 72/98] test mel specto chunks --- examples/voxtral/config.zig | 6 +- examples/voxtral/main.zig | 10 +- examples/voxtral/mel_spectrogram.zig | 27 ++++ examples/voxtral/voxtral.zig | 176 +++++++++++++++------------ 4 files changed, 131 insertions(+), 88 deletions(-) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index f79aca82..285f898b 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -86,6 +86,7 @@ pub const StreamParams = struct { mel_per_step: u32, mel_history: u32, chunk_mel: u32, + chunk_audio: u32, raw_audio_length_per_tok: u32, _hop_length: u32, n_delay_tokens: u32, @@ -105,12 +106,15 @@ pub const StreamParams = struct { const dsf = config.downsample_factor(); const mel_history: u32 = 4; + const window_size = config.audio().window_size; + const chunk_mel = mel_history + dsf * 2; return .{ .dsf = dsf, .mel_per_step = dsf * 2, .mel_history = mel_history, - .chunk_mel = mel_history + dsf * 2, + .chunk_mel = chunk_mel, + .chunk_audio = (chunk_mel - 1) * hop_length + window_size, .raw_audio_length_per_tok = raw_audio_length_per_tok, ._hop_length = hop_length, .n_delay_tokens = n_delay_tokens, diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index b3c32328..7c124da5 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -134,9 +134,7 @@ pub fn main(init: std.process.Init) !void { // Launch concurrent compilation and buffer loading var tokenizer_future = try io.concurrent(voxtral.loadTokenizer, .{ allocator, io, model_dir, &progress }); - // -- Fixed len for now - var compiled_mel_spectrum_future = try io.concurrent(voxtral.compileMelSpectrum, .{ allocator, io, platform, melspectro_model, audio_len, &progress }); - // -- + var compiled_mel_step_future = try io.concurrent(voxtral.compileMelStep, .{ allocator, io, platform, melspectro_model, sp, &progress }); var compiled_conv_stem_future = try io.concurrent(voxtral.compileConvStem, .{ allocator, io, platform, encoder_model, sp.prompt_len * sp.mel_per_step, &progress }); var compiled_conv_stem_step_future = try io.concurrent(voxtral.compileConvStemStep, .{ allocator, io, platform, encoder_model, sp, &progress }); @@ -161,8 +159,8 @@ pub fn main(init: std.process.Init) !void { @memset(tokens[1..], token_streaming_pad); // -- Compiled models - var compiled_mel_spectrum = try compiled_mel_spectrum_future.await(io); - defer compiled_mel_spectrum.deinit(); + var compiled_mel_step = try compiled_mel_step_future.await(io); + defer compiled_mel_step.deinit(); var compiled_conv_stem = try compiled_conv_stem_future.await(io); defer compiled_conv_stem.deinit(); @@ -217,7 +215,7 @@ pub fn main(init: std.process.Init) !void { audio_len, tokens, sp, - &compiled_mel_spectrum, + &compiled_mel_step, &compiled_conv_stem, &compiled_conv_stem_step, &compiled_encoder_prefill, diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig index e492c9d5..482aba0d 100644 --- a/examples/voxtral/mel_spectrogram.zig +++ b/examples/voxtral/mel_spectrogram.zig @@ -63,6 +63,33 @@ pub const LogMelSpectrogram = struct { return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ .mel, .frames }); } + /// Like forward(), but for fixed-size audio chunks with reflect padding done on host. + /// Produces exactly (audio_len - n_fft) / hop_len + 1 mel frames. + /// Output tags: .channels and .time (matching conv stem input). + pub fn melStep(self: LogMelSpectrogram, audio_chunk: Tensor) Tensor { + const dtype = audio_chunk.dtype(); + + const window_weight = self.window.getWeights(self.n_fft, dtype); + const fft_len = window_weight.dim(.samples); + const audio_len: u63 = @intCast(audio_chunk.dim(.samples)); + const num_frames: u63 = @intCast(@divFloor(audio_len - fft_len, self.hop_len) + 1); + + // No reflect padding — done on host + var spectrogram = stft(audio_chunk, window_weight, num_frames, self.hop_len, self.precision); + spectrogram = spectrogram.convert(dtype); + + // Re-weight frequencies for speech + spectrogram = spectrogram.dot(self.mel_filters, .freq_bins); + + spectrogram = spectrogram.maximum(Tensor.constant(dtype.constant(self.mel_floor))); + var log_spec = spectrogram.log().scale(1.0 / @log(10.0)); + + const log_spec_min = Tensor.constant(dtype.constant(self.global_log_mel_max - 8.0)); + log_spec = log_spec.maximum(log_spec_min); + + return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ .mel, .frames }).withTags(.{ .channels, .time }); + } + pub fn load(self: *LogMelSpectrogram, io: std.Io, platform: *zml.Platform) !zml.Bufferized(LogMelSpectrogram) { const mel_filters_data = @embedFile("assets/voxtral_mel_filter.data"); const slice = zml.Slice.init(self.mel_filters.shape(), mel_filters_data); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index e09937bf..3caae932 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -21,12 +21,12 @@ const Adapter = dec.Adapter; const common = @import("common.zig"); const KvCache = common.KvCache; -pub fn compileMelSpectrum(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, padded_audio_len: usize, progress: *std.Progress.Node) !zml.Exe { +pub fn compileMelStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling mel spectrogram...", 1); + var node = progress.start("Compiling mel step...", 1); defer node.end(); - return try platform.compile(allocator, io, model, .forward, .{Tensor.init(.{padded_audio_len}, .f32).withTags(.{.samples})}); + return try platform.compile(allocator, io, model, .melStep, .{Tensor.init(.{sp.chunk_audio}, .f32).withTags(.{.samples})}); } pub fn compileConvStem(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, num_mel_frames: u32, progress: *std.Progress.Node) !zml.Exe { @@ -198,63 +198,80 @@ pub fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32 return result; } -/// Run mel spectrogram and download result to host memory. -/// Returns zero-padded mel data as bytes (channels-major: [128][padded_mel_frames]). Caller owns the memory. -pub fn runMelSpectrogram( +/// Prepend n_fft/2 reflected samples at the start and append n_fft/2 zeros at the end. +/// Left reflect skips boundary sample (index 0) to match Tensor reflect padding in forward(). +/// Right zero-padding ensures STFT windows at the tail have enough samples. +pub fn reflectPadAudio(allocator: std.mem.Allocator, audio: []const f32, pad: usize) ![]f32 { + const result = try allocator.alloc(f32, pad + audio.len + pad); + // Reflect: result[i] = audio[pad - i] for i in 0..pad + // i=0 → audio[pad], i=1 → audio[pad-1], ..., i=pad-1 → audio[1] + for (0..pad) |i| { + result[i] = audio[pad - i]; + } + @memcpy(result[pad..][0..audio.len], audio); + @memset(result[pad + audio.len ..], 0); + return result; +} + +/// Run mel spectrogram chunk-by-chunk and download result to host memory. +/// Returns mel data as bytes (channels-major: [128][prompt_len * mel_per_step]). Caller owns the memory. +pub fn runMelPrefill( allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, sp: StreamParams, padded_audio: []const f32, - audio_len: usize, - compiled_mel_spectrum: *zml.Exe, + compiled_mel_step: *zml.Exe, mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), ) ![]u8 { - const num_frames = sp.numFrames(audio_len); - const padded_mel_frames = sp.paddedMelFrames(audio_len); - const total_steps = sp.totalSteps(audio_len); + const prompt_len = sp.prompt_len; const f32_bytes: usize = @sizeOf(f32); + const prefill_mel_frames: u32 = prompt_len * sp.mel_per_step; - log.info("Running mel spectrogram...", .{}); - var mel_output: zml.Buffer = undefined; - { - var mel_args = try compiled_mel_spectrum.args(allocator); - defer mel_args.deinit(allocator); - var mel_results = try compiled_mel_spectrum.results(allocator); - defer mel_results.deinit(allocator); + log.info("Running mel prefill ({} chunks of {} frames, keeping {} per chunk)...", .{ prompt_len, sp.chunk_mel, sp.mel_per_step }); + + // Allocate host mel buffer: [128][prefill_mel_frames] zero-initialized + const mel_host = try allocator.alloc(u8, @as(usize, 128) * prefill_mel_frames * f32_bytes); + @memset(mel_host, 0); + + var mel_step_args = try compiled_mel_step.args(allocator); + defer mel_step_args.deinit(allocator); + var mel_step_results = try compiled_mel_step.results(allocator); + defer mel_step_results.deinit(allocator); + + var mel_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), .{}); + defer mel_step_output.deinit(); + + for (0..prompt_len) |t| { + // Audio chunk starts at t * mel_per_step * hop_length in the reflect-padded audio + const audio_start: usize = t * sp.mel_per_step * sp._hop_length; + const audio_chunk = padded_audio[audio_start .. audio_start + sp.chunk_audio]; const audio_slice: zml.Slice = .init( - .init(.{audio_len}, .f32), - std.mem.sliceAsBytes(padded_audio), + .init(.{sp.chunk_audio}, .f32), + std.mem.sliceAsBytes(audio_chunk), ); var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); defer audio_buffer.deinit(); - mel_output = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = num_frames }, .f32), .{}); + mel_step_args.set(.{ mel_spectrum_buffers, audio_buffer }); + compiled_mel_step.call(mel_step_args, &mel_step_results); + mel_step_results.fill(.{&mel_step_output}); - mel_args.set(.{ mel_spectrum_buffers, audio_buffer }); - compiled_mel_spectrum.call(mel_args, &mel_results); - mel_results.fill(.{&mel_output}); - } - defer mel_output.deinit(); - log.info("Mel spectrogram done.", .{}); - - // Download mel to host (channels-major: [128][padded_mel_frames] in f32) - const mel_host = try allocator.alloc(u8, @as(usize, 128) * padded_mel_frames * f32_bytes); - @memset(mel_host, 0); // zero-pad beyond num_frames - - { - var download = try mel_output.toSliceAlloc(allocator, io); + // Download chunk and keep first mel_per_step frames for conv stem prefill + var download = try mel_step_output.toSliceAlloc(allocator, io); defer download.free(allocator); const src = download.constData(); - const src_stride = @as(usize, num_frames) * f32_bytes; - const dst_stride = @as(usize, padded_mel_frames) * f32_bytes; - const copy_len = @as(usize, num_frames) * f32_bytes; + + const src_stride = @as(usize, sp.chunk_mel) * f32_bytes; + const dst_stride = @as(usize, prefill_mel_frames) * f32_bytes; + const copy_len = @as(usize, sp.mel_per_step) * f32_bytes; + const dst_offset = t * sp.mel_per_step * f32_bytes; for (0..128) |c| { - @memcpy(mel_host[c * dst_stride ..][0..copy_len], src[c * src_stride ..][0..copy_len]); + @memcpy(mel_host[c * dst_stride + dst_offset ..][0..copy_len], src[c * src_stride ..][0..copy_len]); } } - log.info("Mel downloaded to host ({} frames, padded to {}, {} steps).", .{ num_frames, padded_mel_frames, total_steps }); + log.info("Mel prefill done ({} frames).", .{prefill_mel_frames}); return mel_host; } @@ -268,7 +285,6 @@ pub fn runPrefill( config: Config, sp: StreamParams, mel_host: []const u8, - audio_len: usize, tokens: []const u32, compiled_conv_stem: *zml.Exe, compiled_encoder_prefill: *zml.Exe, @@ -285,30 +301,18 @@ pub fn runPrefill( dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), generated_token_slice: zml.Slice, ) !void { - const padded_mel_frames = sp.paddedMelFrames(audio_len); const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; const f32_bytes: usize = @sizeOf(f32); - // Conv stem prefill: process first prompt_len*mel_per_step mel frames + // Conv stem prefill: mel_host is already [128][prompt_len*mel_per_step] log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * sp.mel_per_step}); var conv_prefill_output: zml.Buffer = undefined; { const prefill_mel_time: u32 = prompt_len * sp.mel_per_step; - const prefill_staging_bytes: usize = @as(usize, 128) * prefill_mel_time * f32_bytes; - const prefill_mel_staging = try allocator.alloc(u8, prefill_staging_bytes); - defer allocator.free(prefill_mel_staging); - - const src_stride = @as(usize, padded_mel_frames) * f32_bytes; - const dst_stride = @as(usize, prefill_mel_time) * f32_bytes; - const copy_len = @as(usize, prefill_mel_time) * f32_bytes; - for (0..128) |c| { - @memcpy(prefill_mel_staging[c * dst_stride ..][0..copy_len], mel_host[c * src_stride ..][0..copy_len]); - } - const mel_slice: zml.Slice = .init( .init(.{ .channels = 128, .time = prefill_mel_time }, .f32), - prefill_mel_staging[0..prefill_staging_bytes], + mel_host[0 .. @as(usize, 128) * prefill_mel_time * f32_bytes], ); var mel_buffer: zml.Buffer = try .fromSlice(io, platform, mel_slice); defer mel_buffer.deinit(); @@ -412,12 +416,14 @@ pub fn runGenerationLoop( config: Config, sp: StreamParams, tokenizer: *zml.tokenizer.Tokenizer, - mel_host: []const u8, + padded_audio: []const f32, audio_len: usize, + compiled_mel_step: *zml.Exe, compiled_conv_stem_step: *zml.Exe, compiled_encoder_step: *zml.Exe, compiled_adapter_step: *zml.Exe, compiled_decoder_decode: *zml.Exe, + mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), encoder_buffers: *zml.Bufferized(Encoder), adapter_buffers: *zml.Bufferized(Adapter), decoder_buffers: *zml.Bufferized(Decoder), @@ -429,11 +435,9 @@ pub fn runGenerationLoop( dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), generated_token_slice: zml.Slice, ) !void { - const padded_mel_frames = sp.paddedMelFrames(audio_len); const total_steps = sp.totalSteps(audio_len); const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; - const f32_bytes: usize = @sizeOf(f32); var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); @@ -445,6 +449,11 @@ pub fn runGenerationLoop( log.info("Running decode loop (steps {}..{})...", .{ prompt_len, total_steps }); const decode_start: std.Io.Timestamp = .now(io, .awake); + var mel_step_args = try compiled_mel_step.args(allocator); + defer mel_step_args.deinit(allocator); + var mel_step_results = try compiled_mel_step.results(allocator); + defer mel_step_results.deinit(allocator); + var conv_step_args = try compiled_conv_stem_step.args(allocator); defer conv_step_args.deinit(allocator); var conv_step_results = try compiled_conv_stem_step.results(allocator); @@ -465,6 +474,9 @@ pub fn runGenerationLoop( var decode_results = try compiled_decoder_decode.results(allocator); defer decode_results.deinit(allocator); + var mel_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), .{}); + defer mel_step_output.deinit(); + var conv_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = sp.dsf, .d = enc_dim }, .bf16), .{}); defer conv_step_output.deinit(); @@ -477,14 +489,6 @@ pub fn runGenerationLoop( var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); defer current_token_buffer.deinit(); - // Mel staging buffer for conv stem step (reused each iteration) - const mel_staging_bytes: usize = @as(usize, 128) * sp.chunk_mel * f32_bytes; - const mel_staging = try allocator.alloc(u8, mel_staging_bytes); - defer allocator.free(mel_staging); - - const mel_src_stride = @as(usize, padded_mel_frames) * f32_bytes; - const mel_dst_stride = @as(usize, sp.chunk_mel) * f32_bytes; - var num_generated: usize = 0; for (prompt_len..total_steps) |t| { const generated_token = generated_token_slice.items(u32)[0]; @@ -501,20 +505,23 @@ pub fn runGenerationLoop( break; } - // Conv stem step: prepare mel chunk with history, run conv stem - const mel_t_start: usize = t * sp.mel_per_step - sp.mel_history; - for (0..128) |c| { - @memcpy(mel_staging[c * mel_dst_stride ..][0..mel_dst_stride], mel_host[c * mel_src_stride + mel_t_start * f32_bytes ..][0..mel_dst_stride]); - } + // Mel step: upload audio chunk → mel spectrogram on device (no host round-trip) + const audio_start: usize = (t * sp.mel_per_step - sp.mel_history) * sp._hop_length; + const audio_chunk = padded_audio[audio_start .. audio_start + sp.chunk_audio]; - const mel_chunk_slice: zml.Slice = .init( - .init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), - mel_staging[0..mel_staging_bytes], + const audio_slice: zml.Slice = .init( + .init(.{sp.chunk_audio}, .f32), + std.mem.sliceAsBytes(audio_chunk), ); - var mel_chunk_buffer: zml.Buffer = try .fromSlice(io, platform, mel_chunk_slice); - defer mel_chunk_buffer.deinit(); + var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); + defer audio_buffer.deinit(); - conv_step_args.set(.{ encoder_buffers, mel_chunk_buffer }); + mel_step_args.set(.{ mel_spectrum_buffers, audio_buffer }); + compiled_mel_step.call(mel_step_args, &mel_step_results); + mel_step_results.fill(.{&mel_step_output}); + + // Conv stem step: mel step output stays on device → conv stem + conv_step_args.set(.{ encoder_buffers, mel_step_output }); compiled_conv_stem_step.call(conv_step_args, &conv_step_results); conv_step_results.fill(.{&conv_step_output}); @@ -560,7 +567,7 @@ pub fn runPipeline( audio_len: usize, tokens: []const u32, sp: StreamParams, - compiled_mel_spectrum: *zml.Exe, + compiled_mel_step: *zml.Exe, compiled_conv_stem: *zml.Exe, compiled_conv_stem_step: *zml.Exe, compiled_encoder_prefill: *zml.Exe, @@ -580,8 +587,14 @@ pub fn runPipeline( ) !void { log.info("Running inference pipeline...", .{}); - // 1. Mel spectrogram - const mel_host = try runMelSpectrogram(allocator, io, platform, sp, padded_audio, audio_len, compiled_mel_spectrum, mel_spectrum_buffers); + // 0. Reflect-pad audio on host (prepend n_fft/2 samples) + const n_fft: usize = config.audio().window_size; + const reflect_pad: usize = n_fft / 2; + const reflect_padded_audio = try reflectPadAudio(allocator, padded_audio, reflect_pad); + defer allocator.free(reflect_padded_audio); + + // 1. Mel prefill (chunk-by-chunk) + const mel_host = try runMelPrefill(allocator, io, platform, sp, reflect_padded_audio, compiled_mel_step, mel_spectrum_buffers); defer allocator.free(mel_host); // 2. Init KV caches + t_cond + RNG @@ -614,7 +627,6 @@ pub fn runPipeline( config, sp, mel_host, - audio_len, tokens, compiled_conv_stem, compiled_encoder_prefill, @@ -640,12 +652,14 @@ pub fn runPipeline( config, sp, tokenizer, - mel_host, + reflect_padded_audio, audio_len, + compiled_mel_step, compiled_conv_stem_step, compiled_encoder_step, compiled_adapter_step, compiled_decoder_decode, + mel_spectrum_buffers, encoder_buffers, adapter_buffers, decoder_buffers, From b5897a127cb661e37a0d8c47fa118398534c3f6b Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 17 Feb 2026 10:14:12 +0100 Subject: [PATCH 73/98] start reading stdin (wrong prefill) --- examples/voxtral/main.zig | 20 +++++++-------- examples/voxtral/voxtral.zig | 48 +++++++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 7c124da5..16cb8761 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -26,7 +26,7 @@ const KvCache = common.KvCache; const voxtral = @import("voxtral.zig"); const CliArgs = struct { - input: []const u8, + input: ?[]const u8 = null, model: []const u8, transcription_delay_ms: f32 = 480.0, backend: ?zml.attention.Backend = null, @@ -60,8 +60,8 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = "Voxtral" }); - const file = try std.Io.Dir.openFile(.cwd(), io, args.input, .{}); - defer file.close(io); + // const file = try std.Io.Dir.openFile(.cwd(), io, args.input, .{}); + // defer file.close(io); const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); @@ -81,12 +81,12 @@ pub fn main(init: std.process.Init) !void { const tokens = try allocator.alloc(u32, sp.prompt_len); defer allocator.free(tokens); - var wav_buffer: [4096]u8 = undefined; - var reader = file.reader(io, &wav_buffer); + // var wav_buffer: [4096]u8 = undefined; + // var reader = file.reader(io, &wav_buffer); - const padded_wav = try wav_utils.loadAndPadWav(allocator, &reader.interface, sp.left_pad, sp.n_right_pad_tokens, sp.raw_audio_length_per_tok); - defer allocator.free(padded_wav); - const audio_len = padded_wav.len; + // const padded_wav = try wav_utils.loadAndPadWav(allocator, &reader.interface, sp.left_pad, sp.n_right_pad_tokens, sp.raw_audio_length_per_tok); + // defer allocator.free(padded_wav); + // const audio_len = padded_wav.len; var platform: *zml.Platform = try .auto(allocator, io, .{ .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.50 } } }, @@ -211,8 +211,8 @@ pub fn main(init: std.process.Init) !void { platform, config, &tokenizer, - padded_wav, - audio_len, + // padded_wav, + // audio_len, tokens, sp, &compiled_mel_step, diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 3caae932..2a5f182a 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -416,8 +416,9 @@ pub fn runGenerationLoop( config: Config, sp: StreamParams, tokenizer: *zml.tokenizer.Tokenizer, - padded_audio: []const f32, - audio_len: usize, + // padded_audio: []const f32, + // audio_len: usize, + stdin_reader: *std.Io.Reader, compiled_mel_step: *zml.Exe, compiled_conv_stem_step: *zml.Exe, compiled_encoder_step: *zml.Exe, @@ -435,7 +436,7 @@ pub fn runGenerationLoop( dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), generated_token_slice: zml.Slice, ) !void { - const total_steps = sp.totalSteps(audio_len); + // const total_steps = sp.totalSteps(audio_len); const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; @@ -446,7 +447,7 @@ pub fn runGenerationLoop( const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); const streaming_word_token = tokenizer.tokenToId("[STREAMING_WORD]"); - log.info("Running decode loop (steps {}..{})...", .{ prompt_len, total_steps }); + // log.info("Running decode loop (steps {}..{})...", .{ prompt_len, total_steps }); const decode_start: std.Io.Timestamp = .now(io, .awake); var mel_step_args = try compiled_mel_step.args(allocator); @@ -490,7 +491,7 @@ pub fn runGenerationLoop( defer current_token_buffer.deinit(); var num_generated: usize = 0; - for (prompt_len..total_steps) |t| { + for (prompt_len..500) |t| { const generated_token = generated_token_slice.items(u32)[0]; num_generated += 1; @@ -506,8 +507,10 @@ pub fn runGenerationLoop( } // Mel step: upload audio chunk → mel spectrogram on device (no host round-trip) - const audio_start: usize = (t * sp.mel_per_step - sp.mel_history) * sp._hop_length; - const audio_chunk = padded_audio[audio_start .. audio_start + sp.chunk_audio]; + // const audio_start: usize = (t * sp.mel_per_step - sp.mel_history) * sp._hop_length; + const audio_chunk = try readStdinChunk(allocator, stdin_reader); + defer allocator.free(audio_chunk); + // const audio_chunk = padded_audio[audio_start .. audio_start + sp.chunk_audio]; const audio_slice: zml.Slice = .init( .init(.{sp.chunk_audio}, .f32), @@ -563,8 +566,8 @@ pub fn runPipeline( platform: *zml.Platform, config: Config, tokenizer: *zml.tokenizer.Tokenizer, - padded_audio: []const f32, - audio_len: usize, + // padded_audio: []const f32, + // audio_len: usize, tokens: []const u32, sp: StreamParams, compiled_mel_step: *zml.Exe, @@ -587,10 +590,17 @@ pub fn runPipeline( ) !void { log.info("Running inference pipeline...", .{}); + var stdin_buff: [2560 * 4]u8 = undefined; + var stdin_reader = std.Io.File.stdin().reader(io, &stdin_buff); + const stdin_reader_interface = &stdin_reader.interface; + + const first_audio_chunk = try readStdinChunk(allocator, stdin_reader_interface); + defer allocator.free(first_audio_chunk); + // 0. Reflect-pad audio on host (prepend n_fft/2 samples) const n_fft: usize = config.audio().window_size; const reflect_pad: usize = n_fft / 2; - const reflect_padded_audio = try reflectPadAudio(allocator, padded_audio, reflect_pad); + const reflect_padded_audio = try reflectPadAudio(allocator, first_audio_chunk, reflect_pad); defer allocator.free(reflect_padded_audio); // 1. Mel prefill (chunk-by-chunk) @@ -652,8 +662,9 @@ pub fn runPipeline( config, sp, tokenizer, - reflect_padded_audio, - audio_len, + // reflect_padded_audio, + // audio_len, + stdin_reader_interface, compiled_mel_step, compiled_conv_stem_step, compiled_encoder_step, @@ -672,3 +683,16 @@ pub fn runPipeline( generated_token_slice, ); } + +fn readStdinChunk(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { + var read: [1280 * 2]u8 = undefined; + var converted = try allocator.alloc(f32, 1280); + try reader.readSliceAll(&read); + + for (0..1280) |i| { + const offset = i * 2; + converted[i] = @as(f32, @floatFromInt(std.mem.bytesToValue(i16, read[offset .. offset + 2]))) / 32768.0; + } + + return converted; +} From 87852ac75e4262a6c547ac8c3ff55780a4c8fa1b Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 17 Feb 2026 10:45:56 +0100 Subject: [PATCH 74/98] good prefill and log information --- examples/voxtral/main.zig | 2 +- examples/voxtral/voxtral.zig | 52 +++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 16cb8761..fc94dfd4 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -89,7 +89,7 @@ pub fn main(init: std.process.Init) !void { // const audio_len = padded_wav.len; var platform: *zml.Platform = try .auto(allocator, io, .{ - .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.50 } } }, + .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.90 } } }, }); defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 2a5f182a..b9ea0d42 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -405,7 +405,7 @@ pub fn runPrefill( generated_token_slice.items(u32)[0] = prefill_output_slice.items(u32)[prompt_len - 1]; log.info("First generated token (from prefill): {}", .{generated_token_slice.items(u32)[0]}); } - log.info("Decoder prefill done.", .{}); + log.info("Decoder prefill done.\n", .{}); } /// Run the streaming decode loop: step-by-step conv_stem -> encoder -> adapter -> decoder. @@ -490,8 +490,10 @@ pub fn runGenerationLoop( var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); defer current_token_buffer.deinit(); + log.info("Start of transcription:\n", .{}); var num_generated: usize = 0; - for (prompt_len..500) |t| { + var t: usize = prompt_len; + while (true) : (t += 1) { const generated_token = generated_token_slice.items(u32)[0]; num_generated += 1; @@ -508,7 +510,14 @@ pub fn runGenerationLoop( // Mel step: upload audio chunk → mel spectrogram on device (no host round-trip) // const audio_start: usize = (t * sp.mel_per_step - sp.mel_history) * sp._hop_length; - const audio_chunk = try readStdinChunk(allocator, stdin_reader); + const audio_chunk = readStdinSamples(allocator, stdin_reader, 1280) catch |err| switch (err) { + error.EndOfStream => { + std.debug.print("\n\n", .{}); + log.info("Audio stream ended at step={}", .{t}); + break; + }, + else => return err, + }; defer allocator.free(audio_chunk); // const audio_chunk = padded_audio[audio_start .. audio_start + sp.chunk_audio]; @@ -594,13 +603,23 @@ pub fn runPipeline( var stdin_reader = std.Io.File.stdin().reader(io, &stdin_buff); const stdin_reader_interface = &stdin_reader.interface; - const first_audio_chunk = try readStdinChunk(allocator, stdin_reader_interface); - defer allocator.free(first_audio_chunk); - - // 0. Reflect-pad audio on host (prepend n_fft/2 samples) + // 0. Read initial audio from stdin and build padded buffer for prefill (like loadAndPadWav) const n_fft: usize = config.audio().window_size; const reflect_pad: usize = n_fft / 2; - const reflect_padded_audio = try reflectPadAudio(allocator, first_audio_chunk, reflect_pad); + + const n_prefill_stdin: usize = @as(usize, sp.n_delay_tokens + 1) * sp.raw_audio_length_per_tok; + const stdin_audio = try readStdinSamples(allocator, stdin_reader_interface, n_prefill_stdin); + defer allocator.free(stdin_audio); + + // Pad: [left_pad zeros][stdin audio][right_pad zeros] + const right_pad: usize = @as(usize, sp.n_right_pad_tokens) * sp.raw_audio_length_per_tok; + const prefill_audio_len: usize = @as(usize, sp.left_pad) + stdin_audio.len + right_pad; + const prefill_audio = try allocator.alloc(f32, prefill_audio_len); + defer allocator.free(prefill_audio); + @memset(prefill_audio, 0); + @memcpy(prefill_audio[@as(usize, sp.left_pad)..][0..stdin_audio.len], stdin_audio); + + const reflect_padded_audio = try reflectPadAudio(allocator, prefill_audio, reflect_pad); defer allocator.free(reflect_padded_audio); // 1. Mel prefill (chunk-by-chunk) @@ -684,15 +703,16 @@ pub fn runPipeline( ); } -fn readStdinChunk(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { - var read: [1280 * 2]u8 = undefined; - var converted = try allocator.alloc(f32, 1280); - try reader.readSliceAll(&read); +fn readStdinSamples(allocator: std.mem.Allocator, reader: *std.Io.Reader, n_samples: usize) ![]f32 { + const byte_count = n_samples * 2; + const raw = try allocator.alloc(u8, byte_count); + defer allocator.free(raw); + try reader.readSliceAll(raw); - for (0..1280) |i| { + const samples = try allocator.alloc(f32, n_samples); + for (0..n_samples) |i| { const offset = i * 2; - converted[i] = @as(f32, @floatFromInt(std.mem.bytesToValue(i16, read[offset .. offset + 2]))) / 32768.0; + samples[i] = @as(f32, @floatFromInt(std.mem.bytesToValue(i16, raw[offset..][0..2]))) / 32768.0; } - - return converted; + return samples; } From 908a97a7ea646856848f7720064e60203dffb4a9 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 17 Feb 2026 14:28:14 +0100 Subject: [PATCH 75/98] correct audio shift --- examples/voxtral/voxtral.zig | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index b9ea0d42..6bb94582 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -416,8 +416,7 @@ pub fn runGenerationLoop( config: Config, sp: StreamParams, tokenizer: *zml.tokenizer.Tokenizer, - // padded_audio: []const f32, - // audio_len: usize, + initial_audio_history: []const f32, stdin_reader: *std.Io.Reader, compiled_mel_step: *zml.Exe, compiled_conv_stem_step: *zml.Exe, @@ -436,9 +435,16 @@ pub fn runGenerationLoop( dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), generated_token_slice: zml.Slice, ) !void { - // const total_steps = sp.totalSteps(audio_len); const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; + const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp._hop_length; + const audio_overlap: usize = sp.chunk_audio - new_audio_per_step; + + // Sliding audio buffer: [overlap from previous step | new samples] + const audio_buf = try allocator.alloc(f32, sp.chunk_audio); + defer allocator.free(audio_buf); + @memcpy(audio_buf[0..audio_overlap], initial_audio_history); + @memset(audio_buf[audio_overlap..], 0); var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); @@ -508,9 +514,8 @@ pub fn runGenerationLoop( break; } - // Mel step: upload audio chunk → mel spectrogram on device (no host round-trip) - // const audio_start: usize = (t * sp.mel_per_step - sp.mel_history) * sp._hop_length; - const audio_chunk = readStdinSamples(allocator, stdin_reader, 1280) catch |err| switch (err) { + // Mel step: read new audio, slide history buffer, upload full chunk + const new_samples = readStdinSamples(allocator, stdin_reader, new_audio_per_step) catch |err| switch (err) { error.EndOfStream => { std.debug.print("\n\n", .{}); log.info("Audio stream ended at step={}", .{t}); @@ -518,12 +523,15 @@ pub fn runGenerationLoop( }, else => return err, }; - defer allocator.free(audio_chunk); - // const audio_chunk = padded_audio[audio_start .. audio_start + sp.chunk_audio]; + defer allocator.free(new_samples); + + // Shift history left, append new samples + std.mem.copyForwards(f32, audio_buf[0..audio_overlap], audio_buf[new_audio_per_step..]); + @memcpy(audio_buf[audio_overlap..], new_samples); const audio_slice: zml.Slice = .init( .init(.{sp.chunk_audio}, .f32), - std.mem.sliceAsBytes(audio_chunk), + std.mem.sliceAsBytes(audio_buf), ); var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); defer audio_buffer.deinit(); @@ -673,7 +681,12 @@ pub fn runPipeline( generated_token_slice, ); - // 4. Generation loop + // 4. Generation loop — extract audio history for sliding buffer + const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp._hop_length; + const audio_overlap: usize = sp.chunk_audio - new_audio_per_step; + const history_start: usize = @as(usize, sp.prompt_len) * sp.mel_per_step * sp._hop_length; + const initial_audio_history = reflect_padded_audio[history_start .. history_start + audio_overlap]; + try runGenerationLoop( allocator, io, @@ -681,8 +694,7 @@ pub fn runPipeline( config, sp, tokenizer, - // reflect_padded_audio, - // audio_len, + initial_audio_history, stdin_reader_interface, compiled_mel_step, compiled_conv_stem_step, From 9a69d1bd7b9a46d4ba1a2d4a0485778ff0ccf311 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 17 Feb 2026 14:42:03 +0100 Subject: [PATCH 76/98] add sliding window --- examples/llama/llama.zig | 2 +- examples/voxtral/decoder.zig | 13 ++++++++++--- examples/voxtral/encoder.zig | 13 ++++++++++--- examples/voxtral/main.zig | 11 +++++------ zml/attention/attention.zig | 8 +++++--- zml/attention/flashattn.zig | 11 ++++++----- 6 files changed, 37 insertions(+), 21 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index f4f3095b..df874ce6 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -436,7 +436,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters, .{}); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 112d5c61..7c932bb6 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -220,12 +220,19 @@ pub const SelfAttention = struct { k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); - // Update KV cache and retrieve cached K/V - const new_kv_cache = kv_cache.update(k, v, pos_index.rename(.{ .s = .k })); + // Update KV cache with modular indexing for sliding window circular buffer + const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); + const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); + const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_out = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + // Cap token_index so seqused_k doesn't exceed cache bounds + const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); + const attn_token_index = token_index.minimum(max_token_index); + const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ + .sliding_window = @intCast(config.sliding_window), + }); // Merge heads and output projection: [q, h, hd] → [s, d] const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index a45b5b06..d69f7422 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -249,12 +249,19 @@ pub const SelfAttention = struct { k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); - // Update KV cache and retrieve cached K/V - const new_kv_cache = kv_cache.update(k, v, pos_index.rename(.{ .s = .k })); + // Update KV cache with modular indexing for sliding window circular buffer + const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); + const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); + const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_out = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + // Cap token_index so seqused_k doesn't exceed cache bounds + const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); + const attn_token_index = token_index.minimum(max_token_index); + const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ + .sliding_window = @intCast(enc.sliding_window), + }); const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index fc94dfd4..8699e7f2 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -30,7 +30,6 @@ const CliArgs = struct { model: []const u8, transcription_delay_ms: f32 = 480.0, backend: ?zml.attention.Backend = null, - max_seqlen: u32 = 1500, }; const n_left_pad_tokens: u32 = 32; @@ -109,12 +108,12 @@ pub fn main(init: std.process.Init) !void { var decoder_model: Decoder = .init(allocator, model_store.view(), config); defer decoder_model.deinit(allocator); - // KV cache shapes for encoder and decoder + // KV cache shapes for encoder and decoder, sized to sliding_window for memory efficiency const enc_cfg = config.encoder(); const enc_dtype = encoder_model.norm.dtype(); const enc_kv_cache: KvCache = .init(.init(.{ .layer = enc_cfg.n_layers, - .k = args.max_seqlen, + .k = enc_cfg.sliding_window, .h = enc_cfg.n_kv_heads, .hd = enc_cfg.head_dim, }, enc_dtype)); @@ -122,13 +121,13 @@ pub fn main(init: std.process.Init) !void { const dec_dtype = decoder_model.tok_embeddings.dtype(); const dec_kv_cache: KvCache = .init(.init(.{ .layer = decoder_model.layers.len, - .k = args.max_seqlen, + .k = config.sliding_window, .h = config.n_kv_heads, .hd = config.head_dim, }, dec_dtype)); - const enc_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(args.max_seqlen))); - const dec_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(args.max_seqlen))); + const enc_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(enc_cfg.sliding_window))); + const dec_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(config.sliding_window))); const attention_parameters: zml.attention.Parameters = .init(.fromBackend(backend)); // Launch concurrent compilation and buffer loading diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 34bf52e0..8b5b1442 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -101,7 +101,9 @@ pub const Metadata = union(Backend) { } }; -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { +pub const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: AttentionOptions) zml.Tensor { return switch (parameters) { .vanilla => b: { // Generate the attention mask. @@ -114,7 +116,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); break :b attn_output; }, - .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), - .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), + .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2, opts), + .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3, opts), }; } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 111b7b1c..a6ee5b3c 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -252,7 +252,7 @@ pub const fa2 = struct { } }; - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: AttentionOptions) zml.Tensor { _ = parameters; // autofix stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); @@ -274,7 +274,7 @@ pub const fa2 = struct { const num_heads_k = k_.dim(.h); const head_size = q_.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; if (seqlenq_ngroups_swapped) { q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); } @@ -294,8 +294,8 @@ pub const fa2 = struct { .{q.shape()}, .{ //.softmax_scale = @as(f32, 1.0), - .is_causal = true, - .window_size_left = @as(i32, -1), + .is_causal = opts.is_causal, + .window_size_left = opts.sliding_window, .window_size_right = @as(i32, -1), .max_seqlen_q = max_seqlen_q, .max_seqlen_k = max_seqlen_k, @@ -435,8 +435,9 @@ pub const fa3 = struct { } }; - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: AttentionOptions) zml.Tensor { _ = parameters; // autofix + _ = opts; // autofix stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa3.attention support for batch size != 1 is not supported yet.", .{}); const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); // TODO(Corendos): replace with cumsum From 719b2ee3475359704b07f792a829b59f80737a8f Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 17 Feb 2026 15:43:52 +0100 Subject: [PATCH 77/98] Add terminalwave --- examples/voxtral/BUILD.bazel | 2 + .../voxtral/terminalwave/terminalwave.zig | 214 ++++++++++++++++++ examples/voxtral/voxtral.zig | 83 ++++++- 3 files changed, 287 insertions(+), 12 deletions(-) create mode 100644 examples/voxtral/terminalwave/terminalwave.zig diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel index a7bcf6f6..85c4ee5b 100644 --- a/examples/voxtral/BUILD.bazel +++ b/examples/voxtral/BUILD.bazel @@ -12,6 +12,7 @@ zig_binary( "encoder.zig", "decoder.zig", "common.zig", + "terminalwave/terminalwave.zig" ], extra_srcs = [ "//examples/voxtral/assets:voxtral_mel_filter", @@ -30,6 +31,7 @@ zig_binary( "encoder.zig", "decoder.zig", "common.zig", + "terminalwave/terminalwave.zig" ], extra_srcs = [ "//examples/voxtral/assets:voxtral_mel_filter", diff --git a/examples/voxtral/terminalwave/terminalwave.zig b/examples/voxtral/terminalwave/terminalwave.zig new file mode 100644 index 00000000..c8e956be --- /dev/null +++ b/examples/voxtral/terminalwave/terminalwave.zig @@ -0,0 +1,214 @@ +const std = @import("std"); + +// ── Public types ───────────────────────────────────────────── + +pub const Color = struct { r: u8, g: u8, b: u8 }; + +pub const Gradient = struct { + start: Color = .{ .r = 140, .g = 20, .b = 8 }, + mid: Color = .{ .r = 230, .g = 120, .b = 5 }, + end: Color = .{ .r = 255, .g = 220, .b = 55 }, +}; + +pub const Config = struct { + num_bars: u16 = 60, + half_height: u16 = 22, + sensitivity: f32 = 12.0, + attack: f32 = 0.6, + decay: f32 = 0.12, + gradient: Gradient = .{}, + padding_left: u16 = 4, + title: []const u8 = "Voxtral Realtime", + show_title: bool = true, +}; + +pub const FrameResult = struct { + level: f32, +}; + +// ── State ──────────────────────────────────────────────────── + +pub const State = struct { + config: Config, + smoothed: f32, + frame_count: u32, + buf: [BUF_SIZE]u8, + pos: usize, + + gradient_lut: [MAX_HALF_HEIGHT]Color, + + const BUF_SIZE = 131072; + const MAX_HALF_HEIGHT = 256; + const Half = enum { upper, lower }; + + pub fn init(config: Config) State { + var state = State{ + .config = config, + .smoothed = 0, + .frame_count = 0, + .gradient_lut = undefined, + .buf = undefined, + .pos = 0, + }; + + state.buildGradientLut(); + + // Hide cursor, clear screen + std.debug.print("\x1b[?25l\x1b[2J", .{}); + + return state; + } + + pub fn deinit(self: *State) void { + _ = self; + // Show cursor, reset scroll region + std.debug.print("\x1b[r\x1b[?25h", .{}); + } + + pub fn render(self: *State, rms: f32) FrameResult { + // Smoothing + const attack_rate: f32 = if (rms > self.smoothed) self.config.attack else self.config.decay; + self.smoothed = self.smoothed * (1.0 - attack_rate) + rms * attack_rate; + const level = @min(self.smoothed * self.config.sensitivity, 1.0); + + // Render + self.pos = 0; + self.put("\x1b[H"); + + if (self.config.show_title) { + self.put("\n\x1b[1;38;2;230;120;5m"); + self.putPadding(); + self.put(self.config.title); + self.put(" "); + self.put("\x1b[0;90m\xe2\x94\x80 live transcription\x1b[0m\n\n"); + } + + const num_bars = self.config.num_bars; + const half_height = self.config.half_height; + const half_term = half_height / 2; + + // Compute bar heights + var heights: [512]f32 = undefined; + const center: f32 = @as(f32, @floatFromInt(num_bars)) / 2.0; + const t: f32 = @as(f32, @floatFromInt(self.frame_count)) * 0.12; + + for (0..num_bars) |i| { + const fi: f32 = @as(f32, @floatFromInt(i)); + const dist = (fi - center) / center; + const envelope = @exp(-dist * dist * 3.5); + + const w1 = 0.28 * @sin(fi * 0.8 + t); + const w2 = 0.18 * @sin(fi * 1.5 - t * 1.6); + const w3 = 0.12 * @sin(fi * 2.4 + t * 0.7); + const w4 = 0.08 * @sin(fi * 3.7 - t * 2.1); + const wobble = w1 + w2 + w3 + w4; + + const h = level * envelope * (1.0 + wobble) * @as(f32, @floatFromInt(half_height)); + heights[i] = @max(@min(h, @as(f32, @floatFromInt(half_height))), 0.0); + } + + self.renderHalf(.upper, half_term, half_height, num_bars, heights[0..num_bars]); + self.renderHalf(.lower, half_term, half_height, num_bars, heights[0..num_bars]); + + self.frame_count +%= 1; + + return .{ .level = level }; + } + + // ── Private helpers ────────────────────────────────────── + + fn buildGradientLut(self: *State) void { + const hh = self.config.half_height; + const g = self.config.gradient; + + for (0..hh) |i| { + const t: f32 = @as(f32, @floatFromInt(i)) / @as(f32, @floatFromInt(hh - 1)); + if (t < 0.5) { + const s = t * 2.0; + self.gradient_lut[i] = .{ + .r = lerpU8(@floatFromInt(g.start.r), @floatFromInt(g.mid.r), s), + .g = lerpU8(@floatFromInt(g.start.g), @floatFromInt(g.mid.g), s), + .b = lerpU8(@floatFromInt(g.start.b), @floatFromInt(g.mid.b), s), + }; + } else { + const s = (t - 0.5) * 2.0; + self.gradient_lut[i] = .{ + .r = lerpU8(@floatFromInt(g.mid.r), @floatFromInt(g.end.r), s), + .g = lerpU8(@floatFromInt(g.mid.g), @floatFromInt(g.end.g), s), + .b = lerpU8(@floatFromInt(g.mid.b), @floatFromInt(g.end.b), s), + }; + } + } + } + + fn renderHalf(self: *State, half: Half, half_term: u16, half_height: u16, num_bars: u16, heights: []const f32) void { + for (0..half_term) |tr| { + self.putPadding(); + const tr16 = @as(u16, @intCast(tr)); + const upper_log, const lower_log = switch (half) { + .upper => .{ half_height - 1 - 2 * tr16, half_height - 2 - 2 * tr16 }, + .lower => .{ 2 * tr16 + 1, 2 * tr16 + 2 }, + }; + for (0..num_bars) |col| { + const h = heights[col]; + const upper_on = h > @as(f32, @floatFromInt(upper_log)); + const lower_on = h > @as(f32, @floatFromInt(lower_log)); + + if (upper_on and lower_on) { + self.cellBoth(self.gradient_lut[upper_log], self.gradient_lut[lower_log]); + } else if (half == .upper and lower_on) { + self.cellLower(self.gradient_lut[lower_log]); + } else if (half == .lower and upper_on) { + self.cellUpper(self.gradient_lut[upper_log]); + } else { + self.put(" "); + } + } + self.put("\n"); + } + } + + // ── Public buffer helpers (for caller to append after render) ── + + pub fn putPadding(self: *State) void { + for (0..self.config.padding_left) |_| { + self.put(" "); + } + } + + pub fn put(self: *State, s: []const u8) void { + if (self.pos + s.len > BUF_SIZE) return; + @memcpy(self.buf[self.pos..][0..s.len], s); + self.pos += s.len; + } + + pub fn fmt(self: *State, comptime f: []const u8, args: anytype) void { + const result = std.fmt.bufPrint(self.buf[self.pos..], f, args) catch return; + self.pos += result.len; + } + + pub fn flush(self: *State) void { + std.debug.print("{s}", .{self.buf[0..self.pos]}); + self.pos = 0; + } + + fn cellBoth(self: *State, upper: Color, lower: Color) void { + self.fmt("\x1b[38;2;{d};{d};{d};48;2;{d};{d};{d}m\xe2\x96\x80\x1b[0m", .{ + upper.r, upper.g, upper.b, lower.r, lower.g, lower.b, + }); + } + + fn cellUpper(self: *State, col: Color) void { + self.fmt("\x1b[38;2;{d};{d};{d}m\xe2\x96\x80\x1b[0m", .{ col.r, col.g, col.b }); + } + + fn cellLower(self: *State, col: Color) void { + self.fmt("\x1b[38;2;{d};{d};{d}m\xe2\x96\x84\x1b[0m", .{ col.r, col.g, col.b }); + } +}; + +// ── Free functions ─────────────────────────────────────────── + +fn lerpU8(a: f32, b: f32, t: f32) u8 { + return @intFromFloat(@max(0.0, @min(255.0, a + (b - a) * t))); +} diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 6bb94582..23819850 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -21,6 +21,8 @@ const Adapter = dec.Adapter; const common = @import("common.zig"); const KvCache = common.KvCache; +const terminalwave = @import("terminalwave/terminalwave.zig"); + pub fn compileMelStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling mel step...", 1); @@ -496,7 +498,30 @@ pub fn runGenerationLoop( var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); defer current_token_buffer.deinit(); - log.info("Start of transcription:\n", .{}); + // Wave visualization + transcript accumulation + var wave = terminalwave.State.init(.{ + .sensitivity = 2.0, + .num_bars = 50, + .half_height = 20, + }); + errdefer wave.deinit(); + + var transcript: std.ArrayListUnmanaged(u8) = .empty; + defer transcript.deinit(allocator); + + // Set up scroll region: wave is fixed at top, transcript scrolls below + const title_rows: u16 = if (wave.config.show_title) 3 else 0; + const transcript_start_row: u16 = title_rows + wave.config.half_height + 4; + + // Render initial empty wave + separator, set scroll region + _ = wave.render(0); + renderSeparator(&wave); + wave.fmt("\x1b[{d};999r", .{transcript_start_row}); + wave.fmt("\x1b[{d};{d}H", .{ transcript_start_row, wave.config.padding_left + 1 }); + wave.put("\x1b[J"); + wave.flush(); + + var last_transcript_len: usize = 0; var num_generated: usize = 0; var t: usize = prompt_len; while (true) : (t += 1) { @@ -505,26 +530,37 @@ pub fn runGenerationLoop( if (generated_token != streaming_pad_token and (streaming_word_token == null or generated_token != streaming_word_token.?)) { if (try tokenizer_decoder.next(generated_token)) |chunk| { - std.debug.print("{s}", .{chunk}); + try transcript.appendSlice(allocator, chunk); } } - if (generated_token == eos_token) { - log.info(" EOS reached at step={}", .{t}); - break; - } + if (generated_token == eos_token) break; // Mel step: read new audio, slide history buffer, upload full chunk const new_samples = readStdinSamples(allocator, stdin_reader, new_audio_per_step) catch |err| switch (err) { - error.EndOfStream => { - std.debug.print("\n\n", .{}); - log.info("Audio stream ended at step={}", .{t}); - break; - }, + error.EndOfStream => break, else => return err, }; defer allocator.free(new_samples); + // Save cursor position (in transcript scroll region) + std.debug.print("\x1b7", .{}); + + // Render wave with current audio level (fixed area above scroll region) + const rms = computeRms(new_samples); + _ = wave.render(rms); + renderSeparator(&wave); + + // Restore cursor and print only new transcript text + wave.put("\x1b8"); + if (transcript.items.len > last_transcript_len) { + wave.put("\x1b[97m"); + wave.put(transcript.items[last_transcript_len..]); + wave.put("\x1b[0m"); + last_transcript_len = transcript.items.len; + } + wave.flush(); + // Shift history left, append new samples std.mem.copyForwards(f32, audio_buf[0..audio_overlap], audio_buf[new_audio_per_step..]); @memcpy(audio_buf[audio_overlap..], new_samples); @@ -568,8 +604,13 @@ pub fn runGenerationLoop( try current_token_buffer.toSlice(io, generated_token_slice); } + + // Reset scroll region and exit wave visualization (returns to main screen) + std.debug.print("\x1b[r", .{}); + wave.deinit(); + const decode_duration = decode_start.untilNow(io, .awake); - std.debug.print("\n", .{}); + std.debug.print("\n\x1b[1mTranscription:\x1b[0m\n{s}\n\n", .{transcript.items}); log.info("Decode done. Generated {} tokens in {D}: {:.3}tok/s", .{ num_generated, stdx.fmt.fmtDuration(decode_duration), @@ -715,6 +756,24 @@ pub fn runPipeline( ); } +fn renderSeparator(wave: *terminalwave.State) void { + wave.put("\n\x1b[38;2;60;60;60m"); + wave.putPadding(); + for (0..wave.config.num_bars) |_| { + wave.put("\xe2\x94\x80"); + } + wave.put("\x1b[0m\n\n"); +} + +fn computeRms(samples: []const f32) f32 { + if (samples.len == 0) return 0; + var sum: f32 = 0; + for (samples) |s| { + sum += s * s; + } + return @sqrt(sum / @as(f32, @floatFromInt(samples.len))); +} + fn readStdinSamples(allocator: std.mem.Allocator, reader: *std.Io.Reader, n_samples: usize) ![]f32 { const byte_count = n_samples * 2; const raw = try allocator.alloc(u8, byte_count); From facfe51ba033020178cab92e811e5f00f6eee287 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Tue, 17 Feb 2026 16:31:38 +0100 Subject: [PATCH 78/98] force temperature and experimental reordering of cicular buffer --- examples/voxtral/decoder.zig | 2 +- examples/voxtral/encoder.zig | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 7c932bb6..fb5efa24 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -129,7 +129,7 @@ pub const Decoder = struct { // Compute logits in f32 for precision const logits = h.convert(.f32).dot(self.tok_embeddings.convert(.f32), .d); - const output_tokens, const new_rng = zml.nn.sampleTokens(logits, .{}, rng); + const output_tokens, const new_rng = zml.nn.sampleTokens(logits, .{ .temperature = 0 }, rng); return .{ output_tokens.convert(.u32), cache, new_rng }; } diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index d69f7422..1e3b1d2c 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -256,6 +256,22 @@ pub const SelfAttention = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); + // EXPERIMENTAL + // Reorder K/V from circular buffer to temporal order for correct causal masking. + // When the cache wraps, physical order != temporal order, which breaks FA2's + // position-based causal mask for multi-token (q_len > 1) attention. + { + const pos_end = token_index.addConstant(x.dim(.s)); + const is_full = pos_end.cmp(.GE, cache_size); + const rotation_start = pos_end.remainder(cache_size); + const safe_start = is_full.select(rotation_start, Tensor.scalar(@as(u32, 0), token_index.dtype())); + const reorder_arange = Tensor.arange(.{ .end = kv_cache.k.dim(.k) }, token_index.dtype()).withTags(.{.kk}); + const reorder_idx = reorder_arange.add(safe_start.broad(reorder_arange.shape())) + .remainder(cache_size.broad(reorder_arange.shape())); + k = k.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); + v = v.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); + } + // Cap token_index so seqused_k doesn't exceed cache bounds const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); const attn_token_index = token_index.minimum(max_token_index); From 65b4ab10dbd02020e04938cf2ed335e2adec7321 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 18 Feb 2026 11:13:59 +0100 Subject: [PATCH 79/98] start of cleanup --- examples/voxtral/common.zig | 113 +++++++- examples/voxtral/config.zig | 28 ++ examples/voxtral/decoder.zig | 105 +------- examples/voxtral/encoder.zig | 118 +-------- examples/voxtral/main.zig | 37 +-- examples/voxtral/mel_spectrogram.zig | 26 +- examples/voxtral/test.zig | 35 +-- examples/voxtral/voxtral.zig | 381 +++++++++++---------------- 8 files changed, 359 insertions(+), 484 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index 684c7ca2..5f524ef0 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -6,6 +6,23 @@ const stdx = zml.stdx; const Tensor = zml.Tensor; const Shape = zml.Shape; +const cfg = @import("config.zig"); + +/// Recursively deinit all Buffer fields in a Bufferized struct. +/// For models with allocated slices (e.g. layers), the caller must also free the slice. +pub fn deinitBufferized(bufferized: anytype) void { + zml.meta.visit((struct { + fn cb(_: void, buf: *zml.Buffer) void { + buf.deinit(); + } + }).cb, {}, bufferized); +} + +pub fn deinitLinear(l: *zml.Bufferized(zml.nn.Linear)) void { + l.weight.deinit(); + if (l.bias) |*b| b.deinit(); +} + pub fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { return .init( store.createTensorWithTags("weight", .{ .dout, .d }), @@ -14,11 +31,6 @@ pub fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { ); } -pub fn deinitLinear(l: *zml.Bufferized(zml.nn.Linear)) void { - l.weight.deinit(); - if (l.bias) |*b| b.deinit(); -} - pub fn rmsNorm(x: Tensor, weight: Tensor, eps: f32) Tensor { return zml.nn.rmsNorm(x, .d, eps).mul(weight.broad(x.shape())); } @@ -56,6 +68,97 @@ pub fn loadModel( }); } +/// Unified self-attention with KV cache, parameterized on whether to reorder +/// the circular KV buffer for correct causal masking (encoder needs this, decoder does not). +pub fn SelfAttention(comptime reorder_kv_cache: bool) type { + return struct { + wq: zml.nn.Linear, + wk: zml.nn.Linear, + wv: zml.nn.Linear, + wo: zml.nn.Linear, + + const Self = @This(); + + pub fn init(store: zml.io.TensorStore.View) Self { + return .{ + .wq = linear(store.withPrefix("wq")), + .wk = linear(store.withPrefix("wk")), + .wv = linear(store.withPrefix("wv")), + .wo = linear(store.withPrefix("wo")), + }; + } + + pub fn unload(self: *zml.Bufferized(Self)) void { + deinitBufferized(self); + } + + /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) + pub fn forward(self: Self, x: Tensor, token_index: Tensor, kv_cache: KvCache, attn_config: cfg.AttentionConfig, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { + const dtype = x.dtype(); + + var q = self.wq.forward(x); + var k = self.wk.forward(x); + var v = self.wv.forward(x); + + q = q.splitAxis(.dout, .{ .h = attn_config.n_heads, .hd = attn_config.head_dim }); + k = k.splitAxis(.dout, .{ .h = attn_config.n_kv_heads, .hd = attn_config.head_dim }); + v = v.splitAxis(.dout, .{ .h = attn_config.n_kv_heads, .hd = attn_config.head_dim }); + + // Position indices for RoPE: token_index + arange(s) + const pos_index = b: { + const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) + .withTags(.{.s}).broad(Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + const rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = attn_config.rope_theta, + }; + + q = zml.nn.rope(q, pos_index, rope_opts); + k = zml.nn.rope(k, pos_index, rope_opts); + + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + // Update KV cache with modular indexing for sliding window circular buffer + const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); + const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); + const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + if (reorder_kv_cache) { + // EXPERIMENTAL + // Reorder K/V from circular buffer to temporal order for correct causal masking. + // When the cache wraps, physical order != temporal order, which breaks FA2's + // position-based causal mask for multi-token (q_len > 1) attention. + const pos_end = token_index.addConstant(x.dim(.s)); + const is_full = pos_end.cmp(.GE, cache_size); + const rotation_start = pos_end.remainder(cache_size); + const safe_start = is_full.select(rotation_start, Tensor.scalar(@as(u32, 0), token_index.dtype())); + const reorder_arange = Tensor.arange(.{ .end = kv_cache.k.dim(.k) }, token_index.dtype()).withTags(.{.kk}); + const reorder_idx = reorder_arange.add(safe_start.broad(reorder_arange.shape())) + .remainder(cache_size.broad(reorder_arange.shape())); + k = k.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); + v = v.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); + } + + // Cap token_index so seqused_k doesn't exceed cache bounds + const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); + const attn_token_index = token_index.minimum(max_token_index); + const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ + .sliding_window = @intCast(attn_config.sliding_window), + }); + + const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; + } + }; +} + /// KV cache for all layers of a transformer. /// Stores K/V tensors with shape {layer, k=max_seq_len, h, hd}. pub const KvCache = struct { diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index 285f898b..5a95863d 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -1,5 +1,13 @@ const std = @import("std"); +pub const AttentionConfig = struct { + n_heads: u32, + n_kv_heads: u32, + head_dim: u32, + rope_theta: f32, + sliding_window: u32, +}; + pub const Config = struct { dim: u32, n_layers: u32, @@ -43,6 +51,16 @@ pub const Config = struct { rope_theta: f32, norm_eps: f32, sliding_window: u32, + + pub fn attentionConfig(self: EncoderArgs) AttentionConfig { + return .{ + .n_heads = self.n_heads, + .n_kv_heads = self.n_kv_heads, + .head_dim = self.head_dim, + .rope_theta = self.rope_theta, + .sliding_window = self.sliding_window, + }; + } }; pub const AudioEncodingArgs = struct { @@ -56,6 +74,16 @@ pub const Config = struct { // -- Shortcuts + pub fn attentionConfig(self: Config) AttentionConfig { + return .{ + .n_heads = self.n_heads, + .n_kv_heads = self.n_kv_heads, + .head_dim = self.head_dim, + .rope_theta = self.rope_theta, + .sliding_window = self.sliding_window, + }; + } + pub fn encoder(self: Config) EncoderArgs { return self.multimodal.whisper_model_args.encoder_args; } diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index fb5efa24..9a9f709f 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -2,7 +2,6 @@ const std = @import("std"); const zml = @import("zml"); const Tensor = zml.Tensor; -const Shape = zml.Shape; const cfg = @import("config.zig"); const Config = cfg.Config; @@ -36,8 +35,7 @@ pub const Adapter = struct { } pub fn unload(self: *zml.Bufferized(Adapter)) void { - common.deinitLinear(&self.proj0); - common.deinitLinear(&self.proj1); + common.deinitBufferized(self); } /// encoder_out: [s, d] → [s/dsf, dim] @@ -94,14 +92,8 @@ pub const Decoder = struct { } pub fn unload(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { - self.tok_embeddings.deinit(); - - for (self.layers) |*layer| { - DecoderLayer.unload(layer); - } - + common.deinitBufferized(self); allocator.free(self.layers); - self.norm.deinit(); } /// Run all decoder layers on input embeddings, sample next tokens and return updated KV cache + RNG. @@ -115,10 +107,11 @@ pub const Decoder = struct { var h = input_embeds; var cache = kv_cache; + const attn_config = self.config.attentionConfig(); for (self.layers, 0..) |layer, i| { const layer_cache = cache.atLayer(i); - const result = layer.forward(h, token_index, layer_cache, t, self.config, attention_metadata, attention_parameters); + const result = layer.forward(h, token_index, layer_cache, t, attn_config, attention_metadata, attention_parameters); h = result[0]; const updated_layer_cache = result[1]; @@ -150,8 +143,7 @@ pub const AdaRmsNorm = struct { } pub fn unload(self: *zml.Bufferized(AdaRmsNorm)) void { - common.deinitLinear(&self.down); - common.deinitLinear(&self.up); + common.deinitBufferized(self); } /// h: [s, d], t_cond: [d] → [s, d] @@ -165,80 +157,7 @@ pub const AdaRmsNorm = struct { pub const KvCache = common.KvCache; -/// Decoder self-attention with KV cache. -pub const SelfAttention = struct { - wq: zml.nn.Linear, - wk: zml.nn.Linear, - wv: zml.nn.Linear, - wo: zml.nn.Linear, - - pub fn init(store: zml.io.TensorStore.View) SelfAttention { - return .{ - .wq = common.linear(store.withPrefix("wq")), - .wk = common.linear(store.withPrefix("wk")), - .wv = common.linear(store.withPrefix("wv")), - .wo = common.linear(store.withPrefix("wo")), - }; - } - - pub fn unload(self: *zml.Bufferized(SelfAttention)) void { - common.deinitLinear(&self.wq); - common.deinitLinear(&self.wk); - common.deinitLinear(&self.wv); - common.deinitLinear(&self.wo); - } - - /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) - pub fn forward(self: SelfAttention, x: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { - const dtype = x.dtype(); - - var q = self.wq.forward(x); - var k = self.wk.forward(x); - var v = self.wv.forward(x); - - // Split into heads: [s, dout] → [s, h, hd] - q = q.splitAxis(.dout, .{ .h = config.n_heads, .hd = config.head_dim }); - k = k.splitAxis(.dout, .{ .h = config.n_kv_heads, .hd = config.head_dim }); - v = v.splitAxis(.dout, .{ .h = config.n_kv_heads, .hd = config.head_dim }); - - // Position indices for RoPE - const pos_index = b: { - const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) - .withTags(.{.s}).broad(Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); - break :b temp.add(token_index.broad(temp.shape())); - }; - - const rope_opts: zml.nn.RopeOpts = .{ - .layout = .interleaved, - .freq_base = config.rope_theta, - }; - - q = zml.nn.rope(q, pos_index, rope_opts); - k = zml.nn.rope(k, pos_index, rope_opts); - - q = q.rename(.{ .s = .q }); - k = k.rename(.{ .s = .k }); - v = v.rename(.{ .s = .k }); - - // Update KV cache with modular indexing for sliding window circular buffer - const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); - const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); - const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); - k = new_kv_cache.keys().convert(dtype); - v = new_kv_cache.values().convert(dtype); - - // Cap token_index so seqused_k doesn't exceed cache bounds - const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); - const attn_token_index = token_index.minimum(max_token_index); - const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ - .sliding_window = @intCast(config.sliding_window), - }); - - // Merge heads and output projection: [q, h, hd] → [s, d] - const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; - } -}; +pub const SelfAttention = common.SelfAttention(false); /// Single decoder transformer layer with AdaRmsNorm time conditioning. pub const DecoderLayer = struct { @@ -261,23 +180,17 @@ pub const DecoderLayer = struct { } pub fn unload(self: *zml.Bufferized(DecoderLayer)) void { - self.attention_norm.deinit(); - SelfAttention.unload(&self.attention); - - self.ffn_norm.deinit(); - SwiGluFfn.unload(&self.feed_forward); - - AdaRmsNorm.unload(&self.ada_norm); + common.deinitBufferized(self); } /// h: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] → ([s, d], KvCache) - pub fn forward(self: DecoderLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, config: Config, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { + pub fn forward(self: DecoderLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, attn_config: cfg.AttentionConfig, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { // Pre-attention norm → attention (with KV cache) → residual const attn_out, const updated_kv_cache = self.attention.forward( rmsNorm(h, self.attention_norm, self.norm_eps), token_index, kv_cache, - config, + attn_config, attention_metadata, attention_parameters, ); diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 1e3b1d2c..7476ee1c 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -48,15 +48,8 @@ pub const Encoder = struct { } pub fn unload(self: *zml.Bufferized(Encoder), allocator: std.mem.Allocator) void { - CausalConv1d.unload(&self.conv0); - CausalConv1d.unload(&self.conv1); - - for (self.layers) |*layer| { - TransformerLayer.unload(layer); - } - + common.deinitBufferized(self); allocator.free(self.layers); - self.norm.deinit(); } /// Streaming conv stem: processes a mel chunk with history prepended. @@ -91,10 +84,11 @@ pub const Encoder = struct { pub fn transformer(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { var h = x; var cache = kv_cache; + const attn_config = self.config.encoder().attentionConfig(); for (self.layers, 0..) |layer, i| { const layer_cache = cache.atLayer(i); - const result = layer.forward(h, token_index, layer_cache, self.config, attention_metadata, attention_parameters); + const result = layer.forward(h, token_index, layer_cache, attn_config, attention_metadata, attention_parameters); h = result[0]; const updated_layer_cache = result[1]; cache = updated_layer_cache.reuseBuffer(layer_cache); @@ -119,8 +113,7 @@ pub const CausalConv1d = struct { } pub fn unload(self: *zml.Bufferized(CausalConv1d)) void { - self.weight.deinit(); - self.bias.deinit(); + common.deinitBufferized(self); } pub fn forward(self: CausalConv1d, input: Tensor) Tensor { @@ -169,20 +162,16 @@ pub const TransformerLayer = struct { } pub fn unload(self: *zml.Bufferized(TransformerLayer)) void { - self.attention_norm.deinit(); - SelfAttention.unload(&self.attention); - - self.ffn_norm.deinit(); - SwiGluFfn.unload(&self.feed_forward); + common.deinitBufferized(self); } /// h: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) - pub fn forward(self: TransformerLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { + pub fn forward(self: TransformerLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, attn_config: cfg.AttentionConfig, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { const attn_out, const updated_kv_cache = self.attention.forward( rmsNorm(h, self.attention_norm, self.norm_eps), token_index, kv_cache, - config, + attn_config, attention_metadata, attention_parameters, ); @@ -195,94 +184,7 @@ pub const TransformerLayer = struct { } }; -pub const SelfAttention = struct { - wq: zml.nn.Linear, - wk: zml.nn.Linear, - wv: zml.nn.Linear, - wo: zml.nn.Linear, - - pub fn init(store: zml.io.TensorStore.View) SelfAttention { - return .{ - .wq = common.linear(store.withPrefix("wq")), - .wk = common.linear(store.withPrefix("wk")), - .wv = common.linear(store.withPrefix("wv")), - .wo = common.linear(store.withPrefix("wo")), - }; - } - - pub fn unload(self: *zml.Bufferized(SelfAttention)) void { - common.deinitLinear(&self.wq); - common.deinitLinear(&self.wk); - common.deinitLinear(&self.wv); - common.deinitLinear(&self.wo); - } - - /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) - pub fn forward(self: SelfAttention, x: Tensor, token_index: Tensor, kv_cache: KvCache, config: Config, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { - const enc = config.encoder(); - const dtype = x.dtype(); - - var q = self.wq.forward(x); - var k = self.wk.forward(x); - var v = self.wv.forward(x); - - q = q.splitAxis(.dout, .{ .h = enc.n_heads, .hd = enc.head_dim }); - k = k.splitAxis(.dout, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); - v = v.splitAxis(.dout, .{ .h = enc.n_kv_heads, .hd = enc.head_dim }); - - // Position indices for RoPE: token_index + arange(s) - const pos_index = b: { - const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) - .withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); - break :b temp.add(token_index.broad(temp.shape())); - }; - - const rope_opts: zml.nn.RopeOpts = .{ - .layout = .interleaved, - .freq_base = enc.rope_theta, - }; - - q = zml.nn.rope(q, pos_index, rope_opts); - k = zml.nn.rope(k, pos_index, rope_opts); - - q = q.rename(.{ .s = .q }); - k = k.rename(.{ .s = .k }); - v = v.rename(.{ .s = .k }); - - // Update KV cache with modular indexing for sliding window circular buffer - const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); - const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); - const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); - k = new_kv_cache.keys().convert(dtype); - v = new_kv_cache.values().convert(dtype); - - // EXPERIMENTAL - // Reorder K/V from circular buffer to temporal order for correct causal masking. - // When the cache wraps, physical order != temporal order, which breaks FA2's - // position-based causal mask for multi-token (q_len > 1) attention. - { - const pos_end = token_index.addConstant(x.dim(.s)); - const is_full = pos_end.cmp(.GE, cache_size); - const rotation_start = pos_end.remainder(cache_size); - const safe_start = is_full.select(rotation_start, Tensor.scalar(@as(u32, 0), token_index.dtype())); - const reorder_arange = Tensor.arange(.{ .end = kv_cache.k.dim(.k) }, token_index.dtype()).withTags(.{.kk}); - const reorder_idx = reorder_arange.add(safe_start.broad(reorder_arange.shape())) - .remainder(cache_size.broad(reorder_arange.shape())); - k = k.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); - v = v.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); - } - - // Cap token_index so seqused_k doesn't exceed cache bounds - const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); - const attn_token_index = token_index.minimum(max_token_index); - const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ - .sliding_window = @intCast(enc.sliding_window), - }); - - const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; - } -}; +pub const SelfAttention = common.SelfAttention(true); pub const SwiGluFfn = struct { w1: zml.nn.Linear, @@ -298,9 +200,7 @@ pub const SwiGluFfn = struct { } pub fn unload(self: *zml.Bufferized(SwiGluFfn)) void { - common.deinitLinear(&self.w1); - common.deinitLinear(&self.w2); - common.deinitLinear(&self.w3); + common.deinitBufferized(self); } /// x: [s, d] -> [s, d] diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 8699e7f2..285497ab 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -30,6 +30,8 @@ const CliArgs = struct { model: []const u8, transcription_delay_ms: f32 = 480.0, backend: ?zml.attention.Backend = null, + enc_kv_size: ?u32 = null, + dec_kv_size: ?u32 = null, }; const n_left_pad_tokens: u32 = 32; @@ -108,12 +110,15 @@ pub fn main(init: std.process.Init) !void { var decoder_model: Decoder = .init(allocator, model_store.view(), config); defer decoder_model.deinit(allocator); - // KV cache shapes for encoder and decoder, sized to sliding_window for memory efficiency const enc_cfg = config.encoder(); + const enc_kv_size = args.enc_kv_size orelse enc_cfg.sliding_window; + const dec_kv_size = args.dec_kv_size orelse config.sliding_window; + + // KV cache shapes for encoder and decoder, sized to sliding_window for memory efficiency const enc_dtype = encoder_model.norm.dtype(); const enc_kv_cache: KvCache = .init(.init(.{ .layer = enc_cfg.n_layers, - .k = enc_cfg.sliding_window, + .k = enc_kv_size, .h = enc_cfg.n_kv_heads, .hd = enc_cfg.head_dim, }, enc_dtype)); @@ -121,13 +126,13 @@ pub fn main(init: std.process.Init) !void { const dec_dtype = decoder_model.tok_embeddings.dtype(); const dec_kv_cache: KvCache = .init(.init(.{ .layer = decoder_model.layers.len, - .k = config.sliding_window, + .k = dec_kv_size, .h = config.n_kv_heads, .hd = config.head_dim, }, dec_dtype)); - const enc_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(enc_cfg.sliding_window))); - const dec_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(config.sliding_window))); + const enc_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(enc_kv_size))); + const dec_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(dec_kv_size))); const attention_parameters: zml.attention.Parameters = .init(.fromBackend(backend)); // Launch concurrent compilation and buffer loading @@ -210,19 +215,19 @@ pub fn main(init: std.process.Init) !void { platform, config, &tokenizer, - // padded_wav, - // audio_len, tokens, sp, - &compiled_mel_step, - &compiled_conv_stem, - &compiled_conv_stem_step, - &compiled_encoder_prefill, - &compiled_encoder_step, - &compiled_adapter, - &compiled_adapter_step, - &compiled_decoder_prefill, - &compiled_decoder_decode, + .{ + .mel_step = &compiled_mel_step, + .conv_stem = &compiled_conv_stem, + .conv_stem_step = &compiled_conv_stem_step, + .encoder_prefill = &compiled_encoder_prefill, + .encoder_step = &compiled_encoder_step, + .adapter = &compiled_adapter, + .adapter_step = &compiled_adapter_step, + .decoder_prefill = &compiled_decoder_prefill, + .decoder_decode = &compiled_decoder_decode, + }, &mel_spectrum_buffers, &encoder_buffers, &adapter_buffers, diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig index 482aba0d..bc5542e8 100644 --- a/examples/voxtral/mel_spectrogram.zig +++ b/examples/voxtral/mel_spectrogram.zig @@ -48,19 +48,8 @@ pub const LogMelSpectrogram = struct { // then drop the last frame to match Whisper convention (stft[..., :-1]). var spectrogram = stft(padded_wav, window_weight, num_frames + 1, self.hop_len, self.precision); spectrogram = spectrogram.slice1d(.frames, .{ .end = -1 }); - spectrogram = spectrogram.convert(dtype); - - // Re-weight frequencies for speech - spectrogram = spectrogram.dot(self.mel_filters, .freq_bins); - spectrogram = spectrogram.maximum(Tensor.constant(dtype.constant(self.mel_floor))); - var log_spec = spectrogram.log().scale(1.0 / @log(10.0)); - - const log_spec_min = Tensor.constant(dtype.constant(self.global_log_mel_max - 8.0)); - log_spec = log_spec.maximum(log_spec_min); - - // "center" the distribution - return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ .mel, .frames }); + return self.postProcess(spectrogram, dtype); } /// Like forward(), but for fixed-size audio chunks with reflect padding done on host. @@ -75,8 +64,14 @@ pub const LogMelSpectrogram = struct { const num_frames: u63 = @intCast(@divFloor(audio_len - fft_len, self.hop_len) + 1); // No reflect padding — done on host - var spectrogram = stft(audio_chunk, window_weight, num_frames, self.hop_len, self.precision); - spectrogram = spectrogram.convert(dtype); + const spectrogram = stft(audio_chunk, window_weight, num_frames, self.hop_len, self.precision); + + return self.postProcess(spectrogram, dtype).withTags(.{ .channels, .time }); + } + + /// Shared post-STFT processing: mel filter dot product, log scale, clamp, normalize. + fn postProcess(self: LogMelSpectrogram, raw_spectrogram: Tensor, dtype: zml.DataType) Tensor { + var spectrogram = raw_spectrogram.convert(dtype); // Re-weight frequencies for speech spectrogram = spectrogram.dot(self.mel_filters, .freq_bins); @@ -87,7 +82,8 @@ pub const LogMelSpectrogram = struct { const log_spec_min = Tensor.constant(dtype.constant(self.global_log_mel_max - 8.0)); log_spec = log_spec.maximum(log_spec_min); - return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ .mel, .frames }).withTags(.{ .channels, .time }); + // "center" the distribution + return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ .mel, .frames }); } pub fn load(self: *LogMelSpectrogram, io: std.Io, platform: *zml.Platform) !zml.Bufferized(LogMelSpectrogram) { diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig index f78aa22e..d268cbc3 100644 --- a/examples/voxtral/test.zig +++ b/examples/voxtral/test.zig @@ -21,30 +21,20 @@ const CliArgs = struct { model: []const u8, }; -pub fn main() !void { +pub fn main(init: std.process.Init) !void { log.info("Start of Voxtral test", .{}); - var dbg = std.heap.DebugAllocator(.{}).init; + var dbg = std.heap.DebugAllocator(.{ .thread_safe = true }).init; defer if (builtin.mode == .Debug) std.debug.assert(dbg.deinit() == .ok); const allocator = switch (builtin.mode) { - .Debug => dbg.allocator(), - else => std.heap.c_allocator, + .Debug => dbg.allocator(), + else => std.heap.c_allocator, }; - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); + const args = stdx.flags.parse(init.minimal.args, CliArgs); - const args = stdx.flags.parseProcessArgs(CliArgs); - - var vfs: zml.io.VFS = try .init(allocator, threaded.io()); - defer vfs.deinit(); - - var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); - defer vfs_file.deinit(); - try vfs.register("file", vfs_file.io()); - - const io = vfs.io(); + const io = init.io; var ref_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.reference); defer ref_registry.deinit(); @@ -226,13 +216,14 @@ const DecoderLayerTestHarness = struct { const h_typed = h.convert(dtype).withTags(.{ .s, .d }); // Create zero KV cache for prefill (empty cache for this layer) + const attn_config = self.config.attentionConfig(); const kv_shape = zml.Shape.init(.{ .layer = 1, .k = h_typed.dim(.s), - .h = @as(i64, @intCast(self.config.n_kv_heads)), - .hd = @as(i64, @intCast(self.config.head_dim)), + .h = @as(i64, @intCast(attn_config.n_kv_heads)), + .hd = @as(i64, @intCast(attn_config.head_dim)), }, dtype); - + const kv_cache = dec.KvCache{ .k = Tensor.zeroes(kv_shape), .v = Tensor.zeroes(kv_shape), @@ -240,8 +231,10 @@ const DecoderLayerTestHarness = struct { }; const token_index = Tensor.scalar(0, .u32); - const result = self.inner.forward(h_typed, token_index, kv_cache, self.t_cond.convert(dtype), self.config); - + const attention_metadata: zml.attention.Metadata = .{ .vanilla = {} }; + const attention_parameters: zml.attention.Parameters = .{ .vanilla = {} }; + const result = self.inner.forward(h_typed, token_index, kv_cache, self.t_cond.convert(dtype), attn_config, attention_metadata, attention_parameters); + return result[0].convert(.f32); } }; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 23819850..bfda8d7c 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -23,94 +23,104 @@ const KvCache = common.KvCache; const terminalwave = @import("terminalwave/terminalwave.zig"); -pub fn compileMelStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling mel step...", 1); - defer node.end(); +pub const CompiledExes = struct { + mel_step: *zml.Exe, + conv_stem: *zml.Exe, + conv_stem_step: *zml.Exe, + encoder_prefill: *zml.Exe, + encoder_step: *zml.Exe, + adapter: *zml.Exe, + adapter_step: *zml.Exe, + decoder_prefill: *zml.Exe, + decoder_decode: *zml.Exe, +}; - return try platform.compile(allocator, io, model, .melStep, .{Tensor.init(.{sp.chunk_audio}, .f32).withTags(.{.samples})}); +pub const ModelBuffers = struct { + mel_spectrum: *zml.Bufferized(LogMelSpectrogram), + encoder: *zml.Bufferized(Encoder), + adapter: *zml.Bufferized(Adapter), + decoder: *zml.Bufferized(Decoder), + enc_kv: *zml.Bufferized(KvCache), + dec_kv: *zml.Bufferized(KvCache), + t_cond: zml.Buffer, + rng: *zml.Bufferized(Tensor.Rng), + enc_attention_metadata: *zml.Bufferized(zml.attention.Metadata), + dec_attention_metadata: *zml.Bufferized(zml.attention.Metadata), +}; + +pub const PipelineContext = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + config: Config, + sp: StreamParams, + exes: CompiledExes, + buffers: ModelBuffers, +}; + +fn compileStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: anytype, comptime method: anytype, inputs: anytype, progress: *std.Progress.Node, comptime label: []const u8) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling " ++ label ++ "...", 1); + defer node.end(); + return try platform.compile(allocator, io, model, method, inputs); +} + +pub fn compileMelStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { + return compileStep(allocator, io, platform, model, .melStep, .{Tensor.init(.{sp.chunk_audio}, .f32).withTags(.{.samples})}, progress, "mel step"); } pub fn compileConvStem(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, num_mel_frames: u32, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling conv stem...", 1); - defer node.end(); - - return try platform.compile(allocator, io, model, .convStem, .{ + return compileStep(allocator, io, platform, model, .convStem, .{ Tensor.init(.{ .channels = 128, .time = num_mel_frames }, .f32), - }); + }, progress, "conv stem"); } pub fn compileConvStemStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling conv stem step...", 1); - defer node.end(); - - return try platform.compile(allocator, io, model, .convStemStep, .{ + return compileStep(allocator, io, platform, model, .convStemStep, .{ Tensor.init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), - }); + }, progress, "conv stem step"); } /// Encoder prefill: processes prompt_len * dsf frames at once, populating the KV cache. pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, prompt_len: u32, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling encoder prefill...", 1); - defer node.end(); - const enc_cfg = model.config.encoder(); const dsf = model.config.downsample_factor(); - - return try platform.compile(allocator, io, model, .transformer, .{ + return compileStep(allocator, io, platform, model, .transformer, .{ Tensor.init(.{ .s = prompt_len * dsf, .d = enc_cfg.dim }, .bf16), Tensor.init(.{}, .u32), enc_kv_cache, attention_metadata, attention_parameters, - }); + }, progress, "encoder prefill"); } /// Encoder step: processes dsf frames with KV cache (for streaming decode). pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling encoder step...", 1); - defer node.end(); - const enc_cfg = model.config.encoder(); const dsf = model.config.downsample_factor(); - - return try platform.compile(allocator, io, model, .transformer, .{ + return compileStep(allocator, io, platform, model, .transformer, .{ Tensor.init(.{ .s = dsf, .d = enc_cfg.dim }, .bf16), Tensor.init(.{}, .u32), enc_kv_cache, attention_metadata, attention_parameters, - }); + }, progress, "encoder step"); } /// Adapter batch: processes prompt_len * dsf encoder frames -> prompt_len embeddings. pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, prompt_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling adapter...", 1); - defer node.end(); - const dsf = config.downsample_factor(); - - return try platform.compile(allocator, io, model, .forward, .{ + return compileStep(allocator, io, platform, model, .forward, .{ Tensor.init(.{ .s = prompt_len * dsf, .d = config.encoder().dim }, .bf16), - }); + }, progress, "adapter"); } /// Adapter step: processes dsf encoder frames -> 1 embedding (for streaming decode). pub fn compileAdapterStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, config: Config, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling adapter step...", 1); - defer node.end(); - const dsf = config.downsample_factor(); - - return try platform.compile(allocator, io, model, .forward, .{ + return compileStep(allocator, io, platform, model, .forward, .{ Tensor.init(.{ .s = dsf, .d = config.encoder().dim }, .bf16), - }); + }, progress, "adapter step"); } /// Decoder prefill (s=prompt_len) + decode (s=1), compiled concurrently. @@ -119,11 +129,7 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const const compilePrefill = struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, s: u32, kv_cache_: KvCache, dim_: u32, attention_metadata_: zml.attention.Metadata, attention_parameters_: zml.attention.Parameters, progress_: *std.Progress.Node) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling decoder prefill...", 1); - defer node_.end(); - - return try platform_.compile(allocator_, io_, model_, .forward, .{ + return compileStep(allocator_, io_, platform_, model_, .forward, .{ Tensor.init(.{ .s = s }, .u32), Tensor.init(.{ .s = s, .d = dim_ }, .bf16), Tensor.init(.{}, .u32), @@ -132,17 +138,13 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const Tensor.Rng.init(), attention_metadata_, attention_parameters_, - }); + }, progress_, "decoder prefill"); } }.call; const compileDecode = struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, kv_cache_: KvCache, dim_: u32, attention_metadata_: zml.attention.Metadata, attention_parameters_: zml.attention.Parameters, progress_: *std.Progress.Node) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling decoder decode...", 1); - defer node_.end(); - - return try platform_.compile(allocator_, io_, model_, .forward, .{ + return compileStep(allocator_, io_, platform_, model_, .forward, .{ Tensor.init(.{ .s = 1 }, .u32), Tensor.init(.{ .s = 1, .d = dim_ }, .bf16), Tensor.init(.{}, .u32), @@ -151,7 +153,7 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const Tensor.Rng.init(), attention_metadata_, attention_parameters_, - }); + }, progress_, "decoder decode"); } }.call; @@ -217,15 +219,16 @@ pub fn reflectPadAudio(allocator: std.mem.Allocator, audio: []const f32, pad: us /// Run mel spectrogram chunk-by-chunk and download result to host memory. /// Returns mel data as bytes (channels-major: [128][prompt_len * mel_per_step]). Caller owns the memory. -pub fn runMelPrefill( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - sp: StreamParams, - padded_audio: []const f32, - compiled_mel_step: *zml.Exe, - mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), -) ![]u8 { +fn runMelPrefill(ctx: *PipelineContext, padded_audio: []const f32) ![]u8 { + // -- + const allocator = ctx.allocator; + const io = ctx.io; + const platform = ctx.platform; + const sp = ctx.sp; + const compiled_mel_step = ctx.exes.mel_step; + const mel_spectrum_buffers = ctx.buffers.mel_spectrum; + // -- + const prompt_len = sp.prompt_len; const f32_bytes: usize = @sizeOf(f32); const prefill_mel_frames: u32 = prompt_len * sp.mel_per_step; @@ -280,29 +283,15 @@ pub fn runMelPrefill( /// Run the prefill phase: conv stem -> encoder -> adapter -> decoder prefill. /// Populates KV caches and produces the first generated token into generated_token_slice. -pub fn runPrefill( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - config: Config, - sp: StreamParams, - mel_host: []const u8, - tokens: []const u32, - compiled_conv_stem: *zml.Exe, - compiled_encoder_prefill: *zml.Exe, - compiled_adapter: *zml.Exe, - compiled_decoder_prefill: *zml.Exe, - encoder_buffers: *zml.Bufferized(Encoder), - adapter_buffers: *zml.Bufferized(Adapter), - decoder_buffers: *zml.Bufferized(Decoder), - enc_kv_buffers: *zml.Bufferized(KvCache), - dec_kv_buffers: *zml.Bufferized(KvCache), - t_cond_buffer: zml.Buffer, - rng_buffers: *zml.Bufferized(Tensor.Rng), - enc_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), - dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), - generated_token_slice: zml.Slice, -) !void { +fn runPrefill(ctx: *PipelineContext, mel_host: []const u8, tokens: []const u32, generated_token_slice: zml.Slice) !void { + // -- + const allocator = ctx.allocator; + const io = ctx.io; + const platform = ctx.platform; + const config = ctx.config; + const sp = ctx.sp; + // -- + const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; const f32_bytes: usize = @sizeOf(f32); @@ -322,13 +311,13 @@ pub fn runPrefill( const prefill_enc_frames = prompt_len * sp.dsf; conv_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_enc_frames, .d = enc_dim }, .bf16), .{}); - var args = try compiled_conv_stem.args(allocator); + var args = try ctx.exes.conv_stem.args(allocator); defer args.deinit(allocator); - var results = try compiled_conv_stem.results(allocator); + var results = try ctx.exes.conv_stem.results(allocator); defer results.deinit(allocator); - args.set(.{ encoder_buffers, mel_buffer }); - compiled_conv_stem.call(args, &results); + args.set(.{ ctx.buffers.encoder, mel_buffer }); + ctx.exes.conv_stem.call(args, &results); results.fill(.{&conv_prefill_output}); } defer conv_prefill_output.deinit(); @@ -344,14 +333,14 @@ pub fn runPrefill( const prefill_frames = prompt_len * sp.dsf; encoder_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_frames, .d = enc_dim }, .bf16), .{}); - var args = try compiled_encoder_prefill.args(allocator); + var args = try ctx.exes.encoder_prefill.args(allocator); defer args.deinit(allocator); - var results = try compiled_encoder_prefill.results(allocator); + var results = try ctx.exes.encoder_prefill.results(allocator); defer results.deinit(allocator); - args.set(.{ encoder_buffers, conv_prefill_output, enc_token_index, enc_kv_buffers, enc_attention_metadata_buffers }); - compiled_encoder_prefill.call(args, &results); - results.fill(.{ &encoder_prefill_output, enc_kv_buffers }); + args.set(.{ ctx.buffers.encoder, conv_prefill_output, enc_token_index, ctx.buffers.enc_kv, ctx.buffers.enc_attention_metadata }); + ctx.exes.encoder_prefill.call(args, &results); + results.fill(.{ &encoder_prefill_output, ctx.buffers.enc_kv }); } defer encoder_prefill_output.deinit(); log.info("Encoder prefill done.", .{}); @@ -362,13 +351,13 @@ pub fn runPrefill( { adapter_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prompt_len, .d = config.dim }, .bf16), .{}); - var args = try compiled_adapter.args(allocator); + var args = try ctx.exes.adapter.args(allocator); defer args.deinit(allocator); - var results = try compiled_adapter.results(allocator); + var results = try ctx.exes.adapter.results(allocator); defer results.deinit(allocator); - args.set(.{ adapter_buffers, encoder_prefill_output }); - compiled_adapter.call(args, &results); + args.set(.{ ctx.buffers.adapter, encoder_prefill_output }); + ctx.exes.adapter.call(args, &results); results.fill(.{&adapter_prefill_output}); } defer adapter_prefill_output.deinit(); @@ -377,9 +366,9 @@ pub fn runPrefill( // Decoder prefill: all prompt tokens + audio embeddings -> first generated token log.info("Running decoder prefill...", .{}); { - var args = try compiled_decoder_prefill.args(allocator); + var args = try ctx.exes.decoder_prefill.args(allocator); defer args.deinit(allocator); - var results = try compiled_decoder_prefill.results(allocator); + var results = try ctx.exes.decoder_prefill.results(allocator); defer results.deinit(allocator); const token_data_slice: zml.Slice = .init( @@ -397,9 +386,9 @@ pub fn runPrefill( var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_output_slice); defer prefill_output.deinit(); - args.set(.{ decoder_buffers, token_buffer, adapter_prefill_output, token_index_buffer, dec_kv_buffers, t_cond_buffer, rng_buffers.*, dec_attention_metadata_buffers }); - compiled_decoder_prefill.call(args, &results); - results.fill(.{ &prefill_output, dec_kv_buffers, rng_buffers }); + args.set(.{ ctx.buffers.decoder, token_buffer, adapter_prefill_output, token_index_buffer, ctx.buffers.dec_kv, ctx.buffers.t_cond, ctx.buffers.rng.*, ctx.buffers.dec_attention_metadata }); + ctx.exes.decoder_prefill.call(args, &results); + results.fill(.{ &prefill_output, ctx.buffers.dec_kv, ctx.buffers.rng }); // Extract last token from prefill output (first generated token) try prefill_output.toSlice(io, prefill_output_slice); @@ -411,32 +400,15 @@ pub fn runPrefill( } /// Run the streaming decode loop: step-by-step conv_stem -> encoder -> adapter -> decoder. -pub fn runGenerationLoop( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - config: Config, - sp: StreamParams, - tokenizer: *zml.tokenizer.Tokenizer, - initial_audio_history: []const f32, - stdin_reader: *std.Io.Reader, - compiled_mel_step: *zml.Exe, - compiled_conv_stem_step: *zml.Exe, - compiled_encoder_step: *zml.Exe, - compiled_adapter_step: *zml.Exe, - compiled_decoder_decode: *zml.Exe, - mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), - encoder_buffers: *zml.Bufferized(Encoder), - adapter_buffers: *zml.Bufferized(Adapter), - decoder_buffers: *zml.Bufferized(Decoder), - enc_kv_buffers: *zml.Bufferized(KvCache), - dec_kv_buffers: *zml.Bufferized(KvCache), - t_cond_buffer: zml.Buffer, - rng_buffers: *zml.Bufferized(Tensor.Rng), - enc_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), - dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), - generated_token_slice: zml.Slice, -) !void { +fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, initial_audio_history: []const f32, stdin_reader: *std.Io.Reader, generated_token_slice: zml.Slice) !void { + // -- + const allocator = ctx.allocator; + const io = ctx.io; + const platform = ctx.platform; + const config = ctx.config; + const sp = ctx.sp; + // -- + const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp._hop_length; @@ -458,29 +430,29 @@ pub fn runGenerationLoop( // log.info("Running decode loop (steps {}..{})...", .{ prompt_len, total_steps }); const decode_start: std.Io.Timestamp = .now(io, .awake); - var mel_step_args = try compiled_mel_step.args(allocator); + var mel_step_args = try ctx.exes.mel_step.args(allocator); defer mel_step_args.deinit(allocator); - var mel_step_results = try compiled_mel_step.results(allocator); + var mel_step_results = try ctx.exes.mel_step.results(allocator); defer mel_step_results.deinit(allocator); - var conv_step_args = try compiled_conv_stem_step.args(allocator); + var conv_step_args = try ctx.exes.conv_stem_step.args(allocator); defer conv_step_args.deinit(allocator); - var conv_step_results = try compiled_conv_stem_step.results(allocator); + var conv_step_results = try ctx.exes.conv_stem_step.results(allocator); defer conv_step_results.deinit(allocator); - var enc_step_args = try compiled_encoder_step.args(allocator); + var enc_step_args = try ctx.exes.encoder_step.args(allocator); defer enc_step_args.deinit(allocator); - var enc_step_results = try compiled_encoder_step.results(allocator); + var enc_step_results = try ctx.exes.encoder_step.results(allocator); defer enc_step_results.deinit(allocator); - var adp_step_args = try compiled_adapter_step.args(allocator); + var adp_step_args = try ctx.exes.adapter_step.args(allocator); defer adp_step_args.deinit(allocator); - var adp_step_results = try compiled_adapter_step.results(allocator); + var adp_step_results = try ctx.exes.adapter_step.results(allocator); defer adp_step_results.deinit(allocator); - var decode_args = try compiled_decoder_decode.args(allocator); + var decode_args = try ctx.exes.decoder_decode.args(allocator); defer decode_args.deinit(allocator); - var decode_results = try compiled_decoder_decode.results(allocator); + var decode_results = try ctx.exes.decoder_decode.results(allocator); defer decode_results.deinit(allocator); var mel_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), .{}); @@ -572,35 +544,35 @@ pub fn runGenerationLoop( var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); defer audio_buffer.deinit(); - mel_step_args.set(.{ mel_spectrum_buffers, audio_buffer }); - compiled_mel_step.call(mel_step_args, &mel_step_results); + mel_step_args.set(.{ ctx.buffers.mel_spectrum, audio_buffer }); + ctx.exes.mel_step.call(mel_step_args, &mel_step_results); mel_step_results.fill(.{&mel_step_output}); // Conv stem step: mel step output stays on device → conv stem - conv_step_args.set(.{ encoder_buffers, mel_step_output }); - compiled_conv_stem_step.call(conv_step_args, &conv_step_results); + conv_step_args.set(.{ ctx.buffers.encoder, mel_step_output }); + ctx.exes.conv_stem_step.call(conv_step_args, &conv_step_results); conv_step_results.fill(.{&conv_step_output}); // Encoder step: conv stem output -> encoded chunk var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)) * sp.dsf, .u32); defer enc_token_index.deinit(); - enc_step_args.set(.{ encoder_buffers, conv_step_output, enc_token_index, enc_kv_buffers, enc_attention_metadata_buffers }); - compiled_encoder_step.call(enc_step_args, &enc_step_results); - enc_step_results.fill(.{ &enc_step_output, enc_kv_buffers }); + enc_step_args.set(.{ ctx.buffers.encoder, conv_step_output, enc_token_index, ctx.buffers.enc_kv, ctx.buffers.enc_attention_metadata }); + ctx.exes.encoder_step.call(enc_step_args, &enc_step_results); + enc_step_results.fill(.{ &enc_step_output, ctx.buffers.enc_kv }); // Adapter step: encoded chunk -> 1 audio embedding - adp_step_args.set(.{ adapter_buffers, enc_step_output }); - compiled_adapter_step.call(adp_step_args, &adp_step_results); + adp_step_args.set(.{ ctx.buffers.adapter, enc_step_output }); + ctx.exes.adapter_step.call(adp_step_args, &adp_step_results); adp_step_results.fill(.{&adapter_step_output}); // Decoder decode: 1 token + 1 audio embedding -> next token var dec_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)), .u32); defer dec_token_index.deinit(); - decode_args.set(.{ decoder_buffers, current_token_buffer, adapter_step_output, dec_token_index, dec_kv_buffers, t_cond_buffer, rng_buffers.*, dec_attention_metadata_buffers }); - compiled_decoder_decode.call(decode_args, &decode_results); - decode_results.fill(.{ ¤t_token_buffer, dec_kv_buffers, rng_buffers }); + decode_args.set(.{ ctx.buffers.decoder, current_token_buffer, adapter_step_output, dec_token_index, ctx.buffers.dec_kv, ctx.buffers.t_cond, ctx.buffers.rng.*, ctx.buffers.dec_attention_metadata }); + ctx.exes.decoder_decode.call(decode_args, &decode_results); + decode_results.fill(.{ ¤t_token_buffer, ctx.buffers.dec_kv, ctx.buffers.rng }); try current_token_buffer.toSlice(io, generated_token_slice); } @@ -624,19 +596,9 @@ pub fn runPipeline( platform: *zml.Platform, config: Config, tokenizer: *zml.tokenizer.Tokenizer, - // padded_audio: []const f32, - // audio_len: usize, tokens: []const u32, sp: StreamParams, - compiled_mel_step: *zml.Exe, - compiled_conv_stem: *zml.Exe, - compiled_conv_stem_step: *zml.Exe, - compiled_encoder_prefill: *zml.Exe, - compiled_encoder_step: *zml.Exe, - compiled_adapter: *zml.Exe, - compiled_adapter_step: *zml.Exe, - compiled_decoder_prefill: *zml.Exe, - compiled_decoder_decode: *zml.Exe, + exes: CompiledExes, mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), encoder_buffers: *zml.Bufferized(Encoder), adapter_buffers: *zml.Bufferized(Adapter), @@ -671,11 +633,7 @@ pub fn runPipeline( const reflect_padded_audio = try reflectPadAudio(allocator, prefill_audio, reflect_pad); defer allocator.free(reflect_padded_audio); - // 1. Mel prefill (chunk-by-chunk) - const mel_host = try runMelPrefill(allocator, io, platform, sp, reflect_padded_audio, compiled_mel_step, mel_spectrum_buffers); - defer allocator.free(mel_host); - - // 2. Init KV caches + t_cond + RNG + // 1. Init KV caches + t_cond + RNG var enc_kv_buffers = try enc_kv_cache.initBuffer(io, platform); defer KvCache.deinitBuffer(&enc_kv_buffers); @@ -694,66 +652,45 @@ pub fn runPipeline( var rng_buffers = try Tensor.Rng.initBuffer(platform, 0, io); defer Tensor.Rng.deinitBuffer(&rng_buffers); + // 2. Assemble pipeline context + var ctx = PipelineContext{ + .allocator = allocator, + .io = io, + .platform = platform, + .config = config, + .sp = sp, + .exes = exes, + .buffers = .{ + .mel_spectrum = mel_spectrum_buffers, + .encoder = encoder_buffers, + .adapter = adapter_buffers, + .decoder = decoder_buffers, + .enc_kv = &enc_kv_buffers, + .dec_kv = &dec_kv_buffers, + .t_cond = t_cond_buffer, + .rng = &rng_buffers, + .enc_attention_metadata = enc_attention_metadata_buffers, + .dec_attention_metadata = dec_attention_metadata_buffers, + }, + }; + + // 3. Mel prefill (chunk-by-chunk) + const mel_host = try runMelPrefill(&ctx, reflect_padded_audio); + defer allocator.free(mel_host); + + // 4. Prefill var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); defer generated_token_slice.free(allocator); - // 3. Prefill - try runPrefill( - allocator, - io, - platform, - config, - sp, - mel_host, - tokens, - compiled_conv_stem, - compiled_encoder_prefill, - compiled_adapter, - compiled_decoder_prefill, - encoder_buffers, - adapter_buffers, - decoder_buffers, - &enc_kv_buffers, - &dec_kv_buffers, - t_cond_buffer, - &rng_buffers, - enc_attention_metadata_buffers, - dec_attention_metadata_buffers, - generated_token_slice, - ); + try runPrefill(&ctx, mel_host, tokens, generated_token_slice); - // 4. Generation loop — extract audio history for sliding buffer + // 5. Generation loop — extract audio history for sliding buffer const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp._hop_length; const audio_overlap: usize = sp.chunk_audio - new_audio_per_step; const history_start: usize = @as(usize, sp.prompt_len) * sp.mel_per_step * sp._hop_length; const initial_audio_history = reflect_padded_audio[history_start .. history_start + audio_overlap]; - try runGenerationLoop( - allocator, - io, - platform, - config, - sp, - tokenizer, - initial_audio_history, - stdin_reader_interface, - compiled_mel_step, - compiled_conv_stem_step, - compiled_encoder_step, - compiled_adapter_step, - compiled_decoder_decode, - mel_spectrum_buffers, - encoder_buffers, - adapter_buffers, - decoder_buffers, - &enc_kv_buffers, - &dec_kv_buffers, - t_cond_buffer, - &rng_buffers, - enc_attention_metadata_buffers, - dec_attention_metadata_buffers, - generated_token_slice, - ); + try runGenerationLoop(&ctx, tokenizer, initial_audio_history, stdin_reader_interface, generated_token_slice); } fn renderSeparator(wave: *terminalwave.State) void { From eecb9139f96a8a03ec90869f73e3af50bb9e4ffc Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 18 Feb 2026 11:16:47 +0100 Subject: [PATCH 80/98] isolate terminalwave code --- .../voxtral/terminalwave/terminalwave.zig | 89 +++++++++++++++---- examples/voxtral/voxtral.zig | 46 ++++------ 2 files changed, 87 insertions(+), 48 deletions(-) diff --git a/examples/voxtral/terminalwave/terminalwave.zig b/examples/voxtral/terminalwave/terminalwave.zig index c8e956be..f1fd68ef 100644 --- a/examples/voxtral/terminalwave/terminalwave.zig +++ b/examples/voxtral/terminalwave/terminalwave.zig @@ -1,5 +1,39 @@ const std = @import("std"); +// ── ANSI escape sequences & Unicode block characters ──────── + +pub const esc = struct { + // Cursor control + pub const hide_cursor = "\x1b[?25l"; + pub const show_cursor = "\x1b[?25h"; + pub const save_cursor = "\x1b7"; + pub const restore_cursor = "\x1b8"; + pub const cursor_home = "\x1b[H"; + + // Screen control + pub const clear_screen = "\x1b[2J"; + pub const clear_to_end = "\x1b[J"; + pub const reset_scroll_region = "\x1b[r"; + + // Text style + pub const reset = "\x1b[0m"; + pub const bold = "\x1b[1m"; + pub const dim_gray = "\x1b[0;90m"; + pub const bright_white = "\x1b[97m"; + + // Unicode box drawing / block elements + pub const horizontal_line = "\xe2\x94\x80"; // ─ + pub const upper_half_block = "\xe2\x96\x80"; // ▀ + pub const lower_half_block = "\xe2\x96\x84"; // ▄ + + // Parameterized format strings (for use with fmt) + pub const fg_rgb_fmt = "\x1b[38;2;{d};{d};{d}m"; + pub const bold_fg_rgb_fmt = "\x1b[1;38;2;{d};{d};{d}m"; + pub const fg_bg_rgb_fmt = "\x1b[38;2;{d};{d};{d};48;2;{d};{d};{d}m"; + pub const set_scroll_region_fmt = "\x1b[{d};999r"; + pub const move_cursor_fmt = "\x1b[{d};{d}H"; +}; + // ── Public types ───────────────────────────────────────────── pub const Color = struct { r: u8, g: u8, b: u8 }; @@ -18,7 +52,7 @@ pub const Config = struct { decay: f32 = 0.12, gradient: Gradient = .{}, padding_left: u16 = 4, - title: []const u8 = "Voxtral Realtime", + title: []const u8, show_title: bool = true, }; @@ -30,6 +64,7 @@ pub const FrameResult = struct { pub const State = struct { config: Config, + writer: *std.Io.Writer, smoothed: f32, frame_count: u32, buf: [BUF_SIZE]u8, @@ -41,9 +76,10 @@ pub const State = struct { const MAX_HALF_HEIGHT = 256; const Half = enum { upper, lower }; - pub fn init(config: Config) State { + pub fn init(config: Config, writer: *std.Io.Writer) State { var state = State{ .config = config, + .writer = writer, .smoothed = 0, .frame_count = 0, .gradient_lut = undefined, @@ -53,16 +89,13 @@ pub const State = struct { state.buildGradientLut(); - // Hide cursor, clear screen - std.debug.print("\x1b[?25l\x1b[2J", .{}); + writer.writeAll(esc.hide_cursor ++ esc.clear_screen) catch {}; return state; } pub fn deinit(self: *State) void { - _ = self; - // Show cursor, reset scroll region - std.debug.print("\x1b[r\x1b[?25h", .{}); + self.writer.writeAll(esc.reset_scroll_region ++ esc.show_cursor) catch {}; } pub fn render(self: *State, rms: f32) FrameResult { @@ -73,14 +106,14 @@ pub const State = struct { // Render self.pos = 0; - self.put("\x1b[H"); + self.put(esc.cursor_home); if (self.config.show_title) { - self.put("\n\x1b[1;38;2;230;120;5m"); + self.put("\n"); + self.fmt(esc.bold_fg_rgb_fmt, .{ 230, 120, 5 }); self.putPadding(); self.put(self.config.title); - self.put(" "); - self.put("\x1b[0;90m\xe2\x94\x80 live transcription\x1b[0m\n\n"); + self.put(" " ++ esc.dim_gray ++ esc.horizontal_line ++ " live transcription" ++ esc.reset ++ "\n\n"); } const num_bars = self.config.num_bars; @@ -115,6 +148,17 @@ pub const State = struct { return .{ .level = level }; } + /// Render a horizontal separator line below the wave visualization. + pub fn renderSeparator(self: *State) void { + self.put("\n"); + self.fmt(esc.fg_rgb_fmt, .{ 60, 60, 60 }); + self.putPadding(); + for (0..self.config.num_bars) |_| { + self.put(esc.horizontal_line); + } + self.put(esc.reset ++ "\n\n"); + } + // ── Private helpers ────────────────────────────────────── fn buildGradientLut(self: *State) void { @@ -188,22 +232,23 @@ pub const State = struct { } pub fn flush(self: *State) void { - std.debug.print("{s}", .{self.buf[0..self.pos]}); + self.writer.writeAll(self.buf[0..self.pos]) catch {}; self.pos = 0; } fn cellBoth(self: *State, upper: Color, lower: Color) void { - self.fmt("\x1b[38;2;{d};{d};{d};48;2;{d};{d};{d}m\xe2\x96\x80\x1b[0m", .{ - upper.r, upper.g, upper.b, lower.r, lower.g, lower.b, - }); + self.fmt(esc.fg_bg_rgb_fmt, .{ upper.r, upper.g, upper.b, lower.r, lower.g, lower.b }); + self.put(esc.upper_half_block ++ esc.reset); } fn cellUpper(self: *State, col: Color) void { - self.fmt("\x1b[38;2;{d};{d};{d}m\xe2\x96\x80\x1b[0m", .{ col.r, col.g, col.b }); + self.fmt(esc.fg_rgb_fmt, .{ col.r, col.g, col.b }); + self.put(esc.upper_half_block ++ esc.reset); } fn cellLower(self: *State, col: Color) void { - self.fmt("\x1b[38;2;{d};{d};{d}m\xe2\x96\x84\x1b[0m", .{ col.r, col.g, col.b }); + self.fmt(esc.fg_rgb_fmt, .{ col.r, col.g, col.b }); + self.put(esc.lower_half_block ++ esc.reset); } }; @@ -212,3 +257,13 @@ pub const State = struct { fn lerpU8(a: f32, b: f32, t: f32) u8 { return @intFromFloat(@max(0.0, @min(255.0, a + (b - a) * t))); } + +/// Compute RMS (root mean square) of audio samples for level metering. +pub fn computeRms(samples: []const f32) f32 { + if (samples.len == 0) return 0; + var sum: f32 = 0; + for (samples) |s| { + sum += s * s; + } + return @sqrt(sum / @as(f32, @floatFromInt(samples.len))); +} diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index bfda8d7c..7569c1d8 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -22,6 +22,7 @@ const common = @import("common.zig"); const KvCache = common.KvCache; const terminalwave = @import("terminalwave/terminalwave.zig"); +const tesc = terminalwave.esc; pub const CompiledExes = struct { mel_step: *zml.Exe, @@ -471,11 +472,13 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, defer current_token_buffer.deinit(); // Wave visualization + transcript accumulation + var stdout = std.Io.File.stdout().writer(io, &.{}); var wave = terminalwave.State.init(.{ + .title = "Voxtral Realtime", .sensitivity = 2.0, .num_bars = 50, .half_height = 20, - }); + }, &stdout.interface); errdefer wave.deinit(); var transcript: std.ArrayListUnmanaged(u8) = .empty; @@ -487,10 +490,10 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, // Render initial empty wave + separator, set scroll region _ = wave.render(0); - renderSeparator(&wave); - wave.fmt("\x1b[{d};999r", .{transcript_start_row}); - wave.fmt("\x1b[{d};{d}H", .{ transcript_start_row, wave.config.padding_left + 1 }); - wave.put("\x1b[J"); + wave.renderSeparator(); + wave.fmt(tesc.set_scroll_region_fmt, .{transcript_start_row}); + wave.fmt(tesc.move_cursor_fmt, .{ transcript_start_row, wave.config.padding_left + 1 }); + wave.put(tesc.clear_to_end); wave.flush(); var last_transcript_len: usize = 0; @@ -516,19 +519,19 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, defer allocator.free(new_samples); // Save cursor position (in transcript scroll region) - std.debug.print("\x1b7", .{}); + wave.writer.writeAll(tesc.save_cursor) catch {}; // Render wave with current audio level (fixed area above scroll region) - const rms = computeRms(new_samples); + const rms = terminalwave.computeRms(new_samples); _ = wave.render(rms); - renderSeparator(&wave); + wave.renderSeparator(); // Restore cursor and print only new transcript text - wave.put("\x1b8"); + wave.put(tesc.restore_cursor); if (transcript.items.len > last_transcript_len) { - wave.put("\x1b[97m"); + wave.put(tesc.bright_white); wave.put(transcript.items[last_transcript_len..]); - wave.put("\x1b[0m"); + wave.put(tesc.reset); last_transcript_len = transcript.items.len; } wave.flush(); @@ -578,11 +581,10 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, } // Reset scroll region and exit wave visualization (returns to main screen) - std.debug.print("\x1b[r", .{}); wave.deinit(); const decode_duration = decode_start.untilNow(io, .awake); - std.debug.print("\n\x1b[1mTranscription:\x1b[0m\n{s}\n\n", .{transcript.items}); + stdout.interface.print("\n" ++ tesc.bold ++ "Transcription:" ++ tesc.reset ++ "\n{s}\n\n", .{transcript.items}) catch {}; log.info("Decode done. Generated {} tokens in {D}: {:.3}tok/s", .{ num_generated, stdx.fmt.fmtDuration(decode_duration), @@ -693,24 +695,6 @@ pub fn runPipeline( try runGenerationLoop(&ctx, tokenizer, initial_audio_history, stdin_reader_interface, generated_token_slice); } -fn renderSeparator(wave: *terminalwave.State) void { - wave.put("\n\x1b[38;2;60;60;60m"); - wave.putPadding(); - for (0..wave.config.num_bars) |_| { - wave.put("\xe2\x94\x80"); - } - wave.put("\x1b[0m\n\n"); -} - -fn computeRms(samples: []const f32) f32 { - if (samples.len == 0) return 0; - var sum: f32 = 0; - for (samples) |s| { - sum += s * s; - } - return @sqrt(sum / @as(f32, @floatFromInt(samples.len))); -} - fn readStdinSamples(allocator: std.mem.Allocator, reader: *std.Io.Reader, n_samples: usize) ![]f32 { const byte_count = n_samples * 2; const raw = try allocator.alloc(u8, byte_count); From f8f94ec30267782d5daa845a4f42216b42fb6c39 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 18 Feb 2026 11:44:36 +0100 Subject: [PATCH 81/98] remove unused code --- examples/voxtral/main.zig | 19 --------- examples/voxtral/mel_spectrogram.zig | 59 +++++++++------------------- 2 files changed, 18 insertions(+), 60 deletions(-) diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 285497ab..a5769306 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -48,22 +48,10 @@ pub fn main(init: std.process.Init) !void { }; const args = stdx.flags.parse(init.minimal.args, CliArgs); - - // var vfs: zml.io.VFS = try .init(allocator, init.io); - // defer vfs.deinit(); - - // var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - // defer vfs_file.deinit(); - // try vfs.register("file", vfs_file.io()); - - // const io = vfs.io(); const io = init.io; var progress = std.Progress.start(io, .{ .root_name = "Voxtral" }); - // const file = try std.Io.Dir.openFile(.cwd(), io, args.input, .{}); - // defer file.close(io); - const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); @@ -82,13 +70,6 @@ pub fn main(init: std.process.Init) !void { const tokens = try allocator.alloc(u32, sp.prompt_len); defer allocator.free(tokens); - // var wav_buffer: [4096]u8 = undefined; - // var reader = file.reader(io, &wav_buffer); - - // const padded_wav = try wav_utils.loadAndPadWav(allocator, &reader.interface, sp.left_pad, sp.n_right_pad_tokens, sp.raw_audio_length_per_tok); - // defer allocator.free(padded_wav); - // const audio_len = padded_wav.len; - var platform: *zml.Platform = try .auto(allocator, io, .{ .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.90 } } }, }); diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig index bc5542e8..a884a8e7 100644 --- a/examples/voxtral/mel_spectrogram.zig +++ b/examples/voxtral/mel_spectrogram.zig @@ -18,45 +18,22 @@ pub const LogMelSpectrogram = struct { precision: zml.DataType = .f32, pub fn init(config: Config) LogMelSpectrogram { - const audio = config.audio(); - - return .{ - .window = .hann, - .mel_filters = Tensor.init(.{201, 128}, .f32).withTags(.{.freq_bins, .mel}), - .hop_len = audio.hop_length, - .n_fft = audio.window_size, - .global_log_mel_max = audio.global_log_mel_max, - }; - } + const audio = config.audio(); - pub fn forward(self: LogMelSpectrogram, waveform: Tensor) Tensor { - const dtype = waveform.dtype(); - - const window_weight = self.window.getWeights(self.n_fft, dtype); - const fft_len = window_weight.dim(.samples); - const num_frames: u63 = @intCast(@divFloor(waveform.dim(.samples), self.hop_len)); - - // Reflect padding - const padded_wav = blk: { - const l = waveform.slice1d(.samples, .{ .start = 1, .end = @divExact(fft_len, 2) + 1 }).reverse(.{.samples}); - const r = waveform.slice1d(.samples, .{ .start = -@divExact(fft_len, 2) - 1, .end = -1 }).reverse(.{.samples}); - break :blk zml.Tensor.concatenate(&.{ l, waveform, r }, .samples); + return .{ + .window = .hann, + .mel_filters = Tensor.init(.{ 201, 128 }, .f32).withTags(.{ .freq_bins, .mel }), + .hop_len = audio.hop_length, + .n_fft = audio.window_size, + .global_log_mel_max = audio.global_log_mel_max, }; - - // Use Short Time Fourier Transform to compute features. - // Generate num_frames+1 (matching torch.stft center=True frame count), - // then drop the last frame to match Whisper convention (stft[..., :-1]). - var spectrogram = stft(padded_wav, window_weight, num_frames + 1, self.hop_len, self.precision); - spectrogram = spectrogram.slice1d(.frames, .{ .end = -1 }); - - return self.postProcess(spectrogram, dtype); } /// Like forward(), but for fixed-size audio chunks with reflect padding done on host. /// Produces exactly (audio_len - n_fft) / hop_len + 1 mel frames. /// Output tags: .channels and .time (matching conv stem input). pub fn melStep(self: LogMelSpectrogram, audio_chunk: Tensor) Tensor { - const dtype = audio_chunk.dtype(); + const dtype = audio_chunk.dtype(); const window_weight = self.window.getWeights(self.n_fft, dtype); const fft_len = window_weight.dim(.samples); @@ -87,16 +64,16 @@ pub const LogMelSpectrogram = struct { } pub fn load(self: *LogMelSpectrogram, io: std.Io, platform: *zml.Platform) !zml.Bufferized(LogMelSpectrogram) { - const mel_filters_data = @embedFile("assets/voxtral_mel_filter.data"); - const slice = zml.Slice.init(self.mel_filters.shape(), mel_filters_data); + const mel_filters_data = @embedFile("assets/voxtral_mel_filter.data"); + const slice = zml.Slice.init(self.mel_filters.shape(), mel_filters_data); - return .{ - .mel_filters = try zml.Buffer.fromSlice(io, platform, slice), - }; + return .{ + .mel_filters = try zml.Buffer.fromSlice(io, platform, slice), + }; } pub fn unload(self: *zml.Bufferized(LogMelSpectrogram)) void { - self.mel_filters.deinit(); + self.mel_filters.deinit(); } }; @@ -104,14 +81,14 @@ pub fn stft(waveform: Tensor, weight: Tensor, num_frames: usize, stride: u63, pr const fft_len = weight.dim(0); const indices = Tensor.arange(.{ .end = @intCast(num_frames * stride), .step = stride }, .i32); - var windows = waveform.gatherSlices(.{fft_len}, indices.appendAxes(.{.coord }), .{ .indices_are_sorted = true }); + var windows = waveform.gatherSlices(.{fft_len}, indices.appendAxes(.{.coord}), .{ .indices_are_sorted = true }); windows = windows.mul(weight.broadcastLeft(windows.shape())); var fft = windows.convert(precision).fft(.{ .kind = .RFFT, .length = &.{fft_len} }); const spectrogram = fft.abs(); - return spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{.frames, .freq_bins}); + return spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{ .frames, .freq_bins }); } pub const AudioWindow = enum { @@ -125,11 +102,11 @@ pub const AudioWindow = enum { .boxcar => Tensor.constant(dtype.one()).withTags(.{.samples}), .hann => { if (len <= 1) return Tensor.constant(dtype.one()); - + const flen: f64 = @floatFromInt(len); const freq = Tensor.constant(dtype.constant(std.math.pi / flen)); const steps = Tensor.arange(.{ .start = -len, .end = len, .step = 2 }, dtype); - + return steps.mul(freq).cos().scale(0.5).addConstant(0.5).convert(dtype).withTags(.{.samples}); }, }; From 9c8d401393d668de8847a9826d776ce36cbcf706 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 18 Feb 2026 11:53:20 +0100 Subject: [PATCH 82/98] make kv cache sequential but keep experimental circular --- examples/voxtral/common.zig | 57 +++++++++++++++++++++++------------- examples/voxtral/decoder.zig | 2 +- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index 5f524ef0..f4dc1eff 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -68,9 +68,12 @@ pub fn loadModel( }); } -/// Unified self-attention with KV cache, parameterized on whether to reorder -/// the circular KV buffer for correct causal masking (encoder needs this, decoder does not). -pub fn SelfAttention(comptime reorder_kv_cache: bool) type { +/// Unified self-attention with KV cache. +/// When `circular_buffer` is true, the KV cache uses modular indexing for an unbounded +/// sliding window and reorders K/V to temporal order for correct causal masking +/// (needed for the encoder's multi-token prefill). +/// When false, the cache uses sequential indexing (matching the reference implementation). +pub fn SelfAttention(comptime circular_buffer: bool) type { return struct { wq: zml.nn.Linear, wk: zml.nn.Linear, @@ -123,15 +126,16 @@ pub fn SelfAttention(comptime reorder_kv_cache: bool) type { k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); - // Update KV cache with modular indexing for sliding window circular buffer const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); - const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); - const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); - k = new_kv_cache.keys().convert(dtype); - v = new_kv_cache.values().convert(dtype); - if (reorder_kv_cache) { - // EXPERIMENTAL + if (circular_buffer) { // Experimental + // Circular buffer: modular indexing for unbounded sliding window. + // Write positions wrap around using pos_index % cache_size. + const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); + const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + // Reorder K/V from circular buffer to temporal order for correct causal masking. // When the cache wraps, physical order != temporal order, which breaks FA2's // position-based causal mask for multi-token (q_len > 1) attention. @@ -144,17 +148,30 @@ pub fn SelfAttention(comptime reorder_kv_cache: bool) type { .remainder(cache_size.broad(reorder_arange.shape())); k = k.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); v = v.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); + + // Cap token_index so seqused_k doesn't exceed cache bounds + const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); + const attn_token_index = token_index.minimum(max_token_index); + const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ + .sliding_window = @intCast(attn_config.sliding_window), + }); + + const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; + } else { + // Sequential: write at pos_index directly (matching reference implementation). + const new_kv_cache = kv_cache.update(k, v, pos_index.rename(.{ .s = .k })); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + const attn_token_index = token_index; + const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ + .sliding_window = @intCast(attn_config.sliding_window), + }); + + const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; } - - // Cap token_index so seqused_k doesn't exceed cache bounds - const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); - const attn_token_index = token_index.minimum(max_token_index); - const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ - .sliding_window = @intCast(attn_config.sliding_window), - }); - - const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; } }; } diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 9a9f709f..31b992ea 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -157,7 +157,7 @@ pub const AdaRmsNorm = struct { pub const KvCache = common.KvCache; -pub const SelfAttention = common.SelfAttention(false); +pub const SelfAttention = common.SelfAttention(true); /// Single decoder transformer layer with AdaRmsNorm time conditioning. pub const DecoderLayer = struct { From 5029738105041cd9f434ffd8924eea98b300eddb Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Wed, 18 Feb 2026 14:33:57 +0100 Subject: [PATCH 83/98] from mel history to state tensors --- examples/voxtral/config.zig | 11 +-- examples/voxtral/encoder.zig | 156 +++++++++++++++++++---------------- examples/voxtral/main.zig | 14 +++- examples/voxtral/voxtral.zig | 39 ++++++--- 4 files changed, 127 insertions(+), 93 deletions(-) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index 5a95863d..56abee32 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -112,7 +112,6 @@ pub fn parseConfig(allocator: std.mem.Allocator, io: std.Io, model_dir: std.Io.D pub const StreamParams = struct { dsf: u32, mel_per_step: u32, - mel_history: u32, chunk_mel: u32, chunk_audio: u32, raw_audio_length_per_tok: u32, @@ -133,16 +132,14 @@ pub const StreamParams = struct { const n_delay_tokens = std.math.divCeil(u32, delay_samples / hop_length, audio_length_per_tok) catch unreachable; const dsf = config.downsample_factor(); - const mel_history: u32 = 4; + const mel_per_step = dsf * 2; const window_size = config.audio().window_size; - const chunk_mel = mel_history + dsf * 2; return .{ .dsf = dsf, - .mel_per_step = dsf * 2, - .mel_history = mel_history, - .chunk_mel = chunk_mel, - .chunk_audio = (chunk_mel - 1) * hop_length + window_size, + .mel_per_step = mel_per_step, + .chunk_mel = mel_per_step, + .chunk_audio = (mel_per_step - 1) * hop_length + window_size, .raw_audio_length_per_tok = raw_audio_length_per_tok, ._hop_length = hop_length, .n_delay_tokens = n_delay_tokens, diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 7476ee1c..8af28ada 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -11,13 +11,32 @@ const rmsNorm = common.rmsNorm; const KvCache = common.KvCache; pub const Encoder = struct { - conv0: CausalConv1d, - conv1: CausalConv1d, + conv0_weight: Tensor, + conv0_bias: Tensor, + conv1_weight: Tensor, + conv1_bias: Tensor, layers: []TransformerLayer, norm: Tensor, norm_eps: f32, config: Config, + pub const ConvState = struct { + conv1: Tensor, // [batch=1, channels=128, time=2] + conv2: Tensor, // [batch=1, channels=enc_dim, time=2] + + pub fn initBuffer(self: ConvState, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvState) { + return .{ + .conv1 = try .uninitialized(io, platform, self.conv1.shape(), .{}), + .conv2 = try .uninitialized(io, platform, self.conv2.shape(), .{}), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(ConvState)) void { + self.conv1.deinit(); + self.conv2.deinit(); + } + }; + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Encoder { const encoder_store = store.withPrefix("mm_streams_embeddings.embedding_module.whisper_encoder"); const conv_store = encoder_store.withPrefix("conv_layers"); @@ -29,9 +48,14 @@ pub const Encoder = struct { layer.* = TransformerLayer.init(transformer_store.withPrefix("layers").withLayer(i), config); } + const conv0_store = conv_store.withLayer(0).withPrefix("conv"); + const conv1_store = conv_store.withLayer(1).withPrefix("conv"); + return .{ - .conv0 = CausalConv1d.init(conv_store.withLayer(0).withPrefix("conv"), 1), - .conv1 = CausalConv1d.init(conv_store.withLayer(1).withPrefix("conv"), 2), + .conv0_weight = conv0_store.createTensorWithTags("weight", .{ .cout, .cin, .k }), + .conv0_bias = conv0_store.createTensorWithTags("bias", .{.channels}), + .conv1_weight = conv1_store.createTensorWithTags("weight", .{ .cout, .cin, .k }), + .conv1_bias = conv1_store.createTensorWithTags("bias", .{.channels}), .layers = layers, .norm = transformer_store.withPrefix("norm").createTensorWithTags("weight", .{.d}), .norm_eps = enc.norm_eps, @@ -52,30 +76,69 @@ pub const Encoder = struct { allocator.free(self.layers); } - /// Streaming conv stem: processes a mel chunk with history prepended. - /// Input: mel_chunk [channels=128, time=mel_history+mel_per_step] - /// Output: [s=dsf, d=1280] — only the new frames (history-derived outputs discarded) - pub fn convStemStep(self: Encoder, mel_chunk: Tensor) Tensor { - const full_output = self.convStem(mel_chunk); - // full_output: [s=(12+1)/2=6, d=1280] when chunk is 12 frames - // Skip first 2 (history-derived), keep last dsf=4 - const dsf: u32 = self.config.downsample_factor(); - const total: u32 = @intCast(full_output.dim(.s)); - return full_output.slice1d(.s, .{ .start = total - dsf }); + /// Prefill conv stem: runs conv on all prefill mel frames and extracts + /// conv states (last kernel_size-1=2 frames of each intermediate) for subsequent streaming. + /// Uses padding={2,0} to match reference zero-initialized state (kernel_size-1=2). + /// mel: [channels=128, time=frames] → ([s=(frames+1)/2, d=enc_dim], ConvState) + pub fn convStemPrefill(self: Encoder, mel: Tensor) struct { Tensor, ConvState } { + const dtype = self.conv0_weight.dtype(); + const h = mel.convert(dtype).insertAxes(.channels, .{.batch}); + + // Conv0 (stride=1): pad kernel_size-1=2 zeros on left (matches zero-initialized state) + var conv0_out = h.conv1d(self.conv0_weight, .{ .window_strides = 1, .padding = &.{ 2, 0 } }); + conv0_out = conv0_out.add(self.conv0_bias.broad(conv0_out.shape())).gelu(); + + // Conv1 (stride=2): pad kernel_size-1=2 zeros on left (matches zero-initialized state) + var conv1_out = conv0_out.conv1d(self.conv1_weight, .{ .window_strides = 2, .padding = &.{ 2, 0 } }); + conv1_out = conv1_out.add(self.conv1_bias.broad(conv1_out.shape())).gelu(); + + // Output: [s=(frames+1)/2, d=enc_dim] + const output = conv1_out.squeeze(.batch) + .transpose(.{ .time, .channels }) + .rename(.{ .time = .s, .channels = .d }); + + // Extract conv states from tails (last kernel_size-1=2 frames) + const h_time: u32 = @intCast(h.dim(.time)); + const conv0_time: u32 = @intCast(conv0_out.dim(.time)); + + return .{ output, .{ + .conv1 = h.slice1d(.time, .{ .start = h_time - 2 }), + .conv2 = conv0_out.slice1d(.time, .{ .start = conv0_time - 2 }), + } }; } - /// Run the convolutional stem: conv0 + gelu + conv1 + gelu + reshape. - /// mel: [channels=128, time=frames] → [s=(frames+1)/2, d=1280] - pub fn convStem(self: Encoder, mel: Tensor) Tensor { - const dtype = self.conv0.weight.dtype(); - var h = mel.convert(dtype).insertAxes(.channels, .{.batch}); + /// Streaming conv stem: processes mel_per_step mel frames with explicit conv states. + /// Concatenates state (2 frames of left context) with new input, runs conv1d (no padding), + /// then saves the tail as new state. Matches the reference ExecuTorch implementation. + /// mel_chunk: [channels=128, time=mel_per_step], conv_state: ConvState + /// Returns: ([s=dsf, d=enc_dim], new ConvState) + pub fn convStemStep(self: Encoder, mel_chunk: Tensor, conv_state: ConvState) struct { Tensor, ConvState } { + const dtype = self.conv0_weight.dtype(); + const mel_with_batch = mel_chunk.convert(dtype).insertAxes(.channels, .{.batch}); - h = self.conv0.forward(h).gelu(); - h = self.conv1.forward(h).gelu(); + // Conv0 (stride=1): concat state with new mel, conv1d with no padding + const conv0_input = Tensor.concatenate(&.{ conv_state.conv1, mel_with_batch }, .time); + var conv0_out = conv0_input.conv1d(self.conv0_weight, .{ .window_strides = 1 }); + conv0_out = conv0_out.add(self.conv0_bias.broad(conv0_out.shape())).gelu(); + const mel_time: u32 = @intCast(mel_with_batch.dim(.time)); + const new_conv1_state = mel_with_batch.slice1d(.time, .{ .start = mel_time - 2 }); - // [batch=1, channels=1280, time'] -> [s, d=1280] - h = h.squeeze(.batch); - return h.transpose(.{ .time, .channels }).rename(.{ .time = .s, .channels = .d }); + // Conv1 (stride=2): concat state with conv0 output, conv1d with no padding + const conv1_input = Tensor.concatenate(&.{ conv_state.conv2, conv0_out }, .time); + var conv1_out = conv1_input.conv1d(self.conv1_weight, .{ .window_strides = 2 }); + conv1_out = conv1_out.add(self.conv1_bias.broad(conv1_out.shape())).gelu(); + const conv0_out_time: u32 = @intCast(conv0_out.dim(.time)); + const new_conv2_state = conv0_out.slice1d(.time, .{ .start = conv0_out_time - 2 }); + + // Reshape to [s=dsf, d=enc_dim] + const output = conv1_out.squeeze(.batch) + .transpose(.{ .time, .channels }) + .rename(.{ .time = .s, .channels = .d }); + + return .{ output, .{ + .conv1 = new_conv1_state, + .conv2 = new_conv2_state, + } }; } /// Process a chunk of conv stem output through all transformer layers with KV cache. @@ -98,51 +161,6 @@ pub const Encoder = struct { } }; -/// Expects input layout [batch, channels, time] and kernel [cout, cin, k]. -pub const CausalConv1d = struct { - weight: Tensor, - bias: Tensor, - stride: i64, - - pub fn init(store: zml.io.TensorStore.View, stride: i64) CausalConv1d { - return .{ - .weight = store.createTensorWithTags("weight", .{ .cout, .cin, .k }), - .bias = store.createTensorWithTags("bias", .{.channels}), - .stride = stride, - }; - } - - pub fn unload(self: *zml.Bufferized(CausalConv1d)) void { - common.deinitBufferized(self); - } - - pub fn forward(self: CausalConv1d, input: Tensor) Tensor { - const kernel_size: i64 = @intCast(self.weight.dim(.k)); - const stride = self.stride; - const input_len: i64 = @intCast(input.dim(.time)); - - // Causal padding: padding_total goes on the left. - // Right padding is computed to align the output length with stride. - const padding_left = kernel_size - stride; - const numerator = input_len - kernel_size + padding_left; - const n_frames = @divTrunc(numerator + stride - 1, stride) + 1; - const target_length = (n_frames - 1) * stride + kernel_size - padding_left; - const padding_right = target_length - input_len; - - const dtype = input.dtype(); - std.debug.assert(dtype == self.weight.dtype()); - - var h = input.conv1d(self.weight, .{ - .window_strides = stride, - .padding = &.{ padding_left, padding_right }, - }); - - // Bias: [channels] broadcasts to [batch, channels, time] - h = h.add(self.bias.broad(h.shape())); - return h; - } -}; - pub const TransformerLayer = struct { attention_norm: Tensor, attention: SelfAttention, diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index a5769306..a1d87a4e 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -121,7 +121,7 @@ pub fn main(init: std.process.Init) !void { var compiled_mel_step_future = try io.concurrent(voxtral.compileMelStep, .{ allocator, io, platform, melspectro_model, sp, &progress }); - var compiled_conv_stem_future = try io.concurrent(voxtral.compileConvStem, .{ allocator, io, platform, encoder_model, sp.prompt_len * sp.mel_per_step, &progress }); + var compiled_conv_stem_prefill_future = try io.concurrent(voxtral.compileConvStemPrefill, .{ allocator, io, platform, encoder_model, sp.prompt_len * sp.mel_per_step, &progress }); var compiled_conv_stem_step_future = try io.concurrent(voxtral.compileConvStemStep, .{ allocator, io, platform, encoder_model, sp, &progress }); var compiled_encoder_prefill_future = try io.concurrent(voxtral.compileEncoderPrefill, .{ allocator, io, platform, encoder_model, sp.prompt_len, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); var compiled_encoder_step_future = try io.concurrent(voxtral.compileEncoderStep, .{ allocator, io, platform, encoder_model, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); @@ -147,8 +147,8 @@ pub fn main(init: std.process.Init) !void { var compiled_mel_step = try compiled_mel_step_future.await(io); defer compiled_mel_step.deinit(); - var compiled_conv_stem = try compiled_conv_stem_future.await(io); - defer compiled_conv_stem.deinit(); + var compiled_conv_stem_prefill = try compiled_conv_stem_prefill_future.await(io); + defer compiled_conv_stem_prefill.deinit(); var compiled_conv_stem_step = try compiled_conv_stem_step_future.await(io); defer compiled_conv_stem_step.deinit(); @@ -190,6 +190,11 @@ pub fn main(init: std.process.Init) !void { progress.end(); + const conv_state: enc.Encoder.ConvState = .{ + .conv1 = Tensor.init(.{ .batch = 1, .channels = config.audio().num_mel_bins, .time = 2 }, enc_dtype), + .conv2 = Tensor.init(.{ .batch = 1, .channels = enc_cfg.dim, .time = 2 }, enc_dtype), + }; + try voxtral.runPipeline( allocator, io, @@ -200,7 +205,7 @@ pub fn main(init: std.process.Init) !void { sp, .{ .mel_step = &compiled_mel_step, - .conv_stem = &compiled_conv_stem, + .conv_stem_prefill = &compiled_conv_stem_prefill, .conv_stem_step = &compiled_conv_stem_step, .encoder_prefill = &compiled_encoder_prefill, .encoder_step = &compiled_encoder_step, @@ -215,6 +220,7 @@ pub fn main(init: std.process.Init) !void { &decoder_buffers, enc_kv_cache, dec_kv_cache, + conv_state, &enc_attention_metadata_buffers, &dec_attention_metadata_buffers, ); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 7569c1d8..c8e2c814 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -26,7 +26,7 @@ const tesc = terminalwave.esc; pub const CompiledExes = struct { mel_step: *zml.Exe, - conv_stem: *zml.Exe, + conv_stem_prefill: *zml.Exe, conv_stem_step: *zml.Exe, encoder_prefill: *zml.Exe, encoder_step: *zml.Exe, @@ -43,6 +43,7 @@ pub const ModelBuffers = struct { decoder: *zml.Bufferized(Decoder), enc_kv: *zml.Bufferized(KvCache), dec_kv: *zml.Bufferized(KvCache), + conv_state: *zml.Bufferized(Encoder.ConvState), t_cond: zml.Buffer, rng: *zml.Bufferized(Tensor.Rng), enc_attention_metadata: *zml.Bufferized(zml.attention.Metadata), @@ -70,15 +71,21 @@ pub fn compileMelStep(allocator: std.mem.Allocator, io: std.Io, platform: *const return compileStep(allocator, io, platform, model, .melStep, .{Tensor.init(.{sp.chunk_audio}, .f32).withTags(.{.samples})}, progress, "mel step"); } -pub fn compileConvStem(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, num_mel_frames: u32, progress: *std.Progress.Node) !zml.Exe { - return compileStep(allocator, io, platform, model, .convStem, .{ +pub fn compileConvStemPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, num_mel_frames: u32, progress: *std.Progress.Node) !zml.Exe { + return compileStep(allocator, io, platform, model, .convStemPrefill, .{ Tensor.init(.{ .channels = 128, .time = num_mel_frames }, .f32), - }, progress, "conv stem"); + }, progress, "conv stem prefill"); } pub fn compileConvStemStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { + const dtype = model.conv0_weight.dtype(); + const enc_dim = model.config.encoder().dim; return compileStep(allocator, io, platform, model, .convStemStep, .{ - Tensor.init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), + Tensor.init(.{ .channels = 128, .time = sp.mel_per_step }, .f32), + Encoder.ConvState{ + .conv1 = Tensor.init(.{ .batch = 1, .channels = 128, .time = 2 }, dtype), + .conv2 = Tensor.init(.{ .batch = 1, .channels = enc_dim, .time = 2 }, dtype), + }, }, progress, "conv stem step"); } @@ -298,6 +305,7 @@ fn runPrefill(ctx: *PipelineContext, mel_host: []const u8, tokens: []const u32, const f32_bytes: usize = @sizeOf(f32); // Conv stem prefill: mel_host is already [128][prompt_len*mel_per_step] + // Also extracts conv states (last 2 frames of each intermediate) for streaming log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * sp.mel_per_step}); var conv_prefill_output: zml.Buffer = undefined; { @@ -312,14 +320,14 @@ fn runPrefill(ctx: *PipelineContext, mel_host: []const u8, tokens: []const u32, const prefill_enc_frames = prompt_len * sp.dsf; conv_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_enc_frames, .d = enc_dim }, .bf16), .{}); - var args = try ctx.exes.conv_stem.args(allocator); + var args = try ctx.exes.conv_stem_prefill.args(allocator); defer args.deinit(allocator); - var results = try ctx.exes.conv_stem.results(allocator); + var results = try ctx.exes.conv_stem_prefill.results(allocator); defer results.deinit(allocator); args.set(.{ ctx.buffers.encoder, mel_buffer }); - ctx.exes.conv_stem.call(args, &results); - results.fill(.{&conv_prefill_output}); + ctx.exes.conv_stem_prefill.call(args, &results); + results.fill(.{ &conv_prefill_output, ctx.buffers.conv_state }); } defer conv_prefill_output.deinit(); log.info("Conv stem prefill done.", .{}); @@ -475,7 +483,7 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, var stdout = std.Io.File.stdout().writer(io, &.{}); var wave = terminalwave.State.init(.{ .title = "Voxtral Realtime", - .sensitivity = 2.0, + .sensitivity = 5.0, .num_bars = 50, .half_height = 20, }, &stdout.interface); @@ -551,10 +559,10 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, ctx.exes.mel_step.call(mel_step_args, &mel_step_results); mel_step_results.fill(.{&mel_step_output}); - // Conv stem step: mel step output stays on device → conv stem - conv_step_args.set(.{ ctx.buffers.encoder, mel_step_output }); + // Conv stem step: mel + conv states → output + updated conv states + conv_step_args.set(.{ ctx.buffers.encoder, mel_step_output, ctx.buffers.conv_state }); ctx.exes.conv_stem_step.call(conv_step_args, &conv_step_results); - conv_step_results.fill(.{&conv_step_output}); + conv_step_results.fill(.{ &conv_step_output, ctx.buffers.conv_state }); // Encoder step: conv stem output -> encoded chunk var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)) * sp.dsf, .u32); @@ -607,6 +615,7 @@ pub fn runPipeline( decoder_buffers: *zml.Bufferized(Decoder), enc_kv_cache: KvCache, dec_kv_cache: KvCache, + conv_state: Encoder.ConvState, enc_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), ) !void { @@ -642,6 +651,9 @@ pub fn runPipeline( var dec_kv_buffers = try dec_kv_cache.initBuffer(io, platform); defer KvCache.deinitBuffer(&dec_kv_buffers); + var conv_state_buffers = try conv_state.initBuffer(io, platform); + defer Encoder.ConvState.deinitBuffer(&conv_state_buffers); + const t_cond_data = computeTimeEmbedding(allocator, @as(f32, @floatFromInt(sp.n_delay_tokens)), config.dim); defer allocator.free(t_cond_data); const t_cond_slice: zml.Slice = .init( @@ -669,6 +681,7 @@ pub fn runPipeline( .decoder = decoder_buffers, .enc_kv = &enc_kv_buffers, .dec_kv = &dec_kv_buffers, + .conv_state = &conv_state_buffers, .t_cond = t_cond_buffer, .rng = &rng_buffers, .enc_attention_metadata = enc_attention_metadata_buffers, From ff7128ac30d43dc2e1223a540711fb19574dee47 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 19 Feb 2026 09:52:18 +0100 Subject: [PATCH 84/98] remove unused old code --- examples/voxtral/BUILD.bazel | 2 - examples/voxtral/common.zig | 5 -- examples/voxtral/decoder.zig | 2 +- examples/voxtral/main.zig | 1 - examples/voxtral/wav.zig | 113 ----------------------------------- 5 files changed, 1 insertion(+), 122 deletions(-) delete mode 100644 examples/voxtral/wav.zig diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel index 85c4ee5b..7abf520f 100644 --- a/examples/voxtral/BUILD.bazel +++ b/examples/voxtral/BUILD.bazel @@ -7,7 +7,6 @@ zig_binary( srcs = [ "voxtral.zig", "mel_spectrogram.zig", - "wav.zig", "config.zig", "encoder.zig", "decoder.zig", @@ -26,7 +25,6 @@ zig_binary( srcs = [ "voxtral.zig", "mel_spectrogram.zig", - "wav.zig", "config.zig", "encoder.zig", "decoder.zig", diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index f4dc1eff..7dfce84b 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -18,11 +18,6 @@ pub fn deinitBufferized(bufferized: anytype) void { }).cb, {}, bufferized); } -pub fn deinitLinear(l: *zml.Bufferized(zml.nn.Linear)) void { - l.weight.deinit(); - if (l.bias) |*b| b.deinit(); -} - pub fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { return .init( store.createTensorWithTags("weight", .{ .dout, .d }), diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 31b992ea..9a9f709f 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -157,7 +157,7 @@ pub const AdaRmsNorm = struct { pub const KvCache = common.KvCache; -pub const SelfAttention = common.SelfAttention(true); +pub const SelfAttention = common.SelfAttention(false); /// Single decoder transformer layer with AdaRmsNorm time conditioning. pub const DecoderLayer = struct { diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index a1d87a4e..055f33e2 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -1,7 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); const log = std.log; -const wav_utils = @import("wav.zig"); const cfg = @import("config.zig"); const Config = cfg.Config; const StreamParams = cfg.StreamParams; diff --git a/examples/voxtral/wav.zig b/examples/voxtral/wav.zig deleted file mode 100644 index d0ffdd78..00000000 --- a/examples/voxtral/wav.zig +++ /dev/null @@ -1,113 +0,0 @@ -const std = @import("std"); - -pub const Chunk = struct { - pub const Id = enum(u32) { - riff = @bitCast(@as([4]u8, "RIFF".*)), - fmt = @bitCast(@as([4]u8, "fmt ".*)), - data = @bitCast(@as([4]u8, "data".*)), - _, - }; - - pub const Header = extern struct { - id: Id align(1), - size: u32 align(1), - }; - - pub const Riff = extern struct { - pub const Format = enum(u32) { - wave = @bitCast(@as([4]u8, "WAVE".*)), - _, - }; - - format: Format align(1), - }; - - pub const Fmt = extern struct { - pub const AudioFormat = enum(u16) { - pcm = 1, - _, - }; - - audio_format: AudioFormat align(1), - num_channels: u16 align(1), - sample_rate: u32 align(1), - byte_rate: u32 align(1), - block_align: u16 align(1), - bits_per_sample: u16 align(1), - }; -}; - -fn readPcmWav(allocator: std.mem.Allocator, reader: *std.Io.Reader, out: *std.ArrayList(u8)) !Chunk.Fmt { - std.debug.assert((try reader.takeStruct(Chunk.Header, .little)).id == .riff); - std.debug.assert((try reader.takeStruct(Chunk.Riff, .little)).format == .wave); - var fmt: ?Chunk.Fmt = null; - var data_read: bool = false; - while (fmt == null or !data_read) { - const hdr = try reader.takeStruct(Chunk.Header, .little); - switch (hdr.id) { - .fmt => { - fmt = try reader.takeStruct(Chunk.Fmt, .little); - if (hdr.size > @sizeOf(Chunk.Fmt)) { - try reader.discardAll(hdr.size - @sizeOf(Chunk.Fmt)); - } - }, - .data => { - try out.resize(allocator, hdr.size); - try reader.readSliceAll(out.items); - data_read = true; - }, - else => try reader.discardAll(hdr.size), - } - } - return fmt.?; -} - -pub fn loadWav(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]const f32 { - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - - const arena = arena_state.allocator(); - var sample_list: std.ArrayList(u8) = .empty; - - const wav_fmt = try readPcmWav(arena, reader, &sample_list); - const byte_per_sample = wav_fmt.bits_per_sample / 8; - const sample_count = sample_list.items.len / (byte_per_sample * wav_fmt.num_channels); - - const samples = try allocator.alloc(f32, sample_count); - for (0..sample_count) |i| { - const offset = i * byte_per_sample * wav_fmt.num_channels; - - const sample = switch(byte_per_sample) { - 1 => (@as(f32, @floatFromInt(std.mem.bytesToValue(u8, sample_list.items[offset .. offset + 1]))) - 128.0) / 128.0, - 2 => @as(f32, @floatFromInt(std.mem.bytesToValue(i16, sample_list.items[offset .. offset + 2]))) / 32768.0, - 3 => @as(f32, @floatFromInt(std.mem.bytesToValue(i24, sample_list.items[offset .. offset + 3]))) / 8388608.0, - 4 => @as(f32, @floatFromInt(std.mem.bytesToValue(i32, sample_list.items[offset .. offset + 4]))) / 2147483648.0, - else => unreachable, - }; - - samples[i] = sample; - } - - return samples; -} - -pub fn loadAndPadWav( - allocator: std.mem.Allocator, - reader: *std.Io.Reader, - left_pad: usize, - right_pad_tokens: usize, - raw_audio_length_per_tok: usize, -) ![]const f32 { - const wav_file = try loadWav(allocator, reader); - defer allocator.free(wav_file); - - const align_pad = (raw_audio_length_per_tok - (wav_file.len % raw_audio_length_per_tok)) % raw_audio_length_per_tok; - const right_pad = align_pad + right_pad_tokens * raw_audio_length_per_tok; - const audio_len = left_pad + wav_file.len + right_pad; - - const padded_audio = try allocator.alloc(f32, audio_len); - @memset(padded_audio, 0); - @memcpy(padded_audio[left_pad .. left_pad + wav_file.len], wav_file); - - return padded_audio; -} From 231dcfb5400a5da2d7da93fad0a299c256b885c2 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Thu, 19 Feb 2026 17:56:52 +0100 Subject: [PATCH 85/98] Shift kv cache --- examples/voxtral/common.zig | 67 ++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index 7dfce84b..5a952e71 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -64,11 +64,12 @@ pub fn loadModel( } /// Unified self-attention with KV cache. -/// When `circular_buffer` is true, the KV cache uses modular indexing for an unbounded -/// sliding window and reorders K/V to temporal order for correct causal masking -/// (needed for the encoder's multi-token prefill). +/// When `shift_buffer` is true, the KV cache uses shift-based overflow handling for +/// unbounded sliding window support. Both paths (normal write vs shifted cache write) +/// are computed and the correct one is selected via cmp/select. The cache stays in +/// temporal order — no reordering needed. /// When false, the cache uses sequential indexing (matching the reference implementation). -pub fn SelfAttention(comptime circular_buffer: bool) type { +pub fn SelfAttention(comptime shift_buffer: bool) type { return struct { wq: zml.nn.Linear, wk: zml.nn.Linear, @@ -121,33 +122,20 @@ pub fn SelfAttention(comptime circular_buffer: bool) type { k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); - const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); + if (shift_buffer) { + const shifted_cache, const clamped_token_index = kv_cache.shiftIfNeeded(token_index, @intCast(x.dim(.s))); - if (circular_buffer) { // Experimental - // Circular buffer: modular indexing for unbounded sliding window. - // Write positions wrap around using pos_index % cache_size. - const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); - const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); + const write_pos = b: { + const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) + .withTags(.{.s}).broad(Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(clamped_token_index.broad(temp.shape())); + }; + + const new_kv_cache = shifted_cache.update(k, v, write_pos.rename(.{ .s = .k })); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - // Reorder K/V from circular buffer to temporal order for correct causal masking. - // When the cache wraps, physical order != temporal order, which breaks FA2's - // position-based causal mask for multi-token (q_len > 1) attention. - const pos_end = token_index.addConstant(x.dim(.s)); - const is_full = pos_end.cmp(.GE, cache_size); - const rotation_start = pos_end.remainder(cache_size); - const safe_start = is_full.select(rotation_start, Tensor.scalar(@as(u32, 0), token_index.dtype())); - const reorder_arange = Tensor.arange(.{ .end = kv_cache.k.dim(.k) }, token_index.dtype()).withTags(.{.kk}); - const reorder_idx = reorder_arange.add(safe_start.broad(reorder_arange.shape())) - .remainder(cache_size.broad(reorder_arange.shape())); - k = k.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); - v = v.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); - - // Cap token_index so seqused_k doesn't exceed cache bounds - const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); - const attn_token_index = token_index.minimum(max_token_index); - const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ + const attn_out = zml.attention.attention(q, k, v, clamped_token_index, attention_metadata, attention_parameters, .{ .sliding_window = @intCast(attn_config.sliding_window), }); @@ -235,6 +223,31 @@ pub const KvCache = struct { cache.scatterSlices(.{ .layer = layer_index }, converted, scatter_opts).reuseBuffer(cache); } + pub fn shiftIfNeeded(self: KvCache, token_index: Tensor, seq_len: u32) struct { KvCache, Tensor } { + const cache_k_size = self.k.dim(.k); + const max_pos = Tensor.scalar(@as(u32, @intCast(cache_k_size - seq_len)), token_index.dtype()); + const would_overflow = token_index.cmp(.GT, max_pos); + + const shift_arange = Tensor.arange(.{ .end = cache_k_size }, token_index.dtype()).withTags(.{.kk}); + const shifted_indices = shift_arange.addConstant(seq_len); + const max_idx = Tensor.scalar(@as(u32, @intCast(cache_k_size - 1)), token_index.dtype()); + const clamped_indices = shifted_indices.minimum(max_idx.broad(shifted_indices.shape())); + + const shifted_k = self.k.gather(.{ .k = clamped_indices }, .{}).rename(.{ .kk = .k }); + const shifted_v = self.v.gather(.{ .k = clamped_indices }, .{}).rename(.{ .kk = .k }); + + const overflow_k = would_overflow.broad(shifted_k.shape()); + const new_k = overflow_k.select(shifted_k, self.k); + const overflow_v = would_overflow.broad(shifted_v.shape()); + const new_v = overflow_v.select(shifted_v, self.v); + + return .{ .{ + .k = new_k.reuseBuffer(self.k), + .v = new_v.reuseBuffer(self.v), + .layer_index = self.layer_index, + }, token_index.minimum(max_pos) }; + } + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { return .{ .k = self.k, From 0bbd034724cd335c9021f2449e5fa866f54db155 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 20 Feb 2026 14:24:18 +0100 Subject: [PATCH 86/98] shift and circular implem --- examples/voxtral/common.zig | 53 +++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index 5a952e71..d5befcc9 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -122,24 +122,61 @@ pub fn SelfAttention(comptime shift_buffer: bool) type { k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); + const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); + if (shift_buffer) { - const shifted_cache, const clamped_token_index = kv_cache.shiftIfNeeded(token_index, @intCast(x.dim(.s))); - const write_pos = b: { - const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) - .withTags(.{.s}).broad(Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); - break :b temp.add(clamped_token_index.broad(temp.shape())); - }; + // -- Circular implementation - const new_kv_cache = shifted_cache.update(k, v, write_pos.rename(.{ .s = .k })); + const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); + const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_out = zml.attention.attention(q, k, v, clamped_token_index, attention_metadata, attention_parameters, .{ + // Reorder K/V from circular buffer to temporal order for correct causal masking. + // When the cache wraps, physical order != temporal order, which breaks FA2's + // position-based causal mask for multi-token (q_len > 1) attention. + const pos_end = token_index.addConstant(x.dim(.s)); + const is_full = pos_end.cmp(.GE, cache_size); + const rotation_start = pos_end.remainder(cache_size); + const safe_start = is_full.select(rotation_start, Tensor.scalar(@as(u32, 0), token_index.dtype())); + const reorder_arange = Tensor.arange(.{ .end = kv_cache.k.dim(.k) }, token_index.dtype()).withTags(.{.kk}); + const reorder_idx = reorder_arange.add(safe_start.broad(reorder_arange.shape())) + .remainder(cache_size.broad(reorder_arange.shape())); + k = k.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); + v = v.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); + + // Cap token_index so seqused_k doesn't exceed cache bounds + const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); + const attn_token_index = token_index.minimum(max_token_index); + const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ .sliding_window = @intCast(attn_config.sliding_window), }); + // -- END Cicrular implem + + // -- Shift implem + + // const shifted_cache, const clamped_token_index = kv_cache.shiftIfNeeded(token_index, @intCast(x.dim(.s))); + + // const write_pos = b: { + // const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) + // .withTags(.{.s}).broad(Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + // break :b temp.add(clamped_token_index.broad(temp.shape())); + // }; + + // const new_kv_cache = shifted_cache.update(k, v, write_pos.rename(.{ .s = .k })); + // k = new_kv_cache.keys().convert(dtype); + // v = new_kv_cache.values().convert(dtype); + + // const attn_out = zml.attention.attention(q, k, v, clamped_token_index, attention_metadata, attention_parameters, .{ + // .sliding_window = @intCast(attn_config.sliding_window), + // }); + + // -- END Shift implem + const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; } else { // Sequential: write at pos_index directly (matching reference implementation). From a21811927f465eb502674fd110a73f81e82f0c6f Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 20 Feb 2026 15:13:17 +0100 Subject: [PATCH 87/98] opti step 1 --- examples/voxtral/decoder.zig | 9 ++- examples/voxtral/encoder.zig | 7 ++ examples/voxtral/main.zig | 10 +++ examples/voxtral/voxtral.zig | 153 +++++++++++++++-------------------- 4 files changed, 91 insertions(+), 88 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 9a9f709f..6119469d 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -126,6 +126,13 @@ pub const Decoder = struct { return .{ output_tokens.convert(.u32), cache, new_rng }; } + + /// Like forward, but also returns the incremented token index (on-device). + pub fn forwardStep(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor.Rng, Tensor } { + const output_tokens, const cache, const new_rng = self.forward(text_tokens, audio_embed, token_index, kv_cache, t_cond, rng, attention_metadata, attention_parameters); + const next_index = token_index.addConstant(1).reuseBuffer(token_index); + return .{ output_tokens, cache, new_rng, next_index }; + } }; /// AdaRmsNorm: per-layer time conditioning. @@ -149,7 +156,7 @@ pub const AdaRmsNorm = struct { /// h: [s, d], t_cond: [d] → [s, d] pub fn forward(self: AdaRmsNorm, h: Tensor, t_cond: Tensor) Tensor { const ada_scale = self.up.forward(self.down.forward(t_cond).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); - const one_plus_scale = ada_scale.broad(h.shape()).add(Tensor.scalar(1.0, h.dtype()).broad(h.shape())); + const one_plus_scale = ada_scale.addConstant(1.0).broad(h.shape()); return h.mul(one_plus_scale); } diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 8af28ada..943c08db 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -159,6 +159,13 @@ pub const Encoder = struct { return .{ rmsNorm(h, self.norm, self.norm_eps), cache }; } + + /// Like transformer, but also returns the incremented token index (on-device). + pub fn transformerStep(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor } { + const output, const cache = self.transformer(x, token_index, kv_cache, attention_metadata, attention_parameters); + const next_index = token_index.addConstant(self.config.downsample_factor()).reuseBuffer(token_index); + return .{ output, cache, next_index }; + } }; pub const TransformerLayer = struct { diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 055f33e2..36c97dc0 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -65,6 +65,11 @@ pub fn main(init: std.process.Init) !void { const sp: StreamParams = .init(config, args.transcription_delay_ms, n_left_pad_tokens); + const delay_ms: u32 = @intFromFloat(args.transcription_delay_ms); + if (delay_ms < 80 or delay_ms > 2400 or delay_ms % 80 != 0) { + std.debug.panic("transcription_delay_ms must be a multiple of 80 in range [80, 2400], got {}", .{delay_ms}); + } + // Build tokens: [BOS] ++ [STREAMING_PAD] * (n_left_pad_tokens + n_delay_tokens) const tokens = try allocator.alloc(u32, sp.prompt_len); defer allocator.free(tokens); @@ -119,6 +124,7 @@ pub fn main(init: std.process.Init) !void { var tokenizer_future = try io.concurrent(voxtral.loadTokenizer, .{ allocator, io, model_dir, &progress }); var compiled_mel_step_future = try io.concurrent(voxtral.compileMelStep, .{ allocator, io, platform, melspectro_model, sp, &progress }); + var compiled_mel_prefill_future = try io.concurrent(voxtral.compileMelPrefill, .{ allocator, io, platform, melspectro_model, sp, &progress }); var compiled_conv_stem_prefill_future = try io.concurrent(voxtral.compileConvStemPrefill, .{ allocator, io, platform, encoder_model, sp.prompt_len * sp.mel_per_step, &progress }); var compiled_conv_stem_step_future = try io.concurrent(voxtral.compileConvStemStep, .{ allocator, io, platform, encoder_model, sp, &progress }); @@ -146,6 +152,9 @@ pub fn main(init: std.process.Init) !void { var compiled_mel_step = try compiled_mel_step_future.await(io); defer compiled_mel_step.deinit(); + var compiled_mel_prefill = try compiled_mel_prefill_future.await(io); + defer compiled_mel_prefill.deinit(); + var compiled_conv_stem_prefill = try compiled_conv_stem_prefill_future.await(io); defer compiled_conv_stem_prefill.deinit(); @@ -204,6 +213,7 @@ pub fn main(init: std.process.Init) !void { sp, .{ .mel_step = &compiled_mel_step, + .mel_prefill = &compiled_mel_prefill, .conv_stem_prefill = &compiled_conv_stem_prefill, .conv_stem_step = &compiled_conv_stem_step, .encoder_prefill = &compiled_encoder_prefill, diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index c8e2c814..711efd17 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -26,6 +26,7 @@ const tesc = terminalwave.esc; pub const CompiledExes = struct { mel_step: *zml.Exe, + mel_prefill: *zml.Exe, conv_stem_prefill: *zml.Exe, conv_stem_step: *zml.Exe, encoder_prefill: *zml.Exe, @@ -71,6 +72,13 @@ pub fn compileMelStep(allocator: std.mem.Allocator, io: std.Io, platform: *const return compileStep(allocator, io, platform, model, .melStep, .{Tensor.init(.{sp.chunk_audio}, .f32).withTags(.{.samples})}, progress, "mel step"); } +pub fn compileMelPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { + const full_audio: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp._hop_length + sp.chunk_audio; + return compileStep(allocator, io, platform, model, .melStep, .{ + Tensor.init(.{full_audio}, .f32).withTags(.{.samples}), + }, progress, "mel prefill"); +} + pub fn compileConvStemPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, num_mel_frames: u32, progress: *std.Progress.Node) !zml.Exe { return compileStep(allocator, io, platform, model, .convStemPrefill, .{ Tensor.init(.{ .channels = 128, .time = num_mel_frames }, .f32), @@ -106,7 +114,7 @@ pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { const enc_cfg = model.config.encoder(); const dsf = model.config.downsample_factor(); - return compileStep(allocator, io, platform, model, .transformer, .{ + return compileStep(allocator, io, platform, model, .transformerStep, .{ Tensor.init(.{ .s = dsf, .d = enc_cfg.dim }, .bf16), Tensor.init(.{}, .u32), enc_kv_cache, @@ -152,7 +160,7 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const const compileDecode = struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, kv_cache_: KvCache, dim_: u32, attention_metadata_: zml.attention.Metadata, attention_parameters_: zml.attention.Parameters, progress_: *std.Progress.Node) !zml.Exe { - return compileStep(allocator_, io_, platform_, model_, .forward, .{ + return compileStep(allocator_, io_, platform_, model_, .forwardStep, .{ Tensor.init(.{ .s = 1 }, .u32), Tensor.init(.{ .s = 1, .d = dim_ }, .bf16), Tensor.init(.{}, .u32), @@ -225,73 +233,44 @@ pub fn reflectPadAudio(allocator: std.mem.Allocator, audio: []const f32, pad: us return result; } -/// Run mel spectrogram chunk-by-chunk and download result to host memory. -/// Returns mel data as bytes (channels-major: [128][prompt_len * mel_per_step]). Caller owns the memory. -fn runMelPrefill(ctx: *PipelineContext, padded_audio: []const f32) ![]u8 { - // -- - const allocator = ctx.allocator; +/// Run mel spectrogram on the full prefill audio in a single kernel call. +/// Returns mel output as a device buffer [channels=128, time=prompt_len*mel_per_step]. +fn runMelPrefill(ctx: *PipelineContext, padded_audio: []const f32) !zml.Buffer { const io = ctx.io; const platform = ctx.platform; const sp = ctx.sp; - const compiled_mel_step = ctx.exes.mel_step; - const mel_spectrum_buffers = ctx.buffers.mel_spectrum; - // -- - const prompt_len = sp.prompt_len; - const f32_bytes: usize = @sizeOf(f32); - const prefill_mel_frames: u32 = prompt_len * sp.mel_per_step; + const full_audio_len: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp._hop_length + sp.chunk_audio; - log.info("Running mel prefill ({} chunks of {} frames, keeping {} per chunk)...", .{ prompt_len, sp.chunk_mel, sp.mel_per_step }); + log.info("Running mel prefill ({} audio samples -> {} mel frames)...", .{ full_audio_len, sp.prompt_len * sp.mel_per_step }); - // Allocate host mel buffer: [128][prefill_mel_frames] zero-initialized - const mel_host = try allocator.alloc(u8, @as(usize, 128) * prefill_mel_frames * f32_bytes); - @memset(mel_host, 0); + const audio_slice: zml.Slice = .init( + .init(.{full_audio_len}, .f32), + std.mem.sliceAsBytes(padded_audio[0..full_audio_len]), + ); + var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); + defer audio_buffer.deinit(); - var mel_step_args = try compiled_mel_step.args(allocator); - defer mel_step_args.deinit(allocator); - var mel_step_results = try compiled_mel_step.results(allocator); - defer mel_step_results.deinit(allocator); + var mel_output: zml.Buffer = try .uninitialized(io, platform, + .init(.{ .channels = 128, .time = sp.prompt_len * sp.mel_per_step }, .f32), .{}); - var mel_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), .{}); - defer mel_step_output.deinit(); + var args = try ctx.exes.mel_prefill.args(ctx.allocator); + defer args.deinit(ctx.allocator); + var results = try ctx.exes.mel_prefill.results(ctx.allocator); + defer results.deinit(ctx.allocator); - for (0..prompt_len) |t| { - // Audio chunk starts at t * mel_per_step * hop_length in the reflect-padded audio - const audio_start: usize = t * sp.mel_per_step * sp._hop_length; - const audio_chunk = padded_audio[audio_start .. audio_start + sp.chunk_audio]; + args.set(.{ ctx.buffers.mel_spectrum, audio_buffer }); + ctx.exes.mel_prefill.call(args, &results); + results.fill(.{&mel_output}); - const audio_slice: zml.Slice = .init( - .init(.{sp.chunk_audio}, .f32), - std.mem.sliceAsBytes(audio_chunk), - ); - var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); - defer audio_buffer.deinit(); + log.info("Mel prefill done ({} frames).", .{sp.prompt_len * sp.mel_per_step}); - mel_step_args.set(.{ mel_spectrum_buffers, audio_buffer }); - compiled_mel_step.call(mel_step_args, &mel_step_results); - mel_step_results.fill(.{&mel_step_output}); - - // Download chunk and keep first mel_per_step frames for conv stem prefill - var download = try mel_step_output.toSliceAlloc(allocator, io); - defer download.free(allocator); - const src = download.constData(); - - const src_stride = @as(usize, sp.chunk_mel) * f32_bytes; - const dst_stride = @as(usize, prefill_mel_frames) * f32_bytes; - const copy_len = @as(usize, sp.mel_per_step) * f32_bytes; - const dst_offset = t * sp.mel_per_step * f32_bytes; - for (0..128) |c| { - @memcpy(mel_host[c * dst_stride + dst_offset ..][0..copy_len], src[c * src_stride ..][0..copy_len]); - } - } - log.info("Mel prefill done ({} frames).", .{prefill_mel_frames}); - - return mel_host; + return mel_output; } /// Run the prefill phase: conv stem -> encoder -> adapter -> decoder prefill. /// Populates KV caches and produces the first generated token into generated_token_slice. -fn runPrefill(ctx: *PipelineContext, mel_host: []const u8, tokens: []const u32, generated_token_slice: zml.Slice) !void { +fn runPrefill(ctx: *PipelineContext, mel_device: zml.Buffer, tokens: []const u32, generated_token_slice: zml.Slice) !void { // -- const allocator = ctx.allocator; const io = ctx.io; @@ -302,21 +281,12 @@ fn runPrefill(ctx: *PipelineContext, mel_host: []const u8, tokens: []const u32, const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; - const f32_bytes: usize = @sizeOf(f32); - // Conv stem prefill: mel_host is already [128][prompt_len*mel_per_step] + // Conv stem prefill: mel_device is already [128][prompt_len*mel_per_step] on device // Also extracts conv states (last 2 frames of each intermediate) for streaming log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * sp.mel_per_step}); var conv_prefill_output: zml.Buffer = undefined; { - const prefill_mel_time: u32 = prompt_len * sp.mel_per_step; - const mel_slice: zml.Slice = .init( - .init(.{ .channels = 128, .time = prefill_mel_time }, .f32), - mel_host[0 .. @as(usize, 128) * prefill_mel_time * f32_bytes], - ); - var mel_buffer: zml.Buffer = try .fromSlice(io, platform, mel_slice); - defer mel_buffer.deinit(); - const prefill_enc_frames = prompt_len * sp.dsf; conv_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_enc_frames, .d = enc_dim }, .bf16), .{}); @@ -325,7 +295,7 @@ fn runPrefill(ctx: *PipelineContext, mel_host: []const u8, tokens: []const u32, var results = try ctx.exes.conv_stem_prefill.results(allocator); defer results.deinit(allocator); - args.set(.{ ctx.buffers.encoder, mel_buffer }); + args.set(.{ ctx.buffers.encoder, mel_device }); ctx.exes.conv_stem_prefill.call(args, &results); results.fill(.{ &conv_prefill_output, ctx.buffers.conv_state }); } @@ -392,7 +362,7 @@ fn runPrefill(ctx: *PipelineContext, mel_host: []const u8, tokens: []const u32, var prefill_output_slice: zml.Slice = try .alloc(allocator, .init(.{prompt_len}, .u32)); defer prefill_output_slice.free(allocator); - var prefill_output: zml.Buffer = try .fromSlice(io, platform, prefill_output_slice); + var prefill_output: zml.Buffer = try .uninitialized(io, platform, .init(.{prompt_len}, .u32), .{}); defer prefill_output.deinit(); args.set(.{ ctx.buffers.decoder, token_buffer, adapter_prefill_output, token_index_buffer, ctx.buffers.dec_kv, ctx.buffers.t_cond, ctx.buffers.rng.*, ctx.buffers.dec_attention_metadata }); @@ -401,9 +371,7 @@ fn runPrefill(ctx: *PipelineContext, mel_host: []const u8, tokens: []const u32, // Extract last token from prefill output (first generated token) try prefill_output.toSlice(io, prefill_output_slice); - log.info("Prefill output tokens: {any}", .{prefill_output_slice.items(u32)[0..prompt_len]}); generated_token_slice.items(u32)[0] = prefill_output_slice.items(u32)[prompt_len - 1]; - log.info("First generated token (from prefill): {}", .{generated_token_slice.items(u32)[0]}); } log.info("Decoder prefill done.\n", .{}); } @@ -479,6 +447,18 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); defer current_token_buffer.deinit(); + // Pre-allocate reusable stdin sample buffers + const raw_buf = try allocator.alloc(u8, new_audio_per_step * 2); + defer allocator.free(raw_buf); + const sample_buf = try allocator.alloc(f32, new_audio_per_step); + defer allocator.free(sample_buf); + + // Persistent on-device token index buffers + var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, prompt_len) * sp.dsf, .u32); + defer enc_token_index.deinit(); + var dec_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, prompt_len), .u32); + defer dec_token_index.deinit(); + // Wave visualization + transcript accumulation var stdout = std.Io.File.stdout().writer(io, &.{}); var wave = terminalwave.State.init(.{ @@ -506,8 +486,7 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, var last_transcript_len: usize = 0; var num_generated: usize = 0; - var t: usize = prompt_len; - while (true) : (t += 1) { + while (true) { const generated_token = generated_token_slice.items(u32)[0]; num_generated += 1; @@ -520,17 +499,16 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, if (generated_token == eos_token) break; // Mel step: read new audio, slide history buffer, upload full chunk - const new_samples = readStdinSamples(allocator, stdin_reader, new_audio_per_step) catch |err| switch (err) { + readStdinSamplesInto(stdin_reader, raw_buf, sample_buf) catch |err| switch (err) { error.EndOfStream => break, else => return err, }; - defer allocator.free(new_samples); // Save cursor position (in transcript scroll region) wave.writer.writeAll(tesc.save_cursor) catch {}; // Render wave with current audio level (fixed area above scroll region) - const rms = terminalwave.computeRms(new_samples); + const rms = terminalwave.computeRms(sample_buf); _ = wave.render(rms); wave.renderSeparator(); @@ -546,13 +524,13 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, // Shift history left, append new samples std.mem.copyForwards(f32, audio_buf[0..audio_overlap], audio_buf[new_audio_per_step..]); - @memcpy(audio_buf[audio_overlap..], new_samples); + @memcpy(audio_buf[audio_overlap..], sample_buf); const audio_slice: zml.Slice = .init( .init(.{sp.chunk_audio}, .f32), std.mem.sliceAsBytes(audio_buf), ); - var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); + var audio_buffer: zml.Buffer = try .fromSliceOpts(io, platform, audio_slice, .{ .wait = false }); defer audio_buffer.deinit(); mel_step_args.set(.{ ctx.buffers.mel_spectrum, audio_buffer }); @@ -565,12 +543,9 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, conv_step_results.fill(.{ &conv_step_output, ctx.buffers.conv_state }); // Encoder step: conv stem output -> encoded chunk - var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)) * sp.dsf, .u32); - defer enc_token_index.deinit(); - enc_step_args.set(.{ ctx.buffers.encoder, conv_step_output, enc_token_index, ctx.buffers.enc_kv, ctx.buffers.enc_attention_metadata }); ctx.exes.encoder_step.call(enc_step_args, &enc_step_results); - enc_step_results.fill(.{ &enc_step_output, ctx.buffers.enc_kv }); + enc_step_results.fill(.{ &enc_step_output, ctx.buffers.enc_kv, &enc_token_index }); // Adapter step: encoded chunk -> 1 audio embedding adp_step_args.set(.{ ctx.buffers.adapter, enc_step_output }); @@ -578,12 +553,9 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, adp_step_results.fill(.{&adapter_step_output}); // Decoder decode: 1 token + 1 audio embedding -> next token - var dec_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, @intCast(t)), .u32); - defer dec_token_index.deinit(); - decode_args.set(.{ ctx.buffers.decoder, current_token_buffer, adapter_step_output, dec_token_index, ctx.buffers.dec_kv, ctx.buffers.t_cond, ctx.buffers.rng.*, ctx.buffers.dec_attention_metadata }); ctx.exes.decoder_decode.call(decode_args, &decode_results); - decode_results.fill(.{ ¤t_token_buffer, ctx.buffers.dec_kv, ctx.buffers.rng }); + decode_results.fill(.{ ¤t_token_buffer, ctx.buffers.dec_kv, ctx.buffers.rng, &dec_token_index }); try current_token_buffer.toSlice(io, generated_token_slice); } @@ -689,15 +661,15 @@ pub fn runPipeline( }, }; - // 3. Mel prefill (chunk-by-chunk) - const mel_host = try runMelPrefill(&ctx, reflect_padded_audio); - defer allocator.free(mel_host); + // 3. Mel prefill (single kernel, stays on device) + var mel_device = try runMelPrefill(&ctx, reflect_padded_audio); // 4. Prefill var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); defer generated_token_slice.free(allocator); - try runPrefill(&ctx, mel_host, tokens, generated_token_slice); + try runPrefill(&ctx, mel_device, tokens, generated_token_slice); + mel_device.deinit(); // 5. Generation loop — extract audio history for sliding buffer const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp._hop_length; @@ -721,3 +693,10 @@ fn readStdinSamples(allocator: std.mem.Allocator, reader: *std.Io.Reader, n_samp } return samples; } + +fn readStdinSamplesInto(reader: *std.Io.Reader, raw_buf: []u8, samples: []f32) !void { + try reader.readSliceAll(raw_buf); + for (0..samples.len) |i| { + samples[i] = @as(f32, @floatFromInt(std.mem.bytesToValue(i16, raw_buf[i * 2 ..][0..2]))) / 32768.0; + } +} From a4b0fa9706abe0031658165c18d6f600e7f7c9b3 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 20 Feb 2026 15:19:35 +0100 Subject: [PATCH 88/98] remove padding and change convert position --- examples/voxtral/decoder.zig | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 6119469d..747124c1 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -43,13 +43,6 @@ pub const Adapter = struct { var h = encoder_out; const dsf = self.downsample_factor; - // Pad encoder output to next multiple of downsample_factor - const seq_len: usize = @intCast(h.dim(.s)); - const remainder = seq_len % dsf; - if (remainder > 0) { - h = h.pad(0, .{ .s = Tensor.Pad{ .high = @as(i64, @intCast(dsf - remainder)) } }); - } - // Reshape [s, d] → [s/dsf, d*dsf] by concatenating consecutive timesteps. h = h.splitAxis(.s, .{ .s = .auto, .group = dsf }) .merge(.{ .d = .{ .group, .d } }); @@ -121,7 +114,7 @@ pub const Decoder = struct { h = rmsNorm(h, self.norm, self.norm_eps); // Compute logits in f32 for precision - const logits = h.convert(.f32).dot(self.tok_embeddings.convert(.f32), .d); + const logits = h.dot(self.tok_embeddings, .d).convert(.f32); const output_tokens, const new_rng = zml.nn.sampleTokens(logits, .{ .temperature = 0 }, rng); return .{ output_tokens.convert(.u32), cache, new_rng }; From 6eb4288593043bf911e0f1947c6096a006be3c7e Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 20 Feb 2026 15:46:34 +0100 Subject: [PATCH 89/98] move swiglu to common --- examples/voxtral/common.zig | 26 ++++++++++++++++++++++++++ examples/voxtral/decoder.zig | 4 +--- examples/voxtral/encoder.zig | 26 +------------------------- examples/voxtral/main.zig | 4 ++-- examples/voxtral/voxtral.zig | 3 +-- 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index d5befcc9..1e412c7f 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -196,6 +196,32 @@ pub fn SelfAttention(comptime shift_buffer: bool) type { }; } +pub const SwiGluFfn = struct { + w1: zml.nn.Linear, + w2: zml.nn.Linear, + w3: zml.nn.Linear, + + pub fn init(store: zml.io.TensorStore.View) SwiGluFfn { + return .{ + .w1 = linear(store.withPrefix("w1")), + .w2 = linear(store.withPrefix("w2")), + .w3 = linear(store.withPrefix("w3")), + }; + } + + pub fn unload(self: *zml.Bufferized(SwiGluFfn)) void { + deinitBufferized(self); + } + + /// x: [s, d] -> [s, d] + pub fn forward(self: SwiGluFfn, x: Tensor) Tensor { + const gate = self.w1.forward(x).silu(); + const up = self.w3.forward(x); + + return self.w2.forward(gate.mul(up).rename(.{ .dout = .d })).rename(.{ .dout = .d }); + } +}; + /// KV cache for all layers of a transformer. /// Stores K/V tensors with shape {layer, k=max_seq_len, h, hd}. pub const KvCache = struct { diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 747124c1..e463d782 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -6,11 +6,9 @@ const Tensor = zml.Tensor; const cfg = @import("config.zig"); const Config = cfg.Config; -const enc = @import("encoder.zig"); -const SwiGluFfn = enc.SwiGluFfn; - const common = @import("common.zig"); const rmsNorm = common.rmsNorm; +const SwiGluFfn = common.SwiGluFfn; /// Adapter: encoder→decoder projection. /// Reshapes encoder output [s, d] → [s/dsf, d*dsf] (concatenate `downsample_factor` consecutive timesteps), diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 943c08db..9271a617 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -211,28 +211,4 @@ pub const TransformerLayer = struct { pub const SelfAttention = common.SelfAttention(true); -pub const SwiGluFfn = struct { - w1: zml.nn.Linear, - w2: zml.nn.Linear, - w3: zml.nn.Linear, - - pub fn init(store: zml.io.TensorStore.View) SwiGluFfn { - return .{ - .w1 = common.linear(store.withPrefix("w1")), - .w2 = common.linear(store.withPrefix("w2")), - .w3 = common.linear(store.withPrefix("w3")), - }; - } - - pub fn unload(self: *zml.Bufferized(SwiGluFfn)) void { - common.deinitBufferized(self); - } - - /// x: [s, d] -> [s, d] - pub fn forward(self: SwiGluFfn, x: Tensor) Tensor { - const gate = self.w1.forward(x).silu(); - const up = self.w3.forward(x); - - return self.w2.forward(gate.mul(up).rename(.{ .dout = .d })).rename(.{ .dout = .d }); - } -}; +pub const SwiGluFfn = common.SwiGluFfn; diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 36c97dc0..f4c6ec9f 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -43,7 +43,7 @@ pub fn main(init: std.process.Init) !void { const allocator = switch (builtin.mode) { .Debug => dbg.allocator(), - else => std.heap.c_allocator, + else => std.heap.smp_allocator, }; const args = stdx.flags.parse(init.minimal.args, CliArgs); @@ -75,7 +75,7 @@ pub fn main(init: std.process.Init) !void { defer allocator.free(tokens); var platform: *zml.Platform = try .auto(allocator, io, .{ - .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.90 } } }, + .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.80 } } }, }); defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 711efd17..a2e7b789 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -251,8 +251,7 @@ fn runMelPrefill(ctx: *PipelineContext, padded_audio: []const f32) !zml.Buffer { var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); defer audio_buffer.deinit(); - var mel_output: zml.Buffer = try .uninitialized(io, platform, - .init(.{ .channels = 128, .time = sp.prompt_len * sp.mel_per_step }, .f32), .{}); + var mel_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.prompt_len * sp.mel_per_step }, .f32), .{}); var args = try ctx.exes.mel_prefill.args(ctx.allocator); defer args.deinit(ctx.allocator); From fe7bffd606ee25ec165c988c34aea016d94b1749 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 20 Feb 2026 16:10:15 +0100 Subject: [PATCH 90/98] remove unused old code --- examples/voxtral/config.zig | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index 56abee32..8677a5c1 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -148,18 +148,4 @@ pub const StreamParams = struct { .prompt_len = 1 + n_left_pad_tokens + n_delay_tokens, }; } - - pub fn numFrames(self: StreamParams, audio_len: usize) u32 { - return @intCast(audio_len / self._hop_length); - } - - pub fn totalSteps(self: StreamParams, audio_len: usize) u32 { - const nf = self.numFrames(audio_len); - const encoder_seq_len = (nf + 1) / 2; - return (encoder_seq_len + self.dsf - 1) / self.dsf; - } - - pub fn paddedMelFrames(self: StreamParams, audio_len: usize) u32 { - return self.totalSteps(audio_len) * self.mel_per_step; - } }; From 6cd9781f50fd368251c97882a751061ea41d10bb Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 20 Feb 2026 16:45:31 +0100 Subject: [PATCH 91/98] one func for prefill and loop --- examples/voxtral/voxtral.zig | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index a2e7b789..a7f007e5 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -601,8 +601,11 @@ pub fn runPipeline( const reflect_pad: usize = n_fft / 2; const n_prefill_stdin: usize = @as(usize, sp.n_delay_tokens + 1) * sp.raw_audio_length_per_tok; - const stdin_audio = try readStdinSamples(allocator, stdin_reader_interface, n_prefill_stdin); + const prefill_raw_buf = try allocator.alloc(u8, n_prefill_stdin * 2); + defer allocator.free(prefill_raw_buf); + const stdin_audio = try allocator.alloc(f32, n_prefill_stdin); defer allocator.free(stdin_audio); + try readStdinSamplesInto(stdin_reader_interface, prefill_raw_buf, stdin_audio); // Pad: [left_pad zeros][stdin audio][right_pad zeros] const right_pad: usize = @as(usize, sp.n_right_pad_tokens) * sp.raw_audio_length_per_tok; @@ -679,20 +682,6 @@ pub fn runPipeline( try runGenerationLoop(&ctx, tokenizer, initial_audio_history, stdin_reader_interface, generated_token_slice); } -fn readStdinSamples(allocator: std.mem.Allocator, reader: *std.Io.Reader, n_samples: usize) ![]f32 { - const byte_count = n_samples * 2; - const raw = try allocator.alloc(u8, byte_count); - defer allocator.free(raw); - try reader.readSliceAll(raw); - - const samples = try allocator.alloc(f32, n_samples); - for (0..n_samples) |i| { - const offset = i * 2; - samples[i] = @as(f32, @floatFromInt(std.mem.bytesToValue(i16, raw[offset..][0..2]))) / 32768.0; - } - return samples; -} - fn readStdinSamplesInto(reader: *std.Io.Reader, raw_buf: []u8, samples: []f32) !void { try reader.readSliceAll(raw_buf); for (0..samples.len) |i| { From 6d0794ad66a6e2d7cf92e6c63e0ebce7b5aa2870 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 20 Feb 2026 16:58:38 +0100 Subject: [PATCH 92/98] inline mel prefill and rename prompt tokens --- examples/voxtral/decoder.zig | 2 +- examples/voxtral/main.zig | 19 ++++++--- examples/voxtral/voxtral.zig | 82 +++++++++++++++--------------------- 3 files changed, 48 insertions(+), 55 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index e463d782..d5b76251 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -155,7 +155,7 @@ pub const AdaRmsNorm = struct { pub const KvCache = common.KvCache; -pub const SelfAttention = common.SelfAttention(false); +pub const SelfAttention = common.SelfAttention(true); /// Single decoder transformer layer with AdaRmsNorm time conditioning. pub const DecoderLayer = struct { diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index f4c6ec9f..191bde19 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -70,12 +70,12 @@ pub fn main(init: std.process.Init) !void { std.debug.panic("transcription_delay_ms must be a multiple of 80 in range [80, 2400], got {}", .{delay_ms}); } - // Build tokens: [BOS] ++ [STREAMING_PAD] * (n_left_pad_tokens + n_delay_tokens) - const tokens = try allocator.alloc(u32, sp.prompt_len); - defer allocator.free(tokens); + // Build prompt_tokens: [BOS] ++ [STREAMING_PAD] * (n_left_pad_tokens + n_delay_tokens) + const prompt_tokens = try allocator.alloc(u32, sp.prompt_len); + defer allocator.free(prompt_tokens); var platform: *zml.Platform = try .auto(allocator, io, .{ - .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.80 } } }, + .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.45 } } }, }); defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); @@ -86,7 +86,10 @@ pub fn main(init: std.process.Init) !void { break :b selected; }; + // -- Models + var melspectro_model: LogMelSpectrogram = .init(config); + var encoder_model: Encoder = .init(allocator, model_store.view(), config); defer encoder_model.deinit(allocator); @@ -95,6 +98,8 @@ pub fn main(init: std.process.Init) !void { var decoder_model: Decoder = .init(allocator, model_store.view(), config); defer decoder_model.deinit(allocator); + // -- END Models + const enc_cfg = config.encoder(); const enc_kv_size = args.enc_kv_size orelse enc_cfg.sliding_window; const dec_kv_size = args.dec_kv_size orelse config.sliding_window; @@ -145,8 +150,8 @@ pub fn main(init: std.process.Init) !void { const token_bos = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); const token_streaming_pad = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); - tokens[0] = token_bos; - @memset(tokens[1..], token_streaming_pad); + prompt_tokens[0] = token_bos; + @memset(prompt_tokens[1..], token_streaming_pad); // -- Compiled models var compiled_mel_step = try compiled_mel_step_future.await(io); @@ -209,7 +214,7 @@ pub fn main(init: std.process.Init) !void { platform, config, &tokenizer, - tokens, + prompt_tokens, sp, .{ .mel_step = &compiled_mel_step, diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index a7f007e5..b3bd391c 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -233,43 +233,9 @@ pub fn reflectPadAudio(allocator: std.mem.Allocator, audio: []const f32, pad: us return result; } -/// Run mel spectrogram on the full prefill audio in a single kernel call. -/// Returns mel output as a device buffer [channels=128, time=prompt_len*mel_per_step]. -fn runMelPrefill(ctx: *PipelineContext, padded_audio: []const f32) !zml.Buffer { - const io = ctx.io; - const platform = ctx.platform; - const sp = ctx.sp; - - const full_audio_len: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp._hop_length + sp.chunk_audio; - - log.info("Running mel prefill ({} audio samples -> {} mel frames)...", .{ full_audio_len, sp.prompt_len * sp.mel_per_step }); - - const audio_slice: zml.Slice = .init( - .init(.{full_audio_len}, .f32), - std.mem.sliceAsBytes(padded_audio[0..full_audio_len]), - ); - var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); - defer audio_buffer.deinit(); - - var mel_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.prompt_len * sp.mel_per_step }, .f32), .{}); - - var args = try ctx.exes.mel_prefill.args(ctx.allocator); - defer args.deinit(ctx.allocator); - var results = try ctx.exes.mel_prefill.results(ctx.allocator); - defer results.deinit(ctx.allocator); - - args.set(.{ ctx.buffers.mel_spectrum, audio_buffer }); - ctx.exes.mel_prefill.call(args, &results); - results.fill(.{&mel_output}); - - log.info("Mel prefill done ({} frames).", .{sp.prompt_len * sp.mel_per_step}); - - return mel_output; -} - -/// Run the prefill phase: conv stem -> encoder -> adapter -> decoder prefill. +/// Run the prefill phase: mel -> conv stem -> encoder -> adapter -> decoder prefill. /// Populates KV caches and produces the first generated token into generated_token_slice. -fn runPrefill(ctx: *PipelineContext, mel_device: zml.Buffer, tokens: []const u32, generated_token_slice: zml.Slice) !void { +fn runPrefill(ctx: *PipelineContext, padded_audio: []const f32, prompt_tokens: []const u32, generated_token_slice: zml.Slice) !void { // -- const allocator = ctx.allocator; const io = ctx.io; @@ -281,7 +247,33 @@ fn runPrefill(ctx: *PipelineContext, mel_device: zml.Buffer, tokens: []const u32 const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; - // Conv stem prefill: mel_device is already [128][prompt_len*mel_per_step] on device + // Mel prefill: single kernel, produces [channels=128, time=prompt_len*mel_per_step] on device + const full_audio_len: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp._hop_length + sp.chunk_audio; + log.info("Running mel prefill ({} audio samples -> {} mel frames)...", .{ full_audio_len, sp.prompt_len * sp.mel_per_step }); + var mel_prefill_output: zml.Buffer = undefined; + { + const audio_slice: zml.Slice = .init( + .init(.{full_audio_len}, .f32), + std.mem.sliceAsBytes(padded_audio[0..full_audio_len]), + ); + var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); + defer audio_buffer.deinit(); + + mel_prefill_output = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.prompt_len * sp.mel_per_step }, .f32), .{}); + + var args = try ctx.exes.mel_prefill.args(allocator); + defer args.deinit(allocator); + var results = try ctx.exes.mel_prefill.results(allocator); + defer results.deinit(allocator); + + args.set(.{ ctx.buffers.mel_spectrum, audio_buffer }); + ctx.exes.mel_prefill.call(args, &results); + results.fill(.{&mel_prefill_output}); + } + defer mel_prefill_output.deinit(); + log.info("Mel prefill done ({} frames).", .{sp.prompt_len * sp.mel_per_step}); + + // Conv stem prefill: mel output [128][prompt_len*mel_per_step] -> encoder frames // Also extracts conv states (last 2 frames of each intermediate) for streaming log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * sp.mel_per_step}); var conv_prefill_output: zml.Buffer = undefined; @@ -294,7 +286,7 @@ fn runPrefill(ctx: *PipelineContext, mel_device: zml.Buffer, tokens: []const u32 var results = try ctx.exes.conv_stem_prefill.results(allocator); defer results.deinit(allocator); - args.set(.{ ctx.buffers.encoder, mel_device }); + args.set(.{ ctx.buffers.encoder, mel_prefill_output }); ctx.exes.conv_stem_prefill.call(args, &results); results.fill(.{ &conv_prefill_output, ctx.buffers.conv_state }); } @@ -351,7 +343,7 @@ fn runPrefill(ctx: *PipelineContext, mel_device: zml.Buffer, tokens: []const u32 const token_data_slice: zml.Slice = .init( .init(.{prompt_len}, .u32), - std.mem.sliceAsBytes(tokens), + std.mem.sliceAsBytes(prompt_tokens), ); var token_buffer: zml.Buffer = try .fromSlice(io, platform, token_data_slice); defer token_buffer.deinit(); @@ -577,7 +569,7 @@ pub fn runPipeline( platform: *zml.Platform, config: Config, tokenizer: *zml.tokenizer.Tokenizer, - tokens: []const u32, + prompt_tokens: []const u32, sp: StreamParams, exes: CompiledExes, mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), @@ -663,17 +655,13 @@ pub fn runPipeline( }, }; - // 3. Mel prefill (single kernel, stays on device) - var mel_device = try runMelPrefill(&ctx, reflect_padded_audio); - - // 4. Prefill + // 3. Prefill (mel -> conv stem -> encoder -> adapter -> decoder) var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); defer generated_token_slice.free(allocator); - try runPrefill(&ctx, mel_device, tokens, generated_token_slice); - mel_device.deinit(); + try runPrefill(&ctx, reflect_padded_audio, prompt_tokens, generated_token_slice); - // 5. Generation loop — extract audio history for sliding buffer + // 4. Generation loop — extract audio history for sliding buffer const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp._hop_length; const audio_overlap: usize = sp.chunk_audio - new_audio_per_step; const history_start: usize = @as(usize, sp.prompt_len) * sp.mel_per_step * sp._hop_length; From d08fe92634a86f1d4e46231b59627638008ff9b6 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Fri, 20 Feb 2026 18:25:59 +0100 Subject: [PATCH 93/98] remove explicit right padding and prefill reflective padding --- examples/voxtral/config.zig | 4 --- examples/voxtral/main.zig | 2 +- examples/voxtral/voxtral.zig | 57 ++++++++++-------------------------- 3 files changed, 17 insertions(+), 46 deletions(-) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index 8677a5c1..5ba24f97 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -112,12 +112,10 @@ pub fn parseConfig(allocator: std.mem.Allocator, io: std.Io, model_dir: std.Io.D pub const StreamParams = struct { dsf: u32, mel_per_step: u32, - chunk_mel: u32, chunk_audio: u32, raw_audio_length_per_tok: u32, _hop_length: u32, n_delay_tokens: u32, - n_right_pad_tokens: u32, left_pad: u32, prompt_len: u32, @@ -138,12 +136,10 @@ pub const StreamParams = struct { return .{ .dsf = dsf, .mel_per_step = mel_per_step, - .chunk_mel = mel_per_step, .chunk_audio = (mel_per_step - 1) * hop_length + window_size, .raw_audio_length_per_tok = raw_audio_length_per_tok, ._hop_length = hop_length, .n_delay_tokens = n_delay_tokens, - .n_right_pad_tokens = (n_delay_tokens + 1) + 10, .left_pad = n_left_pad_tokens * raw_audio_length_per_tok, .prompt_len = 1 + n_left_pad_tokens + n_delay_tokens, }; diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index 191bde19..d2fe8e29 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -75,7 +75,7 @@ pub fn main(init: std.process.Init) !void { defer allocator.free(prompt_tokens); var platform: *zml.Platform = try .auto(allocator, io, .{ - .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.45 } } }, + .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.75 } } }, }); defer platform.deinit(allocator); log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index b3bd391c..4b3dd115 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -218,21 +218,6 @@ pub fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32 return result; } -/// Prepend n_fft/2 reflected samples at the start and append n_fft/2 zeros at the end. -/// Left reflect skips boundary sample (index 0) to match Tensor reflect padding in forward(). -/// Right zero-padding ensures STFT windows at the tail have enough samples. -pub fn reflectPadAudio(allocator: std.mem.Allocator, audio: []const f32, pad: usize) ![]f32 { - const result = try allocator.alloc(f32, pad + audio.len + pad); - // Reflect: result[i] = audio[pad - i] for i in 0..pad - // i=0 → audio[pad], i=1 → audio[pad-1], ..., i=pad-1 → audio[1] - for (0..pad) |i| { - result[i] = audio[pad - i]; - } - @memcpy(result[pad..][0..audio.len], audio); - @memset(result[pad + audio.len ..], 0); - return result; -} - /// Run the prefill phase: mel -> conv stem -> encoder -> adapter -> decoder prefill. /// Populates KV caches and produces the first generated token into generated_token_slice. fn runPrefill(ctx: *PipelineContext, padded_audio: []const f32, prompt_tokens: []const u32, generated_token_slice: zml.Slice) !void { @@ -368,7 +353,7 @@ fn runPrefill(ctx: *PipelineContext, padded_audio: []const f32, prompt_tokens: [ } /// Run the streaming decode loop: step-by-step conv_stem -> encoder -> adapter -> decoder. -fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, initial_audio_history: []const f32, stdin_reader: *std.Io.Reader, generated_token_slice: zml.Slice) !void { +fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, initial_overlap: []const f32, stdin_reader: *std.Io.Reader, generated_token_slice: zml.Slice) !void { // -- const allocator = ctx.allocator; const io = ctx.io; @@ -385,7 +370,7 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, // Sliding audio buffer: [overlap from previous step | new samples] const audio_buf = try allocator.alloc(f32, sp.chunk_audio); defer allocator.free(audio_buf); - @memcpy(audio_buf[0..audio_overlap], initial_audio_history); + @memcpy(audio_buf[0..audio_overlap], initial_overlap); @memset(audio_buf[audio_overlap..], 0); var tokenizer_decoder = try tokenizer.decoder(); @@ -395,7 +380,6 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); const streaming_word_token = tokenizer.tokenToId("[STREAMING_WORD]"); - // log.info("Running decode loop (steps {}..{})...", .{ prompt_len, total_steps }); const decode_start: std.Io.Timestamp = .now(io, .awake); var mel_step_args = try ctx.exes.mel_step.args(allocator); @@ -423,7 +407,7 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, var decode_results = try ctx.exes.decoder_decode.results(allocator); defer decode_results.deinit(allocator); - var mel_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.chunk_mel }, .f32), .{}); + var mel_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.mel_per_step }, .f32), .{}); defer mel_step_output.deinit(); var conv_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = sp.dsf, .d = enc_dim }, .bf16), .{}); @@ -588,27 +572,22 @@ pub fn runPipeline( var stdin_reader = std.Io.File.stdin().reader(io, &stdin_buff); const stdin_reader_interface = &stdin_reader.interface; - // 0. Read initial audio from stdin and build padded buffer for prefill (like loadAndPadWav) - const n_fft: usize = config.audio().window_size; - const reflect_pad: usize = n_fft / 2; - - const n_prefill_stdin: usize = @as(usize, sp.n_delay_tokens + 1) * sp.raw_audio_length_per_tok; + // 0. Read initial audio from stdin and build padded buffer for prefill + // We read the full mel buffer's worth of audio (minus left_pad) so the STFT + // overlap region at the tail has real audio instead of silence. + const full_audio_len: usize = @as(usize, sp.prompt_len - 1) * sp.mel_per_step * sp._hop_length + sp.chunk_audio; + const n_prefill_stdin: usize = full_audio_len - sp.left_pad; const prefill_raw_buf = try allocator.alloc(u8, n_prefill_stdin * 2); defer allocator.free(prefill_raw_buf); const stdin_audio = try allocator.alloc(f32, n_prefill_stdin); defer allocator.free(stdin_audio); try readStdinSamplesInto(stdin_reader_interface, prefill_raw_buf, stdin_audio); - // Pad: [left_pad zeros][stdin audio][right_pad zeros] - const right_pad: usize = @as(usize, sp.n_right_pad_tokens) * sp.raw_audio_length_per_tok; - const prefill_audio_len: usize = @as(usize, sp.left_pad) + stdin_audio.len + right_pad; - const prefill_audio = try allocator.alloc(f32, prefill_audio_len); + // Pad: [left_pad zeros | stdin audio] + const prefill_audio = try allocator.alloc(f32, full_audio_len); defer allocator.free(prefill_audio); - @memset(prefill_audio, 0); - @memcpy(prefill_audio[@as(usize, sp.left_pad)..][0..stdin_audio.len], stdin_audio); - - const reflect_padded_audio = try reflectPadAudio(allocator, prefill_audio, reflect_pad); - defer allocator.free(reflect_padded_audio); + @memset(prefill_audio[0..@as(usize, sp.left_pad)], 0); + @memcpy(prefill_audio[@as(usize, sp.left_pad)..], stdin_audio); // 1. Init KV caches + t_cond + RNG var enc_kv_buffers = try enc_kv_cache.initBuffer(io, platform); @@ -659,15 +638,11 @@ pub fn runPipeline( var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); defer generated_token_slice.free(allocator); - try runPrefill(&ctx, reflect_padded_audio, prompt_tokens, generated_token_slice); + try runPrefill(&ctx, prefill_audio, prompt_tokens, generated_token_slice); - // 4. Generation loop — extract audio history for sliding buffer - const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp._hop_length; - const audio_overlap: usize = sp.chunk_audio - new_audio_per_step; - const history_start: usize = @as(usize, sp.prompt_len) * sp.mel_per_step * sp._hop_length; - const initial_audio_history = reflect_padded_audio[history_start .. history_start + audio_overlap]; - - try runGenerationLoop(&ctx, tokenizer, initial_audio_history, stdin_reader_interface, generated_token_slice); + // 4. Generation loop — seed the overlap with the tail of the prefill audio + const audio_overlap: usize = sp.chunk_audio - @as(usize, sp.mel_per_step) * sp._hop_length; + try runGenerationLoop(&ctx, tokenizer, prefill_audio[full_audio_len - audio_overlap ..], stdin_reader_interface, generated_token_slice); } fn readStdinSamplesInto(reader: *std.Io.Reader, raw_buf: []u8, samples: []f32) !void { From 110c6dab71b33841d0cab0142e073a3f8f487deb Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 23 Feb 2026 11:11:38 +0100 Subject: [PATCH 94/98] start to comment code --- examples/voxtral/common.zig | 31 ++--------------- examples/voxtral/config.zig | 4 +-- examples/voxtral/decoder.zig | 12 ++++--- examples/voxtral/encoder.zig | 8 ++--- examples/voxtral/main.zig | 6 ---- examples/voxtral/mel_spectrogram.zig | 5 +-- examples/voxtral/voxtral.zig | 52 +++++++++++++++++++--------- 7 files changed, 55 insertions(+), 63 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index 1e412c7f..896f294c 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -64,10 +64,9 @@ pub fn loadModel( } /// Unified self-attention with KV cache. -/// When `shift_buffer` is true, the KV cache uses shift-based overflow handling for -/// unbounded sliding window support. Both paths (normal write vs shifted cache write) -/// are computed and the correct one is selected via cmp/select. The cache stays in -/// temporal order — no reordering needed. +/// When `shift_buffer` is true, the KV cache uses a circular (ring) buffer for unbounded +/// sliding window support: keys/values are written at position `pos_index % cache_size`, +/// then reordered into temporal order before attention. /// When false, the cache uses sequential indexing (matching the reference implementation). pub fn SelfAttention(comptime shift_buffer: bool) type { return struct { @@ -126,8 +125,6 @@ pub fn SelfAttention(comptime shift_buffer: bool) type { if (shift_buffer) { - // -- Circular implementation - const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); k = new_kv_cache.keys().convert(dtype); @@ -153,28 +150,6 @@ pub fn SelfAttention(comptime shift_buffer: bool) type { .sliding_window = @intCast(attn_config.sliding_window), }); - // -- END Cicrular implem - - // -- Shift implem - - // const shifted_cache, const clamped_token_index = kv_cache.shiftIfNeeded(token_index, @intCast(x.dim(.s))); - - // const write_pos = b: { - // const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) - // .withTags(.{.s}).broad(Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); - // break :b temp.add(clamped_token_index.broad(temp.shape())); - // }; - - // const new_kv_cache = shifted_cache.update(k, v, write_pos.rename(.{ .s = .k })); - // k = new_kv_cache.keys().convert(dtype); - // v = new_kv_cache.values().convert(dtype); - - // const attn_out = zml.attention.attention(q, k, v, clamped_token_index, attention_metadata, attention_parameters, .{ - // .sliding_window = @intCast(attn_config.sliding_window), - // }); - - // -- END Shift implem - const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index 5ba24f97..e057fa04 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -114,7 +114,7 @@ pub const StreamParams = struct { mel_per_step: u32, chunk_audio: u32, raw_audio_length_per_tok: u32, - _hop_length: u32, + hop_length: u32, n_delay_tokens: u32, left_pad: u32, prompt_len: u32, @@ -138,7 +138,7 @@ pub const StreamParams = struct { .mel_per_step = mel_per_step, .chunk_audio = (mel_per_step - 1) * hop_length + window_size, .raw_audio_length_per_tok = raw_audio_length_per_tok, - ._hop_length = hop_length, + .hop_length = hop_length, .n_delay_tokens = n_delay_tokens, .left_pad = n_left_pad_tokens * raw_audio_length_per_tok, .prompt_len = 1 + n_left_pad_tokens + n_delay_tokens, diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index d5b76251..17023342 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -36,7 +36,8 @@ pub const Adapter = struct { common.deinitBufferized(self); } - /// encoder_out: [s, d] → [s/dsf, dim] + /// Prefill: encoder_out [s=prompt_len*dsf, d=enc_dim] → [s=prompt_len, d=dim] + /// Step: encoder_out [s=dsf, d=enc_dim] → [s=1, d=dim] pub fn forward(self: Adapter, encoder_out: Tensor) Tensor { var h = encoder_out; const dsf = self.downsample_factor; @@ -87,9 +88,9 @@ pub const Decoder = struct { allocator.free(self.layers); } - /// Run all decoder layers on input embeddings, sample next tokens and return updated KV cache + RNG. - /// text_tokens: [s] u32, audio_embed: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d], rng: Rng - /// Returns: (tokens [s] u32, KvCache, Rng) + /// Runs all decoder layers on input embeddings, samples next tokens and returns updated KV cache + RNG. + /// Prefill: text_tokens [s=prompt_len] u32, audio_embed [s=prompt_len, d=dim] → (tokens [s=prompt_len] u32, KvCache, Rng) + /// Step: text_tokens [s=1] u32, audio_embed [s=1, d=dim] → (tokens [s=1] u32, KvCache, Rng) pub fn forward(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor.Rng } { const text_embeds = self.tok_embeddings.gather(.{ .voc = text_tokens }, .{}); const input_embeds = text_embeds.add(audio_embed); @@ -118,7 +119,8 @@ pub const Decoder = struct { return .{ output_tokens.convert(.u32), cache, new_rng }; } - /// Like forward, but also returns the incremented token index (on-device). + /// Like forward (step: text_tokens [s=1], audio_embed [s=1, d=dim]), but also returns the incremented token index. + /// Returns: (tokens [s=1] u32, KvCache, Rng, token_index scalar u32) pub fn forwardStep(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor.Rng, Tensor } { const output_tokens, const cache, const new_rng = self.forward(text_tokens, audio_embed, token_index, kv_cache, t_cond, rng, attention_metadata, attention_parameters); const next_index = token_index.addConstant(1).reuseBuffer(token_index); diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index 9271a617..ec923cf0 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -79,7 +79,7 @@ pub const Encoder = struct { /// Prefill conv stem: runs conv on all prefill mel frames and extracts /// conv states (last kernel_size-1=2 frames of each intermediate) for subsequent streaming. /// Uses padding={2,0} to match reference zero-initialized state (kernel_size-1=2). - /// mel: [channels=128, time=frames] → ([s=(frames+1)/2, d=enc_dim], ConvState) + /// Prefill: mel [channels=128, time=prompt_len*mel_per_step] → ([s=prompt_len*dsf, d=enc_dim] bf16, ConvState) pub fn convStemPrefill(self: Encoder, mel: Tensor) struct { Tensor, ConvState } { const dtype = self.conv0_weight.dtype(); const h = mel.convert(dtype).insertAxes(.channels, .{.batch}); @@ -141,9 +141,9 @@ pub const Encoder = struct { } }; } - /// Process a chunk of conv stem output through all transformer layers with KV cache. - /// x: [s=dsf, d=1280], token_index: scalar, kv_cache: KvCache - /// Returns: ([s=dsf, d=1280], KvCache) + /// Runs all encoder transformer layers on conv stem output with KV cache. + /// Prefill: x [s=prompt_len*dsf, d=enc_dim] → ([s=prompt_len*dsf, d=enc_dim], KvCache) + /// Step: x [s=dsf, d=enc_dim] → ([s=dsf, d=enc_dim], KvCache) pub fn transformer(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { var h = x; var cache = kv_cache; diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig index d2fe8e29..e617e7eb 100644 --- a/examples/voxtral/main.zig +++ b/examples/voxtral/main.zig @@ -86,8 +86,6 @@ pub fn main(init: std.process.Init) !void { break :b selected; }; - // -- Models - var melspectro_model: LogMelSpectrogram = .init(config); var encoder_model: Encoder = .init(allocator, model_store.view(), config); @@ -98,8 +96,6 @@ pub fn main(init: std.process.Init) !void { var decoder_model: Decoder = .init(allocator, model_store.view(), config); defer decoder_model.deinit(allocator); - // -- END Models - const enc_cfg = config.encoder(); const enc_kv_size = args.enc_kv_size orelse enc_cfg.sliding_window; const dec_kv_size = args.dec_kv_size orelse config.sliding_window; @@ -153,7 +149,6 @@ pub fn main(init: std.process.Init) !void { prompt_tokens[0] = token_bos; @memset(prompt_tokens[1..], token_streaming_pad); - // -- Compiled models var compiled_mel_step = try compiled_mel_step_future.await(io); defer compiled_mel_step.deinit(); @@ -182,7 +177,6 @@ pub fn main(init: std.process.Init) !void { defer compiled_decoder_prefill.deinit(); defer compiled_decoder_decode.deinit(); - // -- Buffers var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); defer LogMelSpectrogram.unload(&mel_spectrum_buffers); diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig index a884a8e7..c3650170 100644 --- a/examples/voxtral/mel_spectrogram.zig +++ b/examples/voxtral/mel_spectrogram.zig @@ -29,9 +29,10 @@ pub const LogMelSpectrogram = struct { }; } - /// Like forward(), but for fixed-size audio chunks with reflect padding done on host. + /// Computes the log mel spectrogram for a fixed-size audio chunk (reflect padding done on host). /// Produces exactly (audio_len - n_fft) / hop_len + 1 mel frames. - /// Output tags: .channels and .time (matching conv stem input). + /// Prefill: audio_chunk [samples=(prompt_len-1)*mel_per_step*hop_length + chunk_audio] → [channels=128, time=prompt_len*mel_per_step] f32 + /// Step: audio_chunk [samples=chunk_audio] → [channels=128, time=mel_per_step] f32 pub fn melStep(self: LogMelSpectrogram, audio_chunk: Tensor) Tensor { const dtype = audio_chunk.dtype(); diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 4b3dd115..af853373 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -68,23 +68,35 @@ fn compileStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Pl return try platform.compile(allocator, io, model, method, inputs); } +/// Compiles LogMelSpectrogram.melStep for streaming (one chunk at a time). +/// Input: [samples=chunk_audio] f32 +/// Output: [channels=128, time=mel_per_step] f32 pub fn compileMelStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { return compileStep(allocator, io, platform, model, .melStep, .{Tensor.init(.{sp.chunk_audio}, .f32).withTags(.{.samples})}, progress, "mel step"); } +/// Compiles LogMelSpectrogram.melStep for the full prefill audio. +/// Input: [samples=(prompt_len-1)*mel_per_step*hop_length + chunk_audio] f32 +/// Output: [channels=128, time=prompt_len*mel_per_step] f32 pub fn compileMelPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { - const full_audio: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp._hop_length + sp.chunk_audio; + const full_audio: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp.hop_length + sp.chunk_audio; return compileStep(allocator, io, platform, model, .melStep, .{ Tensor.init(.{full_audio}, .f32).withTags(.{.samples}), }, progress, "mel prefill"); } +/// Compiles Encoder.convStemPrefill for the full prefill mel spectrogram. +/// Input: [channels=128, time=num_mel_frames] f32 (num_mel_frames = prompt_len*mel_per_step) +/// Output: ([s=num_mel_frames/2=prompt_len*dsf, d=enc_dim] bf16, ConvState) pub fn compileConvStemPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, num_mel_frames: u32, progress: *std.Progress.Node) !zml.Exe { return compileStep(allocator, io, platform, model, .convStemPrefill, .{ Tensor.init(.{ .channels = 128, .time = num_mel_frames }, .f32), }, progress, "conv stem prefill"); } +/// Compiles Encoder.convStemStep for one streaming step. +/// Input: [channels=128, time=mel_per_step] f32, ConvState +/// Output: ([s=dsf, d=enc_dim] bf16, ConvState) pub fn compileConvStemStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { const dtype = model.conv0_weight.dtype(); const enc_dim = model.config.encoder().dim; @@ -97,7 +109,9 @@ pub fn compileConvStemStep(allocator: std.mem.Allocator, io: std.Io, platform: * }, progress, "conv stem step"); } -/// Encoder prefill: processes prompt_len * dsf frames at once, populating the KV cache. +/// Compiles Encoder.transformer for prefill: processes prompt_len*dsf frames, populating the KV cache. +/// Input: x [s=prompt_len*dsf, d=enc_dim] bf16, token_index scalar u32 +/// Output: ([s=prompt_len*dsf, d=enc_dim] bf16, KvCache) pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, prompt_len: u32, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { const enc_cfg = model.config.encoder(); const dsf = model.config.downsample_factor(); @@ -110,7 +124,9 @@ pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: }, progress, "encoder prefill"); } -/// Encoder step: processes dsf frames with KV cache (for streaming decode). +/// Compiles Encoder.transformerStep for streaming: processes dsf frames using the KV cache. +/// Input: x [s=dsf, d=enc_dim] bf16, token_index scalar u32 +/// Output: ([s=dsf, d=enc_dim] bf16, KvCache, token_index scalar u32) pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { const enc_cfg = model.config.encoder(); const dsf = model.config.downsample_factor(); @@ -123,7 +139,9 @@ pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *c }, progress, "encoder step"); } -/// Adapter batch: processes prompt_len * dsf encoder frames -> prompt_len embeddings. +/// Compiles Adapter.forward for prefill: maps prompt_len*dsf encoder frames to prompt_len decoder embeddings. +/// Input: [s=prompt_len*dsf, d=enc_dim] bf16 +/// Output: [s=prompt_len, d=dim] bf16 pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, prompt_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { const dsf = config.downsample_factor(); return compileStep(allocator, io, platform, model, .forward, .{ @@ -131,7 +149,9 @@ pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const }, progress, "adapter"); } -/// Adapter step: processes dsf encoder frames -> 1 embedding (for streaming decode). +/// Compiles Adapter.forward for streaming: maps dsf encoder frames to 1 decoder embedding. +/// Input: [s=dsf, d=enc_dim] bf16 +/// Output: [s=1, d=dim] bf16 pub fn compileAdapterStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, config: Config, progress: *std.Progress.Node) !zml.Exe { const dsf = config.downsample_factor(); return compileStep(allocator, io, platform, model, .forward, .{ @@ -139,7 +159,9 @@ pub fn compileAdapterStep(allocator: std.mem.Allocator, io: std.Io, platform: *c }, progress, "adapter step"); } -/// Decoder prefill (s=prompt_len) + decode (s=1), compiled concurrently. +/// Compiles Decoder.forward (prefill) and Decoder.forwardStep (decode step) concurrently. +/// Prefill: text_tokens [s=prompt_len] u32, audio_embed [s=prompt_len, d=dim] → (tokens [s=prompt_len] u32, KvCache, Rng) +/// Decode: text_tokens [s=1] u32, audio_embed [s=1, d=dim] → (token [s=1] u32, KvCache, Rng, token_index scalar u32) pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, prompt_len: u32, dec_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !struct { zml.Exe, zml.Exe } { const dim = model.config.dim; @@ -221,19 +243,17 @@ pub fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32 /// Run the prefill phase: mel -> conv stem -> encoder -> adapter -> decoder prefill. /// Populates KV caches and produces the first generated token into generated_token_slice. fn runPrefill(ctx: *PipelineContext, padded_audio: []const f32, prompt_tokens: []const u32, generated_token_slice: zml.Slice) !void { - // -- const allocator = ctx.allocator; const io = ctx.io; const platform = ctx.platform; const config = ctx.config; const sp = ctx.sp; - // -- const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; // Mel prefill: single kernel, produces [channels=128, time=prompt_len*mel_per_step] on device - const full_audio_len: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp._hop_length + sp.chunk_audio; + const full_audio_len: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp.hop_length + sp.chunk_audio; log.info("Running mel prefill ({} audio samples -> {} mel frames)...", .{ full_audio_len, sp.prompt_len * sp.mel_per_step }); var mel_prefill_output: zml.Buffer = undefined; { @@ -258,7 +278,7 @@ fn runPrefill(ctx: *PipelineContext, padded_audio: []const f32, prompt_tokens: [ defer mel_prefill_output.deinit(); log.info("Mel prefill done ({} frames).", .{sp.prompt_len * sp.mel_per_step}); - // Conv stem prefill: mel output [128][prompt_len*mel_per_step] -> encoder frames + // Conv stem prefill: [channels=128, time=prompt_len*mel_per_step=prompt_len*(dsf*2)] -> [s=prompt_len*dsf, d=enc_dim] // Also extracts conv states (last 2 frames of each intermediate) for streaming log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * sp.mel_per_step}); var conv_prefill_output: zml.Buffer = undefined; @@ -354,17 +374,15 @@ fn runPrefill(ctx: *PipelineContext, padded_audio: []const f32, prompt_tokens: [ /// Run the streaming decode loop: step-by-step conv_stem -> encoder -> adapter -> decoder. fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, initial_overlap: []const f32, stdin_reader: *std.Io.Reader, generated_token_slice: zml.Slice) !void { - // -- const allocator = ctx.allocator; const io = ctx.io; const platform = ctx.platform; const config = ctx.config; const sp = ctx.sp; - // -- const enc_dim: u32 = config.encoder().dim; const prompt_len = sp.prompt_len; - const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp._hop_length; + const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp.hop_length; const audio_overlap: usize = sp.chunk_audio - new_audio_per_step; // Sliding audio buffer: [overlap from previous step | new samples] @@ -473,7 +491,7 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, if (generated_token == eos_token) break; - // Mel step: read new audio, slide history buffer, upload full chunk + // Read new audio from stdin readStdinSamplesInto(stdin_reader, raw_buf, sample_buf) catch |err| switch (err) { error.EndOfStream => break, else => return err, @@ -575,7 +593,7 @@ pub fn runPipeline( // 0. Read initial audio from stdin and build padded buffer for prefill // We read the full mel buffer's worth of audio (minus left_pad) so the STFT // overlap region at the tail has real audio instead of silence. - const full_audio_len: usize = @as(usize, sp.prompt_len - 1) * sp.mel_per_step * sp._hop_length + sp.chunk_audio; + const full_audio_len: usize = @as(usize, sp.prompt_len - 1) * sp.mel_per_step * sp.hop_length + sp.chunk_audio; const n_prefill_stdin: usize = full_audio_len - sp.left_pad; const prefill_raw_buf = try allocator.alloc(u8, n_prefill_stdin * 2); defer allocator.free(prefill_raw_buf); @@ -589,6 +607,8 @@ pub fn runPipeline( @memset(prefill_audio[0..@as(usize, sp.left_pad)], 0); @memcpy(prefill_audio[@as(usize, sp.left_pad)..], stdin_audio); + // full_audio_len: 50160 | n_prefill_stdin: 9200 + // 1. Init KV caches + t_cond + RNG var enc_kv_buffers = try enc_kv_cache.initBuffer(io, platform); defer KvCache.deinitBuffer(&enc_kv_buffers); @@ -641,7 +661,7 @@ pub fn runPipeline( try runPrefill(&ctx, prefill_audio, prompt_tokens, generated_token_slice); // 4. Generation loop — seed the overlap with the tail of the prefill audio - const audio_overlap: usize = sp.chunk_audio - @as(usize, sp.mel_per_step) * sp._hop_length; + const audio_overlap: usize = sp.chunk_audio - @as(usize, sp.mel_per_step) * sp.hop_length; try runGenerationLoop(&ctx, tokenizer, prefill_audio[full_audio_len - audio_overlap ..], stdin_reader_interface, generated_token_slice); } From e59325497bd1103a084782e07ddd4bc30f725f2d Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 23 Feb 2026 11:15:29 +0100 Subject: [PATCH 95/98] centralize into sp --- examples/voxtral/config.zig | 15 +++++++++++++-- examples/voxtral/voxtral.zig | 14 +++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig index e057fa04..a52947dd 100644 --- a/examples/voxtral/config.zig +++ b/examples/voxtral/config.zig @@ -113,11 +113,15 @@ pub const StreamParams = struct { dsf: u32, mel_per_step: u32, chunk_audio: u32, + new_audio_per_step: u32, // mel_per_step * hop_length: new samples consumed per streaming step + audio_overlap: u32, // chunk_audio - new_audio_per_step: STFT tail reused between steps + prefill_audio_len: u32, // (prompt_len-1)*new_audio_per_step + chunk_audio: total prefill samples raw_audio_length_per_tok: u32, hop_length: u32, n_delay_tokens: u32, left_pad: u32, prompt_len: u32, + enc_dim: u32, pub fn init(config: Config, transcription_delay_ms: f32, n_left_pad_tokens: u32) StreamParams { const sample_rate: f32 = @floatFromInt(config.audio().sampling_rate); @@ -132,16 +136,23 @@ pub const StreamParams = struct { const dsf = config.downsample_factor(); const mel_per_step = dsf * 2; const window_size = config.audio().window_size; + const chunk_audio = (mel_per_step - 1) * hop_length + window_size; + const new_audio_per_step = mel_per_step * hop_length; + const prompt_len = 1 + n_left_pad_tokens + n_delay_tokens; return .{ .dsf = dsf, .mel_per_step = mel_per_step, - .chunk_audio = (mel_per_step - 1) * hop_length + window_size, + .chunk_audio = chunk_audio, + .new_audio_per_step = new_audio_per_step, + .audio_overlap = chunk_audio - new_audio_per_step, + .prefill_audio_len = (prompt_len - 1) * new_audio_per_step + chunk_audio, .raw_audio_length_per_tok = raw_audio_length_per_tok, .hop_length = hop_length, .n_delay_tokens = n_delay_tokens, .left_pad = n_left_pad_tokens * raw_audio_length_per_tok, - .prompt_len = 1 + n_left_pad_tokens + n_delay_tokens, + .prompt_len = prompt_len, + .enc_dim = config.encoder().dim, }; } }; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index af853373..259c64b3 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -249,11 +249,11 @@ fn runPrefill(ctx: *PipelineContext, padded_audio: []const f32, prompt_tokens: [ const config = ctx.config; const sp = ctx.sp; - const enc_dim: u32 = config.encoder().dim; + const enc_dim = sp.enc_dim; const prompt_len = sp.prompt_len; // Mel prefill: single kernel, produces [channels=128, time=prompt_len*mel_per_step] on device - const full_audio_len: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp.hop_length + sp.chunk_audio; + const full_audio_len = sp.prefill_audio_len; log.info("Running mel prefill ({} audio samples -> {} mel frames)...", .{ full_audio_len, sp.prompt_len * sp.mel_per_step }); var mel_prefill_output: zml.Buffer = undefined; { @@ -380,10 +380,10 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, const config = ctx.config; const sp = ctx.sp; - const enc_dim: u32 = config.encoder().dim; + const enc_dim = sp.enc_dim; const prompt_len = sp.prompt_len; - const new_audio_per_step: usize = @as(usize, sp.mel_per_step) * sp.hop_length; - const audio_overlap: usize = sp.chunk_audio - new_audio_per_step; + const new_audio_per_step = sp.new_audio_per_step; + const audio_overlap = sp.audio_overlap; // Sliding audio buffer: [overlap from previous step | new samples] const audio_buf = try allocator.alloc(f32, sp.chunk_audio); @@ -593,7 +593,7 @@ pub fn runPipeline( // 0. Read initial audio from stdin and build padded buffer for prefill // We read the full mel buffer's worth of audio (minus left_pad) so the STFT // overlap region at the tail has real audio instead of silence. - const full_audio_len: usize = @as(usize, sp.prompt_len - 1) * sp.mel_per_step * sp.hop_length + sp.chunk_audio; + const full_audio_len = sp.prefill_audio_len; const n_prefill_stdin: usize = full_audio_len - sp.left_pad; const prefill_raw_buf = try allocator.alloc(u8, n_prefill_stdin * 2); defer allocator.free(prefill_raw_buf); @@ -661,7 +661,7 @@ pub fn runPipeline( try runPrefill(&ctx, prefill_audio, prompt_tokens, generated_token_slice); // 4. Generation loop — seed the overlap with the tail of the prefill audio - const audio_overlap: usize = sp.chunk_audio - @as(usize, sp.mel_per_step) * sp.hop_length; + const audio_overlap = sp.audio_overlap; try runGenerationLoop(&ctx, tokenizer, prefill_audio[full_audio_len - audio_overlap ..], stdin_reader_interface, generated_token_slice); } From 69c2ac8f3c794fa20236ed4264ddd97457c03386 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 23 Feb 2026 11:17:20 +0100 Subject: [PATCH 96/98] replace deprecated function --- examples/voxtral/voxtral.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 259c64b3..37872e83 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -516,7 +516,7 @@ fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, wave.flush(); // Shift history left, append new samples - std.mem.copyForwards(f32, audio_buf[0..audio_overlap], audio_buf[new_audio_per_step..]); + @memmove(audio_buf[0..audio_overlap], audio_buf[new_audio_per_step..]); @memcpy(audio_buf[audio_overlap..], sample_buf); const audio_slice: zml.Slice = .init( From 9f98b994f0f058fd3266521ac9da6a1130e20279 Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 23 Feb 2026 11:29:37 +0100 Subject: [PATCH 97/98] merge step functions --- examples/voxtral/decoder.zig | 19 ++++++------------- examples/voxtral/encoder.zig | 17 ++++++----------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig index 17023342..c72d76b2 100644 --- a/examples/voxtral/decoder.zig +++ b/examples/voxtral/decoder.zig @@ -88,10 +88,10 @@ pub const Decoder = struct { allocator.free(self.layers); } - /// Runs all decoder layers on input embeddings, samples next tokens and returns updated KV cache + RNG. - /// Prefill: text_tokens [s=prompt_len] u32, audio_embed [s=prompt_len, d=dim] → (tokens [s=prompt_len] u32, KvCache, Rng) - /// Step: text_tokens [s=1] u32, audio_embed [s=1, d=dim] → (tokens [s=1] u32, KvCache, Rng) - pub fn forward(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor.Rng } { + /// Runs all decoder layers on input embeddings, samples next tokens and returns updated KV cache + RNG + incremented token index. + /// Prefill: text_tokens [s=prompt_len] u32, audio_embed [s=prompt_len, d=dim] → (tokens [s=prompt_len] u32, KvCache, Rng, next_token_index) + /// Step: text_tokens [s=1] u32, audio_embed [s=1, d=dim] → (tokens [s=1] u32, KvCache, Rng, next_token_index) + pub fn forwardStep(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor.Rng, Tensor } { const text_embeds = self.tok_embeddings.gather(.{ .voc = text_tokens }, .{}); const input_embeds = text_embeds.add(audio_embed); @@ -115,16 +115,9 @@ pub const Decoder = struct { // Compute logits in f32 for precision const logits = h.dot(self.tok_embeddings, .d).convert(.f32); const output_tokens, const new_rng = zml.nn.sampleTokens(logits, .{ .temperature = 0 }, rng); - - return .{ output_tokens.convert(.u32), cache, new_rng }; - } - - /// Like forward (step: text_tokens [s=1], audio_embed [s=1, d=dim]), but also returns the incremented token index. - /// Returns: (tokens [s=1] u32, KvCache, Rng, token_index scalar u32) - pub fn forwardStep(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor.Rng, Tensor } { - const output_tokens, const cache, const new_rng = self.forward(text_tokens, audio_embed, token_index, kv_cache, t_cond, rng, attention_metadata, attention_parameters); const next_index = token_index.addConstant(1).reuseBuffer(token_index); - return .{ output_tokens, cache, new_rng, next_index }; + + return .{ output_tokens.convert(.u32), cache, new_rng, next_index }; } }; diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig index ec923cf0..e1b0091d 100644 --- a/examples/voxtral/encoder.zig +++ b/examples/voxtral/encoder.zig @@ -79,7 +79,7 @@ pub const Encoder = struct { /// Prefill conv stem: runs conv on all prefill mel frames and extracts /// conv states (last kernel_size-1=2 frames of each intermediate) for subsequent streaming. /// Uses padding={2,0} to match reference zero-initialized state (kernel_size-1=2). - /// Prefill: mel [channels=128, time=prompt_len*mel_per_step] → ([s=prompt_len*dsf, d=enc_dim] bf16, ConvState) + /// Prefill: mel [channels=128, time=prompt_len*mel_per_step=prompt_len*(dsf*2)] → ([s=prompt_len*dsf, d=enc_dim] bf16, ConvState) pub fn convStemPrefill(self: Encoder, mel: Tensor) struct { Tensor, ConvState } { const dtype = self.conv0_weight.dtype(); const h = mel.convert(dtype).insertAxes(.channels, .{.batch}); @@ -109,7 +109,7 @@ pub const Encoder = struct { /// Streaming conv stem: processes mel_per_step mel frames with explicit conv states. /// Concatenates state (2 frames of left context) with new input, runs conv1d (no padding), - /// then saves the tail as new state. Matches the reference ExecuTorch implementation. + /// then saves the tail as new state. /// mel_chunk: [channels=128, time=mel_per_step], conv_state: ConvState /// Returns: ([s=dsf, d=enc_dim], new ConvState) pub fn convStemStep(self: Encoder, mel_chunk: Tensor, conv_state: ConvState) struct { Tensor, ConvState } { @@ -142,9 +142,9 @@ pub const Encoder = struct { } /// Runs all encoder transformer layers on conv stem output with KV cache. - /// Prefill: x [s=prompt_len*dsf, d=enc_dim] → ([s=prompt_len*dsf, d=enc_dim], KvCache) - /// Step: x [s=dsf, d=enc_dim] → ([s=dsf, d=enc_dim], KvCache) - pub fn transformer(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { + /// Prefill: x [s=prompt_len*dsf, d=enc_dim] → ([s=prompt_len*dsf, d=enc_dim], KvCache, next_token_index) + /// Step: x [s=dsf, d=enc_dim] → ([s=dsf, d=enc_dim], KvCache, next_token_index) + pub fn transformerStep(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor } { var h = x; var cache = kv_cache; const attn_config = self.config.encoder().attentionConfig(); @@ -157,12 +157,7 @@ pub const Encoder = struct { cache = updated_layer_cache.reuseBuffer(layer_cache); } - return .{ rmsNorm(h, self.norm, self.norm_eps), cache }; - } - - /// Like transformer, but also returns the incremented token index (on-device). - pub fn transformerStep(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor } { - const output, const cache = self.transformer(x, token_index, kv_cache, attention_metadata, attention_parameters); + const output = rmsNorm(h, self.norm, self.norm_eps); const next_index = token_index.addConstant(self.config.downsample_factor()).reuseBuffer(token_index); return .{ output, cache, next_index }; } From 9b7e2caea31a3b5d88afb1604720b085ce75305e Mon Sep 17 00:00:00 2001 From: Raphael NAHOUM Date: Mon, 23 Feb 2026 11:36:43 +0100 Subject: [PATCH 98/98] rename shift to circular and update comment --- examples/voxtral/common.zig | 16 ++++++++-------- examples/voxtral/mel_spectrogram.zig | 3 +-- examples/voxtral/voxtral.zig | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig index 896f294c..00479949 100644 --- a/examples/voxtral/common.zig +++ b/examples/voxtral/common.zig @@ -64,11 +64,11 @@ pub fn loadModel( } /// Unified self-attention with KV cache. -/// When `shift_buffer` is true, the KV cache uses a circular (ring) buffer for unbounded +/// When `circular_buffer` is true, the KV cache uses a circular (ring) buffer for unbounded /// sliding window support: keys/values are written at position `pos_index % cache_size`, /// then reordered into temporal order before attention. -/// When false, the cache uses sequential indexing (matching the reference implementation). -pub fn SelfAttention(comptime shift_buffer: bool) type { +/// When false, the cache uses sequential indexing. +pub fn SelfAttention(comptime circular_buffer: bool) type { return struct { wq: zml.nn.Linear, wk: zml.nn.Linear, @@ -123,16 +123,15 @@ pub fn SelfAttention(comptime shift_buffer: bool) type { const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); - if (shift_buffer) { - + if (circular_buffer) { const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); // Reorder K/V from circular buffer to temporal order for correct causal masking. - // When the cache wraps, physical order != temporal order, which breaks FA2's - // position-based causal mask for multi-token (q_len > 1) attention. + // When the cache wraps, physical order != temporal order, which breaks FA's + // position-based causal mask. const pos_end = token_index.addConstant(x.dim(.s)); const is_full = pos_end.cmp(.GE, cache_size); const rotation_start = pos_end.remainder(cache_size); @@ -154,7 +153,7 @@ pub fn SelfAttention(comptime shift_buffer: bool) type { return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; } else { - // Sequential: write at pos_index directly (matching reference implementation). + // Sequential: write at pos_index directly. const new_kv_cache = kv_cache.update(k, v, pos_index.rename(.{ .s = .k })); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); @@ -261,6 +260,7 @@ pub const KvCache = struct { cache.scatterSlices(.{ .layer = layer_index }, converted, scatter_opts).reuseBuffer(cache); } + // Unused but works for shifted kv cache pub fn shiftIfNeeded(self: KvCache, token_index: Tensor, seq_len: u32) struct { KvCache, Tensor } { const cache_k_size = self.k.dim(.k); const max_pos = Tensor.scalar(@as(u32, @intCast(cache_k_size - seq_len)), token_index.dtype()); diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig index c3650170..866190b6 100644 --- a/examples/voxtral/mel_spectrogram.zig +++ b/examples/voxtral/mel_spectrogram.zig @@ -29,8 +29,7 @@ pub const LogMelSpectrogram = struct { }; } - /// Computes the log mel spectrogram for a fixed-size audio chunk (reflect padding done on host). - /// Produces exactly (audio_len - n_fft) / hop_len + 1 mel frames. + /// Computes the log mel spectrogram for a fixed-size audio chunk. /// Prefill: audio_chunk [samples=(prompt_len-1)*mel_per_step*hop_length + chunk_audio] → [channels=128, time=prompt_len*mel_per_step] f32 /// Step: audio_chunk [samples=chunk_audio] → [channels=128, time=mel_per_step] f32 pub fn melStep(self: LogMelSpectrogram, audio_chunk: Tensor) Tensor { diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig index 37872e83..a3823670 100644 --- a/examples/voxtral/voxtral.zig +++ b/examples/voxtral/voxtral.zig @@ -115,7 +115,7 @@ pub fn compileConvStemStep(allocator: std.mem.Allocator, io: std.Io, platform: * pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, prompt_len: u32, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { const enc_cfg = model.config.encoder(); const dsf = model.config.downsample_factor(); - return compileStep(allocator, io, platform, model, .transformer, .{ + return compileStep(allocator, io, platform, model, .transformerStep, .{ Tensor.init(.{ .s = prompt_len * dsf, .d = enc_cfg.dim }, .bf16), Tensor.init(.{}, .u32), enc_kv_cache, @@ -167,7 +167,7 @@ pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const const compilePrefill = struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, s: u32, kv_cache_: KvCache, dim_: u32, attention_metadata_: zml.attention.Metadata, attention_parameters_: zml.attention.Parameters, progress_: *std.Progress.Node) !zml.Exe { - return compileStep(allocator_, io_, platform_, model_, .forward, .{ + return compileStep(allocator_, io_, platform_, model_, .forwardStep, .{ Tensor.init(.{ .s = s }, .u32), Tensor.init(.{ .s = s, .d = dim_ }, .bf16), Tensor.init(.{}, .u32),