From 7b0a0d844c885974760882867b54e3ab38469466 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Wed, 29 Jan 2025 16:01:39 +0100 Subject: [PATCH] WIP --- examples/MODULE.bazel | 19 +++++++++++++++++++ examples/llama/BUILD.bazel | 18 ++++++++++++++++++ examples/llama/llama.zig | 14 ++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index ea66f7a1..41f0eaa6 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -145,6 +145,25 @@ filegroup( ) use_repo(huggingface, "TinyLlama-120M-scratch") +# QWEN-2.5 + +huggingface.model( + name = "Qwen-2.5-7B", + build_file_content = """\ +package(default_visibility = ["//visibility:public"]) +filegroup( + name = "Qwen-2.5-7B", + srcs = glob(["*.json", "*.safetensors"]), +) +""", + commit = "d149729398750b98c0af14eb82c78cfe92750796", + includes = [ + "*.safetensors", + "*.json", + ], + model = "Qwen/Qwen2.5-7B", +) +use_repo(huggingface, "Qwen-2.5-7B") bazel_dep(name = "rules_rust", version = "0.57.0") rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 8d030233..2cd5698b 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -102,6 +102,24 @@ cc_binary( ], deps = [":llama_lib"], ) + +cc_binary( + name = "Qwen-2.5-7B", + args = [ + "--config=$(location @Qwen-2.5-7B//:config.json)", + "--weights=$(location @Qwen-2.5-7B//:model.safetensors.index.json)", + "--tokenizer=$(location @Qwen-2.5-7B//:tokenizer.json)", + "--no-llama3=true", + ], + data = [ + "@Qwen-2.5-7B", + "@Qwen-2.5-7B//:config.json", + "@Qwen-2.5-7B//:model.safetensors.index.json", + "@Qwen-2.5-7B//:tokenizer.json", + ], + deps = [":llama_lib"], +) + # zig_cc_binary( diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 6f6594f2..8913f227 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -272,6 +272,11 @@ pub const SelfAttn = struct { var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + // 🚀 **LOGGING SHAPES RIGHT AFTER PROJECTION** + log.info("q.shape = {}", .{q.shape()}); + log.info("k.shape = {}", .{k.shape()}); + log.info("v.shape = {}", .{v.shape()}); + // Generate the attention mask. const seq_len = kv_cache.k.dim(.k); var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), null); @@ -292,11 +297,20 @@ pub const SelfAttn = struct { k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); + // 🚀 **LOGGING SHAPES BEFORE COMPUTING ATTENTION** + log.info("q.shape (after RoPE) = {}", .{q.shape()}); + log.info("k.shape (after RoPE) = {}", .{k.shape()}); + log.info("v.shape (after RoPE) = {}", .{v.shape()}); + const dtype = q.dtype(); const new_kv_cache = kv_cache.update(k, v, token_index); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); + // 🚀 **LOGGING SHAPES AFTER KV CACHE UPDATE** + log.info("k.shape (after KV cache) = {}", .{k.shape()}); + log.info("v.shape (after KV cache) = {}", .{v.shape()}); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s });