From e3856be90e818729cc3ba83f79cafb11428152cb Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 28 Jan 2025 11:34:17 +0100 Subject: [PATCH 1/8] llama: introduce good TinyLlama-15M model --- examples/MODULE.bazel | 10 +++++----- examples/llama/BUILD.bazel | 17 ++++++++--------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index ea66f7a1..891c939e 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -128,22 +128,22 @@ use_repo(huggingface, "Meta-Llama-3.1-70B-Instruct") huggingface.model( - name = "TinyLlama-120M-scratch", + name = "TinyLlama-15M", build_file_content = """\ package(default_visibility = ["//visibility:public"]) filegroup( - name = "TinyLlama-120M-scratch", + name = "TinyLlama-15M", srcs = glob(["*.json", "*.safetensors"]), ) """, - commit = "89c1bb4ea00861ddaa26c55f102ccb25e161feee", + commit = "ace7596607754a175c6219989545c722c61aded8", includes = [ "*.safetensors", "*.json", ], - model = "Hoyeon/TinyLlama-120M-scratch", + model = "AnirudhRajagopalan1201/TinyLlama-15M", ) -use_repo(huggingface, "TinyLlama-120M-scratch") +use_repo(huggingface, "TinyLlama-15M") bazel_dep(name = "rules_rust", version = "0.57.0") diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 8d030233..3ac344ac 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -19,21 +19,20 @@ zig_cc_binary( "@zml//zml", ], ) - cc_binary( - name = "TinyLlama-120M-scratch", + name = "TinyLlama-15M", args = [ - "--config=$(location @TinyLlama-120M-scratch//:config.json)", - "--weights=$(location @TinyLlama-120M-scratch//:model.safetensors)", - "--tokenizer=$(location @TinyLlama-120M-scratch//:tokenizer.json)", + "--config=$(location @TinyLlama-15M//:config.json)", + "--weights=$(location @TinyLlama-15M//:model.safetensors)", + "--tokenizer=$(location @TinyLlama-15M//:tokenizer.json)", "--no-llama3=true", # don't do llama3 template prompt encoding "--sharding=false", # don't shard this ], data = [ - "@TinyLlama-120M-scratch", - "@TinyLlama-120M-scratch//:config.json", - "@TinyLlama-120M-scratch//:model.safetensors", - "@TinyLlama-120M-scratch//:tokenizer.json", + "@TinyLlama-15M", + "@TinyLlama-15M//:config.json", + "@TinyLlama-15M//:model.safetensors", + "@TinyLlama-15M//:tokenizer.json", ], deps = [":llama_lib"], ) From b1a44f8b03fc1c8894edf02bb8189338e760047d Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 28 Jan 2025 11:38:23 +0100 Subject: [PATCH 2/8] llama:tinyllama-15m: better default prompt --- examples/llama/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 3ac344ac..28954890 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -19,6 +19,7 @@ zig_cc_binary( "@zml//zml", ], ) + cc_binary( name = "TinyLlama-15M", args = [ @@ -27,6 +28,7 @@ cc_binary( "--tokenizer=$(location @TinyLlama-15M//:tokenizer.json)", "--no-llama3=true", # don't do llama3 template prompt encoding "--sharding=false", # don't shard this + "--prompt='Once upon a time,'", ], data = [ "@TinyLlama-15M", From 9c98537a8474352a03a4a1890366ed322aeb4a01 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 28 Jan 2025 11:43:11 +0100 Subject: [PATCH 3/8] llama: add tinyllama back into the docs --- README.md | 12 ++++++++++++ docs/tutorials/getting_started.md | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/README.md b/README.md index 43877544..fba8567f 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,18 @@ bazel run -c opt //mnist ./bazel.sh run -c opt //mnist ``` +### TinyLlama, Stories 15M + +Our LLM examples start with a small model trained specifically on children's +history books. This model has been trained by [Andrej +Karpathy](https://x.com/karpathy); you can read more about it on his +[GitHub](https://github.com/karpathy/llama2.c). + +``` +cd examples +bazel run -c opt //llama:TinyLlama-Stories-15M +bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon" +``` ### Meta Llama 3.1 8B diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index 03070a9c..763f245b 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -59,6 +59,19 @@ Llama is a family of "Large Language Models", trained to generate text, based on the beginning of a sentence/book/article. This "beginning" is generally referred to as the "prompt". +#### TinyLlama, Stories 15M + +To start, you can use a small model trained specifically on children's history +books. This model has been trained by [Andrej Karpathy](https://x.com/karpathy); +you can read more about it on his +[Github](https://github.com/karpathy/llama2.c). + +``` +cd examples +bazel run -c opt //llama:TinyLlama-Stories-15M +bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon" +``` + #### Meta Llama 3.1 8B This model has restrictions, see From 6ebe055acca7f835300f7e71340c3b547b48461a Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 28 Jan 2025 16:05:22 +0100 Subject: [PATCH 4/8] llama: fix freeing prompt tokens --- examples/llama/main.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 3cf2670a..a4f7c6b3 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -66,7 +66,9 @@ pub fn generateText( defer tokenizer_decoder.deinit(); const prompt_tok: []const u32 = if (skip_llama3_encoding) try tokenizer_encoder.encode(prompt) else try tokenizePromptLlama3(allocator, tokenizer, config, prompt); - defer allocator.free(prompt_tok); + defer { + if (skip_llama3_encoding) tokenizer_encoder.reset() else allocator.free(prompt_tok); + } const dims = llama_.model.shape(); const max_seq_len = dims.s; From 5382b1cd21d7cd43968bc9c25bc07f4d82dd7b8e Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 28 Jan 2025 16:07:33 +0100 Subject: [PATCH 5/8] llama: correct fix of prompt token cleanup --- examples/llama/main.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index a4f7c6b3..5064d0ed 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -67,7 +67,8 @@ pub fn generateText( const prompt_tok: []const u32 = if (skip_llama3_encoding) try tokenizer_encoder.encode(prompt) else try tokenizePromptLlama3(allocator, tokenizer, config, prompt); defer { - if (skip_llama3_encoding) tokenizer_encoder.reset() else allocator.free(prompt_tok); + tokenizer_encoder.reset(); + allocator.free(prompt_tok); } const dims = llama_.model.shape(); From e419b084ff76d1651afb167d5def19cc9c87dbb5 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 28 Jan 2025 16:16:45 +0100 Subject: [PATCH 6/8] llama: comments documenting prompt token cleanup --- examples/llama/main.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 5064d0ed..0e178baf 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -67,8 +67,11 @@ pub fn generateText( const prompt_tok: []const u32 = if (skip_llama3_encoding) try tokenizer_encoder.encode(prompt) else try tokenizePromptLlama3(allocator, tokenizer, config, prompt); defer { + // make the HF tokenizer release the memory tokenizer_encoder.reset(); - allocator.free(prompt_tok); + // free the owned slice in case the llama3 template was used + if (skip_llama3_encoding) + allocator.free(prompt_tok); } const dims = llama_.model.shape(); From 42284706f8c108f46df6550ee0f4ef7a06696a20 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 28 Jan 2025 16:25:48 +0100 Subject: [PATCH 7/8] llama: fix documented version of prompt token cleanup. sorry for the mess. --- examples/llama/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 0e178baf..43e37935 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -70,7 +70,7 @@ pub fn generateText( // make the HF tokenizer release the memory tokenizer_encoder.reset(); // free the owned slice in case the llama3 template was used - if (skip_llama3_encoding) + if (!skip_llama3_encoding) allocator.free(prompt_tok); } From 01fd58b46a7ce683c83cd6a4bdc0f752e5aac0f9 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 4 Feb 2025 14:26:27 +0100 Subject: [PATCH 8/8] fixed example prompts & bazel target name --- README.md | 2 +- docs/tutorials/getting_started.md | 2 +- examples/llama/BUILD.bazel | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fba8567f..746ae80e 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ Karpathy](https://x.com/karpathy); you can read more about it on his ``` cd examples bazel run -c opt //llama:TinyLlama-Stories-15M -bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon" +bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon," ``` diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index 763f245b..43e45d33 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -69,7 +69,7 @@ you can read more about it on his ``` cd examples bazel run -c opt //llama:TinyLlama-Stories-15M -bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon" +bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon," ``` #### Meta Llama 3.1 8B diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 28954890..2769007e 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -21,7 +21,7 @@ zig_cc_binary( ) cc_binary( - name = "TinyLlama-15M", + name = "TinyLlama-Stories-15M", args = [ "--config=$(location @TinyLlama-15M//:config.json)", "--weights=$(location @TinyLlama-15M//:model.safetensors)",