diff --git a/README.md b/README.md index 43877544..746ae80e 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,18 @@ bazel run -c opt //mnist ./bazel.sh run -c opt //mnist ``` +### TinyLlama, Stories 15M + +Our LLM examples start with a small model trained specifically on children's +history books. This model has been trained by [Andrej +Karpathy](https://x.com/karpathy); you can read more about it on his +[GitHub](https://github.com/karpathy/llama2.c). + +``` +cd examples +bazel run -c opt //llama:TinyLlama-Stories-15M +bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon," +``` ### Meta Llama 3.1 8B diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index 03070a9c..43e45d33 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -59,6 +59,19 @@ Llama is a family of "Large Language Models", trained to generate text, based on the beginning of a sentence/book/article. This "beginning" is generally referred to as the "prompt". +#### TinyLlama, Stories 15M + +To start, you can use a small model trained specifically on children's history +books. This model has been trained by [Andrej Karpathy](https://x.com/karpathy); +you can read more about it on his +[Github](https://github.com/karpathy/llama2.c). + +``` +cd examples +bazel run -c opt //llama:TinyLlama-Stories-15M +bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon," +``` + #### Meta Llama 3.1 8B This model has restrictions, see diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index ea66f7a1..891c939e 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -128,22 +128,22 @@ use_repo(huggingface, "Meta-Llama-3.1-70B-Instruct") huggingface.model( - name = "TinyLlama-120M-scratch", + name = "TinyLlama-15M", build_file_content = """\ package(default_visibility = ["//visibility:public"]) filegroup( - name = "TinyLlama-120M-scratch", + name = "TinyLlama-15M", srcs = glob(["*.json", "*.safetensors"]), ) """, - commit = "89c1bb4ea00861ddaa26c55f102ccb25e161feee", + commit = "ace7596607754a175c6219989545c722c61aded8", includes = [ "*.safetensors", "*.json", ], - model = "Hoyeon/TinyLlama-120M-scratch", + model = "AnirudhRajagopalan1201/TinyLlama-15M", ) -use_repo(huggingface, "TinyLlama-120M-scratch") +use_repo(huggingface, "TinyLlama-15M") bazel_dep(name = "rules_rust", version = "0.57.0") diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 8d030233..2769007e 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -21,19 +21,20 @@ zig_cc_binary( ) cc_binary( - name = "TinyLlama-120M-scratch", + name = "TinyLlama-Stories-15M", args = [ - "--config=$(location @TinyLlama-120M-scratch//:config.json)", - "--weights=$(location @TinyLlama-120M-scratch//:model.safetensors)", - "--tokenizer=$(location @TinyLlama-120M-scratch//:tokenizer.json)", + "--config=$(location @TinyLlama-15M//:config.json)", + "--weights=$(location @TinyLlama-15M//:model.safetensors)", + "--tokenizer=$(location @TinyLlama-15M//:tokenizer.json)", "--no-llama3=true", # don't do llama3 template prompt encoding "--sharding=false", # don't shard this + "--prompt='Once upon a time,'", ], data = [ - "@TinyLlama-120M-scratch", - "@TinyLlama-120M-scratch//:config.json", - "@TinyLlama-120M-scratch//:model.safetensors", - "@TinyLlama-120M-scratch//:tokenizer.json", + "@TinyLlama-15M", + "@TinyLlama-15M//:config.json", + "@TinyLlama-15M//:model.safetensors", + "@TinyLlama-15M//:tokenizer.json", ], deps = [":llama_lib"], ) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 3cf2670a..43e37935 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -66,7 +66,13 @@ pub fn generateText( defer tokenizer_decoder.deinit(); const prompt_tok: []const u32 = if (skip_llama3_encoding) try tokenizer_encoder.encode(prompt) else try tokenizePromptLlama3(allocator, tokenizer, config, prompt); - defer allocator.free(prompt_tok); + defer { + // make the HF tokenizer release the memory + tokenizer_encoder.reset(); + // free the owned slice in case the llama3 template was used + if (!skip_llama3_encoding) + allocator.free(prompt_tok); + } const dims = llama_.model.shape(); const max_seq_len = dims.s;