diff --git a/async/async.zig b/async/async.zig index 8e8cf742..19166532 100644 --- a/async/async.zig +++ b/async/async.zig @@ -102,7 +102,8 @@ pub fn callGeneric(comptime func: anytype, args: anytype) FnSignature(func, @Typ .notif = Notification.init() catch @panic("Notification.init failed"), .args = &args, }; - defer newtask.notif.deinit(); + // NOTE(Corentin): Disable this to avoid crash with epoll backend + //defer newtask.notif.deinit(); AsyncThread.current.thread_pool.schedule(xev.ThreadPool.Batch.from(&newtask._task)); newtask.notif.wait() catch @panic("Unable to wait for notification"); diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index 942b2f4b..14000765 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -102,6 +102,30 @@ filegroup( ) use_repo(huggingface, "Meta-Llama-3.1-8B-Instruct") +huggingface.model( + name = "Meta-Llama-3.1-70B-Instruct", + build_file_content = """\ +package(default_visibility = ["//visibility:public"]) +filegroup( + name = "model", + srcs = glob(["*.safetensors"]) + ["model.safetensors.index.json"], +) + +filegroup( + name = "tokenizer", + srcs = ["tokenizer.json"], +) +""", + commit = "945c8663693130f8be2ee66210e062158b2a9693", + includes = [ + "*.safetensors", + "model.safetensors.index.json", + "tokenizer.json", + ], + model = "meta-llama/Meta-Llama-3.1-70B-Instruct", +) +use_repo(huggingface, "Meta-Llama-3.1-70B-Instruct") + huggingface.model( name = "TinyLlama-1.1B-Chat-v1.0", build_file_content = """\ diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 8e86a377..e79034fa 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -34,6 +34,23 @@ native_binary( ], ) +native_binary( + name = "Llama-3.1-70B-Instruct", + src = ":llama", + args = [ + "--model=$(location @Meta-Llama-3.1-70B-Instruct//:model.safetensors.index.json)", + "--tokenizer=$(location @Meta-Llama-3.1-70B-Instruct//:tokenizer)", + "--num-heads=64", + "--num-kv-heads=8", + "--rope-freq-base=500000", + ], + data = [ + "@Meta-Llama-3.1-70B-Instruct//:model", + "@Meta-Llama-3.1-70B-Instruct//:model.safetensors.index.json", + "@Meta-Llama-3.1-70B-Instruct//:tokenizer", + ], +) + native_binary( name = "OpenLLaMA-3B", src = ":llama", diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index f01ab91f..32f15339 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -42,7 +42,17 @@ pub const LlamaLM = struct { layer.self_attn.rope_opts = options.rope_opts; layer.input_layernorm.eps = options.rms_norm_eps; layer.post_attention_layernorm.eps = options.rms_norm_eps; + layer.mlp.up_proj.weight = layer.mlp.up_proj.weight.withSharding(.{0}); + layer.mlp.gate_proj.weight = layer.mlp.gate_proj.weight.withSharding(.{0}); + layer.mlp.down_proj.weight = layer.mlp.down_proj.weight.withSharding(.{1}); + + layer.self_attn.q_proj.weight = layer.self_attn.q_proj.weight.withSharding(.{0}); + layer.self_attn.k_proj.weight = layer.self_attn.k_proj.weight.withSharding(.{0}); + layer.self_attn.v_proj.weight = layer.self_attn.v_proj.weight.withSharding(.{0}); + layer.self_attn.o_proj.weight = layer.self_attn.o_proj.weight.withSharding(.{1}); } + + self.lm_head.weight = self.lm_head.weight.withSharding(.{0}); } /// Predicts the token at `token_index` position. @@ -274,9 +284,9 @@ pub const SelfAttn = struct { ) struct { Tensor, KvCache } { // log.debug("x.shape: {}", .{x.shape()}); const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); - var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); + var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); // Generate the attention mask. const kv_cache = kv_cache_ orelse initKvCache(k.shape()); const seq_len = kv_cache.k.dim(.k); @@ -299,7 +309,7 @@ pub const SelfAttn = struct { const new_kv_cache = kv_cache.update(k, v, token_index orelse Tensor.scalar(0, .i32)); if (token_index) |_| { - std.debug.assert(q.dim(.q) == 1); + meta.assert(q.dim(.q) == 1, "Expected dimension .q to be 1, got {}", .{q.dim(.q)}); k = new_kv_cache.keys(); v = new_kv_cache.values(); } @@ -328,8 +338,8 @@ pub const KvCache = struct { pub fn init(kv_shape: zml.Shape) KvCache { // The KV-cache is initialized with ones to detect reads of uninitialized memory. return .{ - .k = Tensor.constant(kv_shape, kv_shape.dtype().one()), - .v = Tensor.constant(kv_shape, kv_shape.dtype().one()), + .k = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .v = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), .layer_index = Tensor.scalar(-1, .i32), }; } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 2577f9b2..9dcc8649 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -145,6 +145,7 @@ pub fn asyncMain() !void { const compilation_options = zml.CompilationOptions{ .cache_location = "/tmp/zml/llama/cache", .xla_dump_to = "/tmp/zml/llama", + .sharding_enabled = true, }; const platform = context.autoPlatform().withCompilationOptions(compilation_options); @@ -234,7 +235,7 @@ pub fn asyncMain() !void { // To do so, we would just need to add `.b = batch_size` to `token_shape` and `kv_shape`. const tokens_shape = zml.Shape.init(.{ .s = dims.s }, .i32); const token_idx_shape = zml.Shape.init(.{}, .i32); - const kv_shape = zml.Shape.init(.{ .layer = llama.model.layers.len, .h = dims.nkvh, .k = dims.s, .hd = dims.hd }, dtype); + const kv_shape = zml.Shape.init(.{ .layer = llama.model.layers.len, .h = dims.nkvh, .k = dims.s, .hd = dims.hd }, dtype).withSharding(.{.h}); // needs to be optional const kv_cache_shape: ?ShapeOf(KvCache) = KvCache.initShape(kv_shape); const rng_shape = Tensor.Rng.shape(); diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index bf7343b4..aa85b8e1 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -182,8 +182,8 @@ cc_import( http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v0.2.2/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "45e91e8649bcccc43900f90d6dcbf0cfe87d3d2ee76f1763f41263d2ed44d31b", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v0.2.3/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "14f39ffef0c9ac529b1a8957750b0b5f5d2f6d310c0d997436051c53a9eb1618", ) return mctx.extension_metadata( diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch b/third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch new file mode 100644 index 00000000..ed524e67 --- /dev/null +++ b/third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch @@ -0,0 +1,22 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/main.zig b/src/main.zig +index 74f451a..1cfb61c 100644 +--- a/src/main.zig ++++ b/src/main.zig +@@ -36,7 +36,7 @@ pub const Backend = enum { + /// Returns a recommend default backend from inspecting the system. + pub fn default() Backend { + return @as(?Backend, switch (builtin.os.tag) { +- .linux => .io_uring, ++ .linux => .epoll, + .ios, .macos => .kqueue, + .wasi => .wasi_poll, + .windows => .iocp, diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/source.json b/third_party/modules/libxev/20240910.0-a2d9b31/source.json index c56bc6a8..c38d9c27 100644 --- a/third_party/modules/libxev/20240910.0-a2d9b31/source.json +++ b/third_party/modules/libxev/20240910.0-a2d9b31/source.json @@ -5,5 +5,9 @@ "overlay": { "MODULE.bazel": "", "BUILD.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-Change-Backend-Linux.patch": "" } } diff --git a/zml/module.zig b/zml/module.zig index 75fd31fa..fb5e5142 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -859,18 +859,23 @@ pub fn ExeWithWeights(comptime func: anytype) type { pub fn call(self: Self, args: Bufferized(Signature.ArgsT)) Bufferized(Signature.ReturnT) { fillBuffers(&args, self.input_per_device, self.inner.model_buffer_count, self.inner.args_buffer_count); - var event: [1]*pjrt.Event = undefined; + var events: [Platform.MAX_NUM_DEVICES]*pjrt.Event = undefined; + const sharding = self.platform().sharding(); self.inner.exe.execute(self.inner.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.inner.args_buffer_count + self.inner.model_buffer_count, .results = self.output_per_device, - .events = &event, + .events = events[0..sharding.num_partitions], // TODO: this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. .non_donatable_input_indices = &.{}, }) catch unreachable; + for (events[0..sharding.num_partitions]) |e| { + e.await_(self.inner.platform.pjrt_api) catch unreachable; + } + var result: Bufferized(Signature.ReturnT) = undefined; assignRawBuffers(&result, self.inner.platform, self.output_per_device, self.inner.result_buffer_shapes, self.inner.result_buffer_count); return result; diff --git a/zml/platform.zig b/zml/platform.zig index a8ec82f4..f1844e15 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -113,7 +113,8 @@ pub const Platform = struct { }{ .notif = try asynk.Notification.init(), }; - defer ctx.notif.deinit(); + // NOTE(Corentin): Disable this to avoid crash with epoll backend + //defer ctx.notif.deinit(); try event.onReady(self.pjrt_api, &(struct { fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.C) void {