From 95201c3a942b9787aac0d4520dd0e328f7c21491 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sat, 12 Oct 2024 14:18:09 +0000 Subject: [PATCH 1/7] fix: Wait for events when calling a module --- zml/module.zig | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 69f2c952..f87ad8d6 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -849,18 +849,23 @@ pub fn ExeWithWeights(comptime func: anytype) type { pub fn call(self: Self, args: Bufferized(Signature.ArgsT)) Bufferized(Signature.ReturnT) { fillBuffers(&args, self.input_per_device, self.inner.model_buffer_count, self.inner.args_buffer_count); - var event: [1]*pjrt.Event = undefined; + var events: [Platform.MAX_NUM_DEVICES]*pjrt.Event = undefined; + const sharding = self.platform().sharding(); self.inner.exe.execute(self.inner.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.inner.args_buffer_count + self.inner.model_buffer_count, .results = self.output_per_device, - .events = &event, + .events = events[0..sharding.num_partitions], // TODO: this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. .non_donatable_input_indices = &.{}, }) catch unreachable; + for (events[0..sharding.num_partitions]) |e| { + e.await_(self.inner.platform.pjrt_api) catch unreachable; + } + var result: Bufferized(Signature.ReturnT) = undefined; assignRawBuffers(&result, self.inner.platform, self.output_per_device, self.inner.result_buffer_shapes, self.inner.result_buffer_count); return result; From eda4ea046caaca5b9877f7a31bb81b8b0290c889 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sat, 12 Oct 2024 14:14:26 +0000 Subject: [PATCH 2/7] temp: libxev epoll backend + temporary fixes --- async/async.zig | 3 ++- .../patches/0001-Change-Backend-Linux.patch | 22 +++++++++++++++++++ .../libxev/20240910.0-a2d9b31/source.json | 4 ++++ zml/platform.zig | 3 ++- 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch diff --git a/async/async.zig b/async/async.zig index 8e8cf742..19166532 100644 --- a/async/async.zig +++ b/async/async.zig @@ -102,7 +102,8 @@ pub fn callGeneric(comptime func: anytype, args: anytype) FnSignature(func, @Typ .notif = Notification.init() catch @panic("Notification.init failed"), .args = &args, }; - defer newtask.notif.deinit(); + // NOTE(Corentin): Disable this to avoid crash with epoll backend + //defer newtask.notif.deinit(); AsyncThread.current.thread_pool.schedule(xev.ThreadPool.Batch.from(&newtask._task)); newtask.notif.wait() catch @panic("Unable to wait for notification"); diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch b/third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch new file mode 100644 index 00000000..ed524e67 --- /dev/null +++ b/third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch @@ -0,0 +1,22 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/main.zig b/src/main.zig +index 74f451a..1cfb61c 100644 +--- a/src/main.zig ++++ b/src/main.zig +@@ -36,7 +36,7 @@ pub const Backend = enum { + /// Returns a recommend default backend from inspecting the system. + pub fn default() Backend { + return @as(?Backend, switch (builtin.os.tag) { +- .linux => .io_uring, ++ .linux => .epoll, + .ios, .macos => .kqueue, + .wasi => .wasi_poll, + .windows => .iocp, diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/source.json b/third_party/modules/libxev/20240910.0-a2d9b31/source.json index c56bc6a8..c38d9c27 100644 --- a/third_party/modules/libxev/20240910.0-a2d9b31/source.json +++ b/third_party/modules/libxev/20240910.0-a2d9b31/source.json @@ -5,5 +5,9 @@ "overlay": { "MODULE.bazel": "", "BUILD.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-Change-Backend-Linux.patch": "" } } diff --git a/zml/platform.zig b/zml/platform.zig index a8ec82f4..f1844e15 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -113,7 +113,8 @@ pub const Platform = struct { }{ .notif = try asynk.Notification.init(), }; - defer ctx.notif.deinit(); + // NOTE(Corentin): Disable this to avoid crash with epoll backend + //defer ctx.notif.deinit(); try event.onReady(self.pjrt_api, &(struct { fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.C) void { From d40c8733986424bf722b26622018b8cf28bdabca Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 14 Oct 2024 10:51:00 +0000 Subject: [PATCH 3/7] misc: bump PJRT CUDA Plugin --- runtimes/cuda/cuda.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index bf7343b4..aa85b8e1 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -182,8 +182,8 @@ cc_import( http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v0.2.2/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "45e91e8649bcccc43900f90d6dcbf0cfe87d3d2ee76f1763f41263d2ed44d31b", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v0.2.3/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "14f39ffef0c9ac529b1a8957750b0b5f5d2f6d310c0d997436051c53a9eb1618", ) return mctx.extension_metadata( From 0dea4e12b380b227c07e70986b049979906a8175 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 14 Oct 2024 10:51:26 +0000 Subject: [PATCH 4/7] misc: batched benchmark --- examples/benchmark/main.zig | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index cbbe3302..2d091d41 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -14,7 +14,7 @@ pub const std_options = .{ const Benchmark = struct { pub fn forward(self: Benchmark, a: zml.Tensor, b: zml.Tensor) zml.Tensor { _ = self; - return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); + return a.dot(b, .{.k}).withSharding(.{.b}); } }; @@ -30,6 +30,7 @@ pub fn asyncMain() !void { \\ benchmark --size=4096 --dtype=f16 ; size: usize = 4096, + batch_size: usize = 2, dtype: zml.DataType = .f16, }; @@ -83,8 +84,8 @@ pub fn asyncMain() !void { var args = std.process.args(); const cli_args = flags.parse(&args, CliArgs); - const a_shape = zml.Shape.init(.{ cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .m, .k }).withSharding(.{.k}); - const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + const a_shape = zml.Shape.init(.{ cli_args.batch_size, cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .b, .m, .k }).withSharding(.{.b}); + const b_shape = a_shape.withTags(.{ .b, .k, .n }).withSharding(.{.b}); var timer = try std.time.Timer.start(); std.debug.print("\nCompiling model to MLIR....\n", .{}); @@ -130,7 +131,7 @@ pub fn asyncMain() !void { std.debug.print("\n✅ Benchmark done!\n\n", .{}); - const floating_op_count = 2 * cli_args.size * cli_args.size * cli_args.size; + const floating_op_count = 2 * cli_args.batch_size * cli_args.size * cli_args.size * cli_args.size; const flops = @as(f64, @floatFromInt(floating_op_count)) / elapsed_s; std.debug.print("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {d:.3}ms - {d:.3} GFLOP/s\n\n", .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), elapsed_ms, flops / 1_000_000_000 }); } From 54cd7da481c0722300eeb4a2687561f84e8bab61 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 15 Oct 2024 14:51:30 +0000 Subject: [PATCH 5/7] misc: improved benchmark --- examples/benchmark/main.zig | 43 ++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 2d091d41..7b17b476 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -14,7 +14,7 @@ pub const std_options = .{ const Benchmark = struct { pub fn forward(self: Benchmark, a: zml.Tensor, b: zml.Tensor) zml.Tensor { _ = self; - return a.dot(b, .{.k}).withSharding(.{.b}); + return a.dot(b, .{.k}).withSharding(.{.n}); } }; @@ -32,6 +32,8 @@ pub fn asyncMain() !void { size: usize = 4096, batch_size: usize = 2, dtype: zml.DataType = .f16, + iterations: usize = 1, + sharded: bool = false, }; // Short lived allocations @@ -44,13 +46,17 @@ pub fn asyncMain() !void { defer arena_state.deinit(); const arena = arena_state.allocator(); + var args = std.process.args(); + const cli_args = flags.parse(&args, CliArgs); + var context = try zml.Context.init(); defer context.deinit(); // Auto-select platform const platform = context.autoPlatform().withCompilationOptions(.{ - .sharding_enabled = true, + .sharding_enabled = cli_args.sharded, }); + { // List available targets std.debug.print("Available Platforms:\n", .{}); @@ -81,11 +87,8 @@ pub fn asyncMain() !void { std.debug.print("\n", .{}); } - var args = std.process.args(); - const cli_args = flags.parse(&args, CliArgs); - - const a_shape = zml.Shape.init(.{ cli_args.batch_size, cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .b, .m, .k }).withSharding(.{.b}); - const b_shape = a_shape.withTags(.{ .b, .k, .n }).withSharding(.{.b}); + const a_shape = zml.Shape.init(.{ cli_args.batch_size, cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .b, .m, .k }); + const b_shape = a_shape.withTags(.{ .b, .k, .n }).withSharding(.{.n}); var timer = try std.time.Timer.start(); std.debug.print("\nCompiling model to MLIR....\n", .{}); @@ -122,18 +125,32 @@ pub fn asyncMain() !void { } // call our executable module - timer.reset(); - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); - defer result.deinit(); - const elapsed_ns = timer.lap(); + var elapsed_ns: u64 = 0; + for (0..cli_args.iterations) |_| { + timer.reset(); + var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + elapsed_ns += timer.lap(); + defer result.deinit(); + } const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; + const avg_elapsed_ms = elapsed_ms / @as(f64, @floatFromInt(cli_args.iterations)); const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; std.debug.print("\n✅ Benchmark done!\n\n", .{}); - const floating_op_count = 2 * cli_args.batch_size * cli_args.size * cli_args.size * cli_args.size; + const floating_op_count = 2 * cli_args.batch_size * cli_args.size * cli_args.size * cli_args.size * cli_args.iterations; const flops = @as(f64, @floatFromInt(floating_op_count)) / elapsed_s; - std.debug.print("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {d:.3}ms - {d:.3} GFLOP/s\n\n", .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), elapsed_ms, flops / 1_000_000_000 }); + std.debug.print( + \\ Dot product size: {d}x{d} + \\ Datatype: {s} + \\ Iterations: {d} + \\ Total elapsed: {d:.3}ms (avg: {d:.3}ms/it) + \\ FLOPs: {d:.3}GFLOP/s + \\ + \\ + , + .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), cli_args.iterations, elapsed_ms, avg_elapsed_ms, flops / 1_000_000_000} + ); } fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { From 90af2de03dec534906765fefa7f117215c23aa78 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 10 Oct 2024 16:34:36 +0200 Subject: [PATCH 6/7] initial sharding --- examples/llama/llama.zig | 14 ++++++++++++-- examples/llama/main.zig | 3 ++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index f01ab91f..126ca3e6 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -43,6 +43,16 @@ pub const LlamaLM = struct { layer.input_layernorm.eps = options.rms_norm_eps; layer.post_attention_layernorm.eps = options.rms_norm_eps; } + + zml.meta.visit(struct { + fn cb(_: void, layer: *zml.nn.Linear) void { + layer.weight = layer.weight.withSharding(.{-1}); + if (layer.bias) |*b| { + b.* = b.withSharding(.{-1}); + } + } + }.cb, {}, &self.model); + self.lm_head.weight = self.lm_head.weight.withSharding(.{0}); } /// Predicts the token at `token_index` position. @@ -328,8 +338,8 @@ pub const KvCache = struct { pub fn init(kv_shape: zml.Shape) KvCache { // The KV-cache is initialized with ones to detect reads of uninitialized memory. return .{ - .k = Tensor.constant(kv_shape, kv_shape.dtype().one()), - .v = Tensor.constant(kv_shape, kv_shape.dtype().one()), + .k = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .v = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), .layer_index = Tensor.scalar(-1, .i32), }; } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 2577f9b2..9dcc8649 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -145,6 +145,7 @@ pub fn asyncMain() !void { const compilation_options = zml.CompilationOptions{ .cache_location = "/tmp/zml/llama/cache", .xla_dump_to = "/tmp/zml/llama", + .sharding_enabled = true, }; const platform = context.autoPlatform().withCompilationOptions(compilation_options); @@ -234,7 +235,7 @@ pub fn asyncMain() !void { // To do so, we would just need to add `.b = batch_size` to `token_shape` and `kv_shape`. const tokens_shape = zml.Shape.init(.{ .s = dims.s }, .i32); const token_idx_shape = zml.Shape.init(.{}, .i32); - const kv_shape = zml.Shape.init(.{ .layer = llama.model.layers.len, .h = dims.nkvh, .k = dims.s, .hd = dims.hd }, dtype); + const kv_shape = zml.Shape.init(.{ .layer = llama.model.layers.len, .h = dims.nkvh, .k = dims.s, .hd = dims.hd }, dtype).withSharding(.{.h}); // needs to be optional const kv_cache_shape: ?ShapeOf(KvCache) = KvCache.initShape(kv_shape); const rng_shape = Tensor.Rng.shape(); From 33dcdd42e9485a5b589af037f15eaf13ecbeece5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 16 Oct 2024 15:29:41 +0000 Subject: [PATCH 7/7] feat: self attn sharding --- examples/llama/BUILD.bazel | 12 ++ examples/llama/llama.zig | 36 ++-- examples/llama/main.zig | 2 +- examples/sharding/BUILD.bazel | 172 ++++++++++++++++++ examples/sharding/main.zig | 332 ++++++++++++++++++++++++++++++++++ tools/zls.sh | 2 +- 6 files changed, 540 insertions(+), 16 deletions(-) create mode 100644 examples/sharding/BUILD.bazel create mode 100644 examples/sharding/main.zig diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 8e86a377..b3ad3fb3 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -3,6 +3,7 @@ load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_library") zig_cc_binary( name = "llama", @@ -17,6 +18,17 @@ zig_cc_binary( ], ) +zig_library( + name = "llamalib", + copts = ["-lc"], + main = "llama.zig", + visibility = ["//visibility:public"], + deps = [ + "@zml//async", + "@zml//zml", + ], +) + native_binary( name = "Llama-3.1-8B-Instruct", src = ":llama", diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 126ca3e6..c7065716 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -42,16 +42,24 @@ pub const LlamaLM = struct { layer.self_attn.rope_opts = options.rope_opts; layer.input_layernorm.eps = options.rms_norm_eps; layer.post_attention_layernorm.eps = options.rms_norm_eps; + layer.mlp.up_proj.weight = layer.mlp.up_proj.weight.withSharding(.{0}); + layer.mlp.gate_proj.weight = layer.mlp.gate_proj.weight.withSharding(.{0}); + layer.mlp.down_proj.weight = layer.mlp.down_proj.weight.withSharding(.{1}); + + layer.self_attn.q_proj.weight = layer.self_attn.q_proj.weight.withSharding(.{0}); + layer.self_attn.k_proj.weight = layer.self_attn.k_proj.weight.withSharding(.{0}); + layer.self_attn.v_proj.weight = layer.self_attn.v_proj.weight.withSharding(.{0}); + layer.self_attn.o_proj.weight = layer.self_attn.o_proj.weight.withSharding(.{1}); } - zml.meta.visit(struct { - fn cb(_: void, layer: *zml.nn.Linear) void { - layer.weight = layer.weight.withSharding(.{-1}); - if (layer.bias) |*b| { - b.* = b.withSharding(.{-1}); - } - } - }.cb, {}, &self.model); + //zml.meta.visit(struct { + // fn cb(_: void, layer: *zml.nn.Linear) void { + // layer.weight = layer.weight.withSharding(.{-1}); + // if (layer.bias) |*b| { + // b.* = b.withSharding(.{-1}); + // } + // } + //}.cb, {}, &self.model); self.lm_head.weight = self.lm_head.weight.withSharding(.{0}); } @@ -196,7 +204,7 @@ pub const Llama = struct { const dims = self.shape(); var kv_shape = embed_shape.insert(0, .{ .layer = dims.layer }).rename(.{ .s = .k }).splitAxes(.{ .d = .{ .h = dims.nkvh, .hd = dims.hd } }); const perm = kv_shape.contiguousPerm(.{ .h, .k, .hd }); - kv_shape = kv_shape.transpose(perm.constSlice()); + kv_shape = kv_shape.transpose(perm.constSlice()).withSharding(.{.h}); return KvCache.init(kv_shape); } }; @@ -246,7 +254,7 @@ const RmsNorm = struct { } }; -const Mlp = struct { +pub const Mlp = struct { up_proj: zml.nn.Linear, // (dim -> hidden_dim) gate_proj: zml.nn.Linear, // (dim -> hidden_dim) down_proj: zml.nn.Linear, // (hidden_dim -> dim) @@ -284,9 +292,9 @@ pub const SelfAttn = struct { ) struct { Tensor, KvCache } { // log.debug("x.shape: {}", .{x.shape()}); const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); - var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); + var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); // Generate the attention mask. const kv_cache = kv_cache_ orelse initKvCache(k.shape()); const seq_len = kv_cache.k.dim(.k); @@ -309,7 +317,7 @@ pub const SelfAttn = struct { const new_kv_cache = kv_cache.update(k, v, token_index orelse Tensor.scalar(0, .i32)); if (token_index) |_| { - std.debug.assert(q.dim(.q) == 1); + meta.assert(q.dim(.q) == 1, "Expected dimension .q to be 1, got {}", .{q.dim(.q)}); k = new_kv_cache.keys(); v = new_kv_cache.values(); } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 9dcc8649..454ab06f 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -143,7 +143,7 @@ pub fn asyncMain() !void { defer context.deinit(); const compilation_options = zml.CompilationOptions{ - .cache_location = "/tmp/zml/llama/cache", + //.cache_location = "/tmp/zml/llama/cache", .xla_dump_to = "/tmp/zml/llama", .sharding_enabled = true, }; diff --git a/examples/sharding/BUILD.bazel b/examples/sharding/BUILD.bazel new file mode 100644 index 00000000..dfff1ab7 --- /dev/null +++ b/examples/sharding/BUILD.bazel @@ -0,0 +1,172 @@ +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "sharding", + srcs = [], + main = "main.zig", + deps = [ + "//third_party/tigerbeetle:flags", + "@zml//async", + "@zml//zml", + "//llama:llamalib", + ], +) + +native_binary( + name = "Llama-3.1-8B-Instruct", + src = ":sharding", + args = [ + "--model=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", + "--num-heads=32", + "--num-kv-heads=8", + "--rope-freq-base=500000", + ], + data = [ + "@Meta-Llama-3.1-8B-Instruct//:model", + "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", + ], +) + +#native_binary( +# name = "Llama-3.1-8B-Instruct", +# src = ":llama", +# args = [ +# "--model=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", +# "--tokenizer=$(location @Meta-Llama-3.1-8B-Instruct//:tokenizer)", +# "--num-heads=32", +# "--num-kv-heads=8", +# "--rope-freq-base=500000", +# ], +# data = [ +# "@Meta-Llama-3.1-8B-Instruct//:model", +# "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", +# "@Meta-Llama-3.1-8B-Instruct//:tokenizer", +# ], +#) +# +#native_binary( +# name = "OpenLLaMA-3B", +# src = ":llama", +# args = [ +# "--model=$(location @OpenLM-Research-OpenLLaMA-3B//:model)", +# "--tokenizer=$(location @OpenLM-Research-OpenLLaMA-3B//:tokenizer)", +# "--num-heads=32", +# "--num-kv-heads=32", +# "--rope-freq-base=10000", +# ], +# data = [ +# "@OpenLM-Research-OpenLLaMA-3B//:model", +# "@OpenLM-Research-OpenLLaMA-3B//:tokenizer", +# ], +#) +# +#native_binary( +# name = "TinyLlama-1.1B-Chat", +# src = ":llama", +# args = [ +# "--model=$(location @TinyLlama-1.1B-Chat-v1.0//:model.safetensors)", +# "--tokenizer=$(location @TinyLlama-1.1B-Chat-v1.0//:tokenizer)", +# "--num-heads=32", +# "--num-kv-heads=4", +# "--rope-freq-base=10000", +# ], +# data = [ +# "@TinyLlama-1.1B-Chat-v1.0//:model.safetensors", +# "@TinyLlama-1.1B-Chat-v1.0//:tokenizer", +# ], +#) +# +#native_binary( +# name = "TinyLlama-Stories-110M", +# src = ":llama", +# args = [ +# "--model=$(location @Karpathy-TinyLlama-Stories//:stories110M)", +# "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", +# ], +# data = [ +# "@Karpathy-TinyLlama-Stories//:stories110M", +# "@Karpathy-TinyLlama-Tokenizer//file", +# ], +#) +# +#native_binary( +# name = "TinyLlama-Stories-15M", +# src = ":llama", +# args = [ +# "--model=$(location @Karpathy-TinyLlama-Stories//:stories15M)", +# "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", +# ], +# data = [ +# "@Karpathy-TinyLlama-Stories//:stories15M", +# "@Karpathy-TinyLlama-Tokenizer//file", +# ], +#) +# +#zig_cc_binary( +# name = "test-implementation", +# srcs = ["llama.zig"], +# args = [ +# "--model=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", +# "--num-heads=32", +# "--num-kv-heads=8", +# "--rope-freq-base=500000", +# ], +# data = [ +# "@Meta-Llama-3.1-8B-Instruct//:model", +# "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", +# ], +# main = "test.zig", +# deps = [ +# "//third_party/tigerbeetle:flags", +# "@zml//async", +# "@zml//zml", +# ], +#) +# +#mtree_spec( +# name = "mtree", +# srcs = [":llama"], +#) +# +#tar( +# name = "archive", +# srcs = [":llama"], +# args = [ +# "--options", +# "zstd:compression-level=9", +# ], +# compress = "zstd", +# mtree = ":mtree", +#) +# +#oci_image( +# name = "image_", +# base = "@distroless_cc_debian12", +# entrypoint = ["./{}/llama".format(package_name())], +# tars = [":archive"], +#) +# +#platform_transition_filegroup( +# name = "image", +# srcs = [":image_"], +# target_platform = "@zml//platforms:linux_amd64", +#) +# +#oci_load( +# name = "load", +# image = ":image", +# repo_tags = [ +# "distroless/llama:latest", +# ], +#) +# +#oci_push( +# name = "push", +# image = ":image", +# remote_tags = ["latest"], +# repository = "index.docker.io/steeve/llama", +#) diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig new file mode 100644 index 00000000..32e846ac --- /dev/null +++ b/examples/sharding/main.zig @@ -0,0 +1,332 @@ +const std = @import("std"); + +const zml = @import("zml"); +const meta = zml.meta; +const asynk = @import("async"); +const flags = @import("tigerbeetle/flags"); +const llama_mod = @import("llamalib"); + +const async_ = asynk.async_; + +const LlamaLM = llama_mod.LlamaLM; +const Llama = llama_mod.Llama; +const KvCache = llama_mod.KvCache; +const TransformerLayer = llama_mod.TransformerLayer; +const SelfAttn = llama_mod.SelfAttn; +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; + +const log = std.log.scoped(.llama); + +// set this to false to disable the verbose logging +const show_mlir = true; + +pub const std_options = .{ + .log_level = .err, + .log_scope_levels = &[_]std.log.ScopeLevel{ + .{ .scope = .pjrt, .level = if (show_mlir) .debug else .err }, + .{ .scope = .zml_module, .level = if (show_mlir) .debug else .err }, + .{ .scope = .zml, .level = if (show_mlir) .debug else .err }, + .{ .scope = .llama, .level = if (show_mlir) .debug else .info }, + }, +}; + +pub const MlpModule = struct { + mlp: llama_mod.Mlp, + + pub fn forward(self: MlpModule, x: zml.Tensor) zml.Tensor { + return zml.call(self.mlp, .forward, .{x}); + } +}; + +pub const SelfAttnModule = struct { + self_attn: llama_mod.SelfAttn, + + pub fn forward(self: SelfAttnModule, x: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache) struct { zml.Tensor, KvCache } { + const y, const updated_kv_cache = self.self_attn.forward(x, token_index, kv_cache); + return .{ y, updated_kv_cache.reuseBuffer(kv_cache) }; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain, .{}); +} + +const CliArgs = struct { + pub const help = + \\ llama --model=llama3.7B.safetensors --tokenizer=vocab.json --num_layers=2 + ; + model: []const u8, + layer_start: u8 = 0, + num_layers: ?u8 = null, + seq_len: u32 = 256, + topk: u32 = 2, + temperature: u32 = 1, + num_heads: ?i64 = null, + num_kv_heads: ?i64 = null, + rope_freq_base: ?i64 = null, + prompt: ?[]const u8 = null, + test_activations: ?[]const u8 = null, + seed: ?u128 = null, + iterations: usize = 1, +}; + +pub fn asyncMain() !void { + log.info(" LLama was compiled with {}", .{@import("builtin").mode}); + + var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + const tmp = try std.fs.openDirAbsolute("/tmp", .{}); + try tmp.makePath("zml/sharding/cache"); + var context = try zml.Context.init(); + defer context.deinit(); + + const compilation_options = zml.CompilationOptions{ + .xla_dump_to = "/tmp/zml/sharding", + .sharding_enabled = true, + }; + + const platform = context.autoPlatform().withCompilationOptions(compilation_options); + { + // List available targets + std.debug.print("\nSupported Platforms:\n", .{}); + const selected_prefix = "✅"; + const not_selected_prefix = "• "; + const selected_postfix = "(AUTO-SELECTED)\n"; + const not_selected_postfix = "\n"; + for (zml.platform.available_targets) |target| { + std.debug.print(" {s} {s} {s}", .{ + if (target == platform.target) selected_prefix else not_selected_prefix, + @tagName(target), + if (target == platform.target) selected_postfix else not_selected_postfix, + }); + + // now the platform's devices + if (context.platforms.get(target)) |pfm| { + for (pfm.getDevices(), 0..) |device, index| { + const deviceKind = device.getDescription(platform.pjrt_api).getKind(platform.pjrt_api); + std.debug.print(" ◦ #{d}: {s}\n", .{ + index, + deviceKind, + }); + // we only list 1 CPU device + if (target == .cpu) break; + } + } + } + std.debug.print("\n", .{}); + } + + var args = std.process.args(); + const cli_args = flags.parse(&args, CliArgs); + const model_file = cli_args.model; + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const model_arena = arena_state.allocator(); + + log.info("Model file: {s}", .{model_file}); + + var ts = try zml.aio.detectFormatAndOpen(allocator, model_file); + defer ts.deinit(); + + var llama: LlamaLM = try zml.aio.populateModel(LlamaLM, model_arena, ts); + const num_heads: i64 = cli_args.num_heads orelse ts.metadata("num_heads", .int64) orelse @panic("--num_heads is required for this model"); + const num_kv_heads: i64 = cli_args.num_kv_heads orelse ts.metadata("num_kv_heads", .int64) orelse num_heads; + + const rope_impl = if (ts.metadata("rope_impl", .string)) |val| + std.meta.stringToEnum(zml.nn.RopeOpts.Implementation, val).? + else + .sequential; + + const llama_options: llama_mod.LlamaOptions = .{ + .max_seq_len = cli_args.seq_len, + .num_kv_heads = num_kv_heads, + .num_heads = num_heads, + .gen_opts = .{ + .topk = cli_args.topk, + .temperature = @floatFromInt(cli_args.temperature), + }, + .rms_norm_eps = @floatCast(ts.metadata("rms_norm_eps", .float64) orelse 1e-5), + .rope_opts = .{ + .impl = rope_impl, + .freq_base = @floatCast(ts.metadata("rope_freq_base", .float64) orelse @as(f32, @floatFromInt(cli_args.rope_freq_base orelse 10_000))), + }, + }; + log.info("✅ Parsed llama config: {}", .{llama_options}); + llama.init(llama_options); + + try runSelfAttnBenchmark(cli_args, &llama, llama_options, ts, model_arena, platform); +} + +fn runMlpBenchmark(cli_args: CliArgs, llama: *LlamaLM, llama_options: llama_mod.LlamaOptions, ts: zml.aio.BufferStore, arena: std.mem.Allocator, platform: zml.Platform) !void { + const dims = llama.model.shape(); + const S: usize = @intCast(cli_args.seq_len); + const D: usize = @intCast(dims.nh * dims.hd); + const H: usize = @intCast(llama.model.layers[0].mlp.up_proj.weight.dim(0)); + const dtype = llama.lm_head.weight.dtype(); + + log.info( + \\ + \\S: {} + \\D: {} + \\H: {} + \\ + , .{ S, D, H }); + + const x_shape = zml.Shape.init(.{ .s = cli_args.seq_len, .d = dims.nh * dims.hd }, dtype); + + var compilation = try async_(zml.compileModel, .{ arena, MlpModule{ .mlp = llama.model.layers[0].mlp }, .forward, .{x_shape}, platform }); + + var llama_weights = try zml.aio.loadBuffers(LlamaLM, .{llama_options}, ts, arena, platform); + defer zml.aio.unloadBuffers(&llama_weights); + + const compiled = try compilation.await_(); + var executable = try compiled.prepare(arena, .{ .mlp = llama_weights.model.layers[0].mlp }); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + var x_buffer = try createRandomBuffer(arena, platform, x_shape, random); + defer x_buffer.deinit(); + std.debug.print("\nRunning benchmark....\n", .{}); + + // Ignore first run + { + var result: zml.Buffer = executable.call(.{x_buffer}); + defer result.deinit(); + } + + // call our executable module + var elapsed_ns: u64 = 0; + var timer = try std.time.Timer.start(); + for (0..cli_args.iterations) |_| { + timer.reset(); + var result: zml.Buffer = executable.call(.{x_buffer}); + elapsed_ns += timer.lap(); + defer result.deinit(); + } + const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; + const avg_elapsed_ms = elapsed_ms / @as(f64, @floatFromInt(cli_args.iterations)); + + std.debug.print("\n✅ Benchmark done!\n\n", .{}); + + std.debug.print( + \\ x_shape: {} + \\ Datatype: {s} + \\ Iterations: {d} + \\ Total elapsed: {d:.3}ms (avg: {d:.3}ms/it) + \\ + \\ + , .{ x_shape, @tagName(x_shape.dtype()), cli_args.iterations, elapsed_ms, avg_elapsed_ms }); +} + +fn runSelfAttnBenchmark(cli_args: CliArgs, llama: *LlamaLM, llama_options: llama_mod.LlamaOptions, ts: zml.aio.BufferStore, arena: std.mem.Allocator, platform: zml.Platform) !void { + const dims = llama.model.shape(); + const dtype = llama.lm_head.weight.dtype(); + const x_shape = zml.Shape.init(.{ .s = 1, .d = dims.nh * dims.hd }, dtype); + const token_idx_shape = zml.Shape.init(.{}, .i32); + const kv_shape = zml.Shape.init(.{ .layer = llama.model.layers.len, .h = dims.nkvh, .k = dims.s, .hd = dims.hd }, dtype).withSharding(.{.h}); + // needs to be optional + const kv_cache_shape: ShapeOf(KvCache) = KvCache.initShape(kv_shape); + + var compilation = try async_(zml.compileModel, .{ arena, SelfAttnModule{ .self_attn = llama.model.layers[0].self_attn }, .forward, .{ x_shape, token_idx_shape, kv_cache_shape }, platform }); + + var llama_weights = try zml.aio.loadBuffers(LlamaLM, .{llama_options}, ts, arena, platform); + defer zml.aio.unloadBuffers(&llama_weights); + + const compiled = try compilation.await_(); + var executable = try compiled.prepare(arena, .{ .self_attn = llama_weights.model.layers[0].self_attn }); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + var x_buffer = try createRandomBuffer(arena, platform, x_shape, random); + defer x_buffer.deinit(); + var token_index_buffer: zml.Buffer = try zml.Buffer.fromSlice(platform, .{}, &[1]i32{1}); + defer token_index_buffer.deinit(); + var k_cache_buffer = try createRandomBuffer(arena, platform, kv_shape, random); + defer k_cache_buffer.deinit(); + var v_cache_buffer = try createRandomBuffer(arena, platform, kv_shape, random); + defer v_cache_buffer.deinit(); + var layer_index_buffer = try zml.Buffer.fromSlice(platform, .{}, &[1]i32{0}); + defer layer_index_buffer.deinit(); + var updated_kv_cache: zml.Bufferized(KvCache) = .{ .k = k_cache_buffer, .v = v_cache_buffer, .layer_index = layer_index_buffer }; + std.debug.print("\nRunning benchmark....\n", .{}); + + // Ignore first run + { + var x: zml.Buffer, updated_kv_cache = executable.call(.{ x_buffer, token_index_buffer, updated_kv_cache }); + defer x.deinit(); + } + + // call our executable module + var elapsed_ns: u64 = 0; + var timer = try std.time.Timer.start(); + for (0..cli_args.iterations) |_| { + timer.reset(); + var x: zml.Buffer, updated_kv_cache = executable.call(.{ x_buffer, token_index_buffer, updated_kv_cache }); + elapsed_ns += timer.lap(); + defer x.deinit(); + } + const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; + const avg_elapsed_ms = elapsed_ms / @as(f64, @floatFromInt(cli_args.iterations)); + + std.debug.print("\n✅ Benchmark done!\n\n", .{}); + + std.debug.print( + \\ x_shape: {} + \\ Datatype: {s} + \\ Iterations: {d} + \\ Total elapsed: {d:.3}ms (avg: {d:.3}ms/it) + \\ + \\ + , .{ x_shape, @tagName(x_shape.dtype()), cli_args.iterations, elapsed_ms, avg_elapsed_ms }); +} + +fn computeFloatingOpCount(S: usize, D: usize, H: usize) usize { + const up_proj_flops = 2 * S * D * H; + const gate_proj_flops = 2 * S * D * H; + const silu_flops = 4 * S * H; + const up_gate_mul_flops = S * H; + const down_proj_flops = 2 * S * H * D; + return up_proj_flops + gate_proj_flops + silu_flops + up_gate_mul_flops + down_proj_flops; +} + +fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const data = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = random.float(f64); + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return zml.Buffer.from(platform, host_buffer); +} diff --git a/tools/zls.sh b/tools/zls.sh index 9fd45149..9fefc368 100755 --- a/tools/zls.sh +++ b/tools/zls.sh @@ -1,3 +1,3 @@ #!/bin/bash cd "$(dirname "${BASH_SOURCE[0]}")" -exec bazel run --config=silent @zml//third_party/zls:zls -- zls "${@}" +exec ../examples/bazel.sh run --config=silent @zml//third_party/zls:zls -- zls "${@}"