From 10f86e93f1d4f501aa42d038cd9d9ee67c75855f Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 5 Jun 2025 16:00:47 +0000 Subject: [PATCH 01/51] explo partitioning --- examples/llama/BUILD.bazel | 2 +- examples/llama/llama.zig | 257 ++++++--- examples/llama/main.zig | 98 +++- examples/partitioning/BUILD.bazel | 11 + examples/partitioning/main.zig | 136 +++++ pjrt/pjrt.zig | 45 +- zml/BUILD.bazel | 84 +++ zml/aio.zig | 39 +- zml/buffer.zig | 214 +++++--- zml/context.zig | 4 +- zml/exe.zig | 36 +- zml/hostbuffer.zig | 21 +- zml/meta.zig | 2 +- zml/module.zig | 188 ++++--- zml/nn.zig | 22 +- zml/ops.zig | 91 ++-- zml/partitioning.zig | 848 ++++++++++++++++++++++++++++++ zml/pjrtx.zig | 5 +- zml/platform.zig | 30 +- zml/shape.zig | 741 ++++++++++++++++++++++++-- zml/slice.zig | 161 ++++++ zml/tensor.zig | 351 +++++++------ zml/testing.zig | 12 +- zml/torch.zig | 2 +- zml/zml.zig | 10 +- 25 files changed, 2838 insertions(+), 572 deletions(-) create mode 100644 examples/partitioning/BUILD.bazel create mode 100644 examples/partitioning/main.zig create mode 100644 zml/partitioning.zig create mode 100644 zml/slice.zig diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index d657f4a1..45b5b66b 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -105,7 +105,7 @@ cc_binary( "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", "--prompt='Once upon a time, there was a little girl named Lily.'", "--no-llama3=1", # don't do template prompt encoding, I'm a simple model - "--sharding=false", # don't shard me, I'm so small + "--sharding=true", # don't shard me, I'm so small ], data = [ ":tinyllama_stories15M_json", diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index ee9c852f..8d859f3f 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -40,38 +40,18 @@ pub const LlamaLM = struct { gen_opts: zml.nn.SamplingStrategy = .{}, config: Config, - pub fn init(self: *LlamaLM, config: Config, options: Options) void { + pub fn init(self: *LlamaLM, mesh: zml.Mesh, config: Config, options: Options) void { self.config = config; self.gen_opts = options.sampling_strategy orelse .{}; - self.model.max_seq_len = @intCast(options.max_seq_len); - self.model.num_heads = @intCast(config.num_attention_heads); - self.model.num_kv_heads = @intCast(config.num_key_value_heads); - self.model.rope_opts = .{ - .layout = if (config.hf_rope_impl) .sequential else .interleaved, - .freq_base = config.rope_theta, - .scaling = config.rope_scaling, - }; - self.model.norm.eps = config.rms_norm_eps; - for (self.model.layers) |*layer| { - layer.self_attn.num_heads = self.model.num_heads; - layer.self_attn.num_kv_heads = self.model.num_kv_heads; - layer.self_attn.rope_opts = self.model.rope_opts; - layer.input_layernorm.eps = config.rms_norm_eps; - layer.post_attention_layernorm.eps = config.rms_norm_eps; - layer.mlp.up_proj.weight = layer.mlp.up_proj.weight.withSharding(.{0}); - layer.mlp.gate_proj.weight = layer.mlp.gate_proj.weight.withSharding(.{0}); - layer.mlp.down_proj.weight = layer.mlp.down_proj.weight.withSharding(.{1}); - layer.self_attn.q_proj.weight = layer.self_attn.q_proj.weight.withSharding(.{0}); - layer.self_attn.k_proj.weight = layer.self_attn.k_proj.weight.withSharding(.{0}); - layer.self_attn.v_proj.weight = layer.self_attn.v_proj.weight.withSharding(.{0}); - layer.self_attn.o_proj.weight = layer.self_attn.o_proj.weight.withSharding(.{1}); - } + const vocab_mesh = mesh.flatten(.all_devices); + + self.model.init(config, options.max_seq_len, mesh, vocab_mesh); // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. // It currently crashes/compilation fails if (self.gen_opts.topk == 1 and self.lm_head != null) { - self.lm_head.?.weight = self.lm_head.?.weight.withSharding(.{0}); + self.lm_head.?.weight = self.lm_head.?.weight.withTags(.{ .d, .vocab }).withMesh(vocab_mesh).withSharding(.{ .vocab = .all_devices }); } } @@ -89,7 +69,7 @@ pub const LlamaLM = struct { ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); + const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } @@ -105,7 +85,7 @@ pub const LlamaLM = struct { var logits = blk: { if (lm_head_) |lm_head| { - break :blk zml.call(lm_head, .forward, .{out}); + break :blk lm_head.forward(out); } else { break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); } @@ -146,36 +126,75 @@ pub const Llama = struct { }; pub fn shape(self: Llama) Shape { - const key_dim = self.layers[0].self_attn.k_proj.weight.dim(0); + const d_model = self.embed_tokens.weight.dim(1); const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + const num_q_heads = self.num_heads; + + const head_dim = @divExact(d_model, num_q_heads); return .{ .s = self.max_seq_len, .layer = @intCast(self.layers.len), - .hd = @intCast(@divExact(key_dim, num_kv_heads)), + .hd = @intCast(head_dim), .nh = @intCast(self.num_heads), .nkvh = @intCast(num_kv_heads), .dtype = self.embed_tokens.weight.dtype(), }; } + pub fn init(self: *Llama, config: LlamaLM.Config, max_seq_len: usize, compute_mesh: zml.Mesh, vocab_mesh: zml.Mesh) void { + self.num_heads = @intCast(config.num_attention_heads); + self.num_kv_heads = @intCast(config.num_key_value_heads); + self.max_seq_len = @intCast(max_seq_len); + self.rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }; + + self.norm.init(config, compute_mesh); + + self.embed_tokens.weight = self.embed_tokens.weight + .withTags(.{ .vocab, .d }) + .withMesh(vocab_mesh) + .withSharding(.{ .vocab = .all_devices }); + + for (self.layers) |*layer| { + layer.init(config, compute_mesh); + } + } + /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { + log.info(">>>>>>>>>>>>>>> Llama.forward(tokens={}, token_index={}, kv_cache={} vocab_mesh={any})", .{ tokens, token_index, kv_cache, self.embed_tokens.weight._mesh }); + const vocab_mesh = self.embed_tokens.weight._mesh.?; + zml.pushMesh(vocab_mesh); const embeds = embed(self.embed_tokens, tokens); - var hidden = embeds; + zml.popMesh(); + + zml.pushMesh(self.layers[0].self_attn.q_proj.weight._mesh.?); + + var hidden = embeds.withSharding(.{ .s = .replicated, .d = .model }); + + if (embeds.dim(.s) > 1) { + hidden = hidden.withSharding(.{ .s = .data, .d = .model }); + } var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { - hidden, updated_kv_cache = zml.call(layer, .forward, .{ hidden, token_index, updated_kv_cache.atLayer(i) }); + hidden, updated_kv_cache = layer.forward(hidden, token_index, updated_kv_cache.atLayer(i)); } - const output = zml.call(self.norm, .forward, .{hidden}); + const output = self.norm.forward(hidden); + + zml.popMesh(); return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; } pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { - return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); + return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); } fn initKvCache(self: Llama, embed_shape: zml.Shape) KvCache { @@ -193,23 +212,38 @@ pub const TransformerLayer = struct { post_attention_layernorm: RmsNorm, mlp: Mlp, + pub fn init(self: *TransformerLayer, config: LlamaLM.Config, mesh: zml.Mesh) void { + self.input_layernorm.init(config, mesh); + self.self_attn.init(config, mesh); + self.post_attention_layernorm.init(config, mesh); + self.mlp.init(mesh); + } + pub fn forward( self: TransformerLayer, - x0: Tensor, + x0_: Tensor, token_index: Tensor, kv_cache: KvCache, ) struct { Tensor, KvCache } { - // Self Attention - //log.debug("TransformerLayer({}) -> {}", .{ x0, self.input_layernorm.forward(x0) }); + // const x0 = x0_.withTags(.{ .s, .d }).withSharding(.{ .s = .data, .d = .model }); + const x0 = x0_; stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); - const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); + // Self Attention + const x0_normalized = self.input_layernorm.forward(x0); + const delta0, const updated_kv_cache = self.self_attn.forward( + x0_normalized, + token_index, + kv_cache, + ); + log.warn("TransformerLayer.forward: delta0={}", .{delta0}); const x1 = x0.add(delta0); // Fully Connected - const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - const x2 = zml.call(self.mlp, .forward, .{x1_normalized}).add(x1); + const x1_normalized = self.post_attention_layernorm.forward(x1); + // FIX: Add the MLP output to the Attention output (x1). + const delta1 = self.mlp.forward(x1_normalized); + const x2 = x1.add(delta1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; } @@ -219,9 +253,19 @@ const RmsNorm = struct { weight: Tensor, eps: f32 = 1e-5, + pub fn init(self: *RmsNorm, config: LlamaLM.Config, mesh: zml.Mesh) void { + self.eps = config.rms_norm_eps; + self.weight = self.weight.withTags(.{.d}).withMesh(mesh); + + if (mesh.topology.hasTag(.model) != null) { + self.weight = self.weight.withSharding(.{ .d = .model }); + } + } + /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: Tensor) Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + log.warn("RmsNorm.forward({})", .{x}); const normalized = zml.nn.rmsNorm(x, .d, self.eps); return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); } @@ -232,11 +276,27 @@ const Mlp = struct { gate_proj: zml.nn.Linear, // (dim -> hidden_dim) down_proj: zml.nn.Linear, // (hidden_dim -> dim) + pub fn init(self: *Mlp, mesh: zml.Mesh) void { + const has_model_axis = mesh.topology.hasTag(.model) != null; + + self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh); + self.up_proj.weight = self.up_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh); + self.down_proj.weight = self.down_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh); + + if (has_model_axis) { + self.gate_proj.weight = self.gate_proj.weight.withSharding(.{ .hidden = .model }); + self.up_proj.weight = self.up_proj.weight.withSharding(.{ .hidden = .model }); + self.down_proj.weight = self.down_proj.weight.withSharding(.{ .d = .model }); + } + } + pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); + const proj = self.up_proj.forward(x); + var output: zml.Tensor = self.gate_proj.forward(x); output = output.silu().mul(proj); - return zml.call(self.down_proj, .forward, .{output}); + output = self.down_proj.forward(output); + + return output; } }; @@ -250,6 +310,30 @@ pub const SelfAttn = struct { num_kv_heads: i64 = 0, rope_opts: zml.nn.RopeOpts = undefined, + pub fn init(self: *SelfAttn, config: LlamaLM.Config, mesh: zml.Mesh) void { + self.num_heads = @intCast(config.num_attention_heads); + self.num_kv_heads = @intCast(config.num_key_value_heads); + self.rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }; + + const has_model_axis = mesh.topology.hasTag(.model) != null; + + self.q_proj.weight = self.q_proj.weight.withTags(.{ .d, .q_hidden }).withMesh(mesh); + self.k_proj.weight = self.k_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh); + self.v_proj.weight = self.v_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh); + self.o_proj.weight = self.o_proj.weight.withTags(.{ .o_hidden, .d }).withMesh(mesh); + + if (has_model_axis) { + self.q_proj.weight = self.q_proj.weight.withSharding(.{ .q_hidden = .model }); + self.k_proj.weight = self.k_proj.weight.withSharding(.{ .kv_hidden = .model }); + self.v_proj.weight = self.v_proj.weight.withSharding(.{ .kv_hidden = .model }); + self.o_proj.weight = self.o_proj.weight.withSharding(.{ .o_hidden = .model }); + } + } + /// Self Attention. /// - If token_index is set, x is assumed to be the representation of one new token, /// and kv_cache will be read for the previous tokens. @@ -264,22 +348,23 @@ pub const SelfAttn = struct { kv_cache: KvCache, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); - var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); - var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + + const q_proj_out = self.q_proj.forward(x); + const k_proj_out = self.k_proj.forward(x); + const v_proj_out = self.v_proj.forward(x); + log.warn("SelfAttn.forward: q_proj_out={}, k_proj_out={}, v_proj_out={}", .{ q_proj_out, k_proj_out, v_proj_out }); + var q = q_proj_out.splitAxis(.d, .{ .h = self.num_heads, .hd = .auto }); + var k = k_proj_out.splitAxis(.d, .{ .h = num_kv_heads, .hd = .auto }); + var v = v_proj_out.splitAxis(.d, .{ .h = num_kv_heads, .hd = .auto }); // Generate the attention mask. const seq_len = kv_cache.k.dim(.k); var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), null); + attn_mask = attn_mask.gatherSlices(.{ .q = x.dim(.s) }, token_index.reshape(.{ .coord = 1 }), .{}); - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, - // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = x.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); - - // In self-attention, .s axis is used both for keys and queries. - const pos_index = b: { + const pos_index = blk: { const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); - break :b temp.add(token_index.broad(temp.shape())); + break :blk temp.add(token_index.broad(temp.shape())); }; q = zml.nn.rope(q, pos_index, self.rope_opts); @@ -290,13 +375,30 @@ pub const SelfAttn = struct { const dtype = q.dtype(); const new_kv_cache = kv_cache.update(k, v, token_index); - k = new_kv_cache.keys().convert(dtype); - v = new_kv_cache.values().convert(dtype); + var k_cached = new_kv_cache.keys().convert(dtype); + var v_cached = new_kv_cache.values().convert(dtype); + + if (self.num_kv_heads > 0 and self.num_heads > self.num_kv_heads) { + const num_groups = @divExact(self.num_heads, self.num_kv_heads); + k_cached = k_cached.repeat1d(.h, @intCast(num_groups)); + v_cached = v_cached.repeat1d(.h, @intCast(num_groups)); + } + + const attn_output = zml.nn.sdpa(q, k_cached, v_cached, .{ .attn_mask = attn_mask, .allow_cudnn = true }); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); - // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; + + var output = self.o_proj.forward(attn).rename(.{ .o_hidden = .d }); + log.warn("SelfAttn.forward: attn_output={}, output={}", .{ attn_output, output }); + if (x.dim(.s) > 1) { + // Prefill phase: Shard the sequence dimension. + output = output.withSharding(.{ .s = .data, .d = .model }); + } else { + // Generation phase (s=1): Replicate the sequence dimension. + output = output.withSharding(.{ .s = .replicated, .d = .model }); + } + + return .{ output, new_kv_cache }; } fn initKvCache(key_shape: zml.Shape) KvCache { @@ -315,28 +417,36 @@ pub const KvCache = struct { v: Tensor, layer_index: Tensor, - pub fn init(kv_shape: zml.Shape) KvCache { - // The KV-cache is initialized with ones to detect reads of uninitialized memory. + pub fn init(kv_shape: zml.Shape, mesh: zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(.{ .k = .data, .h = .model }); + return .{ - .k = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), - .v = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .k = Tensor.constant(sharded_shape, kv_shape.dtype().one()).withMesh(mesh), + .v = Tensor.constant(sharded_shape, kv_shape.dtype().one()).withMesh(mesh), .layer_index = Tensor.scalar(-1, .u32), }; } - pub fn initShape(kv_shape: zml.Shape) ShapeOf(KvCache) { + pub fn initShape( + kv_shape: zml.Shape, + ) ShapeOf(KvCache) { + const sharded_shape = kv_shape.withPartitioning(.{ .k = .data, .h = .model }); + return .{ - .k = kv_shape, - .v = kv_shape, + .k = sharded_shape, + .v = sharded_shape, .layer_index = zml.Shape.init(.{}, .u32), }; } - pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { + pub fn initBuffer(allocator: std.mem.Allocator, kv_shape: zml.Shape, mesh: zml.Mesh, platform: zml.Platform) !zml.Bufferized(KvCache) { + const sharding = zml.Sharding.init(mesh, kv_shape.withPartitioning(.{ .k = .data, .h = .model })); + const slice = try zml.slice.fill(f32, allocator, kv_shape, 1.0); + defer allocator.free(slice); return .{ - .k = try zml.Buffer.constant(platform, kv_shape, 1), - .v = try zml.Buffer.constant(platform, kv_shape, 1), - .layer_index = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0), + .k = try zml.Buffer.from(platform, sharding, slice, .{ .wait = true }), + .v = try zml.Buffer.from(platform, sharding, slice, .{ .wait = true }), + .layer_index = try zml.Buffer.constant(platform, .init(mesh, zml.Shape.init(.{}, .u32)), 0), }; } @@ -349,31 +459,30 @@ pub const KvCache = struct { } pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { - const k_shape = self.k.shape().drop(.layer); var layer = self.layer_index; layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; return if (token_index) |idx| .{ .k = self.k.scatterSlices( .{ .layer = layer, .k = idx }, - new_k.convert(self.k.dtype()).transpose(k_shape), + new_k.convert(self.k.dtype()), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.k), .v = self.v.scatterSlices( .{ .layer = layer, .k = idx }, - new_v.convert(self.v.dtype()).transpose(k_shape), + new_v.convert(self.v.dtype()), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.v), .layer_index = self.layer_index, } else .{ .k = self.k.scatterSlices( .{ .layer = layer }, - new_k.convert(self.k.dtype()).transpose(k_shape), + new_k.convert(self.k.dtype()), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.k), .v = self.v.scatterSlices( .{ .layer = layer }, - new_v.convert(self.v.dtype()).transpose(k_shape), + new_v.convert(self.v.dtype()), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.v), .layer_index = self.layer_index, diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 0b121e65..5996dad4 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -61,6 +61,7 @@ pub fn generateText( kv_cache_: zml.Bufferized(llama.KvCache), tokenizer: zml.tokenizer.Tokenizer, allocator: std.mem.Allocator, + mesh: zml.Mesh, seed: u128, prompt: []const u8, skip_llama3_encoding: bool, @@ -75,7 +76,7 @@ pub fn generateText( const max_seq_len = llama_.model.shape().s; // init RNG and buffers - var rng = try zml.Tensor.Rng.init(platform, seed); + var rng = try zml.Tensor.Rng.init(platform, mesh, seed); var generated_token_buffer = [_]u32{undefined}; var kv_cache = prefill: { @@ -83,13 +84,14 @@ pub fn generateText( const prefill_buffer = try allocator.alloc(u32, max_seq_len); @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); - var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); + var prefill_tokens = try zml.Buffer.fromSlice(platform, mesh, .{max_seq_len}, prefill_buffer); defer prefill_tokens.deinit(); - var prefill_token_pos = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0); + const sharding2: zml.Sharding = .init(mesh, zml.Shape.init(.{}, .u32)); + var prefill_token_pos = try zml.Buffer.constant(platform, sharding2, 0); defer prefill_token_pos.deinit(); const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); - _ = try prefilled_tokens.toHost(std.mem.sliceAsBytes(prefill_buffer)); + _ = try prefilled_tokens.toHostBuffer2(std.mem.sliceAsBytes(prefill_buffer)); generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; break :prefill kv_cache; }; @@ -97,7 +99,7 @@ pub fn generateText( // Prepare for token-by-token generation, // start with the token generated based on the full prompt. - var current_token = try zml.Buffer.fromSlice(platform, .{1}, &generated_token_buffer); + var current_token = try zml.Buffer.fromSlice(platform, mesh, .{1}, &generated_token_buffer); defer current_token.deinit(); // Here we collect the generated text @@ -131,14 +133,14 @@ pub fn generateText( // current token pos needs to go into a zml.Buffer const token_pos_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; - const token_pos = try zml.Buffer.fromSlice(platform, .{}, token_pos_buffer); + const token_pos = try zml.Buffer.fromSlice(platform, mesh, .{}, token_pos_buffer); defer token_pos.deinit(); // call to generate the next token current_token, kv_cache, rng = mod_generate.call(.{ current_token, token_pos, kv_cache, rng }); // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + _ = try current_token.toHostBuffer2(std.mem.sliceAsBytes(&generated_token_buffer)); } const end = std.time.microTimestamp(); const duration = stdx.math.divFloat(f64, end - start, std.time.us_per_s); @@ -217,7 +219,7 @@ pub fn asyncMain() !void { defer context.deinit(); const compilation_options = zml.CompilationOptions{ - .xla_dump_to = "/tmp/zml/llama", + .xla_dump_to = "/tmp/zml/llama-new-sharding", .sharding_enabled = res.args.sharding orelse true, }; @@ -225,10 +227,48 @@ pub fn asyncMain() !void { // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' const create_opts_json = res.args.@"create-options" orelse "{}"; const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); - const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); + const platform = context.autoPlatform(.{ .cpu = .{ .cpu_device_count = 4 } }).withCompilationOptions(compilation_options); create_opts.deinit(); context.printAvailablePlatforms(platform); + log.info("\n\n\n\n", .{}); + + const num_devices = platform.getDevices().len; + const num_heads = config.num_attention_heads; + const num_kv_heads = config.num_key_value_heads; + + var tp_degree: usize = 1; + const search_limit = if (num_devices < num_heads) num_devices else num_heads; + + var i: usize = search_limit; + while (i > 1) : (i -= 1) { + if ((num_devices % i == 0) and (num_heads % i == 0) and (num_kv_heads % i == 0)) { + tp_degree = i; + break; + } + } + + const dp_degree = @divExact(num_devices, tp_degree); + + const mesh = if (num_devices == 1) + zml.Mesh.single() + else if (tp_degree == 1) blk: { + log.info("Strategy: Pure Data Parallelism", .{}); + break :blk zml.Mesh.init(.{ .data = dp_degree }); + } else if (dp_degree == 1) blk: { + log.info("Strategy: Pure Tensor Parallelism", .{}); + break :blk zml.Mesh.init(.{ .model = tp_degree }); + } else blk: { + log.info("Strategy: Hybrid Data + Tensor Parallelism", .{}); + break :blk zml.Mesh.init(.{ .data = dp_degree, .model = tp_degree }); + }; + + log.info( + "Model heads: {d}, KV heads: {d}, Devices: {d} => Using {d}-way Data Parallelism and {d}-way Tensor Parallelism", + .{ num_heads, num_kv_heads, num_devices, dp_degree, tp_degree }, + ); + log.info("Constructed mesh: {}", .{mesh}); + var ts = try zml.aio.detectFormatAndOpen(allocator, res.args.weights.?); defer ts.deinit(); @@ -242,58 +282,70 @@ pub fn asyncMain() !void { .temperature = 1.0, }, }; - model_instance.init(config, llama_options); + model_instance.init(mesh, config, llama_options); const dims = model_instance.model.shape(); const dtype = model_instance.model.embed_tokens.weight.dtype(); - const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); - const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); + const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32).withPartitioning(.{ .s = .data }); + log.info("tokens_shape_prefill: {}", .{tokens_shape_prefill}); + + const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32).withPartitioning(.{ .s = .replicated }); + log.info("tokens_shape: {}", .{tokens_shape}); + const token_idx_shape = zml.Shape.init(.{}, .u32); - const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype).withSharding(.{.h}); + const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype); + log.info("kv_shape: {}", .{kv_shape}); const kv_cache_shape: zml.ShapeOf(llama.KvCache) = llama.KvCache.initShape(kv_shape); + log.info("kv_cache_shape: {}", .{kv_cache_shape}); + const rng_shape = zml.Tensor.Rng.shape(); + log.info("rng_shape: {}", .{rng_shape}); + + log.info("\n\n\n\n", .{}); var start = try std.time.Timer.start(); var fut_mod_prefill = try asynk.asyncc(zml.compile, .{ - allocator, llama.LlamaLM.forward, .{ config, llama_options }, + allocator, llama.LlamaLM.forward, .{ mesh, config, llama_options }, .{ tokens_shape_prefill, token_idx_shape, kv_cache_shape, rng_shape, }, - ts, - platform, + ts, platform, + mesh, }); + const mod_prefill = try fut_mod_prefill.awaitt(); + log.warn(">>>>>>>>>>>>>>> conmpiled mod_prefill", .{}); var fut_mod = try asynk.asyncc(zml.compile, .{ - allocator, llama.LlamaLM.forward, .{ config, llama_options }, + allocator, llama.LlamaLM.forward, .{ mesh, config, llama_options }, .{ tokens_shape, token_idx_shape, kv_cache_shape, rng_shape, }, - ts, - platform, + ts, platform, + mesh, }); log.info("\tLoading Llama weights from {?s}...", .{res.args.weights}); - var llama_weights = try zml.aio.loadBuffers(llama.LlamaLM, .{ config, llama_options }, ts, model_arena.allocator(), platform); + var llama_weights = try zml.aio.loadBuffers(llama.LlamaLM, .{ mesh, config, llama_options }, ts, model_arena.allocator(), platform, mesh); defer zml.aio.unloadBuffers(&llama_weights); log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); - var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_weights); + var llama_module_prefill = mod_prefill.prepare(llama_weights); defer llama_module_prefill.deinit(); var llama_module = (try fut_mod.awaitt()).prepare(llama_weights); defer llama_module.deinit(); log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); log.info("Creating KvCache", .{}); - const kv_cache = try llama.KvCache.initBuffer(kv_shape, platform); + const kv_cache = try llama.KvCache.initBuffer(allocator, kv_shape, mesh, platform); var tokenizer = blk: { if (res.args.tokenizer) |tok| { @@ -314,7 +366,7 @@ pub fn asyncMain() !void { const seed = res.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); const skip_llama3_encoding = res.args.@"no-llama3" orelse false; - const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); + const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, mesh, seed, prompt[0..], skip_llama3_encoding); // generated text will be printed token by token. defer allocator.free(generated_text); } diff --git a/examples/partitioning/BUILD.bazel b/examples/partitioning/BUILD.bazel new file mode 100644 index 00000000..1c38d91f --- /dev/null +++ b/examples/partitioning/BUILD.bazel @@ -0,0 +1,11 @@ +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "partitioning", + main = "main.zig", + deps = [ + "@zml//async", + "@zml//stdx", + "@zml//zml", + ], +) diff --git a/examples/partitioning/main.zig b/examples/partitioning/main.zig new file mode 100644 index 00000000..129865f6 --- /dev/null +++ b/examples/partitioning/main.zig @@ -0,0 +1,136 @@ +const std = @import("std"); +const zml = @import("zml"); +const stdx = @import("stdx"); +const asynk = @import("async"); + +// set log level to debug to print the generated IR +pub const std_options: std.Options = .{ + .log_level = .debug, + .logFn = asynk.logFn(std.log.defaultLog), + .log_scope_levels = &[_]std.log.ScopeLevel{ + .{ .scope = .@"zml/async", .level = .info }, + }, +}; + +const log = std.log.scoped(.@"examples/partitioning"); + +const ThirdPartyModule = struct { + weight: zml.Tensor, + + pub fn init(self: *ThirdPartyModule, mesh: zml.Mesh) void { + self.weight = self.weight.withTags(.{ .rows, .cols }).withMesh(mesh).withSharding(.{ .rows = .x }); + } + + pub fn forward(self: ThirdPartyModule, a: zml.Tensor) zml.Tensor { + log.debug("Forwarding a: {} and weight: {}", .{ a, self.weight }); + + zml.pushMesh(a.mesh().flatten(.x)); + const x = a.withSharding(.{ .m = .x }).add(self.weight.withSharding(.{ .rows = .x })); + zml.popMesh(); + + return x.withSharding(.{ .m = .x, .n = .y }); + } +}; + +const Model = struct { + module_with_weights: ThirdPartyModule, + mesh: zml.Mesh = undefined, + + pub fn init(self: *Model, main_mesh: zml.Mesh, third_party_mesh: zml.Mesh) void { + self.mesh = main_mesh; + self.module_with_weights.init(third_party_mesh); + } + + pub fn forward(self: Model, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + const a_add_b = a.add(b); + const weighted_a: zml.Tensor = zml.call(self.module_with_weights, .forward, .{a}); + return a_add_b.add(weighted_a); + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{ .cpu = .{ .cpu_device_count = 12 } }).withCompilationOptions(.{ + .sharding_enabled = true, + .xla_dump_to = "/tmp/zml/partitioning", + }); + context.printAvailablePlatforms(platform); + + // const mesh: zml.Mesh = .init(.{ .x = 4, .y = 3 }); + const mesh: zml.Mesh = .auto(platform); + // const mesh: zml.Mesh = .single(); + log.info("{}", .{mesh}); + + const shape = zml.Shape.init(.{ .m = 12, .n = 6 }, .i32); + + const a_shape = shape.withTags(.{ .m, .n }).withPartitioning(.{ .m = .x, .n = .y }); + const b_shape = shape.withTags(.{ .n, .m }).withPartitioning(.{ .n = .x, .m = .y }); + + var buffers: zml.aio.BufferStore.Buffers = .{}; + const weight = try zml.slice.arange(allocator, shape, .{}); + defer allocator.free(weight); + try buffers.put(arena, "module_with_weights.weight", zml.HostBuffer.fromBytes(shape, weight)); + + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + var model_shapes = try zml.aio.populateModel(Model, allocator, buffer_store); + model_shapes.init(mesh, mesh.flatten(.x)); + + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Model.forward, model_shapes, .{ a_shape, b_shape }, mesh, platform }); + + var model_weights = try zml.aio.loadModelBuffers(Model, model_shapes, buffer_store, arena, platform, mesh); + defer zml.aio.unloadBuffers(&model_weights); + log.info("✅ Loaded weights", .{}); + + const compiled = try compilation.awaitt(); + var executable = compiled.prepare(model_weights); + defer executable.deinit(); + log.info("✅ Compiled model", .{}); + + const a_sharding: zml.Sharding = .init(mesh, a_shape); + log.info("a_sharding: {}", .{a_sharding}); + + const a_buffer_slice = try zml.slice.arange(allocator, a_shape, .{}); + defer allocator.free(a_buffer_slice); + log.debug("a_buffer_slice: {}", .{zml.slice.pretty(a_shape, a_buffer_slice)}); + + const a_buffer = try zml.Buffer.from(platform, a_sharding, a_buffer_slice, .{}); + defer a_buffer.deinit(); + + const b_sharding: zml.Sharding = .init(mesh, a_shape); + log.info("b_sharding: {}", .{b_sharding}); + + const b_buffer_slice = try zml.slice.arange(allocator, b_shape, .{}); + defer allocator.free(b_buffer_slice); + log.debug("b_buffer_slice: {}", .{zml.slice.pretty(b_shape, b_buffer_slice)}); + + const b_buffer = try zml.Buffer.from(platform, b_sharding, b_buffer_slice, .{}); + defer b_buffer.deinit(); + + log.info("✅ Running model....", .{}); + var result_buffer: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + defer result_buffer.deinit(); + + const result = try result_buffer.toHost(allocator); + defer allocator.free(result); + + log.info("Result buffer: {}", .{zml.slice.pretty(result_buffer.shape(), result)}); +} diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 58d9ac29..2bd5a8fc 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -393,6 +393,7 @@ pub const Client = opaque { /// Not all platform have a profiling api, for those the profiler object will do nothing. /// Platforms with known profiler extensions: cuda, xpu pub fn getProfiler(self: *const Client, api: *const Api, options: Profiler.Options) Profiler { + log.warn(">>>>>>>>>>>>>>>> getProfiler version: {d}\n", .{api.version().minor}); if (api.version().minor >= 45) { if (api.lookupExtension(c.PJRT_Profiler_Extension, c.PJRT_Extension_Type_Profiler)) |ext| { return Profiler.init(ext.profiler_api.*, options); @@ -921,11 +922,51 @@ pub const Buffer = opaque { return ret.is_on_cpu; } - pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { + // pub const struct_PJRT_Buffer_ToHostBuffer_Args = extern struct { + // struct_size: usize = @import("std").mem.zeroes(usize), + // extension_start: [*c]PJRT_Extension_Base = @import("std").mem.zeroes([*c]PJRT_Extension_Base), + // src: ?*PJRT_Buffer = @import("std").mem.zeroes(?*PJRT_Buffer), + // host_layout: [*c]PJRT_Buffer_MemoryLayout = @import("std").mem.zeroes([*c]PJRT_Buffer_MemoryLayout), + // dst: ?*anyopaque = @import("std").mem.zeroes(?*anyopaque), + // dst_size: usize = @import("std").mem.zeroes(usize), + // event: ?*PJRT_Event = @import("std").mem.zeroes(?*PJRT_Event), + // }; + + // struct PJRT_Buffer_ToHostBuffer_Args { + // size_t struct_size; + // PJRT_Extension_Base* extension_start; + // PJRT_Buffer* src; + + // // The caller can specify an optional host layout. If nullptr, the layout of + // // the src buffer will be used. The caller is responsible to keep the data + // // (tiled or strides) in the host_layout alive during the call. + // PJRT_Buffer_MemoryLayout* host_layout; + // // `dst` can be nullptr to query required size which will be set into + // // `dst_size`. + // void* dst; // in/out + // // Size of `dst` in bytes. If `dst` is nullptr, then `dst_size` is set to the + // // size needed. Otherwise, `dst_size` must be greater than or equal to the + // // needed size. + // size_t dst_size; // in/out + + // // Event that signals when the copy has completed. + // PJRT_Event* event; // out + // }; + + pub const ToHostBufferOpts = struct { + host_layout: ?MemoryLayout = null, + dst_size: ?usize = null, + }; + + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8, opts: ToHostBufferOpts) ApiError!?*Event { + if (opts.host_layout) |l| { + std.debug.print(">> layout: {any}", .{l}); + } const ret = try api.call(.PJRT_Buffer_ToHostBuffer, .{ .src = self.inner(), + .host_layout = if (opts.host_layout) |layout| @ptrCast(@constCast(&layout.toCStruct())) else null, .dst = @ptrCast(dst.ptr), - .dst_size = dst.len, + .dst_size = if (opts.dst_size) |size| size else dst.len, }); return @ptrCast(ret.event); } diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 218c484f..caaaadc9 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -57,6 +57,90 @@ zig_cc_test( deps = [":zml"], ) +zig_cc_test( + name = "test_partitioning", + data = [ + "aio/torch/simple.pt", + "aio/torch/simple_test_4.pickle", + ], + test_runner = ":test_runner", + deps = [":zml_partitioning"], +) + +zig_library( + name = "zml_partitioning", + srcs = [ + "dtype.zig", + "partitioning.zig", + "shape.zig", + "slice.zig", + "test_runner.zig", + ], + copts = ["-lc"], + main = "partitioning.zig", + visibility = ["//visibility:public"], + deps = [ + ":posix", + ":sentencepiece_model_proto", + ":xla_proto", + "//async", + "//mlir", + "//mlir/dialects", + "//pjrt", + "//runtimes", + "//stdx", + "//zml/tokenizer", + "//zml/tools", + "@rules_zig//zig/lib:libc", + "@rules_zig//zig/runfiles", + "@zig-yaml//:zig-yaml", + ], +) + +# zig_cc_test( +# name = "test_partitioning", +# data = [ +# "aio/torch/simple.pt", +# "aio/torch/simple_test_4.pickle", +# ], +# test_runner = ":test_runner", +# deps = [":zml_partitioning"], +# ) + +# zig_library( +# name = "zml_partitioning", +# srcs = [ +# "context.zig", +# "dtype.zig", +# "floats.zig", +# "meta.zig", +# "partitioning.zig", +# "pjrtx.zig", +# "platform.zig", +# "shape.zig", +# "test_runner.zig", +# ], +# copts = ["-lc"], +# main = "partitioning.zig", +# visibility = ["//visibility:public"], +# deps = [ +# ":posix", +# ":sentencepiece_model_proto", +# ":xla_proto", +# "//async", +# "//mlir", +# "//mlir/dialects", +# "//pjrt", +# "//runtimes", +# "//stdx", +# "//zml/tokenizer", +# "//zml/tools", +# "@rules_zig//zig/lib:libc", +# "@rules_zig//zig/runfiles", +# "@zig-yaml//:zig-yaml", +# ], +# ) + filegroup( name = "test_runner", srcs = ["test_runner.zig"], diff --git a/zml/aio.zig b/zml/aio.zig index 467def71..92ffca53 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -504,8 +504,9 @@ pub fn loadBuffers( buffer_store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, + mesh: zml.Mesh, ) !zml.Bufferized(Model) { - return loadBuffersWithPrefix(Model, init_args, buffer_store, allocator, platform, ""); + return loadBuffersWithPrefix(Model, init_args, buffer_store, allocator, platform, mesh, ""); } /// Creates a bufferized version of a Model from the given BufferStore with a specified prefix. @@ -522,6 +523,7 @@ pub fn loadBuffersWithPrefix( buffer_store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, + mesh: zml.Mesh, prefix: []const u8, ) !zml.Bufferized(Model) { var arena_state = std.heap.ArenaAllocator.init(allocator); @@ -536,7 +538,7 @@ pub fn loadBuffersWithPrefix( @call(.auto, Model.init, .{&model} ++ init_args); } - return loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, prefix); + return loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, mesh, prefix); } /// Creates a bufferized version of a Model from the given BufferStore. For details about @@ -551,8 +553,9 @@ pub fn loadModelBuffers( buffer_store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, + mesh: zml.Mesh, ) !zml.Bufferized(Model) { - return try loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, ""); + return try loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, mesh, ""); } /// Creates a bufferized version of a Model from the given BufferStore and the given prefix. @@ -567,6 +570,7 @@ pub fn loadModelBuffersWithPrefix( buffer_store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, + mesh: zml.Mesh, prefix: []const u8, ) !zml.Bufferized(Model) { // Allocate the bufferized version. @@ -575,7 +579,7 @@ pub fn loadModelBuffersWithPrefix( var res: zml.Bufferized(Model) = undefined; try zml.meta.mapAlloc(struct { pub fn initBuffer(_: void, tensor: zml.Tensor) zml.Buffer { - return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined }; + return .{ ._shape = tensor.shape(), ._mesh = tensor._mesh, ._api = undefined, ._shards = undefined }; } }.initBuffer, allocator, {}, model, &res); @@ -583,7 +587,7 @@ pub fn loadModelBuffersWithPrefix( try prefix_builder.push(allocator, prefix); defer prefix_builder.deinit(allocator); - try visitStructAndLoadBuffer(allocator, &prefix_builder, buffer_store, &res, platform); + try visitStructAndLoadBuffer(allocator, &prefix_builder, buffer_store, &res, platform, mesh); return res; } @@ -658,7 +662,14 @@ pub fn awaitAll(buffers: anytype) !void { }).cb, {}, buffers); } -fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, buffer_store: BufferStore, obj: anytype, platform: zml.Platform) !void { +fn visitStructAndLoadBuffer( + allocator: std.mem.Allocator, + prefix_builder: *PrefixBuilder, + buffer_store: BufferStore, + obj: anytype, + platform: zml.Platform, + mesh: zml.Mesh, +) !void { const err_msg = "visitStructAndLoadBuffer must be called with a pointer to type. Received "; const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { .pointer => |ptr_info| switch (ptr_info.size) { @@ -676,7 +687,13 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from(platform, buf_with_metadata, .{}); + const m = obj._mesh orelse mesh; + const sharding: zml.Sharding = .init(m, obj._shape); + // log.warn("load {} with sharding: {}", .{ obj, sharding }); + log.warn(" - {s} - sharding: {any} - {s}", .{ prefix, sharding, sharding.getShardingAttr().constSlice() }); + + const data_slice = buf_with_metadata._data[0..buf_with_metadata.shape().byteSize()]; + obj.* = try zml.Buffer.from(platform, sharding, data_slice, .{}); } else { log.err("Buffer not found: {s}", .{prefix}); @@ -693,7 +710,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform, mesh); } } else stdx.debug.compileError("type not supported by visitStructAndLoadBuffer: {}", .{T}); }, @@ -701,7 +718,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi for (obj, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform, mesh); } }, @@ -711,12 +728,12 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi try prefix_builder.push(allocator, field.name); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, &@field(obj, field.name), platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, &@field(obj, field.name), platform, mesh); } }, .optional => { if (obj.*) |*obj_val| { - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, obj_val, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, obj_val, platform, mesh); } }, else => {}, diff --git a/zml/buffer.zig b/zml/buffer.zig index 36f61255..a7c6ff50 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -1,6 +1,7 @@ const std = @import("std"); const asynk = @import("async"); +const runtimes = @import("runtimes"); const stdx = @import("stdx"); const DataType = @import("dtype.zig").DataType; @@ -8,6 +9,9 @@ const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const partitioning = @import("partitioning.zig"); +const Mesh = partitioning.Mesh; +const Sharding = partitioning.Sharding; test { std.testing.refAllDecls(@This()); @@ -43,61 +47,47 @@ pub const Buffer = struct { _shape: Shape, _api: *const pjrt.Api, _shards: Shards, + _mesh: ?Mesh = null, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); pub const FromOptions = struct { wait: bool = true, - memory: ?pjrt.Memory.Kind = null, + memory: ?Memory = null, }; /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(platform: Platform, host_buffer: HostBuffer, opts: FromOptions) !Buffer { + pub fn from(platform: Platform, sharding: Sharding, data: []const u8, opts: FromOptions) !Buffer { + var sharding_devices = sharding.iterator(platform.getDevices()); + var res: Buffer = .{ ._api = platform.pjrt_api, - ._shape = host_buffer.shape(), + ._shape = sharding.global_shape, ._shards = .{}, }; - // We shard only on the first axis so that the chunks are still contiguous. - // TODO: support more advanced sharding specs - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - const sharding_ax: ?u3 = std.simd.firstTrue(host_buffer.shape()._sharding_info); - const n_partitions = platform.sharding().num_partitions; - const chunk_size = if (sharding_ax) |ax| cs: { - // This kind of sharding error should be detected earlier on. - stdx.debug.assert(@rem(host_buffer.dim(ax), n_partitions) == 0, "Buffer.from({}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ host_buffer, ax, n_partitions }); - break :cs @divExact(host_buffer.dim(ax), n_partitions); - } else 0; - - const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); - const byte_strides = host_buffer.strides(); - - const devices = platform.getDevices(); - for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. - const buf = if (sharding_ax) |ax| buf: { - const start: i64 = @as(i64, @intCast(i)) * chunk_size; - break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); - } else host_buffer; + while (sharding_devices.next()) |shard| { + const specs = shard.specs(); var args = pjrt.Client.BufferFromHostBufferArgs{ - .data = buf._data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, + .data = data[specs.start_offset..].ptr, + .buffer_type = bufferTypeFromDtype(shard.shard_shape.dtype()), + .dims = specs.dims[0..specs.num_dims], + .byte_strides = specs.byte_strides[0..specs.num_dims], + .device = shard.device, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; + if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); + const memories = try shard.device.addressableMemories(platform.pjrt_api); const memory = for (memories) |m| { const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; + if (kind == memory_kind.toPjrtMemory()) break m; } else return error.NotFound; args.memory = memory; } else { - args.device = devices[i]; + args.device = shard.device; } const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); @@ -115,7 +105,6 @@ pub const Buffer = struct { return res; } - pub fn awaitt(self: Buffer) !Buffer { for (self._shards.constSlice()) |buffer| { if (buffer.getReadyEvent(self._api)) |ev| { @@ -127,36 +116,41 @@ pub const Buffer = struct { } /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { + pub fn fromPjrtBuffers(platform: Platform, sharding: Sharding, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); var shards: Shards = .{}; shards.appendSliceAssumeCapacity(pjrt_buffers); return .{ ._api = platform.pjrt_api, - ._shape = shape_, + ._shape = sharding.global_shape, ._shards = shards, + ._mesh = sharding.mesh, }; } /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. - pub fn fromSlice(platform: Platform, dimz: anytype, s: anytype) !Buffer { - const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); - return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s)), .{}); + pub fn fromSlice(platform: Platform, mesh: Mesh, dimz: anytype, s: anytype) !Buffer { + const sharding: Sharding = .init(mesh, .init(dimz, DataType.fromSliceElementType(s))); + return from(platform, sharding, std.mem.sliceAsBytes(s), .{}); } /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromBytes(platform: Platform, sh: Shape, data: []const u8) !Buffer { - return from(platform, HostBuffer.fromBytes(sh, data), .{}); + const mesh = Mesh.single(); + const sharding: Sharding = .init(mesh, sh); + return from(platform, sharding, data, .{}); } /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { const host_buffer = HostBuffer.fromArray(&arr); - return try from(platform, host_buffer, .{ .wait = true }); + const mesh = Mesh.single(); + const sharding: Sharding = .init(mesh, host_buffer.shape()); + return try from(platform, sharding, host_buffer.bytes(), .{ .wait = true }); } /// Copies the given Zig slice to the accelerator memory and @@ -169,7 +163,9 @@ pub const Buffer = struct { /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromBytesOpts(platform: Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { - return from(platform, HostBuffer.fromBytes(sh, data), opts); + const mesh = Mesh.single(); + const sharding: Sharding = .init(mesh, sh); + return from(platform, sharding, data, opts); } /// Copies the given Zig array to the accelerator memory and @@ -191,11 +187,12 @@ pub const Buffer = struct { pub fn scalar(platform: Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); const host_buffer = HostBuffer.fromBytes(Shape.init(.{}, dtype_), x.constSlice()); - return try from(platform, host_buffer, .{ .wait = true }); + return try from(platform, .init(Mesh.single(), host_buffer.shape()), host_buffer.bytes(), .{ .wait = true }); } /// Creates a Buffer with a single element repeated manytime. - pub fn constant(platform: Platform, shape_: Shape, val: anytype) !Buffer { + pub fn constant(platform: Platform, sharding: Sharding, val: anytype) !Buffer { + const shape_ = sharding.global_shape; var start = try std.time.Timer.start(); defer { const duration_ms = stdx.math.divFloat(f32, start.read(), std.time.ns_per_ms); @@ -212,12 +209,7 @@ pub const Buffer = struct { // Naive version for scalars and buffers with long last axis. if (shape_.rank() < 1 or byte_size * shape_.dim(-1) > max_bytes) { - const host_buffer: HostBuffer = .{ - ._shape = shape_, - ._strides = @splat(0), - ._data = x.constSlice().ptr, - }; - return try from(platform, host_buffer, .{ .wait = true }); + return try from(platform, sharding, x.constSlice(), .{ .wait = true }); } // To speed up copies, duplicate the scalar value into a vector, @@ -239,15 +231,15 @@ pub const Buffer = struct { }, else => unreachable, } - const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, ._data = &bytes }; - return try from(platform, host_buffer, .{ .wait = true }); + const data: []const u8 = bytes[0..max_bytes]; + return try from(platform, sharding, data, .{ .wait = true }); } test constant { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x = try constant(platform, Shape.init(.{ 4, 3, 2 }, .u16), 42); + const x = try constant(platform, .init(zml.Mesh.single(), Shape.init(.{ 4, 3, 2 }, .u16)), 42); const y = try x.getValue([4 * 3 * 2]u16); try std.testing.expectEqual([_]u16{42} ** (4 * 3 * 2), y); } @@ -304,35 +296,113 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); + const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res), .{}); if (maybe_event) |event| { try event.await_(self._api); } return res; } - /// Copies the content of the Buffer back to host, in the given buffer, - /// and return a new `HostBuffer` object with the same shape. - /// The returned `HostBuffer` doesn't own the memory. - pub fn toHost(self: Buffer, output: []u8) !HostBuffer { - stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); - } - /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. - pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); - stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.bytes())); - if (maybe_event) |event| { - try event.await_(self._api); + pub fn toHostBuffer(self: Buffer, sharding: Sharding, allocator: std.mem.Allocator) !HostBuffer { + var self_with_mesh = self; + self_with_mesh._mesh = sharding.mesh; + const data = try self_with_mesh.toHost(allocator); + return HostBuffer.fromBytes(self_with_mesh.shape(), data); + } + + pub fn toHostBuffer2(self: Buffer, output: []u8) !HostBuffer { + var fba = std.heap.FixedBufferAllocator.init(output); + const allocator = fba.allocator(); + const buffer = try self.toHost(allocator); + return HostBuffer.fromBytes(self.shape(), buffer); + } + + pub fn toHost(self: Buffer, allocator: std.mem.Allocator) ![]u8 { + const sharding: Sharding = .init(self._mesh.?, self.shape()); + stdx.debug.assert(self._shards.len == sharding.mesh.numPartitions(), "Expected {} PJRT buffers, got {}", .{ sharding.mesh.numPartitions(), self._shards.len }); + + const global_buffer = try allocator.alloc(u8, sharding.global_shape.byteSize()); + errdefer allocator.free(global_buffer); + + if (sharding.getType() == .replicated) { + if (self._shards.len > 0) { + const event = try self._shards.get(0).toHostBuffer(self._api, global_buffer, .{}); + if (event) |e| try e.await_(self._api); + } + return global_buffer; } - return output; + + const shard_shape = sharding.shardShape(); + const temp_shard_buffer = try allocator.alloc(u8, shard_shape.byteSize()); + defer allocator.free(temp_shard_buffer); + + const element_size = sharding.global_shape.dtype().sizeOf(); + const global_element_strides = sharding.global_shape.computeElementStrides(); + const rk = sharding.global_shape.rank(); + + var devices: [MAX_NUM_SHARDS]*const pjrt.Device = undefined; + for (self._shards.constSlice(), 0..) |shard, i| { + devices[i] = try shard.getDevice(self._api); + } + + var sharding_iter = sharding.iterator(devices[0..self._shards.len]); + while (sharding_iter.next()) |device_shard| { + const shard_index: usize = @intCast(device_shard.index); + const src_buffer = self._shards.get(shard_index); + + const event = try src_buffer.toHostBuffer(self._api, temp_shard_buffer, .{}); + if (event) |e| try e.await_(self._api); + if (rk == 0) { + if (global_buffer.len > 0) { + @memcpy(global_buffer[0..element_size], temp_shard_buffer[0..element_size]); + } + continue; + } + + const last_dim_size = shard_shape.dim(rk - 1); + const contiguous_row_bytes = @as(usize, @intCast(last_dim_size)) * element_size; + const outer_dims_shape = shard_shape.remove(rk - 1); + + var iter = outer_dims_shape.iterator(); + + while (iter.next()) |item| { + const src_flat_index_start_of_row = item.flat_index * @as(usize, @intCast(last_dim_size)); + const src_byte_offset = src_flat_index_start_of_row * element_size; + + var dest_flat_index_start_of_row: i64 = 0; + + for (0..rk) |dim_idx| { + const shard_coord_in_dim = if (dim_idx < rk - 1) item.coords[dim_idx] else 0; + + const global_coord_for_dim = blk: { + const part_spec = sharding.global_shape.partition(dim_idx); + if (part_spec != .axis) break :blk shard_coord_in_dim; // Replicated axis + const mesh_axis_tag = part_spec.toTag(); + if (device_shard.indices.hasTag(mesh_axis_tag) != null) { + const coord = device_shard.indices.dim(mesh_axis_tag); + break :blk (coord * shard_shape.dim(dim_idx)) + shard_coord_in_dim; + } else { + break :blk shard_coord_in_dim; + } + }; + + dest_flat_index_start_of_row += global_coord_for_dim * global_element_strides.get(dim_idx); + } + + const dest_byte_offset = @as(usize, @intCast(dest_flat_index_start_of_row)) * element_size; + + if (contiguous_row_bytes > 0) { + @memcpy( + global_buffer[dest_byte_offset..][0..contiguous_row_bytes], + temp_shard_buffer[src_byte_offset..][0..contiguous_row_bytes], + ); + } + } + } + + return global_buffer; } /// Frees the accelerator memory. @@ -391,7 +461,7 @@ pub const Buffer = struct { fn hasShardedAxis(self: Buffer) bool { if (self._shards.len == 1) return false; - return @reduce(.Or, self._shape._sharding_info); + return self._shape.hasAtLeastOnePartitionedAxis(); } pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { diff --git a/zml/context.zig b/zml/context.zig index bde867f9..f8d33d9e 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -176,7 +176,7 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); + // try CustomCall.registerZmlCustomCalls(p); self.platforms.set(target, p); return p; @@ -206,7 +206,7 @@ pub const Context = struct { deviceKind, }); // we only list 1 CPU device - if (target == .cpu) break; + // if (target == .cpu) break; } } } diff --git a/zml/exe.zig b/zml/exe.zig index 0923f17c..afc00c7e 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -3,12 +3,15 @@ const std = @import("std"); const stdx = @import("stdx"); const aio = @import("aio.zig"); +const partitioning = @import("partitioning.zig"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("tensor.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const Mesh = partitioning.Mesh; +const Sharding = partitioning.Sharding; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; @@ -30,8 +33,9 @@ pub fn compile( args_shapes: ShapeOf(ModuleSignature(func).ArgsT), buffer_store: aio.BufferStore, platform: Platform, + mesh: Mesh, ) !FnExe(func) { - return compileWithPrefix(allocator, func, init_args, args_shapes, buffer_store, platform, ""); + return compileWithPrefix(allocator, func, init_args, args_shapes, buffer_store, platform, mesh, ""); } /// Compiles a Model struct with the given configuration and shapes, for the given platform. @@ -47,6 +51,7 @@ pub fn compileWithPrefix( args_shapes: ShapeOf(ModuleSignature(func).ArgsT), buffer_store: aio.BufferStore, platform: Platform, + mesh: Mesh, prefix: []const u8, ) !FnExe(func) { const ModelT = ModuleSignature(func).ModelT; @@ -62,7 +67,7 @@ pub fn compileWithPrefix( @call(.auto, ModelT.init, .{@as(*ModelT, &model)} ++ init_args); } - return compileModel(allocator, func, model, args_shapes, platform); + return compileModel(allocator, func, model, args_shapes, mesh, platform); } /// Compiles a Model struct with the given configuration and shapes, for the given platform. @@ -72,16 +77,17 @@ pub fn compileModel( comptime func: anytype, model: ModuleSignature(func).ModelT, args_shapes: ShapeOf(ModuleSignature(func).ArgsT), + mesh: Mesh, platform: Platform, ) !FnExe(func) { const ModelT = ModuleSignature(func).ModelT; const name = @typeName(ModelT) ++ ".forward"; log.info("Compiling {s} with {}", .{ name, args_shapes }); - var context = try CompilationContext.init(allocator, name, platform); + var context = try CompilationContext.init(allocator, name, mesh, platform); defer context.deinit(); - return .{ .inner = try context.compileInternal(allocator, func, .{model} ++ args_shapes) }; + return .{ .inner = try context.compileInternal(allocator, func, .{model} ++ args_shapes, mesh) }; } /// Compiles a function with the given configuration and shapes, for the given platform. @@ -90,14 +96,15 @@ pub fn compileFn( allocator: std.mem.Allocator, comptime func: anytype, args: ShapeOf(stdx.meta.FnArgs(func)), + mesh: Mesh, platform: Platform, ) !FnExe(func) { var pretty_name = try prettyFnName(func, allocator); defer pretty_name.deinit(allocator); - var context = try CompilationContext.init(allocator, pretty_name.items, platform); + var context = try CompilationContext.init(allocator, pretty_name.items, mesh, platform); defer context.deinit(); - return .{ .inner = try context.compileInternal(allocator, func, args) }; + return .{ .inner = try context.compileInternal(allocator, func, args, mesh) }; } pub fn FnExe(comptime func: anytype) type { @@ -174,12 +181,15 @@ pub const BaseExe = struct { /// Num devices used (>1 for sharded executable) num_devices: u8, + mesh: Mesh, + /// Allocator backing memory _arena: std.heap.ArenaAllocator, pub fn init( parent_allocator: std.mem.Allocator, platform: Platform, + mesh: Mesh, exe: *pjrt.LoadedExecutable, args: struct { input_shapes: []const Shape, result_shapes: []const Shape, n_devices: u8 }, ) !BaseExe { @@ -215,6 +225,7 @@ pub const BaseExe = struct { .output_per_device = output_per_device, .input_shapes = all_shapes[0..n_in], .result_shapes = all_shapes[n_in..], + .mesh = mesh, ._arena = arena, }; } @@ -233,13 +244,12 @@ pub const BaseExe = struct { pub fn _unsafeCall(self: BaseExe) void { var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; - const sharding = self.platform.sharding(); self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.input_buffer_count, .results = self.output_per_device, - .events = events[0..sharding.num_partitions], + .events = events[0..@intCast(self.mesh.numPartitions())], // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? @@ -249,7 +259,7 @@ pub const BaseExe = struct { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - for (events[0..sharding.num_partitions]) |e| { + for (events[0..@intCast(self.mesh.numPartitions())]) |e| { if (e) |ev| { ev.await_(self.platform.pjrt_api) catch unreachable; } @@ -260,12 +270,14 @@ pub const BaseExe = struct { const LocalContext = struct { index: u32, platform: Platform, + mesh: Mesh, outputs: []const [*]*pjrt.Buffer, output_shapes: []Shape, }; var local_ctx: LocalContext = .{ .index = 0, .platform = self.platform, + .mesh = self.mesh, .outputs = self.output_per_device, .output_shapes = self.result_shapes, }; @@ -279,7 +291,8 @@ pub const BaseExe = struct { for (ctx.outputs) |buff| { shards.appendAssumeCapacity(buff[i]); } - buffer.* = Buffer.fromPjrtBuffers(ctx.platform, ctx.output_shapes[i], shards.constSlice()); + const sharding: Sharding = .init(ctx.mesh, ctx.output_shapes[i]); + buffer.* = Buffer.fromPjrtBuffers(ctx.platform, sharding, shards.constSlice()); } }).cb, &local_ctx, result); stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); @@ -309,7 +322,8 @@ pub const BaseExe = struct { shards.appendAssumeCapacity(dev_out[i]); } - return Buffer.fromPjrtBuffers(self.platform, self.result_shapes[i], shards.constSlice()); + const sharding: Sharding = .init(self.mesh, self.result_shapes[i]); + return Buffer.fromPjrtBuffers(self.platform, sharding, shards.constSlice()); } pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 8f6fcb1c..2459110a 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -6,6 +6,7 @@ const Buffer = @import("buffer.zig").Buffer; const DataType = @import("dtype.zig").DataType; const floats = @import("floats.zig"); const Platform = @import("platform.zig").Platform; +const Mesh = @import("partitioning.zig").Mesh; const Shape = @import("shape.zig").Shape; test { @@ -30,7 +31,7 @@ pub const HostBuffer = struct { pub fn empty(allocator: std.mem.Allocator, sh: Shape) error{OutOfMemory}!HostBuffer { return .{ ._shape = sh, - ._strides = sh.computeStrides().buffer, + ._strides = sh.computeByteStrides().buffer, ._data = (try allocator.alignedAlloc(u8, 64, sh.byteSize())).ptr, ._memory = .{ .managed = .@"64" }, }; @@ -43,7 +44,7 @@ pub const HostBuffer = struct { stdx.debug.assert(shape_.byteSize() == data_.len, "shape {} and data {} don't match", .{ shape_.byteSize(), data_.len }); return .{ ._shape = shape_, - ._strides = shape_.computeStrides().buffer, + ._strides = shape_.computeByteStrides().buffer, ._data = data_.ptr, ._memory = .unmanaged, }; @@ -67,7 +68,7 @@ pub const HostBuffer = struct { std.debug.assert(shape_.byteSize() == raw_bytes.len); return .{ ._shape = shape_, - ._strides = shape_.computeStrides().buffer, + ._strides = shape_.computeByteStrides().buffer, ._data = raw_bytes.ptr, ._memory = .unmanaged, }; @@ -99,7 +100,7 @@ pub const HostBuffer = struct { std.debug.assert(sh.byteSize() == @sizeOf(T)); return .{ ._shape = sh, - ._strides = sh.computeStrides().buffer, + ._strides = sh.computeByteStrides().buffer, ._data = @ptrCast(arr_ptr), ._memory = .unmanaged, }; @@ -156,13 +157,13 @@ pub const HostBuffer = struct { } /// Copies this HostBuffer to the given accelerator. - pub fn toDevice(self: HostBuffer, platform_: Platform) !Buffer { - return try self.toDeviceOpts(platform_, .{}); + pub fn toDevice(self: HostBuffer, platform_: Platform, mesh: Mesh) !Buffer { + return try Buffer.from(platform_, .init(mesh, self.shape()), self.bytes(), .{}); } /// Copies this HostBuffer to the given accelerator (with options). - pub fn toDeviceOpts(self: HostBuffer, platform_: Platform, opts: Buffer.FromOptions) !Buffer { - return try Buffer.from(platform_, self, opts); + pub fn toDeviceOpts(self: HostBuffer, platform_: Platform, mesh: Mesh, opts: Buffer.FromOptions) !Buffer { + return try Buffer.from(platform_, .init(mesh, self.shape()), self.bytes(), opts); } /// Interpret the underlying data as a contiguous slice. @@ -225,7 +226,7 @@ pub const HostBuffer = struct { pub fn isContiguous(self: HostBuffer) bool { const _strides = self._strides; - const cont_strides = self._shape.computeStrides(); + const cont_strides = self._shape.computeByteStrides(); for (self._shape.dims(), _strides[0..self.rank()], cont_strides.constSlice()) |d, stride, cont_stride| { if (d != 1 and stride != cont_stride) return false; } @@ -236,7 +237,7 @@ pub const HostBuffer = struct { stdx.debug.assert(self.isContiguous(), "reshape expects a contiguous tensor, got: {}", .{self}); var res = self; res._shape = self._shape.reshape(shape_); - res._strides = res._shape.computeStrides().buffer; + res._strides = res._shape.computeByteStrides().buffer; return res; } diff --git a/zml/meta.zig b/zml/meta.zig index ad50fe05..a36b8c41 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -187,7 +187,7 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam @field(from, field.name), &@field(to, field.name), ); - } else if (field.default_value_ptr) |_| { + } else if (field.defaultValue()) |_| { @field(to, field.name) = null; } else { stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); diff --git a/zml/module.zig b/zml/module.zig index f031e815..e3b093e2 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -13,6 +13,10 @@ const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const partitioning = @import("partitioning.zig"); +const Mesh = partitioning.Mesh; +const Partition = partitioning.Partition; +const Sharding = partitioning.Sharding; const Shape = @import("shape.zig").Shape; const Target = @import("platform.zig").Target; const Tensor = @import("tensor.zig").Tensor; @@ -24,8 +28,40 @@ test { std.testing.refAllDecls(@This()); } +pub fn pushMesh(mesh: Mesh) void { + const ctx = CompilationContext.current(); + + if (ctx._current_mesh) |m| { + if (m.eql(mesh)) { + stdx.debug.panic("Mesh already pushed: {}", .{m}); + } + + stdx.debug.panic("Mesh already exists: {}", .{m}); + } + + ctx._current_mesh = mesh; + log.info("Pushed mesh: {}", .{mesh}); +} + +pub fn popMesh() void { + const ctx = CompilationContext.current(); + + if (ctx._current_mesh) |m| { + ctx._current_mesh = null; + log.info("Poped mesh: {}", .{m}); + } else { + stdx.debug.panic("No mesh to pop, main mesh: {}", .{ctx._main_mesh}); + } +} + +pub fn currentMesh() Mesh { + const ctx = CompilationContext.current(); + return ctx._current_mesh orelse ctx._main_mesh; +} + pub const MlirFn = struct { name: []const u8, + mesh: Mesh, args_shapes: []Shape, res_tensors: *const anyopaque, res_types: []mlir.Type, @@ -49,6 +85,8 @@ pub const CompilationContext = struct { _mlir_canonicalizer: mlir.PassManager, _module: mlir.Module, + _main_mesh: Mesh, + _current_mesh: ?Mesh, _blocks: std.BoundedArray(TaggedBlock, 64) = .{}, _fn_cache: FnCache = .{}, @@ -64,7 +102,7 @@ pub const CompilationContext = struct { const TensorToBlockArg = std.AutoHashMapUnmanaged(Tensor._Id, struct { mlir.Value, Tensor._Donation }); const AttributeList = std.BoundedArray(mlir.NamedAttribute, 3); - pub fn init(allocator_: std.mem.Allocator, full_name: []const u8, platform: Platform) !CompilationContext { + pub fn init(allocator_: std.mem.Allocator, full_name: []const u8, mesh: Mesh, platform: Platform) !CompilationContext { const mlir_registry = mlir.Registry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); @@ -97,6 +135,8 @@ pub const CompilationContext = struct { return .{ ._platform = platform, ._name = try arena.allocator().dupe(u8, name), + ._main_mesh = mesh, + ._current_mesh = null, ._mlir_ctx = mlir_ctx, ._mlir_registry = mlir_registry, ._mlir_canonicalizer = canonicalizer, @@ -133,6 +173,10 @@ pub const CompilationContext = struct { return _current.?; } + pub fn currentMesh(self: *const CompilationContext) Mesh { + return self._current_mesh orelse self._main_mesh; + } + pub fn target(self: *const CompilationContext) Target { return self._platform.target; } @@ -155,20 +199,20 @@ pub const CompilationContext = struct { allocator_: std.mem.Allocator, comptime func: anytype, args: anytype, + mesh: Mesh, ) !BaseExe { const arena = self.allocator(); var timer = std.time.Timer.start() catch null; - const tensor_args = try self.tensorFromShapes(stdx.meta.FnArgs(func), arena, args); + const tensor_args = try self.tensorFromArgs(stdx.meta.FnArgs(func), arena, args); // Run in a dedicated thread because compilation relies on `threadlocal`. const f = try asynk.callBlocking(CompilationContext.emitMlir, .{ self, func, &tensor_args, CompilationContext.EmitMlirOpts{ .name = "main", .kind = .main } }); const module = self._module; module.getBody().appendOperation(f.mlir_fn); - const sharding = self._platform.sharding(); const mlir_ctx = self._mlir_ctx; - module.op().setAttributeByName("mhlo.num_replicas", .int(mlir_ctx, .i32, sharding.num_replicas)); - module.op().setAttributeByName("mhlo.num_partitions", .int(mlir_ctx, .i32, sharding.num_partitions)); + module.op().setAttributeByName("mhlo.num_replicas", mlir.IntegerAttribute(.i32).init(mlir_ctx, mesh.numReplicas()).asAttr()); + module.op().setAttributeByName("mhlo.num_partitions", mlir.IntegerAttribute(.i32).init(mlir_ctx, mesh.numPartitions()).asAttr()); const module_hash = computeModuleHash(self._platform, module); var module_dir: ?[]const u8 = null; @@ -203,7 +247,7 @@ pub const CompilationContext = struct { } } - const loaded_executable = compileModuleToPjrtExecutable(arena, self._platform, module, module_dir) catch |err| { + const loaded_executable = compileModuleToPjrtExecutable(arena, self._platform, mesh, module, module_dir) catch |err| { log.err("pjrt-{s} failed to compile: {}", .{ @tagName(self._platform.target), err }); if (module_dir) |dir| log.err("mlir can be found at {s}/module.mlir", .{dir}); return err; @@ -228,11 +272,12 @@ pub const CompilationContext = struct { return BaseExe.init( allocator_, self._platform, + mesh, loaded_executable, .{ .input_shapes = f.args_shapes, .result_shapes = f.res_shapes, - .n_devices = sharding.num_replicas * sharding.num_partitions, + .n_devices = @intCast(mesh.numDevices()), }, ); } @@ -339,7 +384,17 @@ pub const CompilationContext = struct { @memset(locations, mlir.Location.unknown(mlir_ctx)); var input_shapes = try std.ArrayList(Shape).initCapacity(res_allocator, tensor_count); - meta.collect(Tensor.shape, {}, &input_shapes, args) catch unreachable; + var input_meshes = try std.ArrayList(?Mesh).initCapacity(res_allocator, tensor_count); + meta.collect(struct { + pub fn shape(tensor: Tensor) Shape { + return tensor.shape(); + } + }.shape, {}, &input_shapes, args) catch unreachable; + meta.collect(struct { + pub fn mesh(tensor: Tensor) ?Mesh { + return tensor._mesh; + } + }.mesh, {}, &input_meshes, args) catch unreachable; stdx.debug.internalAssert(input_shapes.items.len == tensor_count, "args have changed ?", .{}); const input_types = try arena.alloc(mlir.Type, tensor_count); @@ -392,14 +447,16 @@ pub const CompilationContext = struct { const res_attrs = try arena.alloc(AttributeList, out_tensor_count); @memset(res_attrs, .{}); + const mesh = self.currentMesh(); + if (opts.kind == .main) { self.addDonationsAttributes(arg_attrs, fn_res_donations); self.addOutputMemoryKindAttributes(res_attrs, fn_res_output_memory_kind); - if (self._platform.sharding().num_partitions > 1) { - self.addShardingAttributes(arg_attrs, res_attrs, input_shapes.items, fn_res_shapes); - } } + log.warn("{s} add sharding attributes: {} {any}", .{ opts.name, mesh, input_shapes.items }); + self.addShardingAttributes(mesh, arg_attrs, res_attrs, input_shapes.items, input_meshes.items, fn_res_shapes); + const mlir_fn = dialect.func.func(self.mlirCtx(), .{ .sym_name = opts.name, .args = input_types, @@ -423,6 +480,7 @@ pub const CompilationContext = struct { return .{ .mlir_fn = mlir_fn, + .mesh = mesh, .name = opts.name, .args_shapes = input_shapes.items, .res_tensors = fn_res, @@ -498,7 +556,7 @@ pub const CompilationContext = struct { .bias = zml.Tensor{ ._shape = s, ._id = .{ .buffer_id = 0 } }, }; - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); var tensor_args = .{ model, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1234 } }, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1235 } } }; const f = try comp.emitMlir(Local._fwd, &tensor_args, .{ .name = "test.emitMlir.Local.forward", .kind = .main }); @@ -522,71 +580,44 @@ pub const CompilationContext = struct { } } - fn addShardingAttributes(self: CompilationContext, arg_attrs: []AttributeList, res_attrs: []AttributeList, input_shapes: []const Shape, output_shapes: []const Shape) void { + fn addShardingAttributes( + self: CompilationContext, + mesh: Mesh, + arg_attrs: []AttributeList, + res_attrs: []AttributeList, + input_shapes: []const Shape, + input_meshes: []const ?Mesh, + output_shapes: []const Shape, + ) void { + if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. + const ctx = self.mlirCtx(); if (!self._platform.compilation_options.sharding_enabled) return; const default_layout = mlir.NamedAttribute.named(ctx, "mhlo.layout_mode", .string(ctx, "default")); - for (arg_attrs, input_shapes) |*attr, shape| { + for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { attr.appendAssumeCapacity(default_layout); - attr.appendAssumeCapacity(.named(ctx, "mhlo.sharding", self.getShardingAttr(shape))); + if (input_mesh) |m| { + // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); + attr.appendAssumeCapacity(.named(ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); + } else { + // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); + attr.appendAssumeCapacity(.named(ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + } } for (res_attrs, output_shapes) |*attr, shape| { attr.appendAssumeCapacity(default_layout); - attr.appendAssumeCapacity(.named(ctx, "mhlo.sharding", self.getShardingAttr(shape))); + attr.appendAssumeCapacity(.named(ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); } } - pub fn numPartitions(self: CompilationContext) u8 { - return self._platform.sharding().num_partitions; - } - - pub fn getShardingAttr(self: CompilationContext, shape: Shape) mlir.Attribute { + pub fn getShardingAttr(self: CompilationContext, sharding: Sharding) mlir.Attribute { const ctx = self.mlirCtx(); - const num_partitions = self.numPartitions(); var sharding_str: std.BoundedArray(u8, 128) = .{}; - writeShardingRepresentation(shape, num_partitions, sharding_str.writer()) catch unreachable; + sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; return mlir.Attribute.string(ctx, sharding_str.constSlice()); } - fn writeShardingRepresentation(shape: Shape, num_partitions: u8, writer: anytype) @TypeOf(writer).Error!void { - const n_sharded: u8 = @popCount(@as(u8, @bitCast(shape._sharding_info))); - if (n_sharded == 0 or num_partitions == 1) { - try writer.writeAll("{replicated}"); - return; - } - try writer.writeAll("{devices=["); - for (0..shape.rank()) |i| { - try writer.print("{d}", .{if (shape._sharding_info[i]) num_partitions else 1}); - if (i < shape.rank() - 1) try writer.writeByte(','); - } - try writer.print("]<=[{d}]}}", .{num_partitions}); - } - - test writeShardingRepresentation { - var rule: [64]u8 = undefined; - const x = Shape.init(.{ 16, 8 }, .f32); - - // By default tensors are replicated. - { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x, 4, fbs.writer()); - try std.testing.expectEqualStrings("{replicated}", fbs.getWritten()); - } - // Shard along first axis. - { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x.withSharding(.{0}), 4, fbs.writer()); - try std.testing.expectEqualStrings("{devices=[4,1]<=[4]}", fbs.getWritten()); - } - // Also shard along second axis. - { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x.withSharding(.{ 0, 1 }), 2, fbs.writer()); - try std.testing.expectEqualStrings("{devices=[2,2]<=[2]}", fbs.getWritten()); - } - } - fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: mlir.Context, attributes: []AttributeList) ![]mlir.Attribute { const res = try allocator_.alloc(mlir.Attribute, attributes.len); for (res, attributes) |*r, attr| { @@ -614,7 +645,8 @@ pub const CompilationContext = struct { // first, do the "compile" and check the bytecode // the result of this will also have the correct tags of the result shapes - const args_hash = hashArgs(args); + const mesh = self.currentMesh(); + const args_hash = hashArgs(args) % hashArgs(mesh); const key: FnKey = .{ .fn_ptr = &func, .input_hash = args_hash }; const function = self._fn_cache.get(key) orelse b: { @@ -629,7 +661,7 @@ pub const CompilationContext = struct { fn cb(arg_id_: *u16, x: Tensor) Tensor { const a = arg_id_.*; arg_id_.* += 1; - return Tensor{ ._shape = x._shape, ._id = .{ .arg_id = a }, ._donation = .{ .arg = a } }; + return Tensor{ ._shape = x._shape, ._id = .{ .arg_id = a }, ._donation = .{ .arg = a }, ._mesh = x._mesh }; } }.cb, arena, &arg_id, args, &tensor_args); @@ -713,9 +745,9 @@ pub const CompilationContext = struct { /// Create tensor from the given shapes. /// Each created tensor will receive a unique id, local to this CompilationContext. - pub fn tensorFromShapes(self: *CompilationContext, ArgsT: type, allocator_: std.mem.Allocator, args_shapes: anytype) !ArgsT { + pub fn tensorFromArgs(self: *CompilationContext, ArgsT: type, allocator_: std.mem.Allocator, args: anytype) !ArgsT { const Local = struct { - fn tensorFromShape(arg_id: *u64, shape: Shape) Tensor { + fn tensorFromArg(arg_id: *u64, shape: Shape) Tensor { defer arg_id.* += 1; return Tensor{ ._shape = shape, @@ -725,7 +757,7 @@ pub const CompilationContext = struct { } }; var tensor_args: ArgsT = undefined; - try meta.mapAlloc(Local.tensorFromShape, allocator_, &self._unique_id, args_shapes, &tensor_args); + try meta.mapAlloc(Local.tensorFromArg, allocator_, &self._unique_id, args, &tensor_args); return tensor_args; } @@ -834,31 +866,30 @@ fn storePjrtExecutable(platform: Platform, loaded_executable: *pjrt.LoadedExecut try loaded_executable_file.writeAll(serialize_result.bytes); } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module: mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, mesh: Mesh, module: mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { const tracer = Tracer.init("ai.zml.compilation"); const compile_frame = tracer.frameStart("pjrt compilation"); defer tracer.frameEnd(compile_frame, "pjrt compilation"); - const sharding = platform.sharding(); - var options: xla_pb.CompileOptionsProto = .{ .executable_build_options = .{ .device_ordinal = -1, - .num_replicas = sharding.num_replicas, - .num_partitions = sharding.num_partitions, - .use_spmd_partitioning = sharding.num_partitions > 1 or sharding.num_replicas > 1, + .num_replicas = @intCast(mesh.numReplicas()), + .num_partitions = @intCast(mesh.numPartitions()), + .use_spmd_partitioning = !mesh.isSinglePartition(), .device_assignment = .{ - .replica_count = sharding.num_replicas, - .computation_count = sharding.num_partitions, + .replica_count = @intCast(mesh.numReplicas()), + .computation_count = mesh.numPartitions(), // Filled below. .computation_devices = .{}, }, }, }; + const computation_devices = &options.executable_build_options.?.device_assignment.?.computation_devices; - try computation_devices.ensureTotalCapacity(arena, sharding.num_partitions); - const replica_device_ids = try arena.alloc(i64, sharding.num_partitions); - for (0..sharding.num_partitions) |i| { + try computation_devices.ensureTotalCapacity(arena, @intCast(mesh.numPartitions())); + const replica_device_ids = try arena.alloc(i64, @intCast(mesh.numPartitions())); + for (0..@intCast(mesh.numPartitions())) |i| { replica_device_ids[i] = @intCast(i); computation_devices.appendAssumeCapacity(.{ .replica_device_ids = .fromOwnedSlice(replica_device_ids[i .. i + 1]) }); } @@ -1106,7 +1137,7 @@ pub fn hashShape(hasher: *std.hash.Wyhash, shape: Shape) void { // Note: if we enforced 0-init dims then we could hash dims instead. hashArray(hasher, shape.dims(), .Shallow); hash(hasher, shape._dtype, .Shallow); - hash(hasher, shape._sharding_info, .Shallow); + hash(hasher, shape._partitioning, .Shallow); for (shape.tags()) |tag| { hash(hasher, @intFromPtr(tag), .Shallow); } @@ -1120,6 +1151,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str const Key = @TypeOf(key); if (Key == Tensor) return hashShape(hasher, key.shape()); if (Key == Shape) return hashShape(hasher, key); + if (Key == Mesh) return hashShape(hasher, key.topology); if (strat == .Shallow and std.meta.hasUniqueRepresentation(Key)) { hasher.update(std.mem.asBytes(&key)); diff --git a/zml/nn.zig b/zml/nn.zig index ffdb7a9c..65756098 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -399,7 +399,7 @@ test "real/img" { // now try compiling and calling ourselves { - const mod = try zml.compileFn(std.testing.allocator, Fns.testSplitSeq, .{}, platform); + const mod = try zml.compileFn(std.testing.allocator, Fns.testSplitSeq, .{}, zml.Mesh.single(), platform); defer mod.deinit(); const ret = mod.call({}); try testing.expectEqual(20, ret.getValue(i32)); @@ -651,7 +651,7 @@ test resizeBilinear { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -713,7 +713,7 @@ test resizeBicubic { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -1100,10 +1100,10 @@ test sdpaMemEfficient { // Note we use small input vectors to have the tests run reasonably fast, // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 10, 512, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); + const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 10, 512, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, zml.Mesh.single(), platform); defer rng.deinit(); - const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); + const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, zml.Mesh.single(), platform); defer rng_mask.deinit(); // Note: we pass void here, cause Rng.normal doesn't take any runtime inputs. @@ -1158,10 +1158,10 @@ test "sdpaMemEfficient transposed" { // Note we use small input vectors to have the tests run reasonably fast, // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 512, 10, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); + const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 512, 10, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, zml.Mesh.single(), platform); defer rng.deinit(); - const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); + const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, zml.Mesh.single(), platform); defer rng_mask.deinit(); // Note: we pass void here, cause Rng.normal doesn't take any runtime inputs. @@ -1252,10 +1252,10 @@ test sampleTokens { const allocator = std.testing.allocator; const inf = std.math.inf(f32); - var rng_buff = try zml.Tensor.Rng.init(platform, 0xdeadbeef); + var rng_buff = try zml.Tensor.Rng.init(platform, zml.Mesh.single(), 0xdeadbeef); defer rng_buff._state.deinit(); - const mod = try zml.compileFn(allocator, sampleTokens, .{ Shape.init(.{ .voc = 4 }, .f32), .{ .topk = 4, .temperature = 2.0 }, zml.Tensor.Rng.shape() }, platform); + const mod = try zml.compileFn(allocator, sampleTokens, .{ Shape.init(.{ .voc = 4 }, .f32), .{ .topk = 4, .temperature = 2.0 }, zml.Tensor.Rng.shape() }, zml.Mesh.single(), platform); defer mod.deinit(); inline for (.{ @@ -1377,7 +1377,7 @@ test sampleTokensDynamic { const logits = [_]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }; const top_k_indices = [_]i32{ 2, 3, 0, 1 }; const logits_buff = try zml.Buffer.fromArray(platform, logits); - const mod = try zml.compileFn(allocator, fixupLogits, .{ Shape.init(.{ .voc = logits.len }, .f32), DynamicSamplingStrategy.shapes(.f32, 0) }, platform); + const mod = try zml.compileFn(allocator, fixupLogits, .{ Shape.init(.{ .voc = logits.len }, .f32), DynamicSamplingStrategy.shapes(.f32, 0) }, zml.Mesh.single(), platform); defer mod.deinit(); const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; @@ -1405,7 +1405,7 @@ test sampleTokensDynamic { // Similar but use bf16, and uses infinity to trigger nans after the softmax. const bf16 = zml.floats.BFloat16; - const mod_bf16 = try zml.compileFn(allocator, fixupLogits, .{ Shape.init(.{ .voc = logits.len }, .bf16), DynamicSamplingStrategy.shapes(.bf16, 0) }, platform); + const mod_bf16 = try zml.compileFn(allocator, fixupLogits, .{ Shape.init(.{ .voc = logits.len }, .bf16), DynamicSamplingStrategy.shapes(.bf16, 0) }, zml.Mesh.single(), platform); defer mod_bf16.deinit(); const boost = bf16.inf; const nerf = bf16.minus_inf; diff --git a/zml/ops.zig b/zml/ops.zig index e0af64e8..5856a98a 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -9,6 +9,7 @@ const CompilationContext = @import("module.zig").CompilationContext; const Context = @import("context.zig").Context; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); +const Mesh = @import("partitioning.zig").Mesh; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; @@ -500,7 +501,7 @@ test "if" { const pred = Shape.init(.{}, .i32); const a = Shape.init(.{ 4, 4 }, .f32); const b = Shape.init(.{ 4, 4 }, .f32); - const mod = try zml.compileFn(allocator, IfMod._fwd, .{ pred, a, b }, platform); + const mod = try zml.compileFn(allocator, IfMod._fwd, .{ pred, a, b }, zml.Mesh.single(), platform); defer mod.deinit(); } } @@ -755,7 +756,7 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base // copy tags and sharding info over // some ops can change dims eg reduceWindow, so we trust mlir here. new._shape._tags = tensor._shape._tags; - new._shape._sharding_info = tensor._shape._sharding_info; + new._shape._partitioning = tensor._shape._partitioning; tensor.* = new; inner_ctx.index += 1; } @@ -857,56 +858,56 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T return outputs_; } -test "triton" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); +// test "triton" { +// const zml = @import("zml.zig"); +// const platform = zml.testing.env(); - if (platform.target != .cuda and platform.target != .rocm) return error.SkipZigTest; +// if (platform.target != .cuda and platform.target != .rocm) return error.SkipZigTest; - const ir = - \\ module { - \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { - \\ %0 = tt.get_program_id x : i32 - \\ %1 = tt.load %arg0 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr - \\ %2 = tt.load %arg1 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr - \\ %cst = arith.constant 1.000000e+00 : f32 - \\ %3 = arith.addf %1, %cst : f32 - \\ tt.store %arg2, %3 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr - \\ tt.store %arg3, %2 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr - \\ tt.return - \\ } - \\ } - ; +// const ir = +// \\ module { +// \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { +// \\ %0 = tt.get_program_id x : i32 +// \\ %1 = tt.load %arg0 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr +// \\ %2 = tt.load %arg1 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr +// \\ %cst = arith.constant 1.000000e+00 : f32 +// \\ %3 = arith.addf %1, %cst : f32 +// \\ tt.store %arg2, %3 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr +// \\ tt.store %arg3, %2 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr +// \\ tt.return +// \\ } +// \\ } +// ; - const TritonMod = struct { - pub fn forward(a: Tensor, b: Tensor) [2]Tensor { - return triton(.{ a, b }, .{ a.shape(), b.shape() }, .{ - .debug = false, - .name = "add_one", - .ir = ir, - .grid = .{ 1, 1, 1 }, - .num_stages = 1, - .num_warps = 1, - }); - } - }; +// const TritonMod = struct { +// pub fn forward(a: Tensor, b: Tensor) [2]Tensor { +// return triton(.{ a, b }, .{ a.shape(), b.shape() }, .{ +// .debug = false, +// .name = "add_one", +// .ir = ir, +// .grid = .{ 1, 1, 1 }, +// .num_stages = 1, +// .num_warps = 1, +// }); +// } +// }; - const a = try zml.Buffer.fromSlice(platform, .{}, &[1]f32{1}); - const b = try zml.Buffer.fromSlice(platform, .{}, &[1]f32{3}); +// const a = try zml.Buffer.fromSlice(platform, .{}, &[1]f32{1}); +// const b = try zml.Buffer.fromSlice(platform, .{}, &[1]f32{3}); - const results = try zml.testing.compileAndCall(platform, TritonMod.forward, .{ a, b }); +// const results = try zml.testing.compileAndCall(platform, TritonMod.forward, .{ a, b }); - var cpu_result_0 = try results[0].toHostAlloc(std.testing.allocator); - defer cpu_result_0.deinit(std.testing.allocator); - var cpu_result_1 = try results[1].toHostAlloc(std.testing.allocator); - defer cpu_result_1.deinit(std.testing.allocator); +// var cpu_result_0 = try results[0].toHostAlloc(std.testing.allocator); +// defer cpu_result_0.deinit(std.testing.allocator); +// var cpu_result_1 = try results[1].toHostAlloc(std.testing.allocator); +// defer cpu_result_1.deinit(std.testing.allocator); - const expected_result_a: f32 = 2.0; - const expected_result_b: f32 = 3.0; +// const expected_result_a: f32 = 2.0; +// const expected_result_b: f32 = 3.0; - try std.testing.expectEqual(expected_result_a, cpu_result_0.items(f32)[0]); - try std.testing.expectEqual(expected_result_b, cpu_result_1.items(f32)[0]); -} +// try std.testing.expectEqual(expected_result_a, cpu_result_0.items(f32)[0]); +// try std.testing.expectEqual(expected_result_b, cpu_result_1.items(f32)[0]); +// } /// Generalized version of scatter to many inputs. /// See `zml.Tensor.scatterSlices` for documentation on scatter. @@ -1131,7 +1132,7 @@ test scatterConfig { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); diff --git a/zml/partitioning.zig b/zml/partitioning.zig new file mode 100644 index 00000000..c3f34fa7 --- /dev/null +++ b/zml/partitioning.zig @@ -0,0 +1,848 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const stdx = @import("stdx"); + +const pjrtx = @import("pjrtx.zig"); +const slice = @import("slice.zig"); +const Shaped = slice.Shaped; +const Buffer = @import("buffer.zig").Buffer; +const Context = @import("context.zig").Context; +const DataType = @import("dtype.zig").DataType; +const Device = @import("pjrtx.zig").Device; +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; + +const log = std.log.scoped(.@"zml/partitioning"); + +test { + std.testing.refAllDecls(Mesh); + std.testing.refAllDecls(DeviceShard); + std.testing.refAllDecls(Sharding); +} + +pub const MaxMeshAxes: u8 = 3; +pub const MaxMeshSize: u8 = MaxMeshAxes * Shape.MAX_RANK; + +/// Iterates through all possible multi-dimensional indices of a topology. +/// The iteration order is row-major. +pub const TopologyIndicesIterator = struct { + index: usize = 0, + topology: Shape, + indices: Shape, + + pub fn next(self: *TopologyIndicesIterator) ?Shape { + if (self.index >= self.topology.count()) return null; + + const current_indices = self.indices; + + // Prepare for the next iteration + self.index += 1; + + // Use a standard row-major increment logic + if (self.index < self.topology.count()) { + const rank = self.topology.rank(); + var i: usize = rank; + + while (i > 0) { + i -= 1; + const dim: u4 = @intCast(i); + + const current_value = self.indices.dim(dim); + if (current_value < self.topology.dim(dim) - 1) { + self.indices = self.indices.setDim(dim, current_value + 1); + return current_indices; + } else { + self.indices = self.indices.setDim(dim, 0); + } + } + } + + return current_indices; + } +}; + +pub const Mesh = struct { + topology: Shape, + + pub fn init(topology: anytype) Mesh { + const self: Mesh = .{ + .topology = .init(topology, .u8), + }; + + if (self.rank() == 0) { + stdx.debug.panic("Mesh must have at least one tagged axis defined, got: {}", .{topology}); + } + + if (!self.topology.isFullyTagged()) { + stdx.debug.panic("Mesh must be fully tagged, got: {}", .{topology}); + } + + if (self.rank() > MaxMeshAxes) { + stdx.debug.panic("Mesh rank ({}) exceeds maximum allowed axes ({})", .{ self.rank(), MaxMeshAxes }); + } + + return self; + } + + pub fn single() Mesh { + const topology: Shape = .init(.{ .x = 1 }, .u8); + return .init(topology); + } + + pub fn auto(platform: Platform) Mesh { + const num_devices = platform.getDevices().len; + + if (num_devices == 0) { + stdx.debug.panic("No devices available in the platform: {}", .{platform.target}); + } + + if (num_devices > MaxMeshSize) { + stdx.debug.panic("Too many devices ({}) for a mesh, max is {}", .{ num_devices, MaxMeshSize }); + } + + const topology: Shape = .init(.{ .x = num_devices }, .u8); + + return .init(topology); + } + + pub fn reshape(self: Mesh, new_shape: anytype) Mesh { + const new_topology = self.topology.reshape(new_shape); + return .init(new_topology); + } + + pub fn flatten(self: Mesh, new_axis_name: anytype) Mesh { + const total_devices = self.numDevices(); + + var new_topology = Shape.init(.{}, .u8); + new_topology = new_topology.appendDim(total_devices, @tagName(new_axis_name)); + return Mesh.init(new_topology); + } + + test flatten { + { + // Flatten a 2D mesh + const mesh = Mesh.init(.{ .x = 2, .y = 4 }); + const flattened = mesh.flatten(.all); + const expected = Mesh.init(.{ .all = 8 }); + + try std.testing.expect(flattened.eql(expected)); + try std.testing.expectEqual(1, flattened.rank()); + try std.testing.expectEqual(8, flattened.numDevices()); + } + + { + // Flatten a 3D mesh + const mesh = Mesh.init(.{ .data = 2, .model = 3, .pipeline = 4 }); + const flattened = mesh.flatten(.devices); + const expected = Mesh.init(.{ .devices = 24 }); + try std.testing.expect(flattened.eql(expected)); + try std.testing.expectEqual(1, flattened.rank()); + try std.testing.expectEqual(24, flattened.numDevices()); + } + { + // Flattening a 1D mesh should just rename the axis + const mesh = Mesh.init(.{ .x = 16 }); + const flattened = mesh.flatten(.y); + const expected = Mesh.init(.{ .y = 16 }); + try std.testing.expect(flattened.eql(expected)); + try std.testing.expectEqual(1, expected.rank()); + } + } + + pub fn rank(self: Mesh) i64 { + return @intCast(self.topology.rank()); + } + + pub fn is1D(self: Mesh) bool { + return self.rank() == 1; + } + + pub fn is2D(self: Mesh) bool { + return self.rank() == 2; + } + + pub fn is3D(self: Mesh) bool { + return self.rank() == 3; + } + + pub fn axis(self: Mesh, ax: anytype) i64 { + return self.topology.dim(ax); + } + + pub fn hasManyPartitions(self: Mesh) bool { + return self.numPartitions() > 1; + } + + pub fn isSinglePartition(self: Mesh) bool { + return self.numPartitions() == 1; + } + + pub fn iterator(self: Mesh) TopologyIndicesIterator { + var indices = self.topology; + + for (0..indices.rank()) |dim| { + indices = indices.setDim(dim, 0); + } + + return .{ + .topology = self.topology, + .indices = indices, + }; + } + + pub fn numPartitions(self: Mesh) u8 { + return @intCast(self.topology.count()); + } + + pub fn numReplicas(_: Mesh) u8 { + return 1; + } + + pub fn numDevices(self: Mesh) u8 { + return self.numPartitions() * self.numReplicas(); + } + + pub fn eql(self: Mesh, other: Mesh) bool { + return self.topology.eql(other.topology); + } + + pub fn format( + self: Mesh, + comptime _: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = options; + try writer.print("Mesh(topology={} rank={d} numDevices={d})", .{ self.topology, self.rank(), self.numDevices() }); + } +}; + +test "Mesh / 1D mesh with 1 partition" { + const mesh: Mesh = .init(.{ .x = 1 }); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / 1D mesh with 8 partitions" { + const mesh: Mesh = .init(.{ .x = 8 }); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 8); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(mesh.hasManyPartitions()); + try std.testing.expect(!mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 8); + try std.testing.expect(mesh.numDevices() == 8); +} + +test "Mesh / 2D mesh with 1 partition" { + const mesh: Mesh = .init(.{ .x = 1, .y = 1 }); + try std.testing.expect(mesh.rank() == 2); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.axis(.y) == 1); + try std.testing.expect(!mesh.is1D()); + try std.testing.expect(mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / 2D mesh with 6 partitions" { + const mesh: Mesh = .init(.{ .x = 2, .y = 3 }); + try std.testing.expect(mesh.rank() == 2); + try std.testing.expect(mesh.axis(.x) == 2); + try std.testing.expect(mesh.axis(.y) == 3); + try std.testing.expect(!mesh.is1D()); + try std.testing.expect(mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(mesh.hasManyPartitions()); + try std.testing.expect(!mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 6); + try std.testing.expect(mesh.numDevices() == 6); +} + +test "Mesh / 3D mesh with 1 partition" { + const mesh: Mesh = .init(.{ .x = 1, .y = 1, .z = 1 }); + try std.testing.expect(mesh.rank() == 3); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.axis(.y) == 1); + try std.testing.expect(mesh.axis(.z) == 1); + try std.testing.expect(!mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / 3D mesh with 64 partitions" { + const mesh_3d: Mesh = .init(.{ .x = 4, .y = 4, .z = 4 }); + try std.testing.expect(mesh_3d.rank() == 3); + try std.testing.expect(mesh_3d.axis(.x) == 4); + try std.testing.expect(mesh_3d.axis(.y) == 4); + try std.testing.expect(mesh_3d.axis(.z) == 4); + try std.testing.expect(!mesh_3d.is1D()); + try std.testing.expect(!mesh_3d.is2D()); + try std.testing.expect(mesh_3d.is3D()); + try std.testing.expect(mesh_3d.hasManyPartitions()); + try std.testing.expect(!mesh_3d.isSinglePartition()); + try std.testing.expect(mesh_3d.numPartitions() == 64); + try std.testing.expect(mesh_3d.numDevices() == 64); +} + +test "Mesh / single" { + const mesh: Mesh = .single(); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / auto with 4 devices" { + const platform = env(.{ .cpu = .{ .cpu_device_count = 4 } }); + const mesh: Mesh = .auto(platform); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 4); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(mesh.hasManyPartitions()); + try std.testing.expect(!mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 4); + try std.testing.expect(mesh.numDevices() == 4); +} + +test "Mesh / 1D mesh iterator" { + const mesh = Mesh.init(.{ .x = 4 }); + var iter = mesh.iterator(); + var count: u8 = 0; + + while (iter.next()) |indices| : (count += 1) { + const expected_shape = Shape.init(.{ .x = count }, .u8); + try std.testing.expect(indices.eqlWithTags(expected_shape)); + } + + try std.testing.expectEqual(4, count); + try std.testing.expect(iter.next() == null); +} + +test "Mesh / 2D mesh iterator" { + const mesh = Mesh.init(.{ .x = 2, .y = 3 }); + var iter = mesh.iterator(); + var count: u8 = 0; + const expected_indices = [_]Shape{ + .init(.{ .x = 0, .y = 0 }, .u8), + .init(.{ .x = 0, .y = 1 }, .u8), + .init(.{ .x = 0, .y = 2 }, .u8), + .init(.{ .x = 1, .y = 0 }, .u8), + .init(.{ .x = 1, .y = 1 }, .u8), + .init(.{ .x = 1, .y = 2 }, .u8), + }; + + while (iter.next()) |indices| : (count += 1) { + try std.testing.expect(indices.eqlWithTags(expected_indices[count])); + } + try std.testing.expectEqual(6, count); + try std.testing.expect(iter.next() == null); +} + +test "Mesh / 3D mesh iterator" { + const mesh = Mesh.init(.{ .x = 2, .y = 2, .z = 2 }); + var iter = mesh.iterator(); + var count: u8 = 0; + const expected_indices = [_]Shape{ + .init(.{ .x = 0, .y = 0, .z = 0 }, .u8), + .init(.{ .x = 0, .y = 0, .z = 1 }, .u8), + .init(.{ .x = 0, .y = 1, .z = 0 }, .u8), + .init(.{ .x = 0, .y = 1, .z = 1 }, .u8), + .init(.{ .x = 1, .y = 0, .z = 0 }, .u8), + .init(.{ .x = 1, .y = 0, .z = 1 }, .u8), + .init(.{ .x = 1, .y = 1, .z = 0 }, .u8), + .init(.{ .x = 1, .y = 1, .z = 1 }, .u8), + }; + + while (iter.next()) |indices| : (count += 1) { + try std.testing.expect(indices.eqlWithTags(expected_indices[count])); + } + + try std.testing.expectEqual(8, count); + try std.testing.expect(iter.next() == null); +} + +pub const DeviceShard = struct { + index: u64, + topology: Shape, + global_shape: Shape, + indices: Shape, + shard_shape: Shape, + device: *const Device, + + pub const SliceSpec = struct { + start_offset: usize, + byte_strides: [Shape.MAX_RANK]i64, + dims: [Shape.MAX_RANK]i64, + num_dims: u4, + }; + + pub fn specs(self: DeviceShard) SliceSpec { + const rank = self.global_shape.rank(); + + const host_byte_strides_ba = self.global_shape.computeByteStrides(); + + var shard_start_offset_bytes: i64 = 0; + for (0..rank) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + + // Check if the device's indices actually have this axis. + // If not, this dimension is effectively replicated for this device. + if (self.indices.hasTag(mesh_axis_tag) != null) { + const device_coord_on_mesh_axis = self.indices.dim(mesh_axis_tag); + const shard_dim_size = self.shard_shape.dim(i); + const start_element_in_dim = device_coord_on_mesh_axis * shard_dim_size; + shard_start_offset_bytes += start_element_in_dim * host_byte_strides_ba.get(i); + } + } + } + + var transfer_dims_buffer: [Shape.MAX_RANK]i64 = undefined; + for (0..rank) |i| { + transfer_dims_buffer[i] = self.shard_shape.dim(i); + } + + return .{ + .start_offset = @intCast(shard_start_offset_bytes), + .byte_strides = host_byte_strides_ba.buffer, + .dims = transfer_dims_buffer, + .num_dims = rank, + }; + } + + pub fn format( + self: DeviceShard, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + const slice_specs = self.specs(); + try writer.print("DeviceShard(index={d}/{d} topology={} indices={} shard_shape={} ({d}B) global_shape={} ({d}B) start_offset={d} byte_strides={any} device={})", .{ + self.index + 1, + self.topology.count(), + self.topology, + self.indices, + self.shard_shape, + self.shard_shape.byteSize(), + self.global_shape, + self.global_shape.byteSize(), + slice_specs.start_offset, + slice_specs.byte_strides[0..slice_specs.num_dims], + self.device, + }); + } +}; + +pub const DeviceShardIterator = struct { + index: usize = 0, + indices_iterator: TopologyIndicesIterator, + sharding: Sharding, + devices: []const *const pjrtx.Device, + + pub fn next(self: *DeviceShardIterator) ?DeviceShard { + const num_partitions = self.sharding.mesh.numPartitions(); // todo proxy numPartitions + + if (self.index >= num_partitions) return null; + + defer self.index += 1; + + return .{ + .index = self.index, + .topology = self.sharding.mesh.topology, + .global_shape = self.sharding.global_shape, + .indices = self.indices_iterator.next().?, + .shard_shape = self.sharding.shardShape(), + .device = self.devices[self.index], + }; + } +}; + +pub const Sharding = struct { + mesh: Mesh, + global_shape: Shape, + + pub const Type = enum { + replicated, + maximal, + manual, + }; + + pub fn init(mesh: Mesh, shape: Shape) Sharding { + // Validate that the shape is divisible by the mesh where partitioned. + for (0..shape.rank()) |dim_idx| { + const part_spec = shape.partition(dim_idx); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + + if (mesh.topology.hasTag(mesh_axis_tag) == null) { + // This partitioning is invalid for this mesh, so we skip the divisibility check. + // The dimension will effectively be replicated. + log.debug("Partitioning axis '{s}' not found in mesh {}. Treating dimension {s} as replicated.", .{ + std.mem.span(mesh_axis_tag), + mesh, + shape.debugTag(dim_idx), + }); + continue; + } + + const global_dim = shape.dim(dim_idx); + const mesh_dim = mesh.topology.dim(mesh_axis_tag); + if (@rem(global_dim, mesh_dim) != 0) { + stdx.debug.panic( + "Tensor dimension {s} (size={d}) is not divisible by mesh axis {s} (size={d})", + .{ shape.debugTag(dim_idx), global_dim, std.mem.span(mesh_axis_tag), mesh_dim }, + ); + } + } + } + + return .{ + .mesh = mesh, + .global_shape = shape, + }; + } + + test "init / non existent mesh axis" { + const mesh: Mesh = .init(.{ .x = 4 }); + const shape: Shape = .init(.{ .m = 8, .n = 12 }, .i32); + const sharding = Sharding.init(mesh, shape.withPartitioning(.{ .m = .x, .n = .y })); + const shard_shape = sharding.shardShape(); + try std.testing.expectEqual(2, shard_shape.dim(.m)); + try std.testing.expectEqual(12, shard_shape.dim(.n)); + } + + pub fn getType(self: Sharding) Type { + var has_any_partitioned_axis = false; + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + has_any_partitioned_axis = true; + break; + } + } + } + + if (!has_any_partitioned_axis) { + return .replicated; + } + + var is_fully_partitioned = true; + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + switch (part_spec) { + .replicated, .open, .unknown => { + is_fully_partitioned = false; + break; + }, + .axis => |tag| { + if (self.mesh.topology.hasTag(tag) == null) { + is_fully_partitioned = false; + break; + } + }, + } + } + + if (is_fully_partitioned) { + return .maximal; + } + + return .manual; + } + + pub fn shardShape(self: Sharding) Shape { + var shard_: Shape = .init(.{}, self.global_shape.dtype()); + const rank = self.global_shape.rank(); + + for (0..rank) |dim| { + const part_spec = self.global_shape.partition(dim); + const global_dim_size = self.global_shape.dim(dim); + const tag = self.global_shape.tag(dim); + + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + + // If the specified mesh axis exists, partition. Otherwise, replicate. + if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + const mesh_dim = self.mesh.topology.dim(mesh_axis_tag); + const shard_dim_size = if (mesh_dim > 1) @divExact(global_dim_size, mesh_dim) else global_dim_size; + shard_ = shard_.appendDim(shard_dim_size, tag); + } else { + // Mesh axis does not exist, so this dimension is replicated. + shard_ = shard_.appendDim(global_dim_size, tag); + } + } else { + // For replicated, open, or unknown, the shard dimension is the full global dimension. + shard_ = shard_.appendDim(global_dim_size, tag); + } + } + + return shard_; + } + + pub fn iterator(self: Sharding, devices: []const *const pjrtx.Device) DeviceShardIterator { + const indices_iterator = self.mesh.iterator(); + + return .{ .indices_iterator = indices_iterator, .sharding = self, .devices = devices }; + } + + pub fn getShardingAttr(self: Sharding) std.BoundedArray(u8, 128) { + var sharding_str: std.BoundedArray(u8, 128) = .{}; + self.writeShardingRepresentation(sharding_str.writer()) catch unreachable; + return sharding_str; + } + + pub fn writeShardingRepresentation(self: Sharding, writer: anytype) !void { + if (self.getType() == .replicated) { + try writer.writeAll("{replicated}"); + return; + } + + var product_of_tensor_tiles: i64 = 1; + var num_partitioned_dims: u32 = 0; + + // Calculate the product of explicitly defined tile dimensions + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + product_of_tensor_tiles *= self.mesh.topology.dim(mesh_axis_tag); + num_partitioned_dims += 1; + } + } + } + + const is_fully_sharded_on_mesh = (product_of_tensor_tiles == self.mesh.numPartitions()); + + if (is_fully_sharded_on_mesh) { + // This is the easy case. The tensor is partitioned across all mesh devices. + // The tile array must have the same rank as the tensor. + try writer.writeAll("{devices=["); + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + var tile_dim: i64 = 1; + if (part_spec == .axis) { + if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + tile_dim = self.mesh.topology.dim(part_spec.toTag()); + } + } + try writer.print("{d}", .{tile_dim}); + if (i < self.global_shape.rank() - 1) { + try writer.writeByte(','); + } + } + try writer.print("]<=[{d}]}}", .{self.mesh.numPartitions()}); + } else { + // This is partial sharding. This is more complex. + // The `devices` array lists the tiling only for the *partitioned* dimensions. + // The remaining devices are handled by `last_tile_dim_replicate`. + try writer.writeAll("{devices=["); + var first = true; + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + if (!first) try writer.writeByte(','); + try writer.print("{d}", .{self.mesh.topology.dim(mesh_axis_tag)}); + first = false; + } + } + } + try writer.print("]<=[{d}] last_tile_dim_replicate}}", .{self.mesh.numPartitions()}); + } + } + + test "Sharding MLIR representation" { + // Case 1: Fully Replicated + const mesh_rep = Mesh.init(.{ .x = 4 }); + const shape_rep = Shape.init(.{ .m = 8, .k = 8 }, .i32); + const sharding_rep = Sharding.init(mesh_rep, shape_rep); + var attr_rep_ba = sharding_rep.getShardingAttr(); + try std.testing.expectEqualStrings("{replicated}", attr_rep_ba.constSlice()); + + // Case 2: Partially partitioned + const mesh_part = Mesh.init(.{ .x = 2, .y = 4 }); + const shape_part = Shape.init(.{ .m = 8, .k = 12 }, .i32).withPartitioning(.{ .m = .x }); + const sharding_part = Sharding.init(mesh_part, shape_part); + var attr_part_ba = sharding_part.getShardingAttr(); + // dim 'm' is partitioned on 'x' (size 2), dim 'k' is replicated (size 1) + try std.testing.expectEqualStrings("{devices=[2,1]<=[8]}", attr_part_ba.constSlice()); + + // Case 3: Fully partitioned + const mesh_full = Mesh.init(.{ .x = 2, .y = 4 }); + const shape_full = Shape.init(.{ .m = 8, .k = 12 }, .i32).withPartitioning(.{ .m = .x, .k = .y }); + const sharding_full = Sharding.init(mesh_full, shape_full); + var attr_full_ba = sharding_full.getShardingAttr(); + // dim 'm' partitioned on 'x' (size 2), dim 'k' partitioned on 'y' (size 4) + try std.testing.expectEqualStrings("{devices=[2,4]<=[8]}", attr_full_ba.constSlice()); + + // Case 4: More complex 3D case with a replicated dimension in the middle + const mesh_3d = Mesh.init(.{ .x = 2, .y = 2, .z = 3 }); + const shape_3d = Shape.init(.{ .b = 4, .h = 8, .w = 6 }, .i32).withPartitioning(.{ .b = .x, .w = .z }); + const sharding_3d = Sharding.init(mesh_3d, shape_3d); + var attr_3d_ba = sharding_3d.getShardingAttr(); + // b -> x (size 2), h -> replicated (size 1), w -> z (size 3) + try std.testing.expectEqualStrings("{devices=[2,1,3]<=[12]}", attr_3d_ba.constSlice()); + } + + pub fn format( + self: Sharding, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + try writer.print("Sharding(global_shape={} mesh={})", .{ self.global_shape, self.mesh }); + } +}; + +fn testShardingCase(allocator: std.mem.Allocator, mesh: Mesh, shape: Shape, partition_spec: anytype, verbose: bool) !void { + _platform = null; // Reset the global platform + const platform = env(.{ .cpu = .{ .cpu_device_count = @intCast(mesh.numDevices()) } }); + + const shape_partitioned = shape.withPartitioning(partition_spec); + const sharding = Sharding.init(mesh, shape_partitioned); + std.debug.print("\n--- Testing Case (verbose={any}) --- {}\n", .{ verbose, sharding }); + + const original_data = try slice.arange(allocator, shape_partitioned, .{}); + defer allocator.free(original_data); + if (verbose) { + std.debug.print("Original full slice of data: {any}\n\n", .{Shaped(i32, shape_partitioned, original_data)}); + } + + // --- SCATTER (H2D) --- + var pjrt_buffers = std.ArrayList(*pjrtx.Buffer).init(allocator); + defer pjrt_buffers.deinit(); + + var sharding_iter = sharding.iterator(platform.getDevices()); + while (sharding_iter.next()) |device_shard| { + if (verbose) std.debug.print("{}\n", .{device_shard}); + const specs = device_shard.specs(); + const args = pjrtx.Client.BufferFromHostBufferArgs{ + .data = original_data[specs.start_offset..].ptr, + .buffer_type = bufferTypeFromDtype(device_shard.shard_shape.dtype()), + .dims = specs.dims[0..specs.num_dims], + .byte_strides = specs.byte_strides[0..specs.num_dims], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .device = device_shard.device, + }; + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); + if (event) |e| try e.await_(platform.pjrt_api); + try pjrt_buffers.append(pjrt_buffer); + + // Optional: verify the transferred data for debugging + if (verbose) { + const shard_shape = device_shard.shard_shape; + const fetched_data = try allocator.alloc(u8, shard_shape.byteSize()); + defer allocator.free(fetched_data); + const fetch_event = try pjrt_buffer.toHostBuffer(platform.pjrt_api, fetched_data, .{}); + if (fetch_event) |e| try e.await_(platform.pjrt_api); + std.debug.print("Fetched shard data: {any}\n\n", .{Shaped(i32, shard_shape, fetched_data)}); + } + } + + // --- GATHER (D2H) --- + if (verbose) std.debug.print("--- Reassembling from shards ---\n", .{}); + const sharded_buffer = Buffer.fromPjrtBuffers(platform, sharding, pjrt_buffers.items); + defer sharded_buffer.deinit(); // This will deinit all the underlying PJRT buffers. + + const reassembled_data = try sharded_buffer.toHost(allocator); + defer allocator.free(reassembled_data); + + if (verbose) std.debug.print("Reassembled full slice of data: {any}\n\n", .{Shaped(i32, sharding.global_shape, reassembled_data)}); + + // --- VERIFY --- + try std.testing.expectEqualSlices(u8, original_data, reassembled_data); + std.debug.print("✅ Verification successful for: {}\n\n", .{sharding}); +} + +test "Sharding All Cases" { + const allocator = std.testing.allocator; + const verbose = true; + + // Fully replicated + try testShardingCase(allocator, .init(.{ .x = 4 }), .init(.{ .m = 8, .k = 8 }, .i32), .{}, verbose); + // 1D sharding on a 1D mesh + try testShardingCase(allocator, .init(.{ .x = 8 }), .init(.{ .m = 16, .k = 16 }, .i32), .{ .k = .x }, verbose); + // 2D sharding on a 2D mesh + try testShardingCase(allocator, .init(.{ .x = 2, .y = 4 }), .init(.{ .m = 8, .k = 12 }, .i32), .{ .m = .x, .k = .y }, verbose); + // 3D tensor on a 2D mesh, partitioned on non-contiguous dimensions + try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .b = 4, .h = 8, .w = 6 }, .i32), .{ .b = .x, .w = .y }, verbose); + // 3D sharding on a 3D mesh + try testShardingCase(allocator, .init(.{ .x = 2, .y = 4, .z = 3 }), .init(.{ .m = 4, .k = 8, .j = 6 }, .i32), .{ .m = .x, .k = .y, .j = .z }, verbose); + // 2D tensor, 2D mesh, but only partitioned on one axis + try testShardingCase(allocator, .init(.{ .x = 2, .y = 3 }), .init(.{ .m = 10, .k = 6 }, .i32), .{ .k = .x }, verbose); + // Replicated on a 2D mesh + try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .m = 4, .k = 4 }, .i32), .{}, verbose); + // 3D tensor, 2D mesh, one replicated dimension in the middle + try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .batch = 4, .seq_len = 10, .features = 8 }, .i32), .{ .batch = .x, .features = .y }, verbose); + // 2D mesh where one dimension is 1 + try testShardingCase(allocator, .init(.{ .x = 1, .y = 4 }), .init(.{ .m = 8, .k = 8 }, .i32), .{ .m = .x, .k = .y }, verbose); + // Partially partitioned, non-divisible on replicated axis + try testShardingCase(allocator, .init(.{ .x = 2, .y = 3 }), .init(.{ .m = 8, .k = 10 }, .i32), .{ .m = .x }, verbose); + // Single device mesh + try testShardingCase(allocator, .init(.{ .x = 1 }), .init(.{ .m = 8, .k = 10 }, .i32), .{ .m = .x }, verbose); + // NEW: 3D tensor, 2D mesh, partitioned on first two axes. Good test for new reassemble logic. + try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .b = 4, .h = 8, .w = 6 }, .i32), .{ .b = .x, .h = .y }, verbose); +} + +// todo: temp +pub fn bufferTypeFromDtype(dt: DataType) pjrtx.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrtx.BufferType, @tagName(tag)), + }; +} + +pub fn dtypeFromBufferType(pjrt_type: pjrtx.BufferType) DataType { + return switch (pjrt_type) { + .invalid => @panic("Found an invalid pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; +} + +// todo: temp (zig deps and compilation story...) +var _platform: ?Platform = null; + +pub fn env(opts: Platform.CreateOptions) Platform { + if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); + if (_platform == null) { + var ctx = Context.init() catch unreachable; + _platform = ctx.autoPlatform(opts).withCompilationOptions(.{ + .xla_dump_to = "/tmp/zml/tests-partitioning/", + .sharding_enabled = true, + }); + } + + return _platform.?; +} diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 1795efc3..16e59484 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -163,8 +163,9 @@ pub const Buffer = opaque { return self.inner().memory(api); } - pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { - return @ptrCast(try self.inner().toHostBuffer(api, dst)); + pub const ToHostBufferOpts = pjrt.Buffer.ToHostBufferOpts; + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8, opts: ToHostBufferOpts) ApiError!?*Event { + return @ptrCast(try self.inner().toHostBuffer(api, dst, opts)); } pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { diff --git a/zml/platform.zig b/zml/platform.zig index a7bc87c9..6008b0c5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -15,7 +15,6 @@ pub const CompilationOptions = struct { xla_dump_fusion_visualization: bool = false, xla_dump_hlo_pass_re: ?[]const u8 = null, sharding_enabled: bool = false, - sharding_axes: std.BoundedArray([*:0]const u8, 8) = .{}, }; pub const Platform = struct { @@ -50,21 +49,6 @@ pub const Platform = struct { return all_devices; } - pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; - - pub fn sharding(self: Platform) Sharding { - // replicas run the same function but with different inputs, - // while partitions contribute to one evaluation over a shared input. - // Inside an inference process, we generally don't want replicas, - // as it's best to fully isolate replicas on different processes. - // For now we hardcode num_replicas = 1. - const num_devices: u8 = @intCast(self.getDevices().len); - return if (self.compilation_options.sharding_enabled) - .{ .num_replicas = 1, .num_partitions = num_devices } - else - .{ .num_replicas = 1, .num_partitions = 1 }; - } - pub fn withCompilationOptions(self: Platform, opts: CompilationOptions) Platform { var res = self; res.compilation_options = opts; @@ -95,10 +79,7 @@ pub const Platform = struct { }; const _CreateOptions = struct { - // XLA CPU client doesn't read options - // https://github.com/openxla/xla/blob/42496a28c374bd35f493cc5dbde74805407245dc/xla/pjrt/c/pjrt_c_api_cpu_internal.cc#L33-L46 - cpu: struct {} = .{}, - + cpu: Cpu = .{}, // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the Cuda driver // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 @@ -107,6 +88,14 @@ const _CreateOptions = struct { tpu: struct {} = .{}, neuron: struct {} = .{}, + pub const Cpu = struct { + cpu_device_count: i64 = 4, + + pub fn writeNamedValues(self: Cpu, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.from("cpu_device_count", self.cpu_device_count)); + } + }; + pub const Cuda = struct { allocator: Allocator = .{ .bfc = .{} }, // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 @@ -155,6 +144,7 @@ const _CreateOptions = struct { var values = std.ArrayListUnmanaged(pjrt.NamedValue).fromOwnedSlice(out); values.shrinkRetainingCapacity(0); switch (target) { + .cpu => self.cpu.writeNamedValues(&values), .cuda => self.cuda.writeNamedValues(&values), inline else => |t| { stdx.debug.assertComptime(@hasField(_CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); diff --git a/zml/shape.zig b/zml/shape.zig index ce16ec93..b1d6802a 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -11,28 +11,159 @@ const EnumLiteral = @TypeOf(.enum_literal); const log = std.log.scoped(.shape); test { - std.testing.refAllDecls(Shape); + testing.refAllDecls(Shape); } /// Represent the shape of a tensor. pub const Shape = struct { pub const MAX_RANK: u8 = 8; + pub const PartitionSpec = union(enum) { + axis: Tag, + open, + replicated, + unknown, + + pub fn init(comptime value: anytype) PartitionSpec { + const T = @TypeOf(value); + + if (T == PartitionSpec) { + return value; + } + + if (comptime T == EnumLiteral) { + const name = @tagName(value); + inline for (std.meta.fields(PartitionSpec)) |field| { + if (field.type == void and std.mem.eql(u8, field.name, name)) { + return @field(PartitionSpec, field.name); + } + } + } + + if (comptime isTagConvertible(T)) { + const tag_ = Shape.toTag(value); + + return .{ .axis = tag_ }; + } + + stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); + } + + test "PartitionSpec.init" { + const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; + try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); + + const spec_axis_open: PartitionSpec = .open; + try testing.expect(spec_axis_open.eql(.init(.open))); + } + + pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { + if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { + return false; + } + + switch (self) { + .axis => |t1| { + const t2 = other.axis; + return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); + }, + else => return true, + } + } + + test "PartitionSpec.eql" { + const spec_axis: PartitionSpec = .init(.a); + const spec_axis_b: PartitionSpec = .init(.b); + const spec_open: PartitionSpec = .open; + const spec_replicated: PartitionSpec = .replicated; + const spec_unknown: PartitionSpec = .unknown; + + try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); + try testing.expect(!spec_axis.eql(spec_axis_b)); + try testing.expect(!spec_axis.eql(.open)); + try testing.expect(!spec_axis.eql(.replicated)); + try testing.expect(!spec_axis.eql(.unknown)); + + try testing.expect(spec_open.eql(.open)); + try testing.expect(!spec_open.eql(spec_axis)); + try testing.expect(!spec_open.eql(.replicated)); + try testing.expect(!spec_open.eql(.unknown)); + + try testing.expect(spec_replicated.eql(.replicated)); + try testing.expect(!spec_replicated.eql(spec_axis)); + try testing.expect(!spec_replicated.eql(.open)); + try testing.expect(!spec_replicated.eql(.unknown)); + + try testing.expect(spec_unknown.eql(.unknown)); + try testing.expect(!spec_unknown.eql(spec_axis)); + try testing.expect(!spec_unknown.eql(.open)); + try testing.expect(!spec_unknown.eql(.replicated)); + } + + pub fn isClosed(self: PartitionSpec) bool { + return switch (self) { + .axis => true, + .open => false, + .replicated => true, + .unknown => false, + }; + } + + test isClosed { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.isClosed()); + + const spec_open: PartitionSpec = .open; + try testing.expect(!spec_open.isClosed()); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.isClosed()); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(!spec_unknown.isClosed()); + } + + pub fn toTag(self: PartitionSpec) Tag { + return switch (self) { + .axis => |t| return t, + .open => TagOpen, + .replicated => TagReplicated, + .unknown => TagUnknown, + }; + } + + test "PartitionSpec.toTag" { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); + + const spec_open: PartitionSpec = .open; + try testing.expect(spec_open.toTag() == TagOpen); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.toTag() == TagReplicated); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(spec_unknown.toTag() == TagUnknown); + } + }; + pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; const TagLast = "last".ptr; + const TagReplicated = "replicated".ptr; + const TagOpen = "open".ptr; pub const DimsArray = std.BoundedArray(i64, MAX_RANK); pub const TagsArray = std.BoundedArray(Tag, MAX_RANK); pub const AxesArray = std.BoundedArray(u3, MAX_RANK); - pub const ShardingInfo = @Vector(MAX_RANK, bool); + pub const PartitionArray = std.BoundedArray(PartitionSpec, MAX_RANK); const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** MAX_RANK }; _dtype: DataType, _dims: DimsArray = .{}, _tags: TagsArray = UnknownTags, - _sharding_info: ShardingInfo = @splat(false), + _partitioning: PartitionArray = .{}, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -129,6 +260,8 @@ pub const Shape = struct { pub fn init(dimz: anytype, dt: DataType) Shape { var res: Shape = .{ ._dtype = dt }; res._dims, res._tags = parseDimensions(dimz); + res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); + return res; } @@ -145,6 +278,7 @@ pub const Shape = struct { }; res._tags.append(TagUnknown) catch unreachable; } + res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -153,17 +287,17 @@ pub const Shape = struct { } pub fn rank(self: Shape) u4 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return @intCast(self._dims.len); } pub fn dim(self: Shape, ax: anytype) i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.get(self.axis(ax)); } pub fn dims(self: *const Shape) []const i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.constSlice(); } @@ -190,12 +324,12 @@ pub const Shape = struct { }; } - inline fn ensureDimsAndTagsAreSync(self: Shape) void { - stdx.debug.assert(self._dims.len == self._tags.len, "Tags and dims have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + inline fn ensureAttributesAreSync(self: Shape) void { + stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); } pub fn tag(self: Shape, ax: anytype) Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.get(self.axis(ax)); } @@ -215,7 +349,7 @@ pub const Shape = struct { } pub fn tags(self: *const Shape) []const Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.constSlice(); } @@ -255,7 +389,7 @@ pub const Shape = struct { } pub fn axis(self: Shape, axis_: anytype) u3 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axis_); if (comptime stdx.meta.isInteger(T)) { @@ -270,7 +404,7 @@ pub const Shape = struct { } pub fn axes(self: Shape, axes_: anytype) AxesArray { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axes_); @@ -405,9 +539,12 @@ pub const Shape = struct { } else { try writer.print("{d}", .{d}); } - if (self._sharding_info[i]) { - try writer.writeByte('!'); + + const part = self._partitioning.get(i); + if (part.toTag() != TagUnknown) { + try writer.print("/{s}", .{part.toTag()}); } + need_comma = true; } if (need_comma) try writer.writeByte(','); @@ -440,6 +577,7 @@ pub const Shape = struct { pub fn reshape(self: Shape, new_shape_: anytype) Shape { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); + new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); new_shape.inferMissingAxis(self.count()); stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {d} to {d}", .{ self.dims(), new_shape.dims() }); return new_shape; @@ -514,6 +652,11 @@ pub const Shape = struct { var res = self; res._dims.insertSlice(ax, dims_.constSlice()) catch unreachable; res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + for (0..dims_.len) |_| { + res._partitioning.insert(ax, .unknown) catch unreachable; + } + return res; } @@ -534,6 +677,8 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; + res._partitioning.insert(ax, .unknown) catch unreachable; + return res; } @@ -542,6 +687,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); + res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); return res; } @@ -563,6 +709,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); + res._partitioning.appendAssumeCapacity(.unknown); return res; } @@ -571,14 +718,15 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); + _ = res._partitioning.orderedRemove(a); return res; } pub const drop = remove; test remove { - try std.testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); - try std.testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); } pub fn removeMany(self: Shape, axes_: anytype) Shape { @@ -596,20 +744,22 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; + sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; + sh._partitioning.len = rk - to_remove.len; return sh; } test removeMany { - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{12}, Shape.init(.{ 10, 11, 12 }, .f32).removeMany(.{ 0, 1 }).dims(), ); - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{ 10, 11 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).removeMany(.{ -1, -2 }).dims(), @@ -620,10 +770,16 @@ pub const Shape = struct { std.debug.assert(self.rank() == permutations.len); const permutations_ = self.axes(permutations); var res = self; + const original_dims = self._dims; + const original_tags = self._tags; + const original_parts = self._partitioning; + for (permutations_.constSlice(), 0..) |permutation, i| { - res._dims.set(i, self.dim(permutation)); - res._tags.set(i, self.tag(permutation)); + res._dims.set(i, original_dims.get(permutation)); + res._tags.set(i, original_tags.get(permutation)); + res._partitioning.set(i, original_parts.get(permutation)); } + return res; } @@ -639,6 +795,10 @@ pub const Shape = struct { Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32).transpose(.{ 0, 2, 1, 3 }), ), ); + + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); } /// Tag each ax of this shape with tags from a tuple. @@ -747,16 +907,233 @@ pub const Shape = struct { return res; } - pub fn withSharding(self: Shape, axes_: anytype) Shape { + pub fn withPartitioning(self: Shape, specs: anytype) Shape { + const T = @TypeOf(specs); + + var res = self.withDefaultPartitioning(); // todo add test for this new change + + if (stdx.meta.isStruct(T)) { + inline for (std.meta.fields(T)) |field| { + const partition_axis = @field(specs, field.name); + const axis_ = res.axisFromTagMaybe(toTag(field)); + + if (axis_) |ax| { + res._partitioning.set(ax, PartitionSpec.init(partition_axis)); + } else { + stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); + } + } + } else { + stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + } + + // Check that no mesh axis is used to partition multiple tensor dimensions. + var used_mesh_axes: std.BoundedArray(Tag, MAX_RANK) = .{ .len = 0 }; + + for (res._partitioning.constSlice()) |spec| { + const axis_tag = spec.toTag(); + + if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { + stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); + } + + switch (spec) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), + } + } + + return res; + } + + test withPartitioning { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); + } + + pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { + const T = @TypeOf(mapping); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); + var res = self; - // Reset sharding. - res._sharding_info = @splat(false); - for (self.axes(axes_).constSlice()) |ax| { - res._sharding_info[ax] = true; + for (res._partitioning.slice()) |*spec| { + switch (spec.*) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => |*old_axis_tag| { + inline for (std.meta.fields(T)) |field| { + const logical_axis_in_map = toTag(field); + + if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { + const physical_axis_from_map = @field(mapping, field.name); + spec.* = PartitionSpec.init(physical_axis_from_map); + break; + } + } + }, + } } return res; } + test mapPartitioningAxes { + const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ + .batch = .data, + .seq = .replicated, + .features = .model, + }); + + const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); + + try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); + try testing.expect(physical_shape.partition(.features).eql(.init(.y))); + try testing.expect(physical_shape.partition(.seq).eql(.replicated)); + + const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); + try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); + try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); + } + + pub fn withDefaultPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withDefaultPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + } + + pub fn withReplicatedPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withReplicatedPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withReplicatedPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); + } + + pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec == .axis) return true; + } + return false; + } + + test hasAtLeastOnePartitionedAxis { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + } + + pub fn isFullyPartitioned(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + switch (spec) { + .replicated, .open, .unknown => return false, + .axis => {}, + } + } + return true; + } + + test isFullyPartitioned { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); + try testing.expect(shape.isFullyPartitioned()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyPartitioned()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.isFullyPartitioned()); + } + + pub fn isFullyReplicated(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec != .replicated and spec != .unknown) { + return false; + } + } + return true; + } + + test isFullyReplicated { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyReplicated()); + } + + pub fn partition(self: Shape, ax: anytype) PartitionSpec { + return self._partitioning.get(self.axis(ax)); + } + + test partition { + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); + try testing.expectEqual(.open, shape.partition(.b)); + try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); + } + + pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { + for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { + if (dim_part.eql(part)) return true; + } + return false; + } + + test containsPartitionSpec { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.open)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(shape.containsPartitionSpec(.unknown)); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.open)); + } + /// Renames some of the tags in this shape. /// Shape.init(.{ .a = 10, .b = 20 }).rename(.{ .b = .batch }); // .{ .a = 10, .batch = 20 }; pub fn rename(self: Shape, renames: anytype) Shape { @@ -785,7 +1162,103 @@ pub const Shape = struct { } } - pub fn computeStrides(self: Shape) std.BoundedArray(i64, MAX_RANK) { + /// Computes the strides of the shape in units of elements. + /// For a shape {d0, d1, d2}, the strides would be {d1*d2, d2, 1}. + pub fn computeElementStrides(self: Shape) std.BoundedArray(i64, MAX_RANK) { + return self.computeStrides(1); + } + + test computeElementStrides { + { + const s = Shape.init(.{ 4, 1024 }, .bf16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 2, 3, 4 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12, 4, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .i16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 6, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{10}, .u8); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{1}, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1, 5 }, .i32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 5, 5, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ .n = 8, .c = 3, .h = 224, .w = 224 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 3 * 224 * 224, 224 * 224, 224, 1 }, strides.constSlice()); + } + } + + /// Computes the strides of the shape in units of bytes. + /// For a shape {d0, d1, d2} of type T, the strides would be + /// {(d1*d2)*sizeof(T), d2*sizeof(T), 1*sizeof(T)}. + pub fn computeByteStrides(self: Shape) std.BoundedArray(i64, MAX_RANK) { + return self.computeStrides(self.dtype().sizeOf()); + } + + test computeByteStrides { + { + const s = Shape.init(.{ 2, 3, 4 }, .f16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12 * 2, 4 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1024 }, .bf16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .c64); // 8 bytes (f32 + f32) + const strides = s.computeByteStrides(); + try std.testing.expectEqual(@sizeOf(std.math.Complex(f32)), 8); + try std.testing.expectEqualSlices(i64, &.{ 6 * 8, 1 * 8 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 128, 512 }, .i8); // 1 byte + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 512, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 1, 4096 }, .u32); // 4 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 4096 * 4, 1 * 4 }, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); // 8 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + } + + fn computeStrides(self: Shape, element_size: i64) std.BoundedArray(i64, MAX_RANK) { const rk = self.rank(); var strides: std.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; if (rk == 0) return strides; @@ -796,7 +1269,7 @@ pub const Shape = struct { // and the element size in bytes. var d: V = @bitCast(self._dims.buffer); d = @select(i64, rank_mask, d, @as(V, @splat(1))); - d = std.simd.shiftElementsLeft(d, 1, self.dtype().sizeOf()); + d = std.simd.shiftElementsLeft(d, 1, element_size); d = std.simd.prefixScan(.Mul, -1, d); strides.buffer = @bitCast(d); @@ -855,6 +1328,12 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; + _ = new_shape._partitioning.orderedRemove(ax); + + for (0..dims_.len) |_| { + new_shape._partitioning.insert(ax, .unknown) catch unreachable; + } + new_shape.inferMissingAxis(self.count()); return new_shape; } @@ -899,22 +1378,43 @@ pub const Shape = struct { /// Merge the given axes into one axis. /// eg: `Shape.init(.{.a1 = 5, .a2 = 2, .b = 3}).merge(.{ .a = .{ .a1, .a2 }); -> .{ .a = 10, .b = 3 }` pub fn mergeAxis(self: Shape, axis_: anytype, axes_: anytype) Shape { - const axes__ = self.axes(axes_); - - const first_axis = axes__.get(0); - const last_axis = axes__.get(axes__.len - 1); + const axes_to_merge = self.axes(axes_); + stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); var new_dim: i64 = 1; - for (axes__.constSlice(), first_axis..) |ax, counter| { - new_dim *= self.dim(ax); - stdx.debug.assert(ax == counter, "Can't merge shape {} along non-contiguous axes {any}", .{ self, axes_ }); + // Merging partitioned axes is complex. If ANY axis being merged is partitioned, + // the resulting sharding is ambiguous. A propagation pass must resolve this. + // Marking it 'open' is the correct signal. If all are replicated/unknown, the + // result is also replicated/unknown. + // rule: .replicated wins over .unknown. .open wins over everything. + var resulting_partition_spec: PartitionSpec = .unknown; + + for (0..axes_to_merge.len) |i| { + const current_axis = axes_to_merge.get(i); + new_dim *= self.dim(current_axis); + + if (i > 0) { + stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {} along non-contiguous axes {any}", .{ self, axes_ }); + } + + switch (self.partition(current_axis)) { + .axis, .open => resulting_partition_spec = .open, + .replicated => if (resulting_partition_spec != .open) { + resulting_partition_spec = .replicated; + }, + .unknown => {}, + } } + const first_axis_to_merge = axes_to_merge.get(0); + const num_axes_to_merge = axes_to_merge.len; + var new_shape = self; - new_shape._dims.set(first_axis, new_dim); - new_shape._dims.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.dims()[last_axis + 1 ..]) catch unreachable; - new_shape._tags.set(first_axis, if (comptime isTagConvertible(@TypeOf(axis_))) toTag(axis_) else TagUnknown); - new_shape._tags.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.tags()[last_axis + 1 ..]) catch unreachable; + + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); + new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); + new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); + return new_shape; } @@ -946,6 +1446,28 @@ pub const Shape = struct { ); } + test "mergeAxis partitioning" { + // Merging a partitioned axis with a replicated one -> open + const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); + const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged1.partition(.ab).eql(.open)); + try testing.expect(merged1.dim(.ab) == 10); + try testing.expect(merged1.rank() == 2); + + // Merging two replicated axes -> replicated + const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); + const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged2.partition(.ab).eql(.replicated)); + try testing.expect(merged2.partition(.c).eql(.init(.y))); + + // Merging two unknown axes -> unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown + const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged3.partition(.ab).eql(.unknown)); + } + pub fn mergeAxes(self: Shape, axes_: anytype) Shape { const T = @TypeOf(axes_); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); @@ -1064,17 +1586,17 @@ pub const Shape = struct { test "comptimeShape" { comptime { const s = Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } // comptime only the shape { const s = comptime Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } } @@ -1093,4 +1615,135 @@ pub const Shape = struct { } return res_shape; } + + pub fn iterator(self: Shape) MultiDimIterator { + return .{ + .shape = self, + .current_coords = .{0} ** Shape.MAX_RANK, + .flat_index = 0, + }; + } }; + +/// Iterates over all coordinates of a shape, yielding the multi-dimensional +/// coordinates and the corresponding flat index for each element. +pub const MultiDimIterator = struct { + shape: Shape, + current_coords: [Shape.MAX_RANK]i64, + flat_index: usize, + + pub const Item = struct { + coords: [Shape.MAX_RANK]i64, + rank: u4, + flat_index: usize, + }; + + pub fn next(self: *MultiDimIterator) ?Item { + if (self.flat_index >= self.shape.count()) { + return null; + } + + const rank = self.shape.rank(); + const item = Item{ + .coords = self.current_coords, + .rank = rank, + .flat_index = self.flat_index, + }; + + self.flat_index += 1; + + var i: usize = rank; + while (i > 0) { + i -= 1; + self.current_coords[i] += 1; + if (self.current_coords[i] < self.shape.dim(i)) { + return item; + } + self.current_coords[i] = 0; + } + + return item; + } +}; + +test "MultiDimIterator / Scalar" { + const shape = Shape.scalar(.f32); + var iter = shape.iterator(); + const item = iter.next().?; + + try std.testing.expectEqual(0, item.rank); + try std.testing.expectEqual(0, item.flat_index); + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / Empty" { + const shape = Shape.init(.{ .x = 5, .y = 0, .z = 2 }, .f32); + var iter = shape.iterator(); + + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / 1D shape" { + const shape = Shape.init(.{5}, .f32); + var iter = shape.iterator(); + var count: i64 = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.coords[0]); + } + try std.testing.expectEqual(5, count); +} + +test "MultiDimIterator / 2D shape" { + const shape = Shape.init(.{ .rows = 2, .cols = 3 }, .f32); + var iter = shape.iterator(); + var items = std.ArrayList(MultiDimIterator.Item).init(std.testing.allocator); + defer items.deinit(); + + while (iter.next()) |item| { + try items.append(item); + } + + try std.testing.expectEqual(6, items.items.len); + try std.testing.expectEqualSlices(i64, &.{ 0, 0 }, items.items[0].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 1 }, items.items[1].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 2 }, items.items[2].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 0 }, items.items[3].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 1 }, items.items[4].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 2 }, items.items[5].coords[0..2]); + try std.testing.expectEqual(5, items.items[5].flat_index); +} + +test "MultiDimIterator / 3D shape" { + const shape = Shape.init(.{ .d = 2, .h = 2, .w = 2 }, .f32); + var iter = shape.iterator(); + const expected_coords = [_][3]i64{ + .{ 0, 0, 0 }, + .{ 0, 0, 1 }, + .{ 0, 1, 0 }, + .{ 0, 1, 1 }, + .{ 1, 0, 0 }, + .{ 1, 0, 1 }, + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + }; + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.flat_index); + try std.testing.expectEqualSlices(i64, &expected_coords[count], item.coords[0..3]); + } + + try std.testing.expectEqual(8, count); +} + +test "MultiDimIterator / Shape with dimension of size 1" { + const shape = Shape.init(.{ .a = 2, .b = 1, .c = 3 }, .f32); + var iter = shape.iterator(); + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(0, item.coords[1]); + } + try std.testing.expectEqual(6, count); +} diff --git a/zml/slice.zig b/zml/slice.zig new file mode 100644 index 00000000..04980212 --- /dev/null +++ b/zml/slice.zig @@ -0,0 +1,161 @@ +pub const std = @import("std"); +pub const stdx = @import("stdx"); + +const floats = @import("floats.zig"); +const Shape = @import("shape.zig").Shape; + +const log = std.log.scoped(.@"zml/slice"); + +/// Creates a slice of type `T` representing the specified `shape`. +pub fn Shaped(T: type, shape: Shape, slice: []u8) []T { + stdx.debug.assert(slice.len == shape.byteSize(), "Slice length does not match shape byte size: {} != {}", .{ slice.len, shape.byteSize() }); + const ptr: [*]T = @alignCast(@constCast(@ptrCast(slice.ptr))); + return ptr[0..shape.count()]; +} + +/// Creates a slice of type `T` representing the specified `shape` and fills it with the provided `value`. +pub fn fill(T: type, allocator: std.mem.Allocator, shape: Shape, value: T) ![]u8 { + const slice = try allocator.alloc(u8, shape.byteSize()); + errdefer allocator.free(slice); + + @memset(Shaped(T, shape, slice), value); + + log.debug("Created filled slice of size {d} ptr {*} bytes for {}", .{ slice.len, slice.ptr, shape }); + return slice; +} + +/// Creates a slice of integers in the specified shape, starting from `start` +/// and incrementing by `step`. The slice is allocated using the provided allocator. +/// TODO: Add support for other data types in the future. +pub const ArangeOpts = struct { + start: i64 = 0, + step: i64 = 1, +}; +pub fn arange(allocator: std.mem.Allocator, shape: Shape, opts: ArangeOpts) ![]u8 { + const slice = try allocator.alloc(u8, shape.byteSize()); + errdefer allocator.free(slice); + + switch (shape.dtype()) { + inline else => |d| if (comptime d.class() != .integer) { + stdx.debug.assert(shape.dtype().class() == .integer, "arange expects type to be integer, got {} instead.", .{shape.dtype()}); + } else { + const Zt = d.toZigType(); + var j: i64 = opts.start; + for (Shaped(Zt, shape, slice)) |*val| { + val.* = @intCast(j); + j +%= opts.step; + } + }, + } + log.debug("Created arange slice of size {d} ptr {*} bytes for {}", .{ slice.len, slice.ptr, shape }); + return slice; +} + +pub const Slice1DOpts = struct { + start: i64 = 0, + end: ?i64 = null, +}; + +/// Slices the input Tensor over the given axis using the given parameters. +pub fn slice1d(shape: Shape, slice: []u8, axis_: anytype, opts: Slice1DOpts) struct { Shape, []u8 } { + const ax = shape.axis(axis_); + const d = shape.dim(ax); + const start: i64 = if (opts.start < 0) opts.start + d else opts.start; + var end = opts.end orelse d; + if (end < 0) end += d; + + stdx.debug.assert(start >= 0 and start < d, "slice1d({}, {*}, {}) expects the slice start to be between 0 and {} got: {}", .{ shape, slice.ptr, ax, d, opts }); + stdx.debug.assert(end >= 1 and end <= d, "slice1d({}, {*}, {}) expects the slice end to be between 1 and {} got: {}", .{ shape, slice.ptr, ax, d, opts }); + stdx.debug.assert(start < end, "slice1d({}, {*}, {}) expects the slice start ({}) to be smaller than the end ({}), got: {}", .{ shape, slice.ptr, ax, start, end, opts }); + + const offset: usize = @intCast(start * shape.computeByteStrides().get(ax)); + const new_shape = shape.set(ax, end - start); + const sliced = slice[offset .. offset + new_shape.byteSize()]; + + return .{ new_shape, sliced }; +} + +// This module provides a way to pretty print a slice in a human-readable format. +pub fn pretty(shape: Shape, slice: []u8) PrettyPrinter { + return .{ .shape = shape, .slice = slice }; +} + +pub const PrettyPrinter = struct { + shape: Shape, + slice: []u8, + + pub fn format(self: PrettyPrinter, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { + const fmt_: stdx.fmt.Fmt = switch (self.shape.dtype().class()) { + .integer => .parse(i32, fmt), + .float => .parse(f32, fmt), + else => .parse(void, fmt), + }; + try prettyPrint(self.shape, self.slice, writer, .{ .fmt = fmt_, .options = options }); + } +}; + +pub fn prettyPrint(shape: Shape, slice: []u8, writer: anytype, options: stdx.fmt.FullFormatOptions) !void { + return prettyPrintIndented(shape, slice, writer, 4, 0, options); +} + +fn prettyPrintIndented(shape: Shape, slice: []u8, writer: anytype, num_rows: u8, indent_level: u8, options: stdx.fmt.FullFormatOptions) !void { + if (shape.rank() == 0) { + // Special case input tensor is a scalar + return switch (shape.dtype()) { + inline else => |dt| { + const val: dt.toZigType() = Shaped(dt.toZigType(), shape, slice)[0]; + return switch (comptime dt.class()) { + // Since we have custom floats, we need to explicitly convert to float32 ourselves. + .float => stdx.fmt.formatFloatValue(floats.floatCast(f32, val), options, writer), + .integer => stdx.fmt.formatIntValue(val, options, writer), + .bool, .complex => stdx.fmt.formatAnyValue(val, options, writer), + }; + }, + }; + } + if (shape.rank() == 1) { + // Print a contiguous slice of items from the buffer in one line. + // The number of items printed is controlled by the user through format syntax. + try writer.writeByteNTimes(' ', indent_level); + switch (shape.dtype()) { + inline else => |dt| { + const values = Shaped(dt.toZigType(), shape, slice); + switch (comptime dt.class()) { + .float => try stdx.fmt.formatFloatSlice(values, options, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, writer), + .bool, .complex => try stdx.fmt.formatAnySlice(values, options, writer), + } + }, + } + try writer.writeByte('\n'); + return; + } + // TODO: consider removing the \n if dim is 1 for this axis. + try writer.writeByteNTimes(' ', indent_level); + _ = try writer.write("{\n"); + defer { + writer.writeByteNTimes(' ', indent_level) catch {}; + _ = writer.write("},\n") catch {}; + } + + // Write first rows + const n: u64 = @intCast(shape.dim(0)); + for (0..@min(num_rows, n)) |d| { + const di: i64 = @intCast(d); + const new_shape, const sliced = slice1d(shape, slice, 0, .{ .start = di, .end = di + 1 }); + try prettyPrintIndented(new_shape.drop(0), sliced, writer, num_rows, indent_level + 2, options); + } + + if (n < num_rows) return; + // Skip middle rows + if (n > 2 * num_rows) { + try writer.writeByteNTimes(' ', indent_level + 2); + _ = try writer.write("...\n"); + } + // Write last rows + for (@max(n - num_rows, num_rows)..n) |d| { + const di: i64 = @intCast(d); + const new_shape, const sliced = slice1d(shape, slice, 0, .{ .start = di, .end = di + 1 }); + try prettyPrintIndented(new_shape.drop(0), sliced, writer, num_rows, indent_level + 2, options); + } +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 74bf1779..2058fec7 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -14,6 +14,9 @@ const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; +const Mesh = @import("partitioning.zig").Mesh; +const Partition = @import("partitioning.zig").Partition; +const Sharding = @import("partitioning.zig").Sharding; const Shape = @import("shape.zig").Shape; const EnumLiteral = @TypeOf(.enum_literal); @@ -40,6 +43,7 @@ pub const Tensor = struct { _id: _Id, _donation: _Donation = .no_buffer, _output_memory_kind: Memory = .device, + _mesh: ?Mesh = null, pub const _Donation = union(enum) { no_buffer, input_buffer, arg: u16 }; pub const _Id = union(enum) { mlir: mlir.Value, buffer_id: u64, arg_id: u64 }; @@ -119,6 +123,7 @@ pub const Tensor = struct { sh._dims.appendAssumeCapacity(ranked_tensor.getDimension(i)); } sh._tags.resize(n) catch unreachable; + sh._partitioning.resize(n) catch unreachable; return .{ ._shape = sh, ._id = .{ .mlir = val } }; } @@ -165,15 +170,35 @@ pub const Tensor = struct { return res; } + pub fn mesh(self: Tensor) Mesh { + const ctx = self.getContext(); + return ctx.currentMesh(); + } + + pub fn replicated(self: Tensor) Tensor { + const ctx = self.getContext(); + const mesh_ = ctx.currentMesh(); + + var res = self; + res._mesh = mesh_; + res._shape = self._shape.withReplicatedPartitioning(); + return res; + } + pub fn withSharding(self: Tensor, axes_: anytype) Tensor { + const partitioned_shape = self._shape.withPartitioning(axes_); + scoped_log.debug("withSharding {}: {} -> {}", .{ axes_, self._shape, partitioned_shape }); + return switch (self._id) { .arg_id, .mlir => { const ctx = self.getContext(); + const mesh_ = ctx.currentMesh(); const mlir_ctx = ctx.mlirCtx(); - var res = self; - res._shape = self._shape.withSharding(axes_); - if (ctx.numPartitions() <= 1) return self; + if (mesh_.isSinglePartition()) return self; + + const sharding: Sharding = .init(mesh_, partitioned_shape); + const op = dialect.stablehlo.custom_call( mlir_ctx, &.{self.value()}, @@ -181,23 +206,29 @@ pub const Tensor = struct { .call_target_name = "Sharding", .has_side_effect = false, .backend_config = null, - .additional_attributes = &.{.{ "mhlo.sharding", ctx.getShardingAttr(res._shape) }}, + .additional_attributes = &.{.{ "mhlo.sharding", ctx.getShardingAttr(sharding) }}, .api_version = .original, }, &.{self.value().getType()}, mlir_ctx.location(@src()), ); - return _result(res._shape, op.result(0)); + return _result(partitioned_shape, op.result(0)); }, .buffer_id => { var res = self; - res._shape = self._shape.withSharding(axes_); + res._shape = partitioned_shape; return res; }, }; } + pub fn withMesh(self: Tensor, mesh_: Mesh) Tensor { + var res = self; + res._mesh = mesh_; + return res; + } + pub fn toMemory(self: Tensor, kind: Memory) Tensor { return switch (self._id) { .arg_id, .mlir => { @@ -538,9 +569,10 @@ pub const Tensor = struct { }; } - pub fn init(platform: Platform, seed: u128) !Bufferized(Rng) { + pub fn init(platform: Platform, mesh_: Mesh, seed: u128) !Bufferized(Rng) { + const sharding: Sharding = .init(mesh_, Rng.shape()._state); return .{ - ._state = try Buffer.fromBytes(platform, Rng.shape()._state, std.mem.asBytes(&seed)), + ._state = try Buffer.from(platform, sharding, std.mem.asBytes(&seed), .{}), }; } @@ -642,7 +674,7 @@ pub const Tensor = struct { platform, Stats.uniformStats, .{ Rng.shape(), zml.Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, - .{try Rng.init(platform, 1234)}, + .{try Rng.init(platform, zml.Mesh.single(), 1234)}, ); // Check the Rng state has been modified. @@ -743,7 +775,7 @@ pub const Tensor = struct { platform, Stats.gumbelStats, .{ Rng.shape(), zml.Shape.init(.{tgt_dist.len}, .f32) }, - .{ try Rng.init(platform, 1234), try .fromArray(platform, tgt_dist) }, + .{ try Rng.init(platform, zml.Mesh.single(), 1234), try .fromArray(platform, tgt_dist) }, ); // Check the Rng state has been modified. try std.testing.expect(try rand._state.getValue(i128) != 1234); @@ -1112,7 +1144,7 @@ pub const Tensor = struct { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); @@ -1868,6 +1900,7 @@ pub const Tensor = struct { mlirx.tensorType(mlir_ctx, res_shape), loc, ); + return _result(res_shape, op.result(0)); } @@ -2285,7 +2318,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2428,7 +2461,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2464,6 +2497,7 @@ pub const Tensor = struct { std.testing.allocator, Local._gatherSlices, .{ x.shape(), slice_shape, idx.shape(), .{ .indices_are_sorted = true } }, + zml.Mesh.single(), platform, ); defer mod.deinit(); @@ -2473,7 +2507,7 @@ pub const Tensor = struct { // Test with actual values. const range = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 2 * 4 * 6 }, .u16); defer range.deinit(std.testing.allocator); - const operand = try range.reshape(.{ .a = 2, .b = 4, .c = 6 }).toDevice(platform); + const operand = try range.reshape(.{ .a = 2, .b = 4, .c = 6 }).toDevice(platform, zml.Mesh.single()); defer operand.deinit(); const start_indices = (try zml.Buffer.fromArray(platform, [2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })).withTags(.{ .n, ._ }); defer start_indices.deinit(); @@ -2635,7 +2669,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2668,7 +2702,8 @@ pub const Tensor = struct { // Test with actual values, no batching. { const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); - const a = (try zml.Buffer.from(platform, a_host.reshape(.{ 3, 3 }), .{})).withTags(.{ .a, .b }); + const sharding: zml.Sharding = .init(zml.Mesh.single(), a_host.shape().reshape(.{ 3, 3 })); + const a = (try zml.Buffer.from(platform, sharding, a_host.bytes(), .{})).withTags(.{ .a, .b }); defer a.deinit(); a_host.deinit(std.testing.allocator); @@ -2687,7 +2722,9 @@ pub const Tensor = struct { // Test with setting individual values (no batching) { const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); - const a = try zml.Buffer.from(platform, a_host, .{}); + const sharding: zml.Sharding = .init(zml.Mesh.single(), a_host.shape()); + + const a = try zml.Buffer.from(platform, sharding, a_host.bytes(), .{}); defer a.deinit(); a_host.deinit(std.testing.allocator); @@ -2703,49 +2740,49 @@ pub const Tensor = struct { try std.testing.expect(a.shape().eql(result.shape())); try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); } - { - // Test with actual values and batching along axis .a - const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0); - defer operand.deinit(); - const start_indices = (try zml.Buffer.fromArray( - platform, - [2][2][3][2]i32{ - .{ - .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, - .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, - }, - .{ - .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, - .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, - }, - }, - )).withTags(.{ .n, .a, .m, .coord }); - defer start_indices.deinit(); + // { + // // Test with actual values and batching along axis .a + // const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0); + // defer operand.deinit(); + // const start_indices = (try zml.Buffer.fromArray( + // platform, + // [2][2][3][2]i32{ + // .{ + // .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, + // .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, + // }, + // .{ + // .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, + // .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, + // }, + // }, + // )).withTags(.{ .n, .a, .m, .coord }); + // defer start_indices.deinit(); - const values = try zml.Buffer.constant( - platform, - Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16), - 1, - ); - defer values.deinit(); + // const values = try zml.Buffer.constant( + // platform, + // Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16), + // 1, + // ); + // defer values.deinit(); - const result = try zml.testing.compileAndCall(platform, Local._scatterCB, .{ operand, start_indices, values }); + // const result = try zml.testing.compileAndCall(platform, Local._scatterCB, .{ operand, start_indices, values }); - const expected = [2][3][4][2]u16{ - .{ - .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, - .{ .{ 0, 0 }, .{ 0, 0 }, .{ 2, 2 }, .{ 2, 2 } }, - .{ .{ 0, 0 }, .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 } }, - }, - .{ - .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, - .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, - .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, - }, - }; - try std.testing.expect(operand.shape().eql(result.shape())); - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); - } + // const expected = [2][3][4][2]u16{ + // .{ + // .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, + // .{ .{ 0, 0 }, .{ 0, 0 }, .{ 2, 2 }, .{ 2, 2 } }, + // .{ .{ 0, 0 }, .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 } }, + // }, + // .{ + // .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, + // .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, + // .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, + // }, + // }; + // try std.testing.expect(operand.shape().eql(result.shape())); + // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + // } } /// Returns a Tensor containing the maximum over a given axis. @@ -2832,7 +2869,7 @@ pub const Tensor = struct { } }; - const argmax = try zml.compileFn(allocator, ArgMaxTest._fwd, .{Shape.init(.{ 1, 5 }, .f32)}, platform); + const argmax = try zml.compileFn(allocator, ArgMaxTest._fwd, .{Shape.init(.{ 1, 5 }, .f32)}, zml.Mesh.single(), platform); defer argmax.deinit(); // Test with tie { @@ -2883,71 +2920,71 @@ pub const Tensor = struct { return self.sort(axis_, .{ .descending = opts.descending }).indices; } - test argsort { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); + // test argsort { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); - const Local = struct { - pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { - return x.argsort(axis_, opts); - } - }; + // const Local = struct { + // pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { + // return x.argsort(axis_, opts); + // } + // }; - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - const allocator = arena_state.allocator(); - // 2D Tensor - dim = 1, ascending - { - const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 }); - const res = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{} }); - const res_cpu = try res.toHostAlloc(allocator); - try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); - } - // 3D Tensor, dim = 1, descending - { - const x = try zml.Buffer.fromSlice(platform, .{ 1, 5, 10 }, &[_]f16{ - -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, - 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, - 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, - 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, - 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, - }); - const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{ .descending = true } }); - const res = try res_dev.toHostAlloc(allocator); - try std.testing.expectEqualSlices(i32, &.{ - 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, - 2, 0, 4, 4, 1, 3, 4, 4, 1, 0, - 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, - 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, - 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, - }, res.items(i32)); - } - // 4D Tensor, dim = 3, ascending - { - const x = try zml.Buffer.fromSlice(platform, .{ 4, 2, 1, 4 }, &[_]i32{ - 89, 31, 22, 42, - 64, 39, 0, 30, - 64, 71, 46, 31, - 89, 82, 78, 86, - 55, 32, 43, 19, - 93, 24, 45, 72, - 64, 86, 62, 88, - 57, 21, 19, 12, - }); - const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 3, .{} }); - const res = try res_dev.toHostAlloc(allocator); - try std.testing.expectEqualSlices(i32, &.{ - 2, 1, 3, 0, - 2, 3, 1, 0, - 3, 2, 0, 1, - 2, 1, 3, 0, - 3, 1, 2, 0, - 1, 2, 3, 0, - 2, 0, 1, 3, - 3, 2, 1, 0, - }, res.items(i32)); - } - } + // var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + // defer arena_state.deinit(); + // const allocator = arena_state.allocator(); + // // 2D Tensor - dim = 1, ascending + // { + // const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 }); + // const res = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{} }); + // const res_cpu = try res.toHostAlloc(allocator); + // try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); + // } + // // 3D Tensor, dim = 1, descending + // { + // const x = try zml.Buffer.fromSlice(platform, .{ 1, 5, 10 }, &[_]f16{ + // -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, + // 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, + // 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, + // 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, + // 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, + // }); + // const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{ .descending = true } }); + // const res = try res_dev.toHostAlloc(allocator); + // try std.testing.expectEqualSlices(i32, &.{ + // 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, + // 2, 0, 4, 4, 1, 3, 4, 4, 1, 0, + // 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, + // 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, + // 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, + // }, res.items(i32)); + // } + // // 4D Tensor, dim = 3, ascending + // { + // const x = try zml.Buffer.fromSlice(platform, .{ 4, 2, 1, 4 }, &[_]i32{ + // 89, 31, 22, 42, + // 64, 39, 0, 30, + // 64, 71, 46, 31, + // 89, 82, 78, 86, + // 55, 32, 43, 19, + // 93, 24, 45, 72, + // 64, 86, 62, 88, + // 57, 21, 19, 12, + // }); + // const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 3, .{} }); + // const res = try res_dev.toHostAlloc(allocator); + // try std.testing.expectEqualSlices(i32, &.{ + // 2, 1, 3, 0, + // 2, 3, 1, 0, + // 3, 2, 0, 1, + // 2, 1, 3, 0, + // 3, 1, 2, 0, + // 1, 2, 3, 0, + // 2, 0, 1, 3, + // 3, 2, 1, 0, + // }, res.items(i32)); + // } + // } /// Returns a Tensor representing the result of Top-K over the given axis. pub fn topK(self: Tensor, k: u32, axis_: anytype, opts: struct { descending: bool = true }) SortRes { @@ -3065,7 +3102,7 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3118,7 +3155,7 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3921,42 +3958,42 @@ test "Tensor.maxPool1d" { ); } -test "Tensor.maxPool2d" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); +// test "Tensor.maxPool2d" { +// const zml = @import("zml.zig"); +// const platform = zml.testing.env(); - const MaxPool = struct { - pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { - return x.maxPool2d(.{ - .window_dimensions = .{ 3, 2 }, - .window_strides = .{ 2, 1 }, - }); - } - }; +// const MaxPool = struct { +// pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { +// return x.maxPool2d(.{ +// .window_dimensions = .{ 3, 2 }, +// .window_strides = .{ 2, 1 }, +// }); +// } +// }; - var data: [100]f32 = undefined; - for (&data, 0..) |*v, i| v.* = @floatFromInt(i); - const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5, 5 }, &data); +// var data: [100]f32 = undefined; +// for (&data, 0..) |*v, i| v.* = @floatFromInt(i); +// const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5, 5 }, &data); - const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); - var buffer: [2][2][2][4]f32 = undefined; - _ = try result.values.toHost(std.mem.asBytes(&buffer)); - try std.testing.expectEqualDeep( - [2][2][2][4]f32{ - .{ - .{ .{ 11, 12, 13, 14 }, .{ 21, 22, 23, 24 } }, - .{ .{ 36, 37, 38, 39 }, .{ 46, 47, 48, 49 } }, - }, - .{ - .{ .{ 61, 62, 63, 64 }, .{ 71, 72, 73, 74 } }, - .{ .{ 86, 87, 88, 89 }, .{ 96, 97, 98, 99 } }, - }, - }, - buffer, - ); -} +// const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); +// try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); +// try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); +// var buffer: [2][2][2][4]f32 = undefined; +// _ = try result.values.toHostAlloc(std.mem.asBytes(&buffer)); +// try std.testing.expectEqualDeep( +// [2][2][2][4]f32{ +// .{ +// .{ .{ 11, 12, 13, 14 }, .{ 21, 22, 23, 24 } }, +// .{ .{ 36, 37, 38, 39 }, .{ 46, 47, 48, 49 } }, +// }, +// .{ +// .{ .{ 61, 62, 63, 64 }, .{ 71, 72, 73, 74 } }, +// .{ .{ 86, 87, 88, 89 }, .{ 96, 97, 98, 99 } }, +// }, +// }, +// buffer, +// ); +// } pub fn Bufferized(comptime T: type) type { // TODO: we should strip out the non-buffer fields. @@ -4151,6 +4188,6 @@ test "unused tensor" { } }; - const mod = try zml.compileFn(std.testing.allocator, Local._fwd, .{Shape.init(.{10}, .f32)}, platform); + const mod = try zml.compileFn(std.testing.allocator, Local._fwd, .{Shape.init(.{10}, .f32)}, zml.Mesh.single(), platform); defer mod.deinit(); } diff --git a/zml/testing.zig b/zml/testing.zig index 0dcf3941..73014239 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -38,12 +38,12 @@ pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; var left: zml.HostBuffer, const should_free_left = if (@TypeOf(left_) == zml.Buffer) - .{ try left_.toHostAlloc(allocator), true } + .{ try left_.toHostBuffer(.init(zml.Mesh.single(), left_.shape()), allocator), true } else .{ left_, false }; var right: zml.HostBuffer, const should_free_right = if (@TypeOf(right_) == zml.Buffer) - .{ try right_.toHostAlloc(allocator), true } + .{ try right_.toHostBuffer(.init(zml.Mesh.single(), right_.shape()), allocator), true } else .{ right_, false }; @@ -149,7 +149,7 @@ pub fn compileAndCall( var shape_args: zml.ShapeOf(stdx.meta.FnArgs(func)) = undefined; try meta.mapAlloc(Local.bufferToShape, arena.allocator(), {}, buffer_and_args, &shape_args); - var mod = try zml.compileFn(allocator, func, shape_args, platform); + var mod = try zml.compileFn(allocator, func, shape_args, zml.Mesh.single(), platform); defer mod.deinit(); // Note: we don't use the type safe API of mod, @@ -176,7 +176,7 @@ pub fn compileAndCallWithTensors( var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const mod = try zml.compileFn(allocator, func, shape_args, platform); + const mod = try zml.compileFn(allocator, func, shape_args, zml.Mesh.single(), platform); defer mod.deinit(); return mod.call(buffer_args); @@ -219,7 +219,7 @@ pub fn testLayerOut( log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ n_in_exp, n_in }); } - const exe = try zml.compileModel(alloc, fwd, layer, input_shapes, platform); + const exe = try zml.compileModel(alloc, fwd, layer, input_shapes, zml.Mesh.single(), platform); const n_out_exp = activations.countLayers(out_name); if (exe.inner.result_shapes.len != n_out_exp) { @@ -243,7 +243,7 @@ pub fn testLayerOut( log.err("Didn't find test input: {s}", .{full_prefix.items}); @panic("Missing test input"); }; - return host.toDevice(ctx.platform) catch unreachable; + return host.toDevice(ctx.platform, zml.Mesh.single()) catch unreachable; } }; diff --git a/zml/torch.zig b/zml/torch.zig index 5889353d..3f4b2ee0 100644 --- a/zml/torch.zig +++ b/zml/torch.zig @@ -45,7 +45,7 @@ pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { test matmul { const platform = zml.testing.env(); - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", zml.Mesh.single(), platform); defer comp.deinit(); comp.activate(); diff --git a/zml/zml.zig b/zml/zml.zig index 45abfccd..eef8687a 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -11,6 +11,9 @@ pub const Data = @import("dtype.zig").Data; pub const DataType = @import("dtype.zig").DataType; pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; pub const Platform = @import("platform.zig").Platform; +pub const Mesh = @import("partitioning.zig").Mesh; +pub const Sharding = @import("partitioning.zig").Sharding; +pub const DeviceShard = @import("partitioning.zig").DeviceShard; pub const Shape = @import("shape.zig").Shape; pub const ShapeOf = @import("tensor.zig").ShapeOf; pub const Target = @import("platform.zig").Target; @@ -24,15 +27,17 @@ pub const helpers = @import("helpers.zig"); pub const nn = @import("nn.zig"); pub const module = @import("module.zig"); pub const meta = @import("meta.zig"); +pub const partitioning = @import("partitioning.zig"); pub const platform = @import("platform.zig"); pub const mlir = @import("mlirx.zig"); pub const pjrt = @import("pjrtx.zig"); +pub const slice = @import("slice.zig"); pub const testing = @import("testing.zig"); pub const torch = @import("torch.zig"); // pub const tokenizer = @import("tokenizer.zig"); pub const tokenizer = @import("zml/tokenizer"); - +pub const Shaped = slice.Shaped; pub const call = ops.call; pub const compile = exe.compile; pub const compileWithPrefix = exe.compileWithPrefix; @@ -41,6 +46,9 @@ pub const compileModel = exe.compileModel; pub const FnExe = exe.FnExe; pub const ModuleExe = exe.ModuleExe; pub const ModuleSignature = exe.ModuleSignature; +pub const currentMesh = module.currentMesh; +pub const pushMesh = module.pushMesh; +pub const popMesh = module.popMesh; pub const ops = @import("ops.zig"); pub const tools = struct { From 69e1d1920594a429ad4ec496ccbbc48d5381a2fa Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 30 Jun 2025 18:34:01 +0200 Subject: [PATCH 02/51] rebase --- examples/llama/llama.zig | 9 +- examples/llama/main.zig | 2 +- zml/buffer.zig | 175 ++++++++++++++++++++++++++------------- 3 files changed, 122 insertions(+), 64 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 8d859f3f..78ace40d 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -439,13 +439,12 @@ pub const KvCache = struct { }; } - pub fn initBuffer(allocator: std.mem.Allocator, kv_shape: zml.Shape, mesh: zml.Mesh, platform: zml.Platform) !zml.Bufferized(KvCache) { + pub fn initBuffer(kv_shape: zml.Shape, mesh: zml.Mesh, platform: zml.Platform) !zml.Bufferized(KvCache) { const sharding = zml.Sharding.init(mesh, kv_shape.withPartitioning(.{ .k = .data, .h = .model })); - const slice = try zml.slice.fill(f32, allocator, kv_shape, 1.0); - defer allocator.free(slice); + return .{ - .k = try zml.Buffer.from(platform, sharding, slice, .{ .wait = true }), - .v = try zml.Buffer.from(platform, sharding, slice, .{ .wait = true }), + .k = try zml.Buffer.uninitialized(platform, sharding, .{}), + .v = try zml.Buffer.uninitialized(platform, sharding, .{}), .layer_index = try zml.Buffer.constant(platform, .init(mesh, zml.Shape.init(.{}, .u32)), 0), }; } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 5996dad4..8e452d19 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -345,7 +345,7 @@ pub fn asyncMain() !void { log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); log.info("Creating KvCache", .{}); - const kv_cache = try llama.KvCache.initBuffer(allocator, kv_shape, mesh, platform); + const kv_cache = try llama.KvCache.initBuffer(kv_shape, mesh, platform); var tokenizer = blk: { if (res.args.tokenizer) |tok| { diff --git a/zml/buffer.zig b/zml/buffer.zig index a7c6ff50..d4c68dd5 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -115,6 +115,65 @@ pub const Buffer = struct { return self; } + pub const UnitializedOptions = struct { + memory: ?Memory = null, + }; + + pub fn uninitialized(platform: Platform, sharding: Sharding, opts: UnitializedOptions) !Buffer { + var sharding_devices = sharding.iterator(platform.getDevices()); + + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._shape = sharding.global_shape, + ._shards = .{}, + }; + + errdefer for (res._shards.slice()) |shard| { + shard.deinit(platform.pjrt_api); + }; + + const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { + var minor_to_major: [Shape.MAX_RANK]i64 = undefined; + for (0..Shape.MAX_RANK) |i| { + minor_to_major[i] = @intCast(Shape.MAX_RANK - i - 1); + } + break :blk minor_to_major; + }; + + while (sharding_devices.next()) |shard| { + const specs = shard.specs(); + + var args = pjrt.Client.CreateUninitializedBufferArgs{ + .dims = specs.dims[0..specs.num_dims], + .element_type = bufferTypeFromDtype(shard.shard_shape.dtype()), + .layout = .{ + .tiled = .{ + .minor_to_major = minor_to_major[Shape.MAX_RANK - shard.shard_shape.rank() ..], + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + }; + + if (opts.memory) |memory_kind| { + const memories = try shard.device.addressableMemories(platform.pjrt_api); + const memory = for (memories) |m| { + const kind = m.kind(platform.pjrt_api); + if (kind == memory_kind.toPjrtMemory()) break m; + } else return error.NotFound; + args.memory = memory; + } else { + args.device = shard.device; + } + + const pjrt_buffer = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); + + res._shards.appendAssumeCapacity(pjrt_buffer); + } + + return res; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, sharding: Sharding, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -474,70 +533,70 @@ pub const Buffer = struct { return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; } - pub const UnitializedOptions = struct { - memory: ?pjrt.Memory.Kind = null, - }; + // pub const UnitializedOptions = struct { + // memory: ?pjrt.Memory.Kind = null, + // }; - pub fn uninitialized(platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { - var res: Buffer = .{ - ._api = platform.pjrt_api, - ._shape = shape_, - ._shards = .{}, - }; - errdefer for (res._shards.slice()) |shard| { - shard.deinit(platform.pjrt_api); - }; + // pub fn uninitialized(platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { + // var res: Buffer = .{ + // ._api = platform.pjrt_api, + // ._shape = shape_, + // ._shards = .{}, + // }; + // errdefer for (res._shards.slice()) |shard| { + // shard.deinit(platform.pjrt_api); + // }; - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var minor_to_major: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - minor_to_major[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk minor_to_major; - }; + // const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { + // var minor_to_major: [Shape.MAX_RANK]i64 = undefined; + // for (0..Shape.MAX_RANK) |i| { + // minor_to_major[i] = @intCast(Shape.MAX_RANK - i - 1); + // } + // break :blk minor_to_major; + // }; - // TODO: support more advanced sharding specs - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); - const n_partitions = platform.sharding().num_partitions; - const shard_shape = if (sharding_ax) |ax| s: { - // This kind of sharding error should be detected earlier on. - stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); - const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); - break :s shard_shape; - } else shape_; + // // TODO: support more advanced sharding specs + // stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); + // const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + // const n_partitions = platform.sharding().num_partitions; + // const shard_shape = if (sharding_ax) |ax| s: { + // // This kind of sharding error should be detected earlier on. + // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); + // const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); + // break :s shard_shape; + // } else shape_; - const buffer_type = bufferTypeFromDtype(shape_.dtype()); - const devices = platform.getDevices(); - for (0..n_partitions) |i| { - var args = pjrt.Client.CreateUninitializedBufferArgs{ - .dims = shard_shape.dims(), - .element_type = buffer_type, - .layout = .{ - .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - }; - if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); - const memory = for (memories) |m| { - const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; - } else return error.NotFound; - args.memory = memory; - } else { - args.device = devices[i]; - } - const pjrt_buffer = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); + // const buffer_type = bufferTypeFromDtype(shape_.dtype()); + // const devices = platform.getDevices(); + // for (0..n_partitions) |i| { + // var args = pjrt.Client.CreateUninitializedBufferArgs{ + // .dims = shard_shape.dims(), + // .element_type = buffer_type, + // .layout = .{ + // .tiled = .{ + // .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], + // .tile_dims = &.{}, + // .tile_dims_sizes = &.{}, + // }, + // }, + // }; + // if (opts.memory) |memory_kind| { + // const memories = try devices[i].addressableMemories(platform.pjrt_api); + // const memory = for (memories) |m| { + // const kind = m.kind(platform.pjrt_api); + // if (kind == memory_kind) break m; + // } else return error.NotFound; + // args.memory = memory; + // } else { + // args.device = devices[i]; + // } + // const pjrt_buffer = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); - res._shards.appendAssumeCapacity(pjrt_buffer); - } + // res._shards.appendAssumeCapacity(pjrt_buffer); + // } - return res; - } + // return res; + // } }; pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { From 27bf92c866eee398aaf01ff47e7036acedd54b7c Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 10:23:56 +0200 Subject: [PATCH 03/51] reduce / scatter --- examples/llama/llama.zig | 21 +++++++++---- zml/ops.zig | 64 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 78ace40d..89bea070 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -229,21 +229,32 @@ pub const TransformerLayer = struct { const x0 = x0_; stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); + const mesh = self.input_layernorm.weight.mesh(); + + const x0_replicated = if (mesh.topology.hasTag(.data) != null) + zml.ops.allGather(x0, .data, mesh) + else + x0; + // Self Attention - const x0_normalized = self.input_layernorm.forward(x0); + const x0_normalized = self.input_layernorm.forward(x0_replicated); const delta0, const updated_kv_cache = self.self_attn.forward( x0_normalized, token_index, kv_cache, ); log.warn("TransformerLayer.forward: delta0={}", .{delta0}); - const x1 = x0.add(delta0); + const x1_replicated = x0_replicated.add(delta0); + + const x1 = if (mesh.topology.hasTag(.data) != null) + zml.ops.reduceScatter(x1_replicated, .data, mesh) + else + x1_replicated; // Fully Connected const x1_normalized = self.post_attention_layernorm.forward(x1); - // FIX: Add the MLP output to the Attention output (x1). - const delta1 = self.mlp.forward(x1_normalized); - const x2 = x1.add(delta1); + const mlp_delta = self.mlp.forward(x1_normalized); + const x2 = x1.add(mlp_delta); return .{ x2.reuseBuffer(x0), updated_kv_cache }; } diff --git a/zml/ops.zig b/zml/ops.zig index 5856a98a..79d1fca3 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -616,6 +616,70 @@ pub fn sort( return res; } +fn collectiveOp( + name: [:0]const u8, + input: Tensor, + axis: i64, + mesh: Mesh, +) Tensor { + _ = mesh; // autofix + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + + const replica_groups = mlir.Attribute.denseElements( + mlir_ctx, + &.{ @intCast(axis), 1 }, + .i64, + &[_]i64{0}, + ); + + const op = mlir.Operation.make(ctx.mlirCtx(), name, .{ + .operands = &.{input.value()}, + .result_type_inference = true, + .attributes = &.{ + .{ "replica_groups", replica_groups }, + .{ "channel_id", .int(ctx.mlirCtx(), .i64, 0) }, + }, + .location = ctx.location(@src(), "{s}({_}, {d})", .{ name, input, axis }), + }); + + return fromMlirOperationWithTags(op, input); +} + +pub fn allGather(input: Tensor, axis: anytype, mesh: Mesh) Tensor { + var res_shape = input.shape(); + + const axis_size = mesh.topology.dim(axis); + const mesh_axis_idx = mesh.topology.hasTag(axis).?; + for (0..res_shape.rank()) |i| { + const part_spec = res_shape.partition(i); + if (part_spec == .axis and part_spec.toTag() == mesh.topology.tag(mesh_axis_idx)) { + const current_dim = res_shape.dim(i); + res_shape = res_shape.setDim(i, current_dim * axis_size); + } + } + + var res = collectiveOp("stablehlo.all_gather", input, mesh.topology.axis(axis), mesh); + res._shape._partitioning = input.shape()._partitioning; + return res; +} + +pub fn reduceScatter(input: Tensor, axis: anytype, mesh: Mesh) Tensor { + _ = mesh; // autofix + // For now, only sum is supported. + const body_block, _ = CompilationContext.current().makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ Tensor.scalar(0, input.dtype()), Tensor.scalar(0, input.dtype()) }); + const op = mlir.Operation.make(input.getContext().mlirCtx(), "stablehlo.reduce_scatter", .{ + .operands = &.{input.value()}, + .result_type_inference = true, + .blocks = &.{body_block}, + .attributes = &.{ + .{ "scatter_dimension", .int(input.getContext().mlirCtx(), .i64, input.shape().axis(axis)) }, + }, + .location = input.getContext().location(@src(), "reduceScatter({_}, {d})", .{ input, input.shape().axis(axis) }), + }); + return fromMlirOperationWithTags(op, input); +} + pub const BlockSignature = struct { Fn: type, BlkCtx: type, From 695f85216e34c2d31a2a983df29f5db2fe6297a2 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 10:42:32 +0200 Subject: [PATCH 04/51] add logs --- examples/llama/llama.zig | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 89bea070..804f5ae2 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -231,10 +231,13 @@ pub const TransformerLayer = struct { const mesh = self.input_layernorm.weight.mesh(); - const x0_replicated = if (mesh.topology.hasTag(.data) != null) - zml.ops.allGather(x0, .data, mesh) - else - x0; + const x0_replicated = if (mesh.topology.hasTag(.data) != null) blk: { + log.warn("<<<<<<<<<<<<<<<<<<< x0_replicated", .{}); + break :blk zml.ops.allGather(x0, .data, mesh); + } else blk: { + log.warn("<<<<<<<<<<<<<<<<<<< x0", .{}); + break :blk x0; + }; // Self Attention const x0_normalized = self.input_layernorm.forward(x0_replicated); From 1c913e4662327c24963f59bdb61f4e38193ebfee Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 10:54:12 +0200 Subject: [PATCH 05/51] megatron --- examples/llama/llama.zig | 11 +++++++++++ zml/ops.zig | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 804f5ae2..1940e696 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -310,6 +310,11 @@ const Mlp = struct { output = output.silu().mul(proj); output = self.down_proj.forward(output); + const mesh = self.down_proj.weight.mesh(); + if (mesh.topology.hasTag(.model) != null) { + output = zml.ops.allReduce(output, .model, mesh); + } + return output; } }; @@ -403,6 +408,12 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); var output = self.o_proj.forward(attn).rename(.{ .o_hidden = .d }); + + const mesh = self.o_proj.weight.mesh(); + if (mesh.topology.hasTag(.model) != null) { + output = zml.ops.allReduce(output, .model, mesh); + } + log.warn("SelfAttn.forward: attn_output={}, output={}", .{ attn_output, output }); if (x.dim(.s) > 1) { // Prefill phase: Shard the sequence dimension. diff --git a/zml/ops.zig b/zml/ops.zig index 79d1fca3..916798b5 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -680,6 +680,40 @@ pub fn reduceScatter(input: Tensor, axis: anytype, mesh: Mesh) Tensor { return fromMlirOperationWithTags(op, input); } +pub fn allReduce(input: Tensor, axis: anytype, mesh: Mesh) Tensor { + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + + // The body of the reduce operation (summation) + const body_block, _ = ctx.makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ + Tensor.scalar(0, input.dtype()), + Tensor.scalar(0, input.dtype()), + }); + + // replica_groups defines which devices participate. For an All-Reduce across + // the 'model' axis, it's a 2D array of shape [num_model_devices, 1]. + const replica_groups_shape: []const i64 = &.{ @intCast(mesh.axis(axis)), 1 }; + var replica_groups_data: [mesh.numPartitions()]i64 = undefined; + for (0..mesh.numPartitions()) |i| { + replica_groups_data[i] = @intCast(i); + } + + const op = mlir.Operation.make(mlir_ctx, "stablehlo.all_reduce", .{ + .operands = &.{input.value()}, + .results = &.{input.value().getType()}, + .blocks = &.{body_block}, + .attributes = &.{ + .{ "replica_groups", .denseElements(mlir_ctx, replica_groups_shape, .i64, replica_groups_data[0..mesh.numPartitions()]) }, + .{ "channel_id", .int(mlir_ctx, .i64, 0) }, // Use a default channel + .{ "use_global_device_ids", .boolean(mlir_ctx, false) }, + }, + .location = ctx.location(@src(), "allReduce({_}, {d})", .{ input, mesh.axis(axis) }), + }); + + // The shape of the output of all_reduce is the same as the input. + return fromMlirOperationWithTags(op, input); +} + pub const BlockSignature = struct { Fn: type, BlkCtx: type, From 2d4202e5bb12f24e9744a55be22be0b651b814d2 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 10:55:50 +0200 Subject: [PATCH 06/51] oupsi --- zml/ops.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/ops.zig b/zml/ops.zig index 916798b5..d4ed9a47 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -693,7 +693,7 @@ pub fn allReduce(input: Tensor, axis: anytype, mesh: Mesh) Tensor { // replica_groups defines which devices participate. For an All-Reduce across // the 'model' axis, it's a 2D array of shape [num_model_devices, 1]. const replica_groups_shape: []const i64 = &.{ @intCast(mesh.axis(axis)), 1 }; - var replica_groups_data: [mesh.numPartitions()]i64 = undefined; + var replica_groups_data: [Shape.MAX_RANK]i64 = undefined; for (0..mesh.numPartitions()) |i| { replica_groups_data[i] = @intCast(i); } From e9022db647f43bdf04ecddb3b9345813ff7f335b Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 10:59:56 +0200 Subject: [PATCH 07/51] ops allReduce --- examples/llama/llama.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 1940e696..4a95c7bf 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -312,6 +312,7 @@ const Mlp = struct { const mesh = self.down_proj.weight.mesh(); if (mesh.topology.hasTag(.model) != null) { + log.warn("<<<<<<<<<<<<>>>>>>>>>> Mlp.forward: zml.ops.allReduce", .{}); output = zml.ops.allReduce(output, .model, mesh); } @@ -411,6 +412,8 @@ pub const SelfAttn = struct { const mesh = self.o_proj.weight.mesh(); if (mesh.topology.hasTag(.model) != null) { + log.warn("<<<<<<<<<<<<>>>>>>>>>> SelfAttn.forward: zml.ops.allReduce", .{}); + output = zml.ops.allReduce(output, .model, mesh); } From 7bfc85baa529199635e007e7a8e6355224cbb190 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 11:31:47 +0200 Subject: [PATCH 08/51] fix MLP --- examples/llama/llama.zig | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 4a95c7bf..910156e5 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -293,14 +293,14 @@ const Mlp = struct { pub fn init(self: *Mlp, mesh: zml.Mesh) void { const has_model_axis = mesh.topology.hasTag(.model) != null; - self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh); - self.up_proj.weight = self.up_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh); - self.down_proj.weight = self.down_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh); + self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .hidden_dim, .model_dim }).withMesh(mesh); + self.up_proj.weight = self.up_proj.weight.withTags(.{ .hidden_dim, .model_dim }).withMesh(mesh); + self.down_proj.weight = self.down_proj.weight.withTags(.{ .model_dim, .hidden_dim }).withMesh(mesh); if (has_model_axis) { - self.gate_proj.weight = self.gate_proj.weight.withSharding(.{ .hidden = .model }); - self.up_proj.weight = self.up_proj.weight.withSharding(.{ .hidden = .model }); - self.down_proj.weight = self.down_proj.weight.withSharding(.{ .d = .model }); + self.gate_proj.weight = self.gate_proj.weight.withSharding(.{ .hidden_dim = .model }); + self.up_proj.weight = self.up_proj.weight.withSharding(.{ .hidden_dim = .model }); + self.down_proj.weight = self.down_proj.weight.withSharding(.{ .hidden_dim = .model }); } } From 980587e011c9d1c5231292d8e14696a25dae9aad Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 11:43:17 +0200 Subject: [PATCH 09/51] restore zml.call --- examples/llama/llama.zig | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 910156e5..ceb14ff4 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -69,7 +69,7 @@ pub const LlamaLM = struct { ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache); + const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } @@ -85,7 +85,7 @@ pub const LlamaLM = struct { var logits = blk: { if (lm_head_) |lm_head| { - break :blk lm_head.forward(out); + break :blk zml.call(lm_head, .forward, .{out}); } else { break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); } @@ -184,9 +184,9 @@ pub const Llama = struct { var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { - hidden, updated_kv_cache = layer.forward(hidden, token_index, updated_kv_cache.atLayer(i)); + hidden, updated_kv_cache = zml.call(layer, .forward, .{ hidden, token_index, updated_kv_cache.atLayer(i) }); } - const output = self.norm.forward(hidden); + const output = zml.call(self.norm, .forward, .{hidden}); zml.popMesh(); @@ -194,7 +194,7 @@ pub const Llama = struct { } pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { - return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); + return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); } fn initKvCache(self: Llama, embed_shape: zml.Shape) KvCache { @@ -240,12 +240,8 @@ pub const TransformerLayer = struct { }; // Self Attention - const x0_normalized = self.input_layernorm.forward(x0_replicated); - const delta0, const updated_kv_cache = self.self_attn.forward( - x0_normalized, - token_index, - kv_cache, - ); + const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0_replicated}); + const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); log.warn("TransformerLayer.forward: delta0={}", .{delta0}); const x1_replicated = x0_replicated.add(delta0); @@ -255,8 +251,8 @@ pub const TransformerLayer = struct { x1_replicated; // Fully Connected - const x1_normalized = self.post_attention_layernorm.forward(x1); - const mlp_delta = self.mlp.forward(x1_normalized); + const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); + const mlp_delta = zml.call(self.mlp, .forward, .{x1_normalized}); const x2 = x1.add(mlp_delta); return .{ x2.reuseBuffer(x0), updated_kv_cache }; @@ -305,10 +301,10 @@ const Mlp = struct { } pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = self.up_proj.forward(x); - var output: zml.Tensor = self.gate_proj.forward(x); + const proj = zml.call(self.up_proj, .forward, .{x}); + var output = zml.call(self.gate_proj, .forward, .{x}); output = output.silu().mul(proj); - output = self.down_proj.forward(output); + output = zml.call(self.down_proj, .forward, .{output}); const mesh = self.down_proj.weight.mesh(); if (mesh.topology.hasTag(.model) != null) { @@ -369,9 +365,10 @@ pub const SelfAttn = struct { ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - const q_proj_out = self.q_proj.forward(x); - const k_proj_out = self.k_proj.forward(x); - const v_proj_out = self.v_proj.forward(x); + const q_proj_out = zml.call(self.q_proj, .forward, .{x}); + const k_proj_out = zml.call(self.k_proj, .forward, .{x}); + const v_proj_out = zml.call(self.v_proj, .forward, .{x}); + log.warn("SelfAttn.forward: q_proj_out={}, k_proj_out={}, v_proj_out={}", .{ q_proj_out, k_proj_out, v_proj_out }); var q = q_proj_out.splitAxis(.d, .{ .h = self.num_heads, .hd = .auto }); var k = k_proj_out.splitAxis(.d, .{ .h = num_kv_heads, .hd = .auto }); @@ -408,7 +405,7 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - var output = self.o_proj.forward(attn).rename(.{ .o_hidden = .d }); + var output = zml.call(self.o_proj, .forward, .{attn}).rename(.{ .o_hidden = .d }); const mesh = self.o_proj.weight.mesh(); if (mesh.topology.hasTag(.model) != null) { From 8380847390f268d9b36921524b5c492a8b920eee Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 16:39:26 +0200 Subject: [PATCH 10/51] wip --- examples/llama/llama.zig | 291 ++++++++++++++------------------------- examples/llama/main.zig | 140 +++++++------------ zml/module.zig | 11 +- zml/ops.zig | 200 +++++++++++++++++---------- 4 files changed, 289 insertions(+), 353 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index ceb14ff4..ee1543ce 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -36,30 +36,25 @@ pub const LlamaLM = struct { lm_head: ?zml.nn.Linear, model: Llama, - // Options controlling generation + // Runtime configuration, not part of the model's weights/arch gen_opts: zml.nn.SamplingStrategy = .{}, config: Config, - pub fn init(self: *LlamaLM, mesh: zml.Mesh, config: Config, options: Options) void { + pub fn init(self: *LlamaLM, compute_mesh: zml.Mesh, vocab_mesh: zml.Mesh, config: Config, options: Options) void { self.config = config; self.gen_opts = options.sampling_strategy orelse .{}; - const vocab_mesh = mesh.flatten(.all_devices); + self.model.init(config, options.max_seq_len, compute_mesh, vocab_mesh); - self.model.init(config, options.max_seq_len, mesh, vocab_mesh); - - // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. - // It currently crashes/compilation fails - if (self.gen_opts.topk == 1 and self.lm_head != null) { - self.lm_head.?.weight = self.lm_head.?.weight.withTags(.{ .d, .vocab }).withMesh(vocab_mesh).withSharding(.{ .vocab = .all_devices }); + // Shard the final output layer across all devices + if (self.lm_head) |*lm_head| { + lm_head.weight = lm_head.weight + .withTags(.{ .d, .vocab }) + .withMesh(vocab_mesh) + .withSharding(.{ .vocab = .model }); } } - /// Predicts the token at `token_index` position. - /// Returns: - /// - updated `tokens`, - /// - updated KV cache - /// - a Rng state to allow for probabilistic generation pub fn forward( self: LlamaLM, tokens_: Tensor, @@ -69,7 +64,12 @@ pub const LlamaLM = struct { ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); + + // The model forward pass uses the compute_mesh + zml.pushMesh(self.model.layers[0].input_layernorm.weight.mesh()); const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); + zml.popMesh(); + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } @@ -81,10 +81,17 @@ pub const LlamaLM = struct { rng: Tensor.Rng, opts: zml.nn.SamplingStrategy, ) struct { Tensor, Tensor.Rng } { + // Output from the model body is on the compute_mesh. + // We need to switch to the vocab_mesh for the final projection. + const vocab_mesh = if (lm_head_) |lm_head| lm_head.weight.mesh() else self.model.embed_tokens.weight.mesh(); + zml.pushMesh(vocab_mesh); + const out = out_.withPartialTags(.{ .s, .d }); var logits = blk: { if (lm_head_) |lm_head| { + // The matmul with lm_head (sharded on vocab/model) is what matters. Since lm_head + // is on the vocab_mesh, this operation will execute on it. break :blk zml.call(lm_head, .forward, .{out}); } else { break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); @@ -94,12 +101,12 @@ pub const LlamaLM = struct { if (logits.shape().hasTag(.voc) == null) logits = logits.rename(.{ .d = .voc }); - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); - return .{ next_tokens, new_rng }; - } + // For sampling, we need the full distribution, so we all-gather across the model axis. + logits = zml.ops.allGather(logits, .model, vocab_mesh); - pub fn increment(_: u8, token_index: Tensor) Tensor { - return token_index.addConstant(1).reuseBuffer(token_index); + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); + zml.popMesh(); + return .{ next_tokens, new_rng }; } }; @@ -108,13 +115,12 @@ pub const Llama = struct { norm: RmsNorm, layers: []TransformerLayer, + // No mesh stored here! + max_seq_len: u32 = 0, num_heads: i64 = 32, num_kv_heads: i64 = 32, - rope_opts: zml.nn.RopeOpts = .{ - .layout = .interleaved, - .freq_base = 10_000, - }, + rope_opts: zml.nn.RopeOpts = .{}, const Shape = struct { s: u32, @@ -154,32 +160,26 @@ pub const Llama = struct { self.norm.init(config, compute_mesh); + // Input embeddings are replicated across all devices on the vocab_mesh. self.embed_tokens.weight = self.embed_tokens.weight .withTags(.{ .vocab, .d }) .withMesh(vocab_mesh) - .withSharding(.{ .vocab = .all_devices }); + .withSharding(.{ .vocab = .replicated, .d = .replicated }); for (self.layers) |*layer| { layer.init(config, compute_mesh); } } - /// Forward one token, using KV cache for previous tokens. - /// Returns result and updated KV cache. pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { - log.info(">>>>>>>>>>>>>>> Llama.forward(tokens={}, token_index={}, kv_cache={} vocab_mesh={any})", .{ tokens, token_index, kv_cache, self.embed_tokens.weight._mesh }); - const vocab_mesh = self.embed_tokens.weight._mesh.?; - zml.pushMesh(vocab_mesh); - const embeds = embed(self.embed_tokens, tokens); - zml.popMesh(); + // This function now assumes the correct mesh is already set in the context by its caller. + log.info(">>>>>>>>>>>>>>> Llama.forward(tokens={}, token_index={}, kv_cache={})", .{ tokens, token_index, kv_cache }); - zml.pushMesh(self.layers[0].self_attn.q_proj.weight._mesh.?); + var hidden = embed(self.embed_tokens, tokens); - var hidden = embeds.withSharding(.{ .s = .replicated, .d = .model }); - - if (embeds.dim(.s) > 1) { - hidden = hidden.withSharding(.{ .s = .data, .d = .model }); - } + // Re-shard activations for the compute mesh. + // `embeds` is on vocab_mesh (replicated), re-sharding for compute_mesh (model-parallel) + hidden = hidden.withSharding(.{ .d = .model }); var updated_kv_cache = kv_cache; @@ -188,21 +188,13 @@ pub const Llama = struct { } const output = zml.call(self.norm, .forward, .{hidden}); - zml.popMesh(); - return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; } pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { - return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); - } - - fn initKvCache(self: Llama, embed_shape: zml.Shape) KvCache { - const dims = self.shape(); - var kv_shape = embed_shape.insert(0, .{ .layer = dims.layer }).rename(.{ .s = .k }).splitAxes(.{ .d = .{ .h = dims.nkvh, .hd = dims.hd } }); - const perm = kv_shape.contiguousPerm(.{ .k, .h, .hd }); - kv_shape = kv_shape.transpose(perm.constSlice()); - return KvCache.init(kv_shape); + // Embeddings are replicated to all devices before entering the transformer blocks + const embeddings = zml.call(embed_tokens_, .forward, .{tokens_}); + return embeddings.withPartialTags(.{.d}); } }; @@ -225,94 +217,66 @@ pub const TransformerLayer = struct { token_index: Tensor, kv_cache: KvCache, ) struct { Tensor, KvCache } { - // const x0 = x0_.withTags(.{ .s, .d }).withSharding(.{ .s = .data, .d = .model }); const x0 = x0_; stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); - const mesh = self.input_layernorm.weight.mesh(); - - const x0_replicated = if (mesh.topology.hasTag(.data) != null) blk: { - log.warn("<<<<<<<<<<<<<<<<<<< x0_replicated", .{}); - break :blk zml.ops.allGather(x0, .data, mesh); - } else blk: { - log.warn("<<<<<<<<<<<<<<<<<<< x0", .{}); - break :blk x0; - }; - - // Self Attention - const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0_replicated}); + const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - log.warn("TransformerLayer.forward: delta0={}", .{delta0}); - const x1_replicated = x0_replicated.add(delta0); + const x1 = x0.add(delta0); - const x1 = if (mesh.topology.hasTag(.data) != null) - zml.ops.reduceScatter(x1_replicated, .data, mesh) - else - x1_replicated; - - // Fully Connected const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - const mlp_delta = zml.call(self.mlp, .forward, .{x1_normalized}); - const x2 = x1.add(mlp_delta); + const delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); + const x2 = x1.add(delta1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; } }; -const RmsNorm = struct { +pub const RmsNorm = struct { weight: Tensor, eps: f32 = 1e-5, pub fn init(self: *RmsNorm, config: LlamaLM.Config, mesh: zml.Mesh) void { self.eps = config.rms_norm_eps; - self.weight = self.weight.withTags(.{.d}).withMesh(mesh); - - if (mesh.topology.hasTag(.model) != null) { - self.weight = self.weight.withSharding(.{ .d = .model }); - } + self.weight = self.weight.withTags(.{.d}).withMesh(mesh).withSharding(.{ .d = .replicated }); } - /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: Tensor) Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); log.warn("RmsNorm.forward({})", .{x}); + // RMSNorm can be computed locally on the sharded `d` dimension. const normalized = zml.nn.rmsNorm(x, .d, self.eps); + // `weight` is replicated, so this multiplication is also local. return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); } }; -const Mlp = struct { - up_proj: zml.nn.Linear, // (dim -> hidden_dim) - gate_proj: zml.nn.Linear, // (dim -> hidden_dim) - down_proj: zml.nn.Linear, // (hidden_dim -> dim) +pub const Mlp = struct { + up_proj: zml.nn.Linear, + gate_proj: zml.nn.Linear, + down_proj: zml.nn.Linear, pub fn init(self: *Mlp, mesh: zml.Mesh) void { - const has_model_axis = mesh.topology.hasTag(.model) != null; - - self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .hidden_dim, .model_dim }).withMesh(mesh); - self.up_proj.weight = self.up_proj.weight.withTags(.{ .hidden_dim, .model_dim }).withMesh(mesh); - self.down_proj.weight = self.down_proj.weight.withTags(.{ .model_dim, .hidden_dim }).withMesh(mesh); - - if (has_model_axis) { - self.gate_proj.weight = self.gate_proj.weight.withSharding(.{ .hidden_dim = .model }); - self.up_proj.weight = self.up_proj.weight.withSharding(.{ .hidden_dim = .model }); - self.down_proj.weight = self.down_proj.weight.withSharding(.{ .hidden_dim = .model }); - } + // Shard weights along the `model` axis + self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(.{ .hidden = .model }); + self.up_proj.weight = self.up_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(.{ .hidden = .model }); + self.down_proj.weight = self.down_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(.{ .hidden = .model }); } pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); - output = output.silu().mul(proj); + // Get the current mesh from the context + const mesh = zml.currentMesh(); + _ = mesh; // autofix + + // Column-parallel part (local) + const gate_out = zml.call(self.gate_proj, .forward, .{x}); + const up_out = zml.call(self.up_proj, .forward, .{x}); + var output = gate_out.silu().mul(up_out); + + // Row-parallel part (needs communication) output = zml.call(self.down_proj, .forward, .{output}); - - const mesh = self.down_proj.weight.mesh(); - if (mesh.topology.hasTag(.model) != null) { - log.warn("<<<<<<<<<<<<>>>>>>>>>> Mlp.forward: zml.ops.allReduce", .{}); - output = zml.ops.allReduce(output, .model, mesh); - } - return output; + // return zml.ops.allReduce(output, .model, mesh); } }; @@ -320,7 +284,6 @@ pub const SelfAttn = struct { q_proj: zml.nn.Linear, k_proj: zml.nn.Linear, v_proj: zml.nn.Linear, - o_proj: zml.nn.Linear, num_heads: i64 = undefined, num_kv_heads: i64 = 0, @@ -335,106 +298,69 @@ pub const SelfAttn = struct { .scaling = config.rope_scaling, }; - const has_model_axis = mesh.topology.hasTag(.model) != null; + // Column-parallel layers. Shard the output dimension. + self.q_proj.weight = self.q_proj.weight.withTags(.{ .d, .q_hidden }).withMesh(mesh).withSharding(.{ .q_hidden = .model }); + self.k_proj.weight = self.k_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); + self.v_proj.weight = self.v_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); - self.q_proj.weight = self.q_proj.weight.withTags(.{ .d, .q_hidden }).withMesh(mesh); - self.k_proj.weight = self.k_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh); - self.v_proj.weight = self.v_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh); - self.o_proj.weight = self.o_proj.weight.withTags(.{ .o_hidden, .d }).withMesh(mesh); - - if (has_model_axis) { - self.q_proj.weight = self.q_proj.weight.withSharding(.{ .q_hidden = .model }); - self.k_proj.weight = self.k_proj.weight.withSharding(.{ .kv_hidden = .model }); - self.v_proj.weight = self.v_proj.weight.withSharding(.{ .kv_hidden = .model }); - self.o_proj.weight = self.o_proj.weight.withSharding(.{ .o_hidden = .model }); - } + // Row-parallel layer. Shard the input dimension. + self.o_proj.weight = self.o_proj.weight.withTags(.{ .o_hidden, .d }).withMesh(mesh).withSharding(.{ .o_hidden = .model }); } - /// Self Attention. - /// - If token_index is set, x is assumed to be the representation of one new token, - /// and kv_cache will be read for the previous tokens. - /// - If token_index is not set, x is assumed to be the representation of all tokens - /// since the beginning of the sequence, and kv_cache won't be read. - /// In both case, kv_cache will be updated with the computed key and value. - /// x: {.b, .s, .d } -> .{.b, .s, .d} pub fn forward( self: SelfAttn, x: Tensor, token_index: Tensor, kv_cache: KvCache, ) struct { Tensor, KvCache } { + const mesh = zml.currentMesh(); + _ = mesh; // autofix + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; const q_proj_out = zml.call(self.q_proj, .forward, .{x}); const k_proj_out = zml.call(self.k_proj, .forward, .{x}); const v_proj_out = zml.call(self.v_proj, .forward, .{x}); - log.warn("SelfAttn.forward: q_proj_out={}, k_proj_out={}, v_proj_out={}", .{ q_proj_out, k_proj_out, v_proj_out }); - var q = q_proj_out.splitAxis(.d, .{ .h = self.num_heads, .hd = .auto }); - var k = k_proj_out.splitAxis(.d, .{ .h = num_kv_heads, .hd = .auto }); - var v = v_proj_out.splitAxis(.d, .{ .h = num_kv_heads, .hd = .auto }); - - // Generate the attention mask. - const seq_len = kv_cache.k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), null); - attn_mask = attn_mask.gatherSlices(.{ .q = x.dim(.s) }, token_index.reshape(.{ .coord = 1 }), .{}); + var q = q_proj_out.splitAxis(.d, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{ .h = .model }); + var k = k_proj_out.splitAxis(.d, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); + var v = v_proj_out.splitAxis(.d, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); const pos_index = blk: { const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); break :blk temp.add(token_index.broad(temp.shape())); }; - q = zml.nn.rope(q, pos_index, self.rope_opts); k = zml.nn.rope(k, pos_index, self.rope_opts); + q = q.rename(.{ .s = .q }); k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); - const dtype = q.dtype(); const new_kv_cache = kv_cache.update(k, v, token_index); - var k_cached = new_kv_cache.keys().convert(dtype); - var v_cached = new_kv_cache.values().convert(dtype); - if (self.num_kv_heads > 0 and self.num_heads > self.num_kv_heads) { - const num_groups = @divExact(self.num_heads, self.num_kv_heads); - k_cached = k_cached.repeat1d(.h, @intCast(num_groups)); - v_cached = v_cached.repeat1d(.h, @intCast(num_groups)); - } + const k_cached = new_kv_cache.keys(); + const v_cached = new_kv_cache.values(); - const attn_output = zml.nn.sdpa(q, k_cached, v_cached, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + const k_transposed = k_cached.transpose(.{ .h, .k, .hd }); + const v_transposed = v_cached.transpose(.{ .h, .k, .hd }); - const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + const attn_mask = if (x.dim(.s) > 1) blk: { + var mask = zml.nn.causalAttnMask(.{ .q = k_transposed.dim(.k), .k = k_transposed.dim(.k) }, x.dtype(), null); + // The mask is replicated and broadcasted to all devices. + break :blk mask.gatherSlices(.{ .q = x.dim(.s) }, token_index.reshape(.{ .coord = 1 }), .{}); + } else null; - var output = zml.call(self.o_proj, .forward, .{attn}).rename(.{ .o_hidden = .d }); + const attn_output = zml.nn.sdpa(q, k_transposed, v_transposed, .{ .attn_mask = attn_mask, .allow_cudnn = true }); - const mesh = self.o_proj.weight.mesh(); - if (mesh.topology.hasTag(.model) != null) { - log.warn("<<<<<<<<<<<<>>>>>>>>>> SelfAttn.forward: zml.ops.allReduce", .{}); + const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); - output = zml.ops.allReduce(output, .model, mesh); - } + const output = zml.call(self.o_proj, .forward, .{attn}); - log.warn("SelfAttn.forward: attn_output={}, output={}", .{ attn_output, output }); - if (x.dim(.s) > 1) { - // Prefill phase: Shard the sequence dimension. - output = output.withSharding(.{ .s = .data, .d = .model }); - } else { - // Generation phase (s=1): Replicate the sequence dimension. - output = output.withSharding(.{ .s = .replicated, .d = .model }); - } + // output = zml.ops.allReduce(output, .model, mesh); return .{ output, new_kv_cache }; } - - fn initKvCache(key_shape: zml.Shape) KvCache { - // When we call initKvCache, we haven't renamed .s to .k yet. - var kv_shape = key_shape.insert(0, .{ .layer = 1 }).rename(.{ .s = .k }); - const perm = kv_shape.contiguousPerm(.{ .h, .k, .hd }); - kv_shape = kv_shape.transpose(perm.constSlice()); - var res = KvCache.init(kv_shape); - res.layer_index = Tensor.scalar(0, .u32); - return res; - } }; pub const KvCache = struct { @@ -442,21 +368,9 @@ pub const KvCache = struct { v: Tensor, layer_index: Tensor, - pub fn init(kv_shape: zml.Shape, mesh: zml.Mesh) KvCache { - const sharded_shape = kv_shape.withPartitioning(.{ .k = .data, .h = .model }); - - return .{ - .k = Tensor.constant(sharded_shape, kv_shape.dtype().one()).withMesh(mesh), - .v = Tensor.constant(sharded_shape, kv_shape.dtype().one()).withMesh(mesh), - .layer_index = Tensor.scalar(-1, .u32), - }; - } - - pub fn initShape( - kv_shape: zml.Shape, - ) ShapeOf(KvCache) { - const sharded_shape = kv_shape.withPartitioning(.{ .k = .data, .h = .model }); - + pub fn initShape(kv_shape: zml.Shape) ShapeOf(KvCache) { + // Shard along the head dimension, replicate sequence dimension + const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); return .{ .k = sharded_shape, .v = sharded_shape, @@ -465,7 +379,7 @@ pub const KvCache = struct { } pub fn initBuffer(kv_shape: zml.Shape, mesh: zml.Mesh, platform: zml.Platform) !zml.Bufferized(KvCache) { - const sharding = zml.Sharding.init(mesh, kv_shape.withPartitioning(.{ .k = .data, .h = .model })); + const sharding = zml.Sharding.init(mesh, kv_shape.withPartitioning(.{ .h = .model })); return .{ .k = try zml.Buffer.uninitialized(platform, sharding, .{}), @@ -474,6 +388,7 @@ pub const KvCache = struct { }; } + //... rest of KvCache is unchanged pub fn keys(self: KvCache) Tensor { return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); } @@ -483,30 +398,32 @@ pub const KvCache = struct { } pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + _ = k_shape; // autofix var layer = self.layer_index; layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; return if (token_index) |idx| .{ .k = self.k.scatterSlices( .{ .layer = layer, .k = idx }, - new_k.convert(self.k.dtype()), + new_k.convert(self.k.dtype()), // .transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.k), .v = self.v.scatterSlices( .{ .layer = layer, .k = idx }, - new_v.convert(self.v.dtype()), + new_v.convert(self.v.dtype()), // .transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.v), .layer_index = self.layer_index, } else .{ .k = self.k.scatterSlices( .{ .layer = layer }, - new_k.convert(self.k.dtype()), + new_k.convert(self.k.dtype()), // .transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.k), .v = self.v.scatterSlices( .{ .layer = layer }, - new_v.convert(self.v.dtype()), + new_v.convert(self.v.dtype()), // .transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.v), .layer_index = self.layer_index, diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 8e452d19..efbb72fe 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -227,47 +227,22 @@ pub fn asyncMain() !void { // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' const create_opts_json = res.args.@"create-options" orelse "{}"; const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); - const platform = context.autoPlatform(.{ .cpu = .{ .cpu_device_count = 4 } }).withCompilationOptions(compilation_options); + const platform = context.autoPlatform(.{ .cpu = .{ .cpu_device_count = 8 } }).withCompilationOptions(compilation_options); create_opts.deinit(); context.printAvailablePlatforms(platform); log.info("\n\n\n\n", .{}); const num_devices = platform.getDevices().len; - const num_heads = config.num_attention_heads; - const num_kv_heads = config.num_key_value_heads; - var tp_degree: usize = 1; - const search_limit = if (num_devices < num_heads) num_devices else num_heads; - - var i: usize = search_limit; - while (i > 1) : (i -= 1) { - if ((num_devices % i == 0) and (num_heads % i == 0) and (num_kv_heads % i == 0)) { - tp_degree = i; - break; - } - } - - const dp_degree = @divExact(num_devices, tp_degree); - - const mesh = if (num_devices == 1) - zml.Mesh.single() - else if (tp_degree == 1) blk: { - log.info("Strategy: Pure Data Parallelism", .{}); - break :blk zml.Mesh.init(.{ .data = dp_degree }); - } else if (dp_degree == 1) blk: { - log.info("Strategy: Pure Tensor Parallelism", .{}); - break :blk zml.Mesh.init(.{ .model = tp_degree }); - } else blk: { - log.info("Strategy: Hybrid Data + Tensor Parallelism", .{}); - break :blk zml.Mesh.init(.{ .data = dp_degree, .model = tp_degree }); - }; - - log.info( - "Model heads: {d}, KV heads: {d}, Devices: {d} => Using {d}-way Data Parallelism and {d}-way Tensor Parallelism", - .{ num_heads, num_kv_heads, num_devices, dp_degree, tp_degree }, - ); - log.info("Constructed mesh: {}", .{mesh}); + stdx.debug.assert(config.num_attention_heads % num_devices == 0 and (config.num_key_value_heads % num_devices == 0 or config.num_key_value_heads == 0), "Number of heads ({d}) and KV heads ({d}) must be divisible by the number of devices ({d}) for this sharding strategy.", .{ config.num_attention_heads, config.num_key_value_heads, num_devices }); + log.info("Using 1D Tensor Parallelism across {d} devices", .{num_devices}); + const compute_mesh = zml.Mesh.init(.{ .model = num_devices }); + // For pure 1D TP, the vocab mesh is the same as the compute mesh. + // It's sharded on 'model' (which is equivalent to 'all_devices' in this case). + const vocab_mesh = compute_mesh; + log.info("Constructed compute mesh: {}", .{compute_mesh}); + log.info("Constructed vocab mesh: {}", .{vocab_mesh}); var ts = try zml.aio.detectFormatAndOpen(allocator, res.args.weights.?); defer ts.deinit(); @@ -282,71 +257,20 @@ pub fn asyncMain() !void { .temperature = 1.0, }, }; - model_instance.init(mesh, config, llama_options); + model_instance.init(compute_mesh, vocab_mesh, config, llama_options); const dims = model_instance.model.shape(); const dtype = model_instance.model.embed_tokens.weight.dtype(); - const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32).withPartitioning(.{ .s = .data }); - log.info("tokens_shape_prefill: {}", .{tokens_shape_prefill}); - - const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32).withPartitioning(.{ .s = .replicated }); - log.info("tokens_shape: {}", .{tokens_shape}); - + const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32).withPartitioning(.{ .s = .replicated }); + const tokens_shape_decode = zml.Shape.init(.{ .s = 1 }, .u32).withPartitioning(.{ .s = .replicated }); const token_idx_shape = zml.Shape.init(.{}, .u32); - const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype); - log.info("kv_shape: {}", .{kv_shape}); - - const kv_cache_shape: zml.ShapeOf(llama.KvCache) = llama.KvCache.initShape(kv_shape); - log.info("kv_cache_shape: {}", .{kv_cache_shape}); - + const kv_cache_shape = llama.KvCache.initShape(kv_shape); const rng_shape = zml.Tensor.Rng.shape(); - log.info("rng_shape: {}", .{rng_shape}); log.info("\n\n\n\n", .{}); - var start = try std.time.Timer.start(); - var fut_mod_prefill = try asynk.asyncc(zml.compile, .{ - allocator, llama.LlamaLM.forward, .{ mesh, config, llama_options }, - .{ - tokens_shape_prefill, - token_idx_shape, - kv_cache_shape, - rng_shape, - }, - ts, platform, - mesh, - }); - const mod_prefill = try fut_mod_prefill.awaitt(); - log.warn(">>>>>>>>>>>>>>> conmpiled mod_prefill", .{}); - - var fut_mod = try asynk.asyncc(zml.compile, .{ - allocator, llama.LlamaLM.forward, .{ mesh, config, llama_options }, - .{ - tokens_shape, - token_idx_shape, - kv_cache_shape, - rng_shape, - }, - ts, platform, - mesh, - }); - - log.info("\tLoading Llama weights from {?s}...", .{res.args.weights}); - var llama_weights = try zml.aio.loadBuffers(llama.LlamaLM, .{ mesh, config, llama_options }, ts, model_arena.allocator(), platform, mesh); - defer zml.aio.unloadBuffers(&llama_weights); - log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); - - var llama_module_prefill = mod_prefill.prepare(llama_weights); - defer llama_module_prefill.deinit(); - var llama_module = (try fut_mod.awaitt()).prepare(llama_weights); - defer llama_module.deinit(); - log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); - - log.info("Creating KvCache", .{}); - const kv_cache = try llama.KvCache.initBuffer(kv_shape, mesh, platform); - var tokenizer = blk: { if (res.args.tokenizer) |tok| { log.info("Loading tokenizer from {s}", .{tok}); @@ -364,9 +288,41 @@ pub fn asyncMain() !void { const prompt = res.args.prompt orelse "What is the capital of France?"; log.info("✅\tPrompt: {s}", .{prompt}); + var start = try std.time.Timer.start(); + var fut_mod_prefill = try asynk.asyncc(zml.compileModel, .{ allocator, llama.LlamaLM.forward, model_instance, .{ + tokens_shape_prefill, + token_idx_shape, + kv_cache_shape, + rng_shape, + }, compute_mesh, platform }); + var fut_mod_decode = try asynk.asyncc(zml.compileModel, .{ allocator, llama.LlamaLM.forward, model_instance, .{ + tokens_shape_decode, + token_idx_shape, + kv_cache_shape, + rng_shape, + }, compute_mesh, platform }); + + log.info("\tLoading Llama weights from {?s}...", .{res.args.weights}); + var llama_weights = try zml.aio.loadModelBuffers(LlamaLM, model_instance, ts, model_arena.allocator(), platform, compute_mesh); + defer zml.aio.unloadBuffers(&llama_weights); + log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); + + const mod_prefill = try fut_mod_prefill.awaitt(); + var llama_module_prefill = mod_prefill.prepare(llama_weights); + defer llama_module_prefill.deinit(); + + const mod_decode = try fut_mod_decode.awaitt(); + var llama_module_decode = mod_decode.prepare(llama_weights); + defer llama_module_decode.deinit(); + log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); + + log.info("Creating KvCache", .{}); + const kv_cache = try llama.KvCache.initBuffer(kv_shape, compute_mesh, platform); + + log.info("✅\tPrompt: {s}", .{prompt}); + const seed = res.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); - const skip_llama3_encoding = res.args.@"no-llama3" orelse false; - const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, mesh, seed, prompt[0..], skip_llama3_encoding); - // generated text will be printed token by token. + const skip_llama3_encoding: bool = res.args.@"no-llama3" orelse false; + const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module_decode, kv_cache, tokenizer, allocator, compute_mesh, seed, prompt, skip_llama3_encoding); defer allocator.free(generated_text); } diff --git a/zml/module.zig b/zml/module.zig index e3b093e2..db5dca33 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -94,6 +94,7 @@ pub const CompilationContext = struct { _block_args: TensorToBlockArg = .{}, _unique_id: u64 = 10000, _tracer: Tracer, + _next_channel_id: u64 = 1, _previous: ?*CompilationContext = null, threadlocal var _current: ?*CompilationContext = null; @@ -261,8 +262,8 @@ pub const CompilationContext = struct { break :blk loaded_executable; }; - log.debug("******** ZML generated MLIR ********", .{}); - log.debug("{}", .{module.op().mlirFormatter(.{})}); + log.warn("******** ZML generated MLIR ********", .{}); + log.warn("{}", .{module.op().mlirFormatter(.{})}); if (timer) |*t| { const time_ms = @divFloor(t.lap(), std.time.ns_per_ms); @@ -282,6 +283,12 @@ pub const CompilationContext = struct { ); } + pub fn getNextChannelId(self: *CompilationContext) u64 { + const id = self._next_channel_id; + self._next_channel_id += 1; + return id; + } + fn currentBlock(self: *const CompilationContext) ?TaggedBlock { return if (self._blocks.len > 0) self._blocks.get(self._blocks.len - 1) else null; } diff --git a/zml/ops.zig b/zml/ops.zig index d4ed9a47..b71dd347 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -147,7 +147,7 @@ pub fn reduce( .blocks = &.{body_block}, .attributes = &.{.{ "dimensions", .dense(ctx.mlirCtx(), .i64, axes) }}, // We can't verify right away, cause the weights captured by the reduce haven't been added yet. - .verify = false, + .verify = true, .location = loc, }); @@ -616,102 +616,158 @@ pub fn sort( return res; } -fn collectiveOp( - name: [:0]const u8, - input: Tensor, - axis: i64, - mesh: Mesh, -) Tensor { - _ = mesh; // autofix - const ctx = CompilationContext.current(); - const mlir_ctx = ctx.mlirCtx(); +var next_available_channel_id: u64 = 1; - const replica_groups = mlir.Attribute.denseElements( - mlir_ctx, - &.{ @intCast(axis), 1 }, - .i64, - &[_]i64{0}, - ); - - const op = mlir.Operation.make(ctx.mlirCtx(), name, .{ - .operands = &.{input.value()}, - .result_type_inference = true, - .attributes = &.{ - .{ "replica_groups", replica_groups }, - .{ "channel_id", .int(ctx.mlirCtx(), .i64, 0) }, - }, - .location = ctx.location(@src(), "{s}({_}, {d})", .{ name, input, axis }), - }); - - return fromMlirOperationWithTags(op, input); +// Helper to get a unique ID. @atomicRmw ensures that even if two compilations +// in parallel threads call this, they will get different, unique IDs. +fn getNextChannelId() u64 { + return @atomicRmw(u64, &next_available_channel_id, .Add, 1, .seq_cst); } -pub fn allGather(input: Tensor, axis: anytype, mesh: Mesh) Tensor { - var res_shape = input.shape(); +// Helper function to create replica groups for single-host SPMD +fn getSpmdReplicaGroups(ctx: mlir.Context, mesh: Mesh) mlir.Attribute { + const allocator = CompilationContext.current().allocator(); + var replica_group_data = std.ArrayList(i64).init(allocator); + errdefer replica_group_data.deinit(); - const axis_size = mesh.topology.dim(axis); - const mesh_axis_idx = mesh.topology.hasTag(axis).?; - for (0..res_shape.rank()) |i| { - const part_spec = res_shape.partition(i); - if (part_spec == .axis and part_spec.toTag() == mesh.topology.tag(mesh_axis_idx)) { - const current_dim = res_shape.dim(i); - res_shape = res_shape.setDim(i, current_dim * axis_size); - } + const num_partitions = mesh.numPartitions(); + replica_group_data.ensureTotalCapacity(@intCast(num_partitions)) catch @panic("OOM"); + for (0..num_partitions) |i| { + replica_group_data.appendAssumeCapacity(@intCast(i)); } - var res = collectiveOp("stablehlo.all_gather", input, mesh.topology.axis(axis), mesh); - res._shape._partitioning = input.shape()._partitioning; - return res; -} - -pub fn reduceScatter(input: Tensor, axis: anytype, mesh: Mesh) Tensor { - _ = mesh; // autofix - // For now, only sum is supported. - const body_block, _ = CompilationContext.current().makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ Tensor.scalar(0, input.dtype()), Tensor.scalar(0, input.dtype()) }); - const op = mlir.Operation.make(input.getContext().mlirCtx(), "stablehlo.reduce_scatter", .{ - .operands = &.{input.value()}, - .result_type_inference = true, - .blocks = &.{body_block}, - .attributes = &.{ - .{ "scatter_dimension", .int(input.getContext().mlirCtx(), .i64, input.shape().axis(axis)) }, - }, - .location = input.getContext().location(@src(), "reduceScatter({_}, {d})", .{ input, input.shape().axis(axis) }), - }); - return fromMlirOperationWithTags(op, input); + return mlir.Attribute.denseElements( + ctx, + &.{ 1, @intCast(num_partitions) }, + .i64, + replica_group_data.items, + ); } pub fn allReduce(input: Tensor, axis: anytype, mesh: Mesh) Tensor { + if (mesh.axis(axis) <= 1) return input; + const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlirCtx(); + const channel_id = getNextChannelId(); + log.warn(">>>>>>>> channel id: {d}", .{channel_id}); - // The body of the reduce operation (summation) const body_block, _ = ctx.makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ Tensor.scalar(0, input.dtype()), Tensor.scalar(0, input.dtype()), }); - // replica_groups defines which devices participate. For an All-Reduce across - // the 'model' axis, it's a 2D array of shape [num_model_devices, 1]. - const replica_groups_shape: []const i64 = &.{ @intCast(mesh.axis(axis)), 1 }; - var replica_groups_data: [Shape.MAX_RANK]i64 = undefined; - for (0..mesh.numPartitions()) |i| { - replica_groups_data[i] = @intCast(i); - } - const op = mlir.Operation.make(mlir_ctx, "stablehlo.all_reduce", .{ .operands = &.{input.value()}, .results = &.{input.value().getType()}, .blocks = &.{body_block}, .attributes = &.{ - .{ "replica_groups", .denseElements(mlir_ctx, replica_groups_shape, .i64, replica_groups_data[0..mesh.numPartitions()]) }, - .{ "channel_id", .int(mlir_ctx, .i64, 0) }, // Use a default channel - .{ "use_global_device_ids", .boolean(mlir_ctx, false) }, + .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, + .{ "channel_id", .int(mlir_ctx, .i64, @intCast(channel_id)) }, + .{ "use_global_device_ids", .unit(mlir_ctx) }, }, - .location = ctx.location(@src(), "allReduce({_}, {d})", .{ input, mesh.axis(axis) }), + .location = ctx.location(@src(), "allReduce({_}, {any})", .{ input, axis }), + .verify = false, }); - // The shape of the output of all_reduce is the same as the input. - return fromMlirOperationWithTags(op, input); + var res = fromMlirOperationWithTags(op, input); + res._shape = res._shape.withReplicatedPartitioning(); + return res; +} + +pub fn allGather(input: Tensor, axis: anytype, mesh: Mesh) Tensor { + if (mesh.axis(axis) <= 1) return input; + + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const channel_id = getNextChannelId(); + log.warn(">>>>>>>> channel id: {d}", .{channel_id}); + const mesh_axis_tag = mesh.topology.tag(axis); + + var res_shape = input.shape(); + const axis_size = mesh.axis(axis); + + var gather_dim: ?u3 = null; + for (0..res_shape.rank()) |i| { + const part_spec = res_shape.partition(i); + if (part_spec == .axis and std.mem.eql(u8, std.mem.span(part_spec.toTag()), std.mem.span(mesh_axis_tag))) { + stdx.debug.assert(gather_dim == null, "Mesh axis '{s}' is used to partition multiple tensor dimensions.", .{mesh_axis_tag}); + gather_dim = @intCast(i); + } + } + + const gather_dim_unwrapped = gather_dim orelse { + return input; + }; + + const old_dim_size = res_shape.dim(gather_dim_unwrapped); + res_shape = res_shape.set(gather_dim_unwrapped, old_dim_size * axis_size); + res_shape = res_shape.withReplicatedPartitioning(); + + const op = mlir.Operation.make(mlir_ctx, "stablehlo.all_gather", .{ + .operands = &.{input.value()}, + .results = &.{mlirx.tensorType(mlir_ctx, res_shape)}, + .attributes = &.{ + .{ "all_gather_dim", .int(mlir_ctx, .i64, gather_dim_unwrapped) }, + .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, + .{ "channel_id", .int(mlir_ctx, .i64, @intCast(channel_id)) }, + .{ "use_global_device_ids", .unit(mlir_ctx) }, + }, + .location = ctx.location(@src(), "all_gather({_}, {any})", .{ input, axis }), + .verify = false, + }); + + return Tensor._result(res_shape, op.result(0)); +} + +pub fn reduceScatter(input: Tensor, axis: anytype, mesh: Mesh) Tensor { + if (mesh.axis(axis) <= 1) return input; + + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const channel_id = getNextChannelId(); + log.warn(">>>>>>>> channel id: {d}", .{channel_id}); + const mesh_axis_tag = mesh.topology.tag(axis); + + var res_shape = input.shape(); + const axis_size = mesh.axis(axis); + + var scatter_dim: ?u3 = null; + for (0..res_shape.rank()) |i| { + if (input.shape().partition(i) == .replicated) { + if (@rem(res_shape.dim(i), axis_size) == 0) { + scatter_dim = @intCast(i); + break; + } + } + } + const scatter_dim_unwrapped = scatter_dim orelse stdx.debug.panic("Could not find a suitable replicated dimension to scatter on for tensor {}", .{input.shape()}); + + const old_dim_size = res_shape.dim(scatter_dim_unwrapped); + stdx.debug.assert(@rem(old_dim_size, axis_size) == 0, "Reduce-scatter dimension size {} must be divisible by mesh axis size {}", .{ old_dim_size, axis_size }); + + res_shape = res_shape.set(scatter_dim_unwrapped, @divExact(old_dim_size, axis_size)); + + const new_partition_spec = Shape.PartitionSpec.init(mesh_axis_tag); + res_shape._partitioning.set(scatter_dim_unwrapped, new_partition_spec); + + const body_block, _ = ctx.makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ Tensor.scalar(0, input.dtype()), Tensor.scalar(0, input.dtype()) }); + + const op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_scatter", .{ + .operands = &.{input.value()}, + .results = &.{mlirx.tensorType(mlir_ctx, res_shape)}, + .blocks = &.{body_block}, + .attributes = &.{ + .{ "scatter_dimension", .int(mlir_ctx, .i64, scatter_dim_unwrapped) }, + .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, + .{ "channel_id", .int(mlir_ctx, .i64, @intCast(channel_id)) }, + .{ "use_global_device_ids", .unit(mlir_ctx) }, + }, + .location = ctx.location(@src(), "reduceScatter({_}, {any})", .{ input, axis }), + .verify = false, + }); + + return Tensor._result(res_shape, op.result(0)); } pub const BlockSignature = struct { From 229a1c31596649cdf6244a777627d2f130c1d605 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 17:21:04 +0200 Subject: [PATCH 11/51] attempt --- examples/llama/llama.zig | 48 +++++++--------------------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index ee1543ce..825681ea 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -1,3 +1,4 @@ +// examples/llama/llama.zig const std = @import("std"); const testing = std.testing; @@ -81,17 +82,14 @@ pub const LlamaLM = struct { rng: Tensor.Rng, opts: zml.nn.SamplingStrategy, ) struct { Tensor, Tensor.Rng } { - // Output from the model body is on the compute_mesh. - // We need to switch to the vocab_mesh for the final projection. const vocab_mesh = if (lm_head_) |lm_head| lm_head.weight.mesh() else self.model.embed_tokens.weight.mesh(); zml.pushMesh(vocab_mesh); + defer zml.popMesh(); const out = out_.withPartialTags(.{ .s, .d }); var logits = blk: { if (lm_head_) |lm_head| { - // The matmul with lm_head (sharded on vocab/model) is what matters. Since lm_head - // is on the vocab_mesh, this operation will execute on it. break :blk zml.call(lm_head, .forward, .{out}); } else { break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); @@ -105,7 +103,6 @@ pub const LlamaLM = struct { logits = zml.ops.allGather(logits, .model, vocab_mesh); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); - zml.popMesh(); return .{ next_tokens, new_rng }; } }; @@ -115,8 +112,6 @@ pub const Llama = struct { norm: RmsNorm, layers: []TransformerLayer, - // No mesh stored here! - max_seq_len: u32 = 0, num_heads: i64 = 32, num_kv_heads: i64 = 32, @@ -160,7 +155,8 @@ pub const Llama = struct { self.norm.init(config, compute_mesh); - // Input embeddings are replicated across all devices on the vocab_mesh. + // Input embeddings are replicated across all devices. This is crucial for the first + // layer to receive a replicated activation. self.embed_tokens.weight = self.embed_tokens.weight .withTags(.{ .vocab, .d }) .withMesh(vocab_mesh) @@ -172,15 +168,8 @@ pub const Llama = struct { } pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { - // This function now assumes the correct mesh is already set in the context by its caller. - log.info(">>>>>>>>>>>>>>> Llama.forward(tokens={}, token_index={}, kv_cache={})", .{ tokens, token_index, kv_cache }); - var hidden = embed(self.embed_tokens, tokens); - // Re-shard activations for the compute mesh. - // `embeds` is on vocab_mesh (replicated), re-sharding for compute_mesh (model-parallel) - hidden = hidden.withSharding(.{ .d = .model }); - var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { @@ -192,7 +181,6 @@ pub const Llama = struct { } pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { - // Embeddings are replicated to all devices before entering the transformer blocks const embeddings = zml.call(embed_tokens_, .forward, .{tokens_}); return embeddings.withPartialTags(.{.d}); } @@ -243,10 +231,7 @@ pub const RmsNorm = struct { pub fn forward(self: RmsNorm, input: Tensor) Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); - log.warn("RmsNorm.forward({})", .{x}); - // RMSNorm can be computed locally on the sharded `d` dimension. const normalized = zml.nn.rmsNorm(x, .d, self.eps); - // `weight` is replicated, so this multiplication is also local. return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); } }; @@ -257,26 +242,16 @@ pub const Mlp = struct { down_proj: zml.nn.Linear, pub fn init(self: *Mlp, mesh: zml.Mesh) void { - // Shard weights along the `model` axis self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(.{ .hidden = .model }); self.up_proj.weight = self.up_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(.{ .hidden = .model }); self.down_proj.weight = self.down_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(.{ .hidden = .model }); } pub fn forward(self: Mlp, x: Tensor) Tensor { - // Get the current mesh from the context - const mesh = zml.currentMesh(); - _ = mesh; // autofix - - // Column-parallel part (local) const gate_out = zml.call(self.gate_proj, .forward, .{x}); const up_out = zml.call(self.up_proj, .forward, .{x}); - var output = gate_out.silu().mul(up_out); - - // Row-parallel part (needs communication) - output = zml.call(self.down_proj, .forward, .{output}); - return output; - // return zml.ops.allReduce(output, .model, mesh); + const output = gate_out.silu().mul(up_out); + return zml.call(self.down_proj, .forward, .{output}); } }; @@ -298,12 +273,9 @@ pub const SelfAttn = struct { .scaling = config.rope_scaling, }; - // Column-parallel layers. Shard the output dimension. self.q_proj.weight = self.q_proj.weight.withTags(.{ .d, .q_hidden }).withMesh(mesh).withSharding(.{ .q_hidden = .model }); self.k_proj.weight = self.k_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); self.v_proj.weight = self.v_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); - - // Row-parallel layer. Shard the input dimension. self.o_proj.weight = self.o_proj.weight.withTags(.{ .o_hidden, .d }).withMesh(mesh).withSharding(.{ .o_hidden = .model }); } @@ -313,9 +285,6 @@ pub const SelfAttn = struct { token_index: Tensor, kv_cache: KvCache, ) struct { Tensor, KvCache } { - const mesh = zml.currentMesh(); - _ = mesh; // autofix - const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; const q_proj_out = zml.call(self.q_proj, .forward, .{x}); @@ -347,7 +316,6 @@ pub const SelfAttn = struct { const attn_mask = if (x.dim(.s) > 1) blk: { var mask = zml.nn.causalAttnMask(.{ .q = k_transposed.dim(.k), .k = k_transposed.dim(.k) }, x.dtype(), null); - // The mask is replicated and broadcasted to all devices. break :blk mask.gatherSlices(.{ .q = x.dim(.s) }, token_index.reshape(.{ .coord = 1 }), .{}); } else null; @@ -355,10 +323,10 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); + // Row-parallel matmul. Output is a partial sum. const output = zml.call(self.o_proj, .forward, .{attn}); - // output = zml.ops.allReduce(output, .model, mesh); - + // The All-Reduce is now implicit, handled by the compiler. return .{ output, new_kv_cache }; } }; From 044d7c47029bcc8aea56a9b482659c5d40031d9c Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 17:27:52 +0200 Subject: [PATCH 12/51] another version --- examples/llama/llama.zig | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 825681ea..50e8157e 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -66,7 +66,6 @@ pub const LlamaLM = struct { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); - // The model forward pass uses the compute_mesh zml.pushMesh(self.model.layers[0].input_layernorm.weight.mesh()); const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); zml.popMesh(); @@ -99,7 +98,6 @@ pub const LlamaLM = struct { if (logits.shape().hasTag(.voc) == null) logits = logits.rename(.{ .d = .voc }); - // For sampling, we need the full distribution, so we all-gather across the model axis. logits = zml.ops.allGather(logits, .model, vocab_mesh); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); @@ -130,7 +128,6 @@ pub const Llama = struct { const d_model = self.embed_tokens.weight.dim(1); const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; const num_q_heads = self.num_heads; - const head_dim = @divExact(d_model, num_q_heads); return .{ @@ -155,8 +152,6 @@ pub const Llama = struct { self.norm.init(config, compute_mesh); - // Input embeddings are replicated across all devices. This is crucial for the first - // layer to receive a replicated activation. self.embed_tokens.weight = self.embed_tokens.weight .withTags(.{ .vocab, .d }) .withMesh(vocab_mesh) @@ -169,7 +164,6 @@ pub const Llama = struct { pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { var hidden = embed(self.embed_tokens, tokens); - var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { @@ -248,9 +242,13 @@ pub const Mlp = struct { } pub fn forward(self: Mlp, x: Tensor) Tensor { + // x is replicated. const gate_out = zml.call(self.gate_proj, .forward, .{x}); const up_out = zml.call(self.up_proj, .forward, .{x}); - const output = gate_out.silu().mul(up_out); + // gate_out and up_out are sharded. + const output = gate_out.silu().mul(up_out); // output is sharded. + // down_proj takes a sharded input and produces a partial result. + // The implicit all_reduce happens because of the residual connection in the parent layer. return zml.call(self.down_proj, .forward, .{output}); } }; @@ -323,10 +321,10 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); - // Row-parallel matmul. Output is a partial sum. + // o_proj takes a sharded input and produces a partial result. + // The implicit all_reduce happens because of the residual connection in the parent layer. const output = zml.call(self.o_proj, .forward, .{attn}); - // The All-Reduce is now implicit, handled by the compiler. return .{ output, new_kv_cache }; } }; @@ -337,7 +335,6 @@ pub const KvCache = struct { layer_index: Tensor, pub fn initShape(kv_shape: zml.Shape) ShapeOf(KvCache) { - // Shard along the head dimension, replicate sequence dimension const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); return .{ .k = sharded_shape, @@ -356,7 +353,6 @@ pub const KvCache = struct { }; } - //... rest of KvCache is unchanged pub fn keys(self: KvCache) Tensor { return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); } @@ -374,24 +370,24 @@ pub const KvCache = struct { return if (token_index) |idx| .{ .k = self.k.scatterSlices( .{ .layer = layer, .k = idx }, - new_k.convert(self.k.dtype()), // .transpose(k_shape), + new_k.convert(self.k.dtype()), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.k), .v = self.v.scatterSlices( .{ .layer = layer, .k = idx }, - new_v.convert(self.v.dtype()), // .transpose(k_shape), + new_v.convert(self.v.dtype()), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.v), .layer_index = self.layer_index, } else .{ .k = self.k.scatterSlices( .{ .layer = layer }, - new_k.convert(self.k.dtype()), // .transpose(k_shape), + new_k.convert(self.k.dtype()), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.k), .v = self.v.scatterSlices( .{ .layer = layer }, - new_v.convert(self.v.dtype()), // .transpose(k_shape), + new_v.convert(self.v.dtype()), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.v), .layer_index = self.layer_index, From 966da15cc2303e54d307ff467a58ff87dc81bbd2 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 17:48:27 +0200 Subject: [PATCH 13/51] correct mlp sharding --- examples/llama/llama.zig | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 50e8157e..0bff7fce 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -236,9 +236,23 @@ pub const Mlp = struct { down_proj: zml.nn.Linear, pub fn init(self: *Mlp, mesh: zml.Mesh) void { - self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(.{ .hidden = .model }); - self.up_proj.weight = self.up_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(.{ .hidden = .model }); - self.down_proj.weight = self.down_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(.{ .hidden = .model }); + self.gate_proj.weight = self.gate_proj.weight + .withTags(.{ .hidden, .d }) // Shape is (hidden_dim, dim) + .withMesh(mesh) + .withSharding(.{ .hidden = .model }); // Shard along hidden_dim + + self.up_proj.weight = self.up_proj.weight + .withTags(.{ .hidden, .d }) // Shape is (hidden_dim, dim) + .withMesh(mesh) + .withSharding(.{ .hidden = .model }); // Shard along hidden_dim + + // Row-parallel layer: Shard the *input* dimension. + // The matmul is `x @ W.T`, so the input dimension of the operation + // corresponds to the first dimension of the weight matrix W. + self.down_proj.weight = self.down_proj.weight + .withTags(.{ .d, .hidden }) // Shape is (dim, hidden_dim) + .withMesh(mesh) + .withSharding(.{ .hidden = .model }); // Shard along hidden_dim } pub fn forward(self: Mlp, x: Tensor) Tensor { From f027119199f0b5781eac91d6671668ad4e1e5579 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 18:41:07 +0200 Subject: [PATCH 14/51] transpose k --- examples/llama/llama.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 0bff7fce..30456d8d 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -323,7 +323,7 @@ pub const SelfAttn = struct { const k_cached = new_kv_cache.keys(); const v_cached = new_kv_cache.values(); - const k_transposed = k_cached.transpose(.{ .h, .k, .hd }); + const k_transposed = k_cached.transpose(.{ .h, .hd, .k }); const v_transposed = v_cached.transpose(.{ .h, .k, .hd }); const attn_mask = if (x.dim(.s) > 1) blk: { From 354c8cfe0adbba02e406cd8b91da8d8716abe91f Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 18:44:10 +0200 Subject: [PATCH 15/51] simpler --- examples/llama/llama.zig | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 30456d8d..93e4e04f 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -323,15 +323,15 @@ pub const SelfAttn = struct { const k_cached = new_kv_cache.keys(); const v_cached = new_kv_cache.values(); - const k_transposed = k_cached.transpose(.{ .h, .hd, .k }); - const v_transposed = v_cached.transpose(.{ .h, .k, .hd }); + // const k_transposed = k_cached.transpose(.{ .h, .hd, .k }); + // const v_transposed = v_cached.transpose(.{ .h, .k, .hd }); const attn_mask = if (x.dim(.s) > 1) blk: { - var mask = zml.nn.causalAttnMask(.{ .q = k_transposed.dim(.k), .k = k_transposed.dim(.k) }, x.dtype(), null); + var mask = zml.nn.causalAttnMask(.{ .q = k_cached.dim(.k), .k = k_cached.dim(.k) }, x.dtype(), null); break :blk mask.gatherSlices(.{ .q = x.dim(.s) }, token_index.reshape(.{ .coord = 1 }), .{}); } else null; - const attn_output = zml.nn.sdpa(q, k_transposed, v_transposed, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + const attn_output = zml.nn.sdpa(q, k_cached, v_cached, .{ .attn_mask = attn_mask, .allow_cudnn = true }); const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); @@ -377,31 +377,30 @@ pub const KvCache = struct { pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { const k_shape = self.k.shape().drop(.layer); - _ = k_shape; // autofix var layer = self.layer_index; layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; return if (token_index) |idx| .{ .k = self.k.scatterSlices( .{ .layer = layer, .k = idx }, - new_k.convert(self.k.dtype()), + new_k.convert(self.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.k), .v = self.v.scatterSlices( .{ .layer = layer, .k = idx }, - new_v.convert(self.v.dtype()), + new_v.convert(self.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.v), .layer_index = self.layer_index, } else .{ .k = self.k.scatterSlices( .{ .layer = layer }, - new_k.convert(self.k.dtype()), + new_k.convert(self.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.k), .v = self.v.scatterSlices( .{ .layer = layer }, - new_v.convert(self.v.dtype()), + new_v.convert(self.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.v), .layer_index = self.layer_index, From 3dfca85029615e881363086b883d0776e67c4090 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 19:11:19 +0200 Subject: [PATCH 16/51] deactivate zml call for fusion --- examples/llama/llama.zig | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 93e4e04f..1db3b30b 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -192,22 +192,28 @@ pub const TransformerLayer = struct { self.post_attention_layernorm.init(config, mesh); self.mlp.init(mesh); } - pub fn forward( self: TransformerLayer, - x0_: Tensor, + x0: Tensor, token_index: Tensor, kv_cache: KvCache, ) struct { Tensor, KvCache } { - const x0 = x0_; stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); - const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); + // --- Self Attention Block --- + const x0_normalized = self.input_layernorm.forward(x0); + const delta0, const updated_kv_cache = self.self_attn.forward(x0_normalized, token_index, kv_cache); const x1 = x0.add(delta0); - const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - const delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); + // --- MLP Block --- + const x1_normalized = self.post_attention_layernorm.forward(x1); + + // Inlined Mlp.forward + const gate_out = self.mlp.gate_proj.forward(x1_normalized); + const up_out = self.mlp.up_proj.forward(x1_normalized); + const mlp_hidden = gate_out.silu().mul(up_out); + const delta1 = self.mlp.down_proj.forward(mlp_hidden); + const x2 = x1.add(delta1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; From 214f78cb43d8899d0d8ea9cdf60e0f86137e38e4 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 1 Jul 2025 19:14:52 +0200 Subject: [PATCH 17/51] restore zml call --- examples/llama/llama.zig | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 1db3b30b..93e4e04f 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -192,28 +192,22 @@ pub const TransformerLayer = struct { self.post_attention_layernorm.init(config, mesh); self.mlp.init(mesh); } + pub fn forward( self: TransformerLayer, - x0: Tensor, + x0_: Tensor, token_index: Tensor, kv_cache: KvCache, ) struct { Tensor, KvCache } { + const x0 = x0_; stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); - // --- Self Attention Block --- - const x0_normalized = self.input_layernorm.forward(x0); - const delta0, const updated_kv_cache = self.self_attn.forward(x0_normalized, token_index, kv_cache); + const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); + const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); const x1 = x0.add(delta0); - // --- MLP Block --- - const x1_normalized = self.post_attention_layernorm.forward(x1); - - // Inlined Mlp.forward - const gate_out = self.mlp.gate_proj.forward(x1_normalized); - const up_out = self.mlp.up_proj.forward(x1_normalized); - const mlp_hidden = gate_out.silu().mul(up_out); - const delta1 = self.mlp.down_proj.forward(mlp_hidden); - + const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); + const delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); const x2 = x1.add(delta1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; From 8b571abbcdea65c361fef5481f269092a68356f4 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 2 Jul 2025 14:11:04 +0200 Subject: [PATCH 18/51] collectives --- examples/partitioning/BUILD.bazel | 10 ++ examples/partitioning/collectives.zig | 126 ++++++++++++++++++++++++++ zml/ops.zig | 71 ++++++--------- 3 files changed, 166 insertions(+), 41 deletions(-) create mode 100644 examples/partitioning/collectives.zig diff --git a/examples/partitioning/BUILD.bazel b/examples/partitioning/BUILD.bazel index 1c38d91f..00ca1b7d 100644 --- a/examples/partitioning/BUILD.bazel +++ b/examples/partitioning/BUILD.bazel @@ -9,3 +9,13 @@ zig_cc_binary( "@zml//zml", ], ) + +zig_cc_binary( + name = "collectives", + main = "collectives.zig", + deps = [ + "@zml//async", + "@zml//stdx", + "@zml//zml", + ], +) diff --git a/examples/partitioning/collectives.zig b/examples/partitioning/collectives.zig new file mode 100644 index 00000000..1b94f2c5 --- /dev/null +++ b/examples/partitioning/collectives.zig @@ -0,0 +1,126 @@ +const std = @import("std"); +const zml = @import("zml"); +const stdx = @import("stdx"); +const asynk = @import("async"); +const builtin = @import("builtin"); + +// Set log level to info to see the output. Use .debug to see the generated IR. +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/collectives"); + +const AllReduceModel = struct { + pub fn forward(x: zml.Tensor) zml.Tensor { + const mesh = x.mesh(); + const reduced = zml.ops.allReduce(x, .x, mesh); + + const replicated = reduced.withSharding(.{}); + return replicated.add(zml.Tensor.constant(replicated.shape(), replicated.dtype().zero())); + } +}; + +const AllGatherModel = struct { + pub fn forward(x_sharded: zml.Tensor) zml.Tensor { + const mesh = x_sharded.mesh(); + const gathered = zml.ops.allGather(x_sharded, .x, mesh); + const replicated = gathered.withSharding(.{}); + return replicated.add(zml.Tensor.constant(replicated.shape(), replicated.dtype().zero())); + } +}; + +const ReduceScatterModel = struct { + pub fn forward(x_replicated: zml.Tensor) zml.Tensor { + const mesh = x_replicated.mesh(); + return zml.ops.reduceScatter(x_replicated, .x, mesh); + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{ .cpu = .{ .cpu_device_count = 8 } }).withCompilationOptions(.{ + .sharding_enabled = true, + .xla_dump_to = "/tmp/zml/collectives", + }); + + const num_devices = platform.getDevices().len; + if (num_devices < 2) { + log.warn("This example requires at least 2 devices. Skipping actual run.", .{}); + return; + } + context.printAvailablePlatforms(platform); + + log.info("Running collectives example on {d} devices", .{num_devices}); + const mesh = zml.Mesh.init(.{ .x = num_devices }); + + { + log.info("--- Running AllReduce ---", .{}); + const shape = zml.Shape.init(.{ .m = 8, .n = num_devices }, .i32).withPartitioning(.{ .n = .x }); + const mod = try zml.compileFn(allocator, AllReduceModel.forward, .{shape}, mesh, platform); + defer mod.deinit(); + + const input_data = try zml.slice.arange(allocator, shape, .{}); + defer allocator.free(input_data); + const input_buffer = try zml.Buffer.from(platform, .init(mesh, shape), input_data, .{}); + defer input_buffer.deinit(); + + const result_buffer = mod.call(.{input_buffer}); + defer result_buffer.deinit(); + + const result_data = try result_buffer.toHost(allocator); + defer allocator.free(result_data); + log.info("AllReduce Result (first 8 elements): {any}", .{result_data[0..@min(result_data.len, 8)]}); + } + + { + log.info("--- Running AllGather ---", .{}); + const global_shape = zml.Shape.init(.{ .m = 8 * num_devices, .n = 2 }, .i32); + const input_shape = global_shape.withPartitioning(.{ .m = .x }); + + const mod = try zml.compileFn(allocator, AllGatherModel.forward, .{input_shape}, mesh, platform); + defer mod.deinit(); + + const input_data = try zml.slice.arange(allocator, global_shape, .{}); + defer allocator.free(input_data); + const input_buffer = try zml.Buffer.from(platform, .init(mesh, input_shape), input_data, .{}); + defer input_buffer.deinit(); + + const result_buffer = mod.call(.{input_buffer}); + defer result_buffer.deinit(); + + const result_data = try result_buffer.toHost(allocator); + defer allocator.free(result_data); + log.info("AllGather Result (first 8 elements): {any}", .{result_data[0..@min(result_data.len, 8)]}); + } + + { + log.info("--- Running ReduceScatter ---", .{}); + const input_shape = zml.Shape.init(.{ .m = 8 * num_devices, .n = 2 }, .i32).withReplicatedPartitioning(); + const mod = try zml.compileFn(allocator, ReduceScatterModel.forward, .{input_shape}, mesh, platform); + defer mod.deinit(); + + const input_data = try zml.slice.arange(allocator, input_shape, .{}); + defer allocator.free(input_data); + const input_buffer = try zml.Buffer.from(platform, .init(mesh, input_shape), input_data, .{}); + defer input_buffer.deinit(); + + const result_buffer = mod.call(.{input_buffer}); + defer result_buffer.deinit(); + + const result_data = try result_buffer.toHost(allocator); + defer allocator.free(result_data); + log.info("ReduceScatter Result (first 8 elements): {any}", .{result_data[0..@min(result_data.len, 8)]}); + } +} diff --git a/zml/ops.zig b/zml/ops.zig index b71dd347..42372481 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1,3 +1,4 @@ +const c = @import("c"); const std = @import("std"); const mlir = @import("mlir"); @@ -616,41 +617,12 @@ pub fn sort( return res; } -var next_available_channel_id: u64 = 1; - -// Helper to get a unique ID. @atomicRmw ensures that even if two compilations -// in parallel threads call this, they will get different, unique IDs. -fn getNextChannelId() u64 { - return @atomicRmw(u64, &next_available_channel_id, .Add, 1, .seq_cst); -} - -// Helper function to create replica groups for single-host SPMD -fn getSpmdReplicaGroups(ctx: mlir.Context, mesh: Mesh) mlir.Attribute { - const allocator = CompilationContext.current().allocator(); - var replica_group_data = std.ArrayList(i64).init(allocator); - errdefer replica_group_data.deinit(); - - const num_partitions = mesh.numPartitions(); - replica_group_data.ensureTotalCapacity(@intCast(num_partitions)) catch @panic("OOM"); - for (0..num_partitions) |i| { - replica_group_data.appendAssumeCapacity(@intCast(i)); - } - - return mlir.Attribute.denseElements( - ctx, - &.{ 1, @intCast(num_partitions) }, - .i64, - replica_group_data.items, - ); -} - pub fn allReduce(input: Tensor, axis: anytype, mesh: Mesh) Tensor { if (mesh.axis(axis) <= 1) return input; const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlirCtx(); - const channel_id = getNextChannelId(); - log.warn(">>>>>>>> channel id: {d}", .{channel_id}); + const channel_id = ctx.getNextChannelId(); const body_block, _ = ctx.makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ Tensor.scalar(0, input.dtype()), @@ -663,11 +635,11 @@ pub fn allReduce(input: Tensor, axis: anytype, mesh: Mesh) Tensor { .blocks = &.{body_block}, .attributes = &.{ .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, - .{ "channel_id", .int(mlir_ctx, .i64, @intCast(channel_id)) }, + .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, .{ "use_global_device_ids", .unit(mlir_ctx) }, }, .location = ctx.location(@src(), "allReduce({_}, {any})", .{ input, axis }), - .verify = false, + .verify = true, }); var res = fromMlirOperationWithTags(op, input); @@ -680,8 +652,7 @@ pub fn allGather(input: Tensor, axis: anytype, mesh: Mesh) Tensor { const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlirCtx(); - const channel_id = getNextChannelId(); - log.warn(">>>>>>>> channel id: {d}", .{channel_id}); + const channel_id = ctx.getNextChannelId(); const mesh_axis_tag = mesh.topology.tag(axis); var res_shape = input.shape(); @@ -710,11 +681,11 @@ pub fn allGather(input: Tensor, axis: anytype, mesh: Mesh) Tensor { .attributes = &.{ .{ "all_gather_dim", .int(mlir_ctx, .i64, gather_dim_unwrapped) }, .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, - .{ "channel_id", .int(mlir_ctx, .i64, @intCast(channel_id)) }, + .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, .{ "use_global_device_ids", .unit(mlir_ctx) }, }, .location = ctx.location(@src(), "all_gather({_}, {any})", .{ input, axis }), - .verify = false, + .verify = true, }); return Tensor._result(res_shape, op.result(0)); @@ -725,8 +696,7 @@ pub fn reduceScatter(input: Tensor, axis: anytype, mesh: Mesh) Tensor { const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlirCtx(); - const channel_id = getNextChannelId(); - log.warn(">>>>>>>> channel id: {d}", .{channel_id}); + const channel_id = ctx.getNextChannelId(); const mesh_axis_tag = mesh.topology.tag(axis); var res_shape = input.shape(); @@ -748,7 +718,7 @@ pub fn reduceScatter(input: Tensor, axis: anytype, mesh: Mesh) Tensor { res_shape = res_shape.set(scatter_dim_unwrapped, @divExact(old_dim_size, axis_size)); - const new_partition_spec = Shape.PartitionSpec.init(mesh_axis_tag); + const new_partition_spec: Shape.PartitionSpec = .{ .axis = mesh_axis_tag }; res_shape._partitioning.set(scatter_dim_unwrapped, new_partition_spec); const body_block, _ = ctx.makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ Tensor.scalar(0, input.dtype()), Tensor.scalar(0, input.dtype()) }); @@ -760,16 +730,35 @@ pub fn reduceScatter(input: Tensor, axis: anytype, mesh: Mesh) Tensor { .attributes = &.{ .{ "scatter_dimension", .int(mlir_ctx, .i64, scatter_dim_unwrapped) }, .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, - .{ "channel_id", .int(mlir_ctx, .i64, @intCast(channel_id)) }, + .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, .{ "use_global_device_ids", .unit(mlir_ctx) }, }, .location = ctx.location(@src(), "reduceScatter({_}, {any})", .{ input, axis }), - .verify = false, + .verify = true, }); return Tensor._result(res_shape, op.result(0)); } +fn getSpmdReplicaGroups(ctx: mlir.Context, mesh: Mesh) mlir.Attribute { + const allocator = CompilationContext.current().allocator(); + var replica_group_data = std.ArrayList(i64).init(allocator); + errdefer replica_group_data.deinit(); + + const num_partitions = mesh.numPartitions(); + replica_group_data.ensureTotalCapacity(@intCast(num_partitions)) catch @panic("OOM"); + for (0..num_partitions) |i| { + replica_group_data.appendAssumeCapacity(@intCast(i)); + } + + return mlir.Attribute.denseElements( + ctx, + &.{ 1, @intCast(num_partitions) }, + .i64, + replica_group_data.items, + ); +} + pub const BlockSignature = struct { Fn: type, BlkCtx: type, From 4874ad497ee4cb72d1b14969fa94a3bfbb862698 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 2 Jul 2025 15:34:11 +0200 Subject: [PATCH 19/51] working llama --- examples/llama/llama.zig | 46 ++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 93e4e04f..2df3043d 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -236,33 +236,28 @@ pub const Mlp = struct { down_proj: zml.nn.Linear, pub fn init(self: *Mlp, mesh: zml.Mesh) void { + // Column-Parallel: Shard on the output/column dimension self.gate_proj.weight = self.gate_proj.weight - .withTags(.{ .hidden, .d }) // Shape is (hidden_dim, dim) + .withTags(.{ .hidden, .d }) .withMesh(mesh) - .withSharding(.{ .hidden = .model }); // Shard along hidden_dim + .withSharding(.{ .hidden = .model }); self.up_proj.weight = self.up_proj.weight - .withTags(.{ .hidden, .d }) // Shape is (hidden_dim, dim) + .withTags(.{ .hidden, .d }) .withMesh(mesh) - .withSharding(.{ .hidden = .model }); // Shard along hidden_dim + .withSharding(.{ .hidden = .model }); - // Row-parallel layer: Shard the *input* dimension. - // The matmul is `x @ W.T`, so the input dimension of the operation - // corresponds to the first dimension of the weight matrix W. + // Row-Parallel: Shard on the INPUT/column dimension self.down_proj.weight = self.down_proj.weight - .withTags(.{ .d, .hidden }) // Shape is (dim, hidden_dim) + .withTags(.{ .d, .hidden }) .withMesh(mesh) - .withSharding(.{ .hidden = .model }); // Shard along hidden_dim + .withSharding(.{ .hidden = .model }); } pub fn forward(self: Mlp, x: Tensor) Tensor { - // x is replicated. const gate_out = zml.call(self.gate_proj, .forward, .{x}); const up_out = zml.call(self.up_proj, .forward, .{x}); - // gate_out and up_out are sharded. - const output = gate_out.silu().mul(up_out); // output is sharded. - // down_proj takes a sharded input and produces a partial result. - // The implicit all_reduce happens because of the residual connection in the parent layer. + const output = gate_out.silu().mul(up_out); return zml.call(self.down_proj, .forward, .{output}); } }; @@ -285,10 +280,10 @@ pub const SelfAttn = struct { .scaling = config.rope_scaling, }; - self.q_proj.weight = self.q_proj.weight.withTags(.{ .d, .q_hidden }).withMesh(mesh).withSharding(.{ .q_hidden = .model }); - self.k_proj.weight = self.k_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); - self.v_proj.weight = self.v_proj.weight.withTags(.{ .d, .kv_hidden }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); - self.o_proj.weight = self.o_proj.weight.withTags(.{ .o_hidden, .d }).withMesh(mesh).withSharding(.{ .o_hidden = .model }); + self.q_proj.weight = self.q_proj.weight.withTags(.{ .q_hidden, .d }).withMesh(mesh).withSharding(.{ .q_hidden = .model }); + self.k_proj.weight = self.k_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); + self.v_proj.weight = self.v_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); + self.o_proj.weight = self.o_proj.weight.withTags(.{ .d, .o_hidden }).withMesh(mesh).withSharding(.{ .o_hidden = .model }); } pub fn forward( @@ -303,9 +298,9 @@ pub const SelfAttn = struct { const k_proj_out = zml.call(self.k_proj, .forward, .{x}); const v_proj_out = zml.call(self.v_proj, .forward, .{x}); - var q = q_proj_out.splitAxis(.d, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{ .h = .model }); - var k = k_proj_out.splitAxis(.d, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); - var v = v_proj_out.splitAxis(.d, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); + var q = q_proj_out.withPartialTags(.{.q_hidden}).splitAxis(.q_hidden, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{ .h = .model }); + var k = k_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); + var v = v_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); const pos_index = blk: { const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); @@ -323,9 +318,6 @@ pub const SelfAttn = struct { const k_cached = new_kv_cache.keys(); const v_cached = new_kv_cache.values(); - // const k_transposed = k_cached.transpose(.{ .h, .hd, .k }); - // const v_transposed = v_cached.transpose(.{ .h, .k, .hd }); - const attn_mask = if (x.dim(.s) > 1) blk: { var mask = zml.nn.causalAttnMask(.{ .q = k_cached.dim(.k), .k = k_cached.dim(.k) }, x.dtype(), null); break :blk mask.gatherSlices(.{ .q = x.dim(.s) }, token_index.reshape(.{ .coord = 1 }), .{}); @@ -335,11 +327,9 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); - // o_proj takes a sharded input and produces a partial result. - // The implicit all_reduce happens because of the residual connection in the parent layer. - const output = zml.call(self.o_proj, .forward, .{attn}); + const partial_output = zml.call(self.o_proj, .forward, .{attn}); - return .{ output, new_kv_cache }; + return .{ partial_output, new_kv_cache }; } }; From 78aecbcb30fce6a1d0a900885896fd2a6c477f26 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 2 Jul 2025 15:43:27 +0200 Subject: [PATCH 20/51] setup pjrt profiler --- examples/llama/main.zig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index efbb72fe..2c2de4eb 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -231,6 +231,16 @@ pub fn asyncMain() !void { create_opts.deinit(); context.printAvailablePlatforms(platform); + const use_pjrt_profiler: bool = blk: { + const env = std.c.getenv("ZML_PJRT_PROFILER") orelse "0"; + break :blk std.mem.eql(u8, std.mem.span(env), "1"); + }; + + var profiler = if (use_pjrt_profiler) platform.getProfiler(null) else undefined; + if (use_pjrt_profiler) { + defer profiler.deinit(); + } + log.info("\n\n\n\n", .{}); const num_devices = platform.getDevices().len; @@ -323,6 +333,16 @@ pub fn asyncMain() !void { const seed = res.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); const skip_llama3_encoding: bool = res.args.@"no-llama3" orelse false; + + if (use_pjrt_profiler) { + profiler.start(); + } + const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module_decode, kv_cache, tokenizer, allocator, compute_mesh, seed, prompt, skip_llama3_encoding); defer allocator.free(generated_text); + + if (use_pjrt_profiler) { + profiler.stop(); + try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), "llama-profile.json"); + } } From c5a87e0ba458e170453f9b412f6f16f2be3c3d71 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 2 Jul 2025 16:15:24 +0200 Subject: [PATCH 21/51] just test nccl --- examples/llama/llama.zig | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 2df3043d..c6007acb 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -203,11 +203,16 @@ pub const TransformerLayer = struct { stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - const x1 = x0.add(delta0); + const partial_delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - const delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); + const x1_unreduced = x0.add(partial_delta0); + const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1_unreduced}); + const partial_delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); + + const delta0 = zml.ops.allReduce(partial_delta0, .model, self.self_attn.o_proj.weight.mesh()); + const delta1 = zml.ops.allReduce(partial_delta1, .model, self.mlp.down_proj.weight.mesh()); + + const x1 = x0.add(delta0); const x2 = x1.add(delta1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; From 4751c3d66640e29907c52af6614666a04c002b21 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 3 Jul 2025 10:51:19 +0200 Subject: [PATCH 22/51] use allGather --- examples/llama/llama.zig | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index c6007acb..91af5fe6 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -195,24 +195,23 @@ pub const TransformerLayer = struct { pub fn forward( self: TransformerLayer, - x0_: Tensor, + x0: Tensor, token_index: Tensor, kv_cache: KvCache, ) struct { Tensor, KvCache } { - const x0 = x0_; stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - const partial_delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - const x1_unreduced = x0.add(partial_delta0); - const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1_unreduced}); - const partial_delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); - - const delta0 = zml.ops.allReduce(partial_delta0, .model, self.self_attn.o_proj.weight.mesh()); - const delta1 = zml.ops.allReduce(partial_delta1, .model, self.mlp.down_proj.weight.mesh()); + const sharded_delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); + const delta0 = zml.ops.allGather(sharded_delta0, .model, self.self_attn.o_proj.weight.mesh()); const x1 = x0.add(delta0); + + const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); + const sharded_delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); + + const delta1 = zml.ops.allGather(sharded_delta1, .model, self.mlp.down_proj.weight.mesh()); const x2 = x1.add(delta1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; @@ -332,9 +331,9 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); - const partial_output = zml.call(self.o_proj, .forward, .{attn}); + const sharded_output = zml.call(self.o_proj, .forward, .{attn}); - return .{ partial_output, new_kv_cache }; + return .{ sharded_output, new_kv_cache }; } }; From c38ea4553c31ba2470c415885ec5053202acd23d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 3 Jul 2025 15:48:38 +0200 Subject: [PATCH 23/51] attempt --- examples/llama/llama.zig | 82 +++++++++++++++++++++++++++++------ zml/ops.zig | 94 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 13 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 91af5fe6..0bb75fa1 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -10,6 +10,49 @@ const ShapeOf = zml.ShapeOf; const log = std.log.scoped(.llama); +pub const Linear = struct { + weight: Tensor, + bias: ?Tensor = null, + + pub fn forward(self: Linear, x: Tensor) Tensor { + var y = x.dotGeneral(self.weight.convert(x.dtype()), &.{.{ -1, -1 }}, &.{}); + + // If self.weight doesn't have tags, preserve tags from x. + if (y.shape().tag(-1) == zml.Shape.TagUnknown) { + y._shape._tags.set(y.rank() - 1, x.shape().tag(-1)); + } + + // New sharding propagation logic + const weight_sharding = self.weight.shape()._partitioning; + const out_dim_idx = y.rank() - 1; + + if (weight_sharding.get(out_dim_idx) != .replicated and weight_sharding.get(out_dim_idx) != .unknown) { + // This looks like row-parallelism (weight sharded on input dim). + // The input `x` should have been sharded. + // The output of the GEMM is partial and needs an All-Reduce. + // Let's assume the mesh for the all-reduce is the same as the weight's mesh. + const mesh = self.weight._mesh orelse zml.currentMesh(); + // The axis to reduce over is the one the weight is sharded on. + const mesh_axis = weight_sharding.get(out_dim_idx).toTag(); + y = zml.ops.allReduce(y, mesh_axis, mesh); + } else if (weight_sharding.get(0) != .replicated and weight_sharding.get(0) != .unknown) { + // This looks like column-parallelism (weight sharded on output dim). + // The output tensor `y` should be sharded along its feature dimension. + var new_sharding = zml.Shape.PartitionArray.init(0) catch unreachable; + new_sharding.appendNTimesAssumeCapacity(.replicated, y.rank()); + new_sharding.set(out_dim_idx, weight_sharding.get(0)); + y._shape._partitioning = new_sharding; + } + + if (self.bias) |bias| { + // Bias is typically replicated and added after any communication. + return y.add(bias.broadcastLeft(y.shape())); + } else { + return y; + } + } +}; + /// Llama architecture, using huggingface transformers naming. /// Dimensions of activations: {.b, .s, .d} pub const LlamaLM = struct { @@ -34,7 +77,7 @@ pub const LlamaLM = struct { max_seq_len: usize, }; - lm_head: ?zml.nn.Linear, + lm_head: ?Linear, model: Llama, // Runtime configuration, not part of the model's weights/arch @@ -76,7 +119,7 @@ pub const LlamaLM = struct { pub fn sampleTokens( self: LlamaLM, - lm_head_: ?zml.nn.Linear, + lm_head_: ?Linear, out_: Tensor, rng: Tensor.Rng, opts: zml.nn.SamplingStrategy, @@ -205,13 +248,23 @@ pub const TransformerLayer = struct { const sharded_delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - const delta0 = zml.ops.allGather(sharded_delta0, .model, self.self_attn.o_proj.weight.mesh()); + // --- MODIFICATION 1: Replace allReduce with reduceScatter -> allGather for Attention Output --- + const attn_mesh = self.self_attn.o_proj.weight.mesh(); + const scattered_delta0 = zml.ops.reduceScatter2(sharded_delta0, .model, attn_mesh); + const delta0 = zml.ops.allGather2(scattered_delta0, .model, attn_mesh); + // --- END MODIFICATION 1 --- + const x1 = x0.add(delta0); const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); const sharded_delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); - const delta1 = zml.ops.allGather(sharded_delta1, .model, self.mlp.down_proj.weight.mesh()); + // --- MODIFICATION 2: Replace allReduce with reduceScatter -> allGather for MLP Output --- + const mlp_mesh = self.mlp.down_proj.weight.mesh(); + const scattered_delta1 = zml.ops.reduceScatter2(sharded_delta1, .model, mlp_mesh); + const delta1 = zml.ops.allGather2(scattered_delta1, .model, mlp_mesh); + // --- END MODIFICATION 2 --- + const x2 = x1.add(delta1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; @@ -235,12 +288,12 @@ pub const RmsNorm = struct { }; pub const Mlp = struct { - up_proj: zml.nn.Linear, - gate_proj: zml.nn.Linear, - down_proj: zml.nn.Linear, + up_proj: Linear, + gate_proj: Linear, + down_proj: Linear, pub fn init(self: *Mlp, mesh: zml.Mesh) void { - // Column-Parallel: Shard on the output/column dimension + // Column-Parallel: output is sharded self.gate_proj.weight = self.gate_proj.weight .withTags(.{ .hidden, .d }) .withMesh(mesh) @@ -251,7 +304,7 @@ pub const Mlp = struct { .withMesh(mesh) .withSharding(.{ .hidden = .model }); - // Row-Parallel: Shard on the INPUT/column dimension + // Row-Parallel: input is sharded on .hidden, output is replicated after all-reduce self.down_proj.weight = self.down_proj.weight .withTags(.{ .d, .hidden }) .withMesh(mesh) @@ -267,10 +320,10 @@ pub const Mlp = struct { }; pub const SelfAttn = struct { - q_proj: zml.nn.Linear, - k_proj: zml.nn.Linear, - v_proj: zml.nn.Linear, - o_proj: zml.nn.Linear, + q_proj: Linear, + k_proj: Linear, + v_proj: Linear, + o_proj: Linear, num_heads: i64 = undefined, num_kv_heads: i64 = 0, rope_opts: zml.nn.RopeOpts = undefined, @@ -284,9 +337,12 @@ pub const SelfAttn = struct { .scaling = config.rope_scaling, }; + // Column-parallel: output is sharded self.q_proj.weight = self.q_proj.weight.withTags(.{ .q_hidden, .d }).withMesh(mesh).withSharding(.{ .q_hidden = .model }); self.k_proj.weight = self.k_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); self.v_proj.weight = self.v_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); + + // Row-parallel: input is sharded on .o_hidden, output is replicated after all-reduce self.o_proj.weight = self.o_proj.weight.withTags(.{ .d, .o_hidden }).withMesh(mesh).withSharding(.{ .o_hidden = .model }); } diff --git a/zml/ops.zig b/zml/ops.zig index 42372481..14f9f959 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -740,6 +740,100 @@ pub fn reduceScatter(input: Tensor, axis: anytype, mesh: Mesh) Tensor { return Tensor._result(res_shape, op.result(0)); } +pub fn allGather2(input: Tensor, axis: anytype, mesh: Mesh) Tensor { + if (mesh.axis(axis) <= 1) return input; + + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const channel_id = ctx.getNextChannelId(); + const mesh_axis_tag = mesh.topology.tag(axis); + + var res_shape = input.shape(); + const axis_size = mesh.axis(axis); + + var gather_dim: ?u3 = null; + for (0..res_shape.rank()) |i| { + const part_spec = res_shape.partition(i); + if (part_spec == .axis and std.mem.eql(u8, std.mem.span(part_spec.toTag()), std.mem.span(mesh_axis_tag))) { + stdx.debug.assert(gather_dim == null, "Mesh axis '{s}' is used to partition multiple tensor dimensions.", .{mesh_axis_tag}); + gather_dim = @intCast(i); + } + } + + const gather_dim_unwrapped = gather_dim orelse { + // If the sharding axis is not found on the tensor, it's a no-op. + return input; + }; + + const old_dim_size = res_shape.dim(gather_dim_unwrapped); + res_shape = res_shape.set(gather_dim_unwrapped, old_dim_size * axis_size); + res_shape = res_shape.withReplicatedPartitioning(); + + const op = mlir.Operation.make(mlir_ctx, "stablehlo.all_gather", .{ + .operands = &.{input.value()}, + .results = &.{mlirx.tensorType(mlir_ctx, res_shape)}, + .attributes = &.{ + .{ "all_gather_dim", .int(mlir_ctx, .i64, gather_dim_unwrapped) }, + .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, + .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, + .{ "use_global_device_ids", .unit(mlir_ctx) }, + }, + .location = ctx.location(@src(), "all_gather({_}, {any})", .{ input, axis }), + .verify = true, + }); + + return Tensor._result(res_shape, op.result(0)); +} + +pub fn reduceScatter2(input: Tensor, axis: anytype, mesh: Mesh) Tensor { + if (mesh.axis(axis) <= 1) return input; + + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const channel_id = ctx.getNextChannelId(); + const mesh_axis_tag = mesh.topology.tag(axis); + + var res_shape = input.shape(); + const axis_size = mesh.axis(axis); + + var scatter_dim: ?u3 = null; + for (0..res_shape.rank()) |i| { + if (input.shape().partition(i) == .replicated) { + if (@rem(res_shape.dim(i), axis_size) == 0) { + scatter_dim = @intCast(i); + break; + } + } + } + const scatter_dim_unwrapped = scatter_dim orelse stdx.debug.panic("Could not find a suitable replicated dimension to scatter on for tensor {}", .{input.shape()}); + + const old_dim_size = res_shape.dim(scatter_dim_unwrapped); + stdx.debug.assert(@rem(old_dim_size, axis_size) == 0, "Reduce-scatter dimension size {} must be divisible by mesh axis size {}", .{ old_dim_size, axis_size }); + + res_shape = res_shape.set(scatter_dim_unwrapped, @divExact(old_dim_size, axis_size)); + + const new_partition_spec: Shape.PartitionSpec = .{ .axis = mesh_axis_tag }; + res_shape._partitioning.set(scatter_dim_unwrapped, new_partition_spec); + + const body_block, _ = ctx.makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ Tensor.scalar(0, input.dtype()), Tensor.scalar(0, input.dtype()) }); + + const op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_scatter", .{ + .operands = &.{input.value()}, + .results = &.{mlirx.tensorType(mlir_ctx, res_shape)}, + .blocks = &.{body_block}, + .attributes = &.{ + .{ "scatter_dimension", .int(mlir_ctx, .i64, scatter_dim_unwrapped) }, + .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, + .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, + .{ "use_global_device_ids", .unit(mlir_ctx) }, + }, + .location = ctx.location(@src(), "reduceScatter({_}, {any})", .{ input, axis }), + .verify = true, + }); + + return Tensor._result(res_shape, op.result(0)); +} + fn getSpmdReplicaGroups(ctx: mlir.Context, mesh: Mesh) mlir.Attribute { const allocator = CompilationContext.current().allocator(); var replica_group_data = std.ArrayList(i64).init(allocator); From 050b98d0de1b3d65f49039b5854c8ef758b8521e Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 3 Jul 2025 16:04:24 +0200 Subject: [PATCH 24/51] wip --- examples/llama/llama.zig | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 0bb75fa1..1885d7a8 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -248,22 +248,14 @@ pub const TransformerLayer = struct { const sharded_delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - // --- MODIFICATION 1: Replace allReduce with reduceScatter -> allGather for Attention Output --- - const attn_mesh = self.self_attn.o_proj.weight.mesh(); - const scattered_delta0 = zml.ops.reduceScatter2(sharded_delta0, .model, attn_mesh); - const delta0 = zml.ops.allGather2(scattered_delta0, .model, attn_mesh); - // --- END MODIFICATION 1 --- + const delta0 = zml.ops.allGather(sharded_delta0, .model, self.self_attn.o_proj.weight.mesh()); const x1 = x0.add(delta0); const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); const sharded_delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); - // --- MODIFICATION 2: Replace allReduce with reduceScatter -> allGather for MLP Output --- - const mlp_mesh = self.mlp.down_proj.weight.mesh(); - const scattered_delta1 = zml.ops.reduceScatter2(sharded_delta1, .model, mlp_mesh); - const delta1 = zml.ops.allGather2(scattered_delta1, .model, mlp_mesh); - // --- END MODIFICATION 2 --- + const delta1 = zml.ops.allGather(sharded_delta1, .model, self.mlp.down_proj.weight.mesh()); const x2 = x1.add(delta1); From a6aa94c6e3bf74df4b9c01c2c0faeda309dbfa8f Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 3 Jul 2025 16:06:33 +0200 Subject: [PATCH 25/51] rollback --- examples/llama/llama.zig | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 1885d7a8..a2faa558 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -77,7 +77,7 @@ pub const LlamaLM = struct { max_seq_len: usize, }; - lm_head: ?Linear, + lm_head: ?zml.nn.Linear, model: Llama, // Runtime configuration, not part of the model's weights/arch @@ -119,7 +119,7 @@ pub const LlamaLM = struct { pub fn sampleTokens( self: LlamaLM, - lm_head_: ?Linear, + lm_head_: ?zml.nn.Linear, out_: Tensor, rng: Tensor.Rng, opts: zml.nn.SamplingStrategy, @@ -280,9 +280,9 @@ pub const RmsNorm = struct { }; pub const Mlp = struct { - up_proj: Linear, - gate_proj: Linear, - down_proj: Linear, + up_proj: zml.nn.Linear, + gate_proj: zml.nn.Linear, + down_proj: zml.nn.Linear, pub fn init(self: *Mlp, mesh: zml.Mesh) void { // Column-Parallel: output is sharded @@ -312,10 +312,10 @@ pub const Mlp = struct { }; pub const SelfAttn = struct { - q_proj: Linear, - k_proj: Linear, - v_proj: Linear, - o_proj: Linear, + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + o_proj: zml.nn.Linear, num_heads: i64 = undefined, num_kv_heads: i64 = 0, rope_opts: zml.nn.RopeOpts = undefined, From 4897398a82ec3245467737578b1af0b68d0cf4ab Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 4 Jul 2025 14:11:56 +0200 Subject: [PATCH 26/51] new sharding --- examples/llama/llama.zig | 133 +++++++++++++++---------------------- examples/llama/main.zig | 32 +++++---- zml/aio.zig | 3 +- zml/buffer.zig | 7 +- zml/module.zig | 7 +- zml/ops.zig | 140 ++++++--------------------------------- 6 files changed, 102 insertions(+), 220 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index a2faa558..c824ef12 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -17,27 +17,18 @@ pub const Linear = struct { pub fn forward(self: Linear, x: Tensor) Tensor { var y = x.dotGeneral(self.weight.convert(x.dtype()), &.{.{ -1, -1 }}, &.{}); - // If self.weight doesn't have tags, preserve tags from x. if (y.shape().tag(-1) == zml.Shape.TagUnknown) { y._shape._tags.set(y.rank() - 1, x.shape().tag(-1)); } - // New sharding propagation logic const weight_sharding = self.weight.shape()._partitioning; const out_dim_idx = y.rank() - 1; if (weight_sharding.get(out_dim_idx) != .replicated and weight_sharding.get(out_dim_idx) != .unknown) { - // This looks like row-parallelism (weight sharded on input dim). - // The input `x` should have been sharded. - // The output of the GEMM is partial and needs an All-Reduce. - // Let's assume the mesh for the all-reduce is the same as the weight's mesh. const mesh = self.weight._mesh orelse zml.currentMesh(); - // The axis to reduce over is the one the weight is sharded on. const mesh_axis = weight_sharding.get(out_dim_idx).toTag(); y = zml.ops.allReduce(y, mesh_axis, mesh); } else if (weight_sharding.get(0) != .replicated and weight_sharding.get(0) != .unknown) { - // This looks like column-parallelism (weight sharded on output dim). - // The output tensor `y` should be sharded along its feature dimension. var new_sharding = zml.Shape.PartitionArray.init(0) catch unreachable; new_sharding.appendNTimesAssumeCapacity(.replicated, y.rank()); new_sharding.set(out_dim_idx, weight_sharding.get(0)); @@ -45,7 +36,6 @@ pub const Linear = struct { } if (self.bias) |bias| { - // Bias is typically replicated and added after any communication. return y.add(bias.broadcastLeft(y.shape())); } else { return y; @@ -93,9 +83,9 @@ pub const LlamaLM = struct { // Shard the final output layer across all devices if (self.lm_head) |*lm_head| { lm_head.weight = lm_head.weight - .withTags(.{ .d, .vocab }) + .withTags(.{ .d, .voc }) .withMesh(vocab_mesh) - .withSharding(.{ .vocab = .model }); + .withSharding(.{ .voc = .model }); } } @@ -114,7 +104,8 @@ pub const LlamaLM = struct { zml.popMesh(); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + const final_tokens = new_tokens.convert(tokens.dtype()); + return .{ final_tokens, updated_kv_cache, new_rng }; } pub fn sampleTokens( @@ -130,7 +121,7 @@ pub const LlamaLM = struct { const out = out_.withPartialTags(.{ .s, .d }); - var logits = blk: { + var logits_sharded = blk: { if (lm_head_) |lm_head| { break :blk zml.call(lm_head, .forward, .{out}); } else { @@ -138,10 +129,10 @@ pub const LlamaLM = struct { } }; - if (logits.shape().hasTag(.voc) == null) - logits = logits.rename(.{ .d = .voc }); + if (logits_sharded.shape().hasTag(.voc) == null) + logits_sharded = logits_sharded.rename(.{ .d = .voc }); - logits = zml.ops.allGather(logits, .model, vocab_mesh); + const logits = zml.ops.allGather(logits_sharded, .voc, vocab_mesh); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); return .{ next_tokens, new_rng }; @@ -193,15 +184,16 @@ pub const Llama = struct { .scaling = config.rope_scaling, }; - self.norm.init(config, compute_mesh); + const replicated_sharding = .{ .d = .replicated }; + self.norm.init(config, compute_mesh, replicated_sharding); self.embed_tokens.weight = self.embed_tokens.weight - .withTags(.{ .vocab, .d }) + .withTags(.{ .voc, .d }) .withMesh(vocab_mesh) - .withSharding(.{ .vocab = .replicated, .d = .replicated }); + .withSharding(.{ .d = .replicated, .voc = .model }); for (self.layers) |*layer| { - layer.init(config, compute_mesh); + layer.init(config, compute_mesh, replicated_sharding); } } @@ -212,14 +204,25 @@ pub const Llama = struct { for (self.layers, 0..) |layer, i| { hidden, updated_kv_cache = zml.call(layer, .forward, .{ hidden, token_index, updated_kv_cache.atLayer(i) }); } + const output = zml.call(self.norm, .forward, .{hidden}); - return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; + return .{ output, updated_kv_cache }; } pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { const embeddings = zml.call(embed_tokens_, .forward, .{tokens_}); - return embeddings.withPartialTags(.{.d}); + var tagged_embeddings = embeddings.withTags(.{ .s, .d }); + + if (tokens_.dim(.s) > 1) { + // prefill: re-shard the replicated embedding output to be sharded along the sequence axis. + return tagged_embeddings.withSharding(.{ .s = .s }); + } else { + // decode: the output of gather is already replicated. We just ensure the + // sharding annotation is explicitly set to replicated for clarity + // and correctness downstream. + return tagged_embeddings.replicated(); + } } }; @@ -229,11 +232,11 @@ pub const TransformerLayer = struct { post_attention_layernorm: RmsNorm, mlp: Mlp, - pub fn init(self: *TransformerLayer, config: LlamaLM.Config, mesh: zml.Mesh) void { - self.input_layernorm.init(config, mesh); - self.self_attn.init(config, mesh); - self.post_attention_layernorm.init(config, mesh); - self.mlp.init(mesh); + pub fn init(self: *TransformerLayer, config: LlamaLM.Config, mesh: zml.Mesh, sharding: anytype) void { + self.input_layernorm.init(config, mesh, sharding); + self.self_attn.init(config, mesh, sharding); + self.post_attention_layernorm.init(config, mesh, sharding); + self.mlp.init(mesh, sharding); } pub fn forward( @@ -248,16 +251,12 @@ pub const TransformerLayer = struct { const sharded_delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - const delta0 = zml.ops.allGather(sharded_delta0, .model, self.self_attn.o_proj.weight.mesh()); - - const x1 = x0.add(delta0); + const x1 = x0.add(sharded_delta0); const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); const sharded_delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); - const delta1 = zml.ops.allGather(sharded_delta1, .model, self.mlp.down_proj.weight.mesh()); - - const x2 = x1.add(delta1); + const x2 = x1.add(sharded_delta1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; } @@ -267,9 +266,9 @@ pub const RmsNorm = struct { weight: Tensor, eps: f32 = 1e-5, - pub fn init(self: *RmsNorm, config: LlamaLM.Config, mesh: zml.Mesh) void { + pub fn init(self: *RmsNorm, config: LlamaLM.Config, mesh: zml.Mesh, sharding: anytype) void { self.eps = config.rms_norm_eps; - self.weight = self.weight.withTags(.{.d}).withMesh(mesh).withSharding(.{ .d = .replicated }); + self.weight = self.weight.withTags(.{.d}).withMesh(mesh).withSharding(sharding); } pub fn forward(self: RmsNorm, input: Tensor) Tensor { @@ -284,23 +283,10 @@ pub const Mlp = struct { gate_proj: zml.nn.Linear, down_proj: zml.nn.Linear, - pub fn init(self: *Mlp, mesh: zml.Mesh) void { - // Column-Parallel: output is sharded - self.gate_proj.weight = self.gate_proj.weight - .withTags(.{ .hidden, .d }) - .withMesh(mesh) - .withSharding(.{ .hidden = .model }); - - self.up_proj.weight = self.up_proj.weight - .withTags(.{ .hidden, .d }) - .withMesh(mesh) - .withSharding(.{ .hidden = .model }); - - // Row-Parallel: input is sharded on .hidden, output is replicated after all-reduce - self.down_proj.weight = self.down_proj.weight - .withTags(.{ .d, .hidden }) - .withMesh(mesh) - .withSharding(.{ .hidden = .model }); + pub fn init(self: *Mlp, mesh: zml.Mesh, sharding: anytype) void { + self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(sharding); + self.up_proj.weight = self.up_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(sharding); + self.down_proj.weight = self.down_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(sharding); } pub fn forward(self: Mlp, x: Tensor) Tensor { @@ -320,7 +306,7 @@ pub const SelfAttn = struct { num_kv_heads: i64 = 0, rope_opts: zml.nn.RopeOpts = undefined, - pub fn init(self: *SelfAttn, config: LlamaLM.Config, mesh: zml.Mesh) void { + pub fn init(self: *SelfAttn, config: LlamaLM.Config, mesh: zml.Mesh, sharding: anytype) void { self.num_heads = @intCast(config.num_attention_heads); self.num_kv_heads = @intCast(config.num_key_value_heads); self.rope_opts = .{ @@ -329,13 +315,10 @@ pub const SelfAttn = struct { .scaling = config.rope_scaling, }; - // Column-parallel: output is sharded - self.q_proj.weight = self.q_proj.weight.withTags(.{ .q_hidden, .d }).withMesh(mesh).withSharding(.{ .q_hidden = .model }); - self.k_proj.weight = self.k_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); - self.v_proj.weight = self.v_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); - - // Row-parallel: input is sharded on .o_hidden, output is replicated after all-reduce - self.o_proj.weight = self.o_proj.weight.withTags(.{ .d, .o_hidden }).withMesh(mesh).withSharding(.{ .o_hidden = .model }); + self.q_proj.weight = self.q_proj.weight.withTags(.{ .q_hidden, .d }).withMesh(mesh).withSharding(sharding); + self.k_proj.weight = self.k_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(sharding); + self.v_proj.weight = self.v_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(sharding); + self.o_proj.weight = self.o_proj.weight.withTags(.{ .d, .o_hidden }).withMesh(mesh).withSharding(sharding); } pub fn forward( @@ -345,14 +328,15 @@ pub const SelfAttn = struct { kv_cache: KvCache, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + const mesh = self.q_proj.weight.mesh(); const q_proj_out = zml.call(self.q_proj, .forward, .{x}); const k_proj_out = zml.call(self.k_proj, .forward, .{x}); const v_proj_out = zml.call(self.v_proj, .forward, .{x}); - var q = q_proj_out.withPartialTags(.{.q_hidden}).splitAxis(.q_hidden, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{ .h = .model }); - var k = k_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); - var v = v_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); + var q = q_proj_out.withPartialTags(.{.q_hidden}).splitAxis(.q_hidden, .{ .h = self.num_heads, .hd = .auto }); + var k = k_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }); + var v = v_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }); const pos_index = blk: { const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); @@ -367,8 +351,8 @@ pub const SelfAttn = struct { const new_kv_cache = kv_cache.update(k, v, token_index); - const k_cached = new_kv_cache.keys(); - const v_cached = new_kv_cache.values(); + const k_cached = zml.ops.allGather(new_kv_cache.keys(), .k, mesh); + const v_cached = zml.ops.allGather(new_kv_cache.values(), .k, mesh); const attn_mask = if (x.dim(.s) > 1) blk: { var mask = zml.nn.causalAttnMask(.{ .q = k_cached.dim(.k), .k = k_cached.dim(.k) }, x.dtype(), null); @@ -400,12 +384,13 @@ pub const KvCache = struct { } pub fn initBuffer(kv_shape: zml.Shape, mesh: zml.Mesh, platform: zml.Platform) !zml.Bufferized(KvCache) { - const sharding = zml.Sharding.init(mesh, kv_shape.withPartitioning(.{ .h = .model })); + const kv_sharding = zml.Sharding.init(mesh, kv_shape.withPartitioning(.{ .h = .model })); + const layer_index_sharding = zml.Sharding.init(mesh, zml.Shape.init(.{}, .u32)); return .{ - .k = try zml.Buffer.uninitialized(platform, sharding, .{}), - .v = try zml.Buffer.uninitialized(platform, sharding, .{}), - .layer_index = try zml.Buffer.constant(platform, .init(mesh, zml.Shape.init(.{}, .u32)), 0), + .k = try zml.Buffer.uninitialized(platform, kv_sharding, .{}), + .v = try zml.Buffer.uninitialized(platform, kv_sharding, .{}), + .layer_index = try zml.Buffer.constant(platform, layer_index_sharding, 0), }; } @@ -456,12 +441,4 @@ pub const KvCache = struct { .layer_index = Tensor.scalar(layer_index, .u32), }; } - - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), - }; - } }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 2c2de4eb..886bec6b 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -80,14 +80,16 @@ pub fn generateText( var generated_token_buffer = [_]u32{undefined}; var kv_cache = prefill: { - // prepare device buffers for the prefill tokens and their positions const prefill_buffer = try allocator.alloc(u32, max_seq_len); @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); + @memset(prefill_buffer[prompt_tok.len..], 0); - var prefill_tokens = try zml.Buffer.fromSlice(platform, mesh, .{max_seq_len}, prefill_buffer); + const prefill_shape = zml.Shape.init(.{ .s = max_seq_len }, .u32).withPartitioning(.{ .s = .s }); + var prefill_tokens = try zml.Buffer.fromSlice(platform, mesh, prefill_shape, prefill_buffer); defer prefill_tokens.deinit(); - const sharding2: zml.Sharding = .init(mesh, zml.Shape.init(.{}, .u32)); - var prefill_token_pos = try zml.Buffer.constant(platform, sharding2, 0); + + const token_pos_sharding: zml.Sharding = .init(mesh, zml.Shape.init(.{}, .u32)); + var prefill_token_pos = try zml.Buffer.constant(platform, token_pos_sharding, 0); defer prefill_token_pos.deinit(); const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); @@ -95,6 +97,7 @@ pub fn generateText( generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; break :prefill kv_cache; }; + defer zml.aio.unloadBuffers(&kv_cache); // Prepare for token-by-token generation, @@ -133,7 +136,8 @@ pub fn generateText( // current token pos needs to go into a zml.Buffer const token_pos_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; - const token_pos = try zml.Buffer.fromSlice(platform, mesh, .{}, token_pos_buffer); + const token_pos_sharding = zml.Sharding.init(mesh, zml.Shape.init(.{}, .u32)); + const token_pos = try zml.Buffer.from(platform, token_pos_sharding, std.mem.sliceAsBytes(token_pos_buffer), .{}); defer token_pos.deinit(); // call to generate the next token @@ -241,18 +245,12 @@ pub fn asyncMain() !void { defer profiler.deinit(); } - log.info("\n\n\n\n", .{}); - const num_devices = platform.getDevices().len; - - stdx.debug.assert(config.num_attention_heads % num_devices == 0 and (config.num_key_value_heads % num_devices == 0 or config.num_key_value_heads == 0), "Number of heads ({d}) and KV heads ({d}) must be divisible by the number of devices ({d}) for this sharding strategy.", .{ config.num_attention_heads, config.num_key_value_heads, num_devices }); - log.info("Using 1D Tensor Parallelism across {d} devices", .{num_devices}); - const compute_mesh = zml.Mesh.init(.{ .model = num_devices }); - // For pure 1D TP, the vocab mesh is the same as the compute mesh. - // It's sharded on 'model' (which is equivalent to 'all_devices' in this case). - const vocab_mesh = compute_mesh; - log.info("Constructed compute mesh: {}", .{compute_mesh}); - log.info("Constructed vocab mesh: {}", .{vocab_mesh}); + const compute_mesh = zml.Mesh.init(.{ .s = num_devices }); + const vocab_mesh = zml.Mesh.init(.{ .model = num_devices }); + log.info("Using Sequence Parallelism across {} devices.", .{num_devices}); + log.info("Compute Mesh (for sequence): {}", .{compute_mesh}); + log.info("Vocab/Weight Mesh: {}", .{vocab_mesh}); var ts = try zml.aio.detectFormatAndOpen(allocator, res.args.weights.?); defer ts.deinit(); @@ -272,7 +270,7 @@ pub fn asyncMain() !void { const dims = model_instance.model.shape(); const dtype = model_instance.model.embed_tokens.weight.dtype(); - const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32).withPartitioning(.{ .s = .replicated }); + const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32).withPartitioning(.{ .s = .s }); const tokens_shape_decode = zml.Shape.init(.{ .s = 1 }, .u32).withPartitioning(.{ .s = .replicated }); const token_idx_shape = zml.Shape.init(.{}, .u32); const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype); diff --git a/zml/aio.zig b/zml/aio.zig index 92ffca53..858dcb9a 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -689,8 +689,7 @@ fn visitStructAndLoadBuffer( buf_with_metadata._shape = obj._shape; const m = obj._mesh orelse mesh; const sharding: zml.Sharding = .init(m, obj._shape); - // log.warn("load {} with sharding: {}", .{ obj, sharding }); - log.warn(" - {s} - sharding: {any} - {s}", .{ prefix, sharding, sharding.getShardingAttr().constSlice() }); + // log.debug(" - {s} - sharding: {any} - {s}", .{ prefix, sharding, sharding.getShardingAttr().constSlice() }); const data_slice = buf_with_metadata._data[0..buf_with_metadata.shape().byteSize()]; obj.* = try zml.Buffer.from(platform, sharding, data_slice, .{}); diff --git a/zml/buffer.zig b/zml/buffer.zig index d4c68dd5..0b240ff6 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -191,7 +191,12 @@ pub const Buffer = struct { /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromSlice(platform: Platform, mesh: Mesh, dimz: anytype, s: anytype) !Buffer { - const sharding: Sharding = .init(mesh, .init(dimz, DataType.fromSliceElementType(s))); + const sh: Shape = if (comptime @TypeOf(dimz) == Shape) + dimz + else + Shape.init(dimz, DataType.fromSliceElementType(s)); + + const sharding: Sharding = .init(mesh, sh); return from(platform, sharding, std.mem.sliceAsBytes(s), .{}); } diff --git a/zml/module.zig b/zml/module.zig index db5dca33..878331e5 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -40,15 +40,16 @@ pub fn pushMesh(mesh: Mesh) void { } ctx._current_mesh = mesh; - log.info("Pushed mesh: {}", .{mesh}); + // log.info("Pushed mesh: {}", .{mesh}); } pub fn popMesh() void { const ctx = CompilationContext.current(); if (ctx._current_mesh) |m| { + _ = m; // autofix ctx._current_mesh = null; - log.info("Poped mesh: {}", .{m}); + // log.info("Poped mesh: {}", .{m}); } else { stdx.debug.panic("No mesh to pop, main mesh: {}", .{ctx._main_mesh}); } @@ -461,7 +462,7 @@ pub const CompilationContext = struct { self.addOutputMemoryKindAttributes(res_attrs, fn_res_output_memory_kind); } - log.warn("{s} add sharding attributes: {} {any}", .{ opts.name, mesh, input_shapes.items }); + // log.warn("{s} add sharding attributes: {} {any}", .{ opts.name, mesh, input_shapes.items }); self.addShardingAttributes(mesh, arg_attrs, res_attrs, input_shapes.items, input_meshes.items, fn_res_shapes); const mlir_fn = dialect.func.func(self.mlirCtx(), .{ diff --git a/zml/ops.zig b/zml/ops.zig index 14f9f959..f22c89ca 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -647,44 +647,40 @@ pub fn allReduce(input: Tensor, axis: anytype, mesh: Mesh) Tensor { return res; } -pub fn allGather(input: Tensor, axis: anytype, mesh: Mesh) Tensor { - if (mesh.axis(axis) <= 1) return input; +pub fn allGather(input: Tensor, gather_dim: anytype, mesh: Mesh) Tensor { + const gather_ax = input.axis(gather_dim); + const partition_spec = input.shape().partition(gather_ax); + + if (partition_spec != .axis) { + log.debug("Skipping all-gather on {any} as it is not sharded.", .{input}); + return input; + } + const mesh_axis_tag = partition_spec.toTag(); + + if (mesh.topology.hasTag(mesh_axis_tag) == null or mesh.axis(mesh_axis_tag) <= 1) { + return input; + } + const axis_size = mesh.axis(mesh_axis_tag); + + var res_shape = input.shape(); + const old_dim_size = res_shape.dim(gather_ax); + res_shape = res_shape.set(gather_ax, old_dim_size * axis_size); + res_shape._partitioning.set(gather_ax, .replicated); const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlirCtx(); const channel_id = ctx.getNextChannelId(); - const mesh_axis_tag = mesh.topology.tag(axis); - - var res_shape = input.shape(); - const axis_size = mesh.axis(axis); - - var gather_dim: ?u3 = null; - for (0..res_shape.rank()) |i| { - const part_spec = res_shape.partition(i); - if (part_spec == .axis and std.mem.eql(u8, std.mem.span(part_spec.toTag()), std.mem.span(mesh_axis_tag))) { - stdx.debug.assert(gather_dim == null, "Mesh axis '{s}' is used to partition multiple tensor dimensions.", .{mesh_axis_tag}); - gather_dim = @intCast(i); - } - } - - const gather_dim_unwrapped = gather_dim orelse { - return input; - }; - - const old_dim_size = res_shape.dim(gather_dim_unwrapped); - res_shape = res_shape.set(gather_dim_unwrapped, old_dim_size * axis_size); - res_shape = res_shape.withReplicatedPartitioning(); const op = mlir.Operation.make(mlir_ctx, "stablehlo.all_gather", .{ .operands = &.{input.value()}, .results = &.{mlirx.tensorType(mlir_ctx, res_shape)}, .attributes = &.{ - .{ "all_gather_dim", .int(mlir_ctx, .i64, gather_dim_unwrapped) }, + .{ "all_gather_dim", .int(mlir_ctx, .i64, gather_ax) }, .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, .{ "use_global_device_ids", .unit(mlir_ctx) }, }, - .location = ctx.location(@src(), "all_gather({_}, {any})", .{ input, axis }), + .location = ctx.location(@src(), "all_gather({_}, {any})", .{ input, gather_dim }), .verify = true, }); @@ -740,100 +736,6 @@ pub fn reduceScatter(input: Tensor, axis: anytype, mesh: Mesh) Tensor { return Tensor._result(res_shape, op.result(0)); } -pub fn allGather2(input: Tensor, axis: anytype, mesh: Mesh) Tensor { - if (mesh.axis(axis) <= 1) return input; - - const ctx = CompilationContext.current(); - const mlir_ctx = ctx.mlirCtx(); - const channel_id = ctx.getNextChannelId(); - const mesh_axis_tag = mesh.topology.tag(axis); - - var res_shape = input.shape(); - const axis_size = mesh.axis(axis); - - var gather_dim: ?u3 = null; - for (0..res_shape.rank()) |i| { - const part_spec = res_shape.partition(i); - if (part_spec == .axis and std.mem.eql(u8, std.mem.span(part_spec.toTag()), std.mem.span(mesh_axis_tag))) { - stdx.debug.assert(gather_dim == null, "Mesh axis '{s}' is used to partition multiple tensor dimensions.", .{mesh_axis_tag}); - gather_dim = @intCast(i); - } - } - - const gather_dim_unwrapped = gather_dim orelse { - // If the sharding axis is not found on the tensor, it's a no-op. - return input; - }; - - const old_dim_size = res_shape.dim(gather_dim_unwrapped); - res_shape = res_shape.set(gather_dim_unwrapped, old_dim_size * axis_size); - res_shape = res_shape.withReplicatedPartitioning(); - - const op = mlir.Operation.make(mlir_ctx, "stablehlo.all_gather", .{ - .operands = &.{input.value()}, - .results = &.{mlirx.tensorType(mlir_ctx, res_shape)}, - .attributes = &.{ - .{ "all_gather_dim", .int(mlir_ctx, .i64, gather_dim_unwrapped) }, - .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, - .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, - .{ "use_global_device_ids", .unit(mlir_ctx) }, - }, - .location = ctx.location(@src(), "all_gather({_}, {any})", .{ input, axis }), - .verify = true, - }); - - return Tensor._result(res_shape, op.result(0)); -} - -pub fn reduceScatter2(input: Tensor, axis: anytype, mesh: Mesh) Tensor { - if (mesh.axis(axis) <= 1) return input; - - const ctx = CompilationContext.current(); - const mlir_ctx = ctx.mlirCtx(); - const channel_id = ctx.getNextChannelId(); - const mesh_axis_tag = mesh.topology.tag(axis); - - var res_shape = input.shape(); - const axis_size = mesh.axis(axis); - - var scatter_dim: ?u3 = null; - for (0..res_shape.rank()) |i| { - if (input.shape().partition(i) == .replicated) { - if (@rem(res_shape.dim(i), axis_size) == 0) { - scatter_dim = @intCast(i); - break; - } - } - } - const scatter_dim_unwrapped = scatter_dim orelse stdx.debug.panic("Could not find a suitable replicated dimension to scatter on for tensor {}", .{input.shape()}); - - const old_dim_size = res_shape.dim(scatter_dim_unwrapped); - stdx.debug.assert(@rem(old_dim_size, axis_size) == 0, "Reduce-scatter dimension size {} must be divisible by mesh axis size {}", .{ old_dim_size, axis_size }); - - res_shape = res_shape.set(scatter_dim_unwrapped, @divExact(old_dim_size, axis_size)); - - const new_partition_spec: Shape.PartitionSpec = .{ .axis = mesh_axis_tag }; - res_shape._partitioning.set(scatter_dim_unwrapped, new_partition_spec); - - const body_block, _ = ctx.makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ Tensor.scalar(0, input.dtype()), Tensor.scalar(0, input.dtype()) }); - - const op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_scatter", .{ - .operands = &.{input.value()}, - .results = &.{mlirx.tensorType(mlir_ctx, res_shape)}, - .blocks = &.{body_block}, - .attributes = &.{ - .{ "scatter_dimension", .int(mlir_ctx, .i64, scatter_dim_unwrapped) }, - .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, - .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, - .{ "use_global_device_ids", .unit(mlir_ctx) }, - }, - .location = ctx.location(@src(), "reduceScatter({_}, {any})", .{ input, axis }), - .verify = true, - }); - - return Tensor._result(res_shape, op.result(0)); -} - fn getSpmdReplicaGroups(ctx: mlir.Context, mesh: Mesh) mlir.Attribute { const allocator = CompilationContext.current().allocator(); var replica_group_data = std.ArrayList(i64).init(allocator); From a6b5e550e370585b4b0ae20f6981c7f25990a714 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 4 Jul 2025 16:14:42 +0200 Subject: [PATCH 27/51] sharding output --- examples/llama/llama.zig | 21 ++++++++++++++------- zml/ops.zig | 25 +++++++++++++++++++++---- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index c824ef12..392939ec 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -215,12 +215,8 @@ pub const Llama = struct { var tagged_embeddings = embeddings.withTags(.{ .s, .d }); if (tokens_.dim(.s) > 1) { - // prefill: re-shard the replicated embedding output to be sharded along the sequence axis. return tagged_embeddings.withSharding(.{ .s = .s }); } else { - // decode: the output of gather is already replicated. We just ensure the - // sharding annotation is explicitly set to replicated for clarity - // and correctness downstream. return tagged_embeddings.replicated(); } } @@ -293,7 +289,14 @@ pub const Mlp = struct { const gate_out = zml.call(self.gate_proj, .forward, .{x}); const up_out = zml.call(self.up_proj, .forward, .{x}); const output = gate_out.silu().mul(up_out); - return zml.call(self.down_proj, .forward, .{output}); + const down_proj_partial = zml.call(self.down_proj, .forward, .{output}); + + const down_proj_final = if (x.dim(.s) > 1) + down_proj_partial.withSharding(.{ .s = .s }) + else + down_proj_partial; + + return down_proj_final; } }; @@ -360,12 +363,16 @@ pub const SelfAttn = struct { } else null; const attn_output = zml.nn.sdpa(q, k_cached, v_cached, .{ .attn_mask = attn_mask, .allow_cudnn = true }); - const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); const sharded_output = zml.call(self.o_proj, .forward, .{attn}); - return .{ sharded_output, new_kv_cache }; + const output_final = if (x.dim(.s) > 1) + sharded_output.withSharding(.{ .s = .s }) + else + sharded_output; + + return .{ output_final, new_kv_cache }; } }; diff --git a/zml/ops.zig b/zml/ops.zig index f22c89ca..b00ee44d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -617,8 +617,10 @@ pub fn sort( return res; } -pub fn allReduce(input: Tensor, axis: anytype, mesh: Mesh) Tensor { - if (mesh.axis(axis) <= 1) return input; +pub fn allReduce(input: Tensor, reduce_axis_name: anytype, mesh: Mesh) Tensor { + const reduce_mesh_axis_tag = mesh.topology.tag(reduce_axis_name); + + if (mesh.axis(reduce_axis_name) <= 1) return input; const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlirCtx(); @@ -638,12 +640,27 @@ pub fn allReduce(input: Tensor, axis: anytype, mesh: Mesh) Tensor { .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, .{ "use_global_device_ids", .unit(mlir_ctx) }, }, - .location = ctx.location(@src(), "allReduce({_}, {any})", .{ input, axis }), + .location = ctx.location(@src(), "allReduce({_}, {any})", .{ input, reduce_mesh_axis_tag }), .verify = true, }); var res = fromMlirOperationWithTags(op, input); - res._shape = res._shape.withReplicatedPartitioning(); + + var new_partitioning = input.shape()._partitioning; + var axis_was_sharded = false; + + for (new_partitioning.slice(), 0..) |spec, i| { + _ = i; // autofix + if (spec.* == .axis and std.mem.eql(u8, std.mem.span(spec.*.toTag()), std.mem.span(reduce_mesh_axis_tag))) { + spec.* = .replicated; // This dimension is no longer sharded + axis_was_sharded = true; + } + } + + if (axis_was_sharded) { + res._shape._partitioning = new_partitioning; + } + return res; } From 3d179606ad36d68c1222affd12e10cf25eb43b10 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 7 Jul 2025 17:21:16 +0200 Subject: [PATCH 28/51] wip --- examples/llama/llama.zig | 149 +++++++++++++++--------------- examples/partitioning/BUILD.bazel | 10 ++ examples/partitioning/overlap.zig | 141 ++++++++++++++++++++++++++++ mlir/mlir.zig | 10 +- zml/mlirx.zig | 31 +++++++ zml/ops.zig | 57 ++++++------ 6 files changed, 290 insertions(+), 108 deletions(-) create mode 100644 examples/partitioning/overlap.zig diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 392939ec..d7b957f9 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -15,25 +15,8 @@ pub const Linear = struct { bias: ?Tensor = null, pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dotGeneral(self.weight.convert(x.dtype()), &.{.{ -1, -1 }}, &.{}); - - if (y.shape().tag(-1) == zml.Shape.TagUnknown) { - y._shape._tags.set(y.rank() - 1, x.shape().tag(-1)); - } - - const weight_sharding = self.weight.shape()._partitioning; - const out_dim_idx = y.rank() - 1; - - if (weight_sharding.get(out_dim_idx) != .replicated and weight_sharding.get(out_dim_idx) != .unknown) { - const mesh = self.weight._mesh orelse zml.currentMesh(); - const mesh_axis = weight_sharding.get(out_dim_idx).toTag(); - y = zml.ops.allReduce(y, mesh_axis, mesh); - } else if (weight_sharding.get(0) != .replicated and weight_sharding.get(0) != .unknown) { - var new_sharding = zml.Shape.PartitionArray.init(0) catch unreachable; - new_sharding.appendNTimesAssumeCapacity(.replicated, y.rank()); - new_sharding.set(out_dim_idx, weight_sharding.get(0)); - y._shape._partitioning = new_sharding; - } + const x_dtype = x.dtype(); + var y = x.dotGeneral(self.weight.convert(x_dtype), &.{.{ -1, -1 }}, &.{}); if (self.bias) |bias| { return y.add(bias.broadcastLeft(y.shape())); @@ -67,7 +50,7 @@ pub const LlamaLM = struct { max_seq_len: usize, }; - lm_head: ?zml.nn.Linear, + lm_head: ?Linear, model: Llama, // Runtime configuration, not part of the model's weights/arch @@ -110,29 +93,36 @@ pub const LlamaLM = struct { pub fn sampleTokens( self: LlamaLM, - lm_head_: ?zml.nn.Linear, + lm_head_: ?Linear, out_: Tensor, rng: Tensor.Rng, opts: zml.nn.SamplingStrategy, ) struct { Tensor, Tensor.Rng } { + // The body of this function from the previous step is correct. + // We only need to fix the signature. const vocab_mesh = if (lm_head_) |lm_head| lm_head.weight.mesh() else self.model.embed_tokens.weight.mesh(); zml.pushMesh(vocab_mesh); defer zml.popMesh(); const out = out_.withPartialTags(.{ .s, .d }); - var logits_sharded = blk: { + var logits = blk: { if (lm_head_) |lm_head| { - break :blk zml.call(lm_head, .forward, .{out}); + const partial_logits = zml.call(lm_head, .forward, .{out}); + break :blk zml.ops.allGather(partial_logits, .model, vocab_mesh); } else { - break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); + const embed_table = self.model.embed_tokens.weight.withTags(.{ .voc, .d }); + const partial_logits = out.dotGeneral( + embed_table.transpose(.{ .d, .voc }), + &.{.{ 1, 0 }}, + &.{}, + ); + break :blk zml.ops.allReduce(partial_logits, .model, vocab_mesh, 3); } }; - if (logits_sharded.shape().hasTag(.voc) == null) - logits_sharded = logits_sharded.rename(.{ .d = .voc }); - - const logits = zml.ops.allGather(logits_sharded, .voc, vocab_mesh); + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); return .{ next_tokens, new_rng }; @@ -193,7 +183,7 @@ pub const Llama = struct { .withSharding(.{ .d = .replicated, .voc = .model }); for (self.layers) |*layer| { - layer.init(config, compute_mesh, replicated_sharding); + layer.init(config, vocab_mesh); } } @@ -212,11 +202,14 @@ pub const Llama = struct { pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { const embeddings = zml.call(embed_tokens_, .forward, .{tokens_}); - var tagged_embeddings = embeddings.withTags(.{ .s, .d }); + var tagged_embeddings = embeddings.withPartialTags(.{.d}); - if (tokens_.dim(.s) > 1) { - return tagged_embeddings.withSharding(.{ .s = .s }); - } else { + // Sequence parallelism shards this dimension. + // Tensor parallelism replicates it. + // We are doing Tensor Parallelism, so it should be replicated. + if (tokens_.dim(.s) > 1) { // Prefill + return tagged_embeddings.withSharding(.{ .s = .replicated }); + } else { // Decode return tagged_embeddings.replicated(); } } @@ -228,11 +221,11 @@ pub const TransformerLayer = struct { post_attention_layernorm: RmsNorm, mlp: Mlp, - pub fn init(self: *TransformerLayer, config: LlamaLM.Config, mesh: zml.Mesh, sharding: anytype) void { - self.input_layernorm.init(config, mesh, sharding); - self.self_attn.init(config, mesh, sharding); - self.post_attention_layernorm.init(config, mesh, sharding); - self.mlp.init(mesh, sharding); + pub fn init(self: *TransformerLayer, config: LlamaLM.Config, mesh: zml.Mesh) void { + self.input_layernorm.init(config, mesh, .{ .d = .replicated }); + self.self_attn.init(config, mesh); + self.post_attention_layernorm.init(config, mesh, .{ .d = .replicated }); + self.mlp.init(mesh); } pub fn forward( @@ -245,14 +238,14 @@ pub const TransformerLayer = struct { const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - const sharded_delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - - const x1 = x0.add(sharded_delta0); + // self_attn now returns the final, reduced tensor. + const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); + const x1 = x0.add(delta0); const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - const sharded_delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); - - const x2 = x1.add(sharded_delta1); + // mlp now returns the final, reduced tensor. + const delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); + const x2 = x1.add(delta1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; } @@ -275,14 +268,17 @@ pub const RmsNorm = struct { }; pub const Mlp = struct { - up_proj: zml.nn.Linear, - gate_proj: zml.nn.Linear, - down_proj: zml.nn.Linear, + up_proj: Linear, + gate_proj: Linear, + down_proj: Linear, - pub fn init(self: *Mlp, mesh: zml.Mesh, sharding: anytype) void { - self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(sharding); - self.up_proj.weight = self.up_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(sharding); - self.down_proj.weight = self.down_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(sharding); + pub fn init(self: *Mlp, mesh: zml.Mesh) void { + // Column-Parallel: Shard on the output/column dimension + self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(.{ .hidden = .model }); + self.up_proj.weight = self.up_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(.{ .hidden = .model }); + + // Row-Parallel: Shard on the INPUT/column dimension + self.down_proj.weight = self.down_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(.{ .hidden = .model }); } pub fn forward(self: Mlp, x: Tensor) Tensor { @@ -290,26 +286,26 @@ pub const Mlp = struct { const up_out = zml.call(self.up_proj, .forward, .{x}); const output = gate_out.silu().mul(up_out); const down_proj_partial = zml.call(self.down_proj, .forward, .{output}); + const down_proj_reduced = zml.ops.allReduce(down_proj_partial, .model, self.down_proj.weight.mesh(), 1); - const down_proj_final = if (x.dim(.s) > 1) - down_proj_partial.withSharding(.{ .s = .s }) + // FIX: Re-shard the replicated output to match the input's sharding scheme for the residual connection. + return if (x.dim(.s) > 1) + down_proj_reduced.withSharding(.{ .s = .s }) else - down_proj_partial; - - return down_proj_final; + down_proj_reduced; } }; pub const SelfAttn = struct { - q_proj: zml.nn.Linear, - k_proj: zml.nn.Linear, - v_proj: zml.nn.Linear, - o_proj: zml.nn.Linear, + q_proj: Linear, + k_proj: Linear, + v_proj: Linear, + o_proj: Linear, num_heads: i64 = undefined, num_kv_heads: i64 = 0, rope_opts: zml.nn.RopeOpts = undefined, - pub fn init(self: *SelfAttn, config: LlamaLM.Config, mesh: zml.Mesh, sharding: anytype) void { + pub fn init(self: *SelfAttn, config: LlamaLM.Config, mesh: zml.Mesh) void { self.num_heads = @intCast(config.num_attention_heads); self.num_kv_heads = @intCast(config.num_key_value_heads); self.rope_opts = .{ @@ -318,10 +314,10 @@ pub const SelfAttn = struct { .scaling = config.rope_scaling, }; - self.q_proj.weight = self.q_proj.weight.withTags(.{ .q_hidden, .d }).withMesh(mesh).withSharding(sharding); - self.k_proj.weight = self.k_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(sharding); - self.v_proj.weight = self.v_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(sharding); - self.o_proj.weight = self.o_proj.weight.withTags(.{ .d, .o_hidden }).withMesh(mesh).withSharding(sharding); + self.q_proj.weight = self.q_proj.weight.withTags(.{ .q_hidden, .d }).withMesh(mesh).withSharding(.{ .q_hidden = .model }); + self.k_proj.weight = self.k_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); + self.v_proj.weight = self.v_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); + self.o_proj.weight = self.o_proj.weight.withTags(.{ .d, .o_hidden }).withMesh(mesh).withSharding(.{ .o_hidden = .model }); } pub fn forward( @@ -331,15 +327,14 @@ pub const SelfAttn = struct { kv_cache: KvCache, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - const mesh = self.q_proj.weight.mesh(); const q_proj_out = zml.call(self.q_proj, .forward, .{x}); const k_proj_out = zml.call(self.k_proj, .forward, .{x}); const v_proj_out = zml.call(self.v_proj, .forward, .{x}); - var q = q_proj_out.withPartialTags(.{.q_hidden}).splitAxis(.q_hidden, .{ .h = self.num_heads, .hd = .auto }); - var k = k_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }); - var v = v_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }); + var q = q_proj_out.withPartialTags(.{.q_hidden}).splitAxis(.q_hidden, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{ .h = .model }); + var k = k_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); + var v = v_proj_out.withPartialTags(.{.kv_hidden}).splitAxis(.kv_hidden, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{ .h = .model }); const pos_index = blk: { const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); @@ -354,8 +349,8 @@ pub const SelfAttn = struct { const new_kv_cache = kv_cache.update(k, v, token_index); - const k_cached = zml.ops.allGather(new_kv_cache.keys(), .k, mesh); - const v_cached = zml.ops.allGather(new_kv_cache.values(), .k, mesh); + const k_cached = new_kv_cache.keys(); + const v_cached = new_kv_cache.values(); const attn_mask = if (x.dim(.s) > 1) blk: { var mask = zml.nn.causalAttnMask(.{ .q = k_cached.dim(.k), .k = k_cached.dim(.k) }, x.dtype(), null); @@ -363,16 +358,18 @@ pub const SelfAttn = struct { } else null; const attn_output = zml.nn.sdpa(q, k_cached, v_cached, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); - const sharded_output = zml.call(self.o_proj, .forward, .{attn}); + const o_proj_partial = zml.call(self.o_proj, .forward, .{attn}); + const sharded_output_reduced = zml.ops.allReduce(o_proj_partial, .model, self.o_proj.weight.mesh(), 2); - const output_final = if (x.dim(.s) > 1) - sharded_output.withSharding(.{ .s = .s }) + const sharded_output = if (x.dim(.s) > 1) + sharded_output_reduced.withSharding(.{ .s = .s }) else - sharded_output; + sharded_output_reduced; - return .{ output_final, new_kv_cache }; + return .{ sharded_output, new_kv_cache }; } }; diff --git a/examples/partitioning/BUILD.bazel b/examples/partitioning/BUILD.bazel index 00ca1b7d..02516397 100644 --- a/examples/partitioning/BUILD.bazel +++ b/examples/partitioning/BUILD.bazel @@ -19,3 +19,13 @@ zig_cc_binary( "@zml//zml", ], ) + +zig_cc_binary( + name = "overlap", + main = "overlap.zig", + deps = [ + "@zml//async", + "@zml//stdx", + "@zml//zml", + ], +) diff --git a/examples/partitioning/overlap.zig b/examples/partitioning/overlap.zig new file mode 100644 index 00000000..87591c5a --- /dev/null +++ b/examples/partitioning/overlap.zig @@ -0,0 +1,141 @@ +const std = @import("std"); +const zml = @import("zml"); +const stdx = @import("stdx"); +const asynk = @import("async"); +const testing = @import("std").testing; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/overlap"); + +/// This module simulates a compute-heavy workload. +const ComputeIntensiveModule = struct { + iterations: zml.Tensor, + + const LoopContext = struct { + iterations: zml.Tensor, + }; + + fn loopCond(ctx: LoopContext, work: zml.Tensor, i: zml.Tensor) zml.Tensor { + _ = work; // The `work` tensor is just a loop-carried variable. + return i.cmp(.LT, ctx.iterations); + } + + fn loopBody(_: LoopContext, work: zml.Tensor, i: zml.Tensor) [2]zml.Tensor { + const prev_work = work; // Shape: {rows, cols}, {M, M} + + // Standard matrix multiplication: work @ work + // Contracts axis 1 (.cols) of `work` with axis 0 (.rows) of `work`. + const work_squared = work.dotGeneral(work, &.{.{ 1, 0 }}, &.{}); + + // The output of dotGeneral is untagged, so we must add the tags back. + const work_squared_tagged = work_squared.withTags(.{ .rows, .cols }); + + const work_activated = work_squared_tagged.tanh(); + const work_residual = work_activated.add(prev_work); + + const next_i = i.addConstant(1); + return .{ work_residual, next_i }; + } + + /// Runs a loop of computations for a dynamic number of iterations. + pub fn forward(self: @This(), work_init: zml.Tensor) zml.Tensor { + const i_init = zml.Tensor.scalar(0, .i32); + const loop_context = LoopContext{ .iterations = self.iterations }; + + // We tag `work_init` here to ensure the loop variable has the correct tags + // for the first iteration. The loop body is responsible for maintaining the tags. + const tagged_work_init = work_init.withTags(.{ .rows, .cols }); + + const final_loop_vars = zml.ops.while_(loopCond, loopBody, loop_context, .{ tagged_work_init, i_init }); + + // The result of the while loop is a tuple. We only care about the first + // element, which is the processed tensor. + return final_loop_vars[0]; + } +}; + +const OverlapModel = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + const mesh = a.mesh(); + const a_reduced = zml.ops.allReduce(a, .x, mesh, null); + + const b_f32 = b.convert(.f32); + + const b_squared = b_f32.dotGeneral(b_f32.transpose(.{ .n, .m }), &.{.{ 1, 0 }}, &.{}); + + const num_iterations = a.choose(.{ .m = 0, .n = 0 }).convert(.i32); + const compute_module: ComputeIntensiveModule = .{ .iterations = num_iterations }; + const b_processed = zml.call(compute_module, .forward, .{b_squared}); + + const a_scalar = a_reduced.sum(0).sum(1).asScalar(); + const result = b_processed.add(a_scalar.convert(.f32).broad(b_processed.shape())); + + return result; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{ .cpu = .{ .cpu_device_count = 8 } }).withCompilationOptions(.{ + .sharding_enabled = true, + .xla_dump_to = "/tmp/zml/collectives_overlap", + .xla_dump_fusion_visualization = true, + }); + + const num_devices = platform.getDevices().len; + if (num_devices < 2) { + log.warn("This example requires at least 2 devices to demonstrate collectives. Exiting.", .{}); + return; + } + context.printAvailablePlatforms(platform); + + const mesh = zml.Mesh.init(.{ .x = num_devices }); + log.info("Using mesh: {}", .{mesh}); + + const M: i64 = 8192; + const N: i64 = 256; + const K: i64 = 1024; + + const shape_a = zml.Shape.init(.{ .m = M, .n = K }, .i32).withPartitioning(.{ .n = .x }); + const shape_b = zml.Shape.init(.{ .m = M, .n = N }, .i32).withPartitioning(.{ .n = .x }); + + log.info("Compiling the OverlapModel...", .{}); + const mod = try zml.compileFn(allocator, OverlapModel.forward, .{ shape_a, shape_b }, mesh, platform); + defer mod.deinit(); + log.info("Compilation complete.", .{}); + + log.info("Preparing input data...", .{}); + const a_data_slice = try zml.slice.arange(allocator, shape_a, .{}); + defer allocator.free(a_data_slice); + const a_buffer = try zml.Buffer.from(platform, .init(mesh, shape_a), a_data_slice, .{}); + defer a_buffer.deinit(); + + const b_data_slice = try zml.slice.arange(allocator, shape_b, .{}); + defer allocator.free(b_data_slice); + const b_buffer = try zml.Buffer.from(platform, .init(mesh, shape_b), b_data_slice, .{}); + defer b_buffer.deinit(); + log.info("Input data prepared and transferred to device.", .{}); + + log.info("Executing the model...", .{}); + var timer = try std.time.Timer.start(); + const result_buffer = mod.call(.{ a_buffer, b_buffer }); + defer result_buffer.deinit(); + const result_data = try result_buffer.toHost(allocator); // Await completion for timing + defer allocator.free(result_data); + log.info("Execution finished in {d:.3}ms.", .{@as(f32, @floatFromInt(timer.read())) / 1e6}); + log.info("Result (first 8 elements): {any}", .{result_data[0..@min(result_data.len, 8)]}); +} diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 10d26c93..511452dc 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1273,7 +1273,11 @@ pub const Type = struct { } pub fn tensor(dimensions: []const i64, elem_type: Type) Type { - return RankedTensorType.init(dimensions, elem_type).asType(); + return RankedTensorType.init(dimensions, elem_type, null).asType(); + } + + pub fn tensorWithEncoding(dimensions: []const i64, elem_type: Type, encoding: ?Attribute) Type { + return RankedTensorType.init(dimensions, elem_type, encoding).asType(); } }; @@ -1516,12 +1520,12 @@ pub const RankedTensorType = struct { pub const eql = Type.eqlAny(RankedTensorType); pub const format = helpers.format(Type, c.mlirTypePrint); - pub fn init(dimensions: []const i64, elemType: Type) RankedTensorType { + pub fn init(dimensions: []const i64, elemType: Type, encoding: ?Attribute) RankedTensorType { return .{ ._inner = c.mlirRankedTensorTypeGet( @intCast(dimensions.len), @ptrCast(dimensions.ptr), elemType._inner, - c.mlirAttributeGetNull(), + if (encoding) |e| e._inner else c.mlirAttributeGetNull(), ) }; } diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 28f40383..3f9b45dc 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -3,6 +3,9 @@ const std = @import("std"); const mlir = @import("mlir"); const dtype = @import("dtype.zig"); +const CompilationContext = @import("module.zig").CompilationContext; +const Sharding = @import("partitioning.zig").Sharding; +const Mesh = @import("partitioning.zig").Mesh; const Shape = @import("shape.zig").Shape; const mlirx = @This(); @@ -12,6 +15,34 @@ pub fn tensorType(ctx: mlir.Context, sh: Shape) mlir.Type { return .tensor(sh.dims(), mlirx.Type.fromDType(ctx, sh.dtype())); } +pub fn tensorWithEncoding(ctx: mlir.Context, sh: Shape, encoding: ?mlir.Attribute) mlir.Type { + return .tensorWithEncoding(sh.dims(), mlirx.Type.fromDType(ctx, sh.dtype()), encoding); +} + +pub fn tensorTypeWithEncoding(ctx: mlir.Context, mesh: Mesh, sh: Shape, allocator: std.mem.Allocator) mlir.Type { + const sharding_attr = blk: { + // No sharding attribute needed for single-device computations. + if (mesh.isSinglePartition()) break :blk null; + + var sharding_builder = std.ArrayList(u8).init(allocator); + defer sharding_builder.deinit(); + + const sharding = Sharding.init(mesh, sh); + sharding.writeShardingRepresentation(sharding_builder.writer()) catch unreachable; + + var sharding_string: []const u8 = "{replicated}"; + if (sharding_builder.items.len > 0) { + sharding_string = sharding_builder.items; + } + + break :blk mlir.DictionaryAttribute.init(ctx, &.{ + mlir.NamedAttribute.named(ctx, "sharding", mlir.Attribute.string(ctx, sharding_string)), + }).asAttr(); + }; + + return tensorWithEncoding(ctx, sh, sharding_attr); +} + pub fn denseElementAttrType(dt: dtype.DataType) ?mlir.DenseElementsAttributeTypes { return switch (dt) { .bool => .bool, diff --git a/zml/ops.zig b/zml/ops.zig index b00ee44d..6b893265 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -14,6 +14,7 @@ const Mesh = @import("partitioning.zig").Mesh; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; +const Sharding = @import("partitioning.zig").Sharding; const EnumLiteral = @TypeOf(.enum_literal); const dialect = struct { @@ -617,51 +618,49 @@ pub fn sort( return res; } -pub fn allReduce(input: Tensor, reduce_axis_name: anytype, mesh: Mesh) Tensor { - const reduce_mesh_axis_tag = mesh.topology.tag(reduce_axis_name); +pub fn allReduce(input: Tensor, reduce_axis_name: anytype, mesh: Mesh, channel_id: ?u64) Tensor { + // The reduce_axis_name is implicitly handled by the mesh and replica_groups. + _ = reduce_axis_name; - if (mesh.axis(reduce_axis_name) <= 1) return input; + // On a single device, all_reduce is an identity operation. + if (mesh.isSinglePartition()) { + return input; + } const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlirCtx(); - const channel_id = ctx.getNextChannelId(); + + // FIX: Use the optional channel ID or get a new one. + const actual_channel_id = if (channel_id) |id| id else ctx.getNextChannelId(); const body_block, _ = ctx.makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ Tensor.scalar(0, input.dtype()), Tensor.scalar(0, input.dtype()), }); + const result_shape = input.shape().withReplicatedPartitioning(); + const result_sharding = Sharding.init(mesh, result_shape); + const sharding_attr = ctx.getShardingAttr(result_sharding); + + const op_attrs: [4]mlir.AttrTuple = .{ + .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, + .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(actual_channel_id), 1)) }, + .{ "use_global_device_ids", .unit(mlir_ctx) }, + .{ "mhlo.sharding", sharding_attr }, + }; + + const result_type = mlirx.tensorType(mlir_ctx, result_shape); + const op = mlir.Operation.make(mlir_ctx, "stablehlo.all_reduce", .{ .operands = &.{input.value()}, - .results = &.{input.value().getType()}, + .results = &.{result_type}, .blocks = &.{body_block}, - .attributes = &.{ - .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, - .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(channel_id), 1)) }, - .{ "use_global_device_ids", .unit(mlir_ctx) }, - }, - .location = ctx.location(@src(), "allReduce({_}, {any})", .{ input, reduce_mesh_axis_tag }), + .attributes = &op_attrs, + .location = ctx.location(@src(), "allReduce({_})", .{input}), .verify = true, }); - var res = fromMlirOperationWithTags(op, input); - - var new_partitioning = input.shape()._partitioning; - var axis_was_sharded = false; - - for (new_partitioning.slice(), 0..) |spec, i| { - _ = i; // autofix - if (spec.* == .axis and std.mem.eql(u8, std.mem.span(spec.*.toTag()), std.mem.span(reduce_mesh_axis_tag))) { - spec.* = .replicated; // This dimension is no longer sharded - axis_was_sharded = true; - } - } - - if (axis_was_sharded) { - res._shape._partitioning = new_partitioning; - } - - return res; + return Tensor._result(result_shape, op.result(0)); } pub fn allGather(input: Tensor, gather_dim: anytype, mesh: Mesh) Tensor { From 933445a76b4bf7fbd66f2945361c03605949862a Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 7 Jul 2025 17:43:20 +0200 Subject: [PATCH 29/51] fix 70B --- examples/llama/llama.zig | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index d7b957f9..98c20fe2 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -66,7 +66,7 @@ pub const LlamaLM = struct { // Shard the final output layer across all devices if (self.lm_head) |*lm_head| { lm_head.weight = lm_head.weight - .withTags(.{ .d, .voc }) + .withTags(.{ .voc, .d }) .withMesh(vocab_mesh) .withSharding(.{ .voc = .model }); } @@ -108,7 +108,12 @@ pub const LlamaLM = struct { var logits = blk: { if (lm_head_) |lm_head| { - const partial_logits = zml.call(lm_head, .forward, .{out}); + var partial_logits = zml.call(lm_head, .forward, .{out}); + + if (partial_logits.shape().hasTag(.voc) == null) { + partial_logits = partial_logits.rename(.{ .d = .voc }); + } + break :blk zml.ops.allGather(partial_logits, .model, vocab_mesh); } else { const embed_table = self.model.embed_tokens.weight.withTags(.{ .voc, .d }); From 585b51db442e5d4f9c1ba4530f02beaa77c4fa58 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 7 Jul 2025 17:44:33 +0200 Subject: [PATCH 30/51] gather right dim --- examples/llama/llama.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 98c20fe2..de68e042 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -114,7 +114,7 @@ pub const LlamaLM = struct { partial_logits = partial_logits.rename(.{ .d = .voc }); } - break :blk zml.ops.allGather(partial_logits, .model, vocab_mesh); + break :blk zml.ops.allGather(partial_logits, .voc, vocab_mesh); } else { const embed_table = self.model.embed_tokens.weight.withTags(.{ .voc, .d }); const partial_logits = out.dotGeneral( From beaf19824bda0c382d63c9bee06b2833ce37edee Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 7 Jul 2025 18:24:05 +0200 Subject: [PATCH 31/51] try --- examples/llama/llama.zig | 54 +++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index de68e042..629260e1 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -93,13 +93,11 @@ pub const LlamaLM = struct { pub fn sampleTokens( self: LlamaLM, - lm_head_: ?Linear, + lm_head_: ?Linear, // This should be the local `Linear` out_: Tensor, rng: Tensor.Rng, opts: zml.nn.SamplingStrategy, ) struct { Tensor, Tensor.Rng } { - // The body of this function from the previous step is correct. - // We only need to fix the signature. const vocab_mesh = if (lm_head_) |lm_head| lm_head.weight.mesh() else self.model.embed_tokens.weight.mesh(); zml.pushMesh(vocab_mesh); defer zml.popMesh(); @@ -241,18 +239,29 @@ pub const TransformerLayer = struct { ) struct { Tensor, KvCache } { stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); - const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); + // --- Attention Block --- + const attn_input = zml.call(self.input_layernorm, .forward, .{x0}); + // self_attn will do the q,k,v projections and return the PARTIAL o_proj result. + const attn_partial_output, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ attn_input, token_index, kv_cache }); - // self_attn now returns the final, reduced tensor. - const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - const x1 = x0.add(delta0); + // Start the all_reduce for the attention block. XLA can schedule this to run in the background. + const attn_reduced_output = zml.ops.allReduce(attn_partial_output, .model, self.self_attn.o_proj.weight.mesh(), 1); - const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - // mlp now returns the final, reduced tensor. - const delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); - const x2 = x1.add(delta1); + // --- MLP Block (starts while attn_reduced_output is being computed) --- + // The input to the MLP block depends on the result of the attention block. + const mlp_input = x0.add(attn_reduced_output); + const mlp_norm_input = zml.call(self.post_attention_layernorm, .forward, .{mlp_input}); - return .{ x2.reuseBuffer(x0), updated_kv_cache }; + // The MLP forward pass computes the partial result. + const mlp_partial_output = zml.call(self.mlp, .forward, .{mlp_norm_input}); + + // Start the all_reduce for the MLP block. + const mlp_reduced_output = zml.ops.allReduce(mlp_partial_output, .model, self.mlp.down_proj.weight.mesh(), 2); + + // Final residual connection + const x_final = mlp_input.add(mlp_reduced_output); + + return .{ x_final.reuseBuffer(x0), updated_kv_cache }; } }; @@ -290,14 +299,8 @@ pub const Mlp = struct { const gate_out = zml.call(self.gate_proj, .forward, .{x}); const up_out = zml.call(self.up_proj, .forward, .{x}); const output = gate_out.silu().mul(up_out); - const down_proj_partial = zml.call(self.down_proj, .forward, .{output}); - const down_proj_reduced = zml.ops.allReduce(down_proj_partial, .model, self.down_proj.weight.mesh(), 1); - - // FIX: Re-shard the replicated output to match the input's sharding scheme for the residual connection. - return if (x.dim(.s) > 1) - down_proj_reduced.withSharding(.{ .s = .s }) - else - down_proj_reduced; + // Return the partial result. The all-reduce is now in TransformerLayer. + return zml.call(self.down_proj, .forward, .{output}); } }; @@ -363,18 +366,11 @@ pub const SelfAttn = struct { } else null; const attn_output = zml.nn.sdpa(q, k_cached, v_cached, .{ .attn_mask = attn_mask, .allow_cudnn = true }); - const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); + // Return the partial result. The all-reduce is now in TransformerLayer. const o_proj_partial = zml.call(self.o_proj, .forward, .{attn}); - const sharded_output_reduced = zml.ops.allReduce(o_proj_partial, .model, self.o_proj.weight.mesh(), 2); - - const sharded_output = if (x.dim(.s) > 1) - sharded_output_reduced.withSharding(.{ .s = .s }) - else - sharded_output_reduced; - - return .{ sharded_output, new_kv_cache }; + return .{ o_proj_partial, new_kv_cache }; } }; From 07c58ff185307638ef2f5f784c2644cf21535780 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 8 Jul 2025 11:32:14 +0200 Subject: [PATCH 32/51] wip --- examples/llama/llama.zig | 2 -- examples/partitioning/overlap.zig | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 629260e1..729f0b98 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -299,7 +299,6 @@ pub const Mlp = struct { const gate_out = zml.call(self.gate_proj, .forward, .{x}); const up_out = zml.call(self.up_proj, .forward, .{x}); const output = gate_out.silu().mul(up_out); - // Return the partial result. The all-reduce is now in TransformerLayer. return zml.call(self.down_proj, .forward, .{output}); } }; @@ -368,7 +367,6 @@ pub const SelfAttn = struct { const attn_output = zml.nn.sdpa(q, k_cached, v_cached, .{ .attn_mask = attn_mask, .allow_cudnn = true }); const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); - // Return the partial result. The all-reduce is now in TransformerLayer. const o_proj_partial = zml.call(self.o_proj, .forward, .{attn}); return .{ o_proj_partial, new_kv_cache }; } diff --git a/examples/partitioning/overlap.zig b/examples/partitioning/overlap.zig index 87591c5a..d92ee1e4 100644 --- a/examples/partitioning/overlap.zig +++ b/examples/partitioning/overlap.zig @@ -106,9 +106,9 @@ pub fn asyncMain() !void { const mesh = zml.Mesh.init(.{ .x = num_devices }); log.info("Using mesh: {}", .{mesh}); - const M: i64 = 8192; - const N: i64 = 256; - const K: i64 = 1024; + const M: i64 = 16384; + const N: i64 = 16384; + const K: i64 = 16384; const shape_a = zml.Shape.init(.{ .m = M, .n = K }, .i32).withPartitioning(.{ .n = .x }); const shape_b = zml.Shape.init(.{ .m = M, .n = N }, .i32).withPartitioning(.{ .n = .x }); From 7d3a2d0441d0855bc03d50ea4e47b5f3e1e332a7 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 10 Jul 2025 12:51:29 +0000 Subject: [PATCH 33/51] wip --- examples/partitioning/overlap.zig | 117 ++++++++++++++++-------------- zml/module.zig | 3 + 2 files changed, 67 insertions(+), 53 deletions(-) diff --git a/examples/partitioning/overlap.zig b/examples/partitioning/overlap.zig index d92ee1e4..ab741247 100644 --- a/examples/partitioning/overlap.zig +++ b/examples/partitioning/overlap.zig @@ -13,68 +13,76 @@ const log = std.log.scoped(.@"examples/overlap"); /// This module simulates a compute-heavy workload. const ComputeIntensiveModule = struct { - iterations: zml.Tensor, + // By making this a compile-time constant, we allow the compiler to fully + // unroll the loop, creating a large, static computation graph that is ideal + // for optimization and overlapping. + comptime num_iterations: u32 = 300, - const LoopContext = struct { - iterations: zml.Tensor, - }; - - fn loopCond(ctx: LoopContext, work: zml.Tensor, i: zml.Tensor) zml.Tensor { - _ = work; // The `work` tensor is just a loop-carried variable. - return i.cmp(.LT, ctx.iterations); - } - - fn loopBody(_: LoopContext, work: zml.Tensor, i: zml.Tensor) [2]zml.Tensor { - const prev_work = work; // Shape: {rows, cols}, {M, M} - - // Standard matrix multiplication: work @ work - // Contracts axis 1 (.cols) of `work` with axis 0 (.rows) of `work`. - const work_squared = work.dotGeneral(work, &.{.{ 1, 0 }}, &.{}); - - // The output of dotGeneral is untagged, so we must add the tags back. - const work_squared_tagged = work_squared.withTags(.{ .rows, .cols }); - - const work_activated = work_squared_tagged.tanh(); - const work_residual = work_activated.add(prev_work); - - const next_i = i.addConstant(1); - return .{ work_residual, next_i }; - } - - /// Runs a loop of computations for a dynamic number of iterations. + /// Runs a loop of computations for a fixed number of iterations. pub fn forward(self: @This(), work_init: zml.Tensor) zml.Tensor { - const i_init = zml.Tensor.scalar(0, .i32); - const loop_context = LoopContext{ .iterations = self.iterations }; + var work = work_init.withTags(.{ .rows, .cols }); - // We tag `work_init` here to ensure the loop variable has the correct tags - // for the first iteration. The loop body is responsible for maintaining the tags. - const tagged_work_init = work_init.withTags(.{ .rows, .cols }); + // A simple Zig `for` loop unrolls into a sequence of MLIR operations, + // which can be efficiently scheduled by the XLA compiler. + for (0..self.num_iterations) |_| { + const prev_work = work; - const final_loop_vars = zml.ops.while_(loopCond, loopBody, loop_context, .{ tagged_work_init, i_init }); + // Standard matrix multiplication: work @ work + // Contracts axis 1 (.cols) of `work` with axis 0 (.rows) of `work`. + const work_squared = work.dotGeneral(work, &.{.{ 1, 0 }}, &.{}); - // The result of the while loop is a tuple. We only care about the first - // element, which is the processed tensor. - return final_loop_vars[0]; + // The output of dotGeneral is untagged, so we must add the tags back. + const work_squared_tagged = work_squared.withTags(.{ .rows, .cols }); + + const work_activated = work_squared_tagged.tanh(); + work = work_activated.add(prev_work); + } + return work; } }; const OverlapModel = struct { + // We simulate a model with multiple layers to create a pipeline. + var num_layers: u32 = 4; + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { const mesh = a.mesh(); - const a_reduced = zml.ops.allReduce(a, .x, mesh, null); + const compute_module: ComputeIntensiveModule = .{}; - const b_f32 = b.convert(.f32); + var a_intermediate = a; + // The result of the b computation is carried over. + // It starts as the original `b` input tensor. + var b_result = b.convert(.f32); - const b_squared = b_f32.dotGeneral(b_f32.transpose(.{ .n, .m }), &.{.{ 1, 0 }}, &.{}); + // --- Pipelined Execution Loop --- + for (0..OverlapModel.num_layers) |i| { + // Stage 1: Communication on the 'a' data path. + // The result of the previous layer's communication is reduced. + const a_reduced = zml.ops.allReduce(a_intermediate, .x, mesh, i + 1); - const num_iterations = a.choose(.{ .m = 0, .n = 0 }).convert(.i32); - const compute_module: ComputeIntensiveModule = .{ .iterations = num_iterations }; - const b_processed = zml.call(compute_module, .forward, .{b_squared}); + // Stage 2: Independent Heavy Computation on the 'b' data path. + // This is the heavy part that does not depend on `a_reduced`. + // The XLA compiler can schedule this to run while `allReduce` is in flight. + // The key fix is that the computation is now based on the *result* of the + // previous layer's `b` computation, maintaining a logical data flow. + const b_squared = b_result.dotGeneral(b_result.transpose(.{ .n, .m }), &.{.{ 1, 0 }}, &.{}); + const b_squared_tagged = b_squared.withTags(.{ .rows, .cols }); + const b_processed = zml.call(compute_module, .forward, .{b_squared_tagged}); - const a_scalar = a_reduced.sum(0).sum(1).asScalar(); - const result = b_processed.add(a_scalar.convert(.f32).broad(b_processed.shape())); + // Stage 3: Dependent Computation (Synchronization Point) + // Use the result of the all-reduce. This forces a wait on the communication. + const a_scalar = a_reduced.sum(0).sum(1).asScalar(); - return result; + // --- Prepare inputs for the next iteration --- + // The result of the communication is the input for the next layer's communication. + a_intermediate = a_reduced; + // The result of this layer's computation is the input for the next layer's computation. + // We give it back the original tags so the next iteration's transpose works. + b_result = b_processed.add(a_scalar.convert(.f32).broad(b_processed.shape())).withTags(.{ .m, .n }); + } + + // Return the final result from the 'b' data path, converted to the original type. + return b_result.convert(.i32); } }; @@ -106,17 +114,19 @@ pub fn asyncMain() !void { const mesh = zml.Mesh.init(.{ .x = num_devices }); log.info("Using mesh: {}", .{mesh}); - const M: i64 = 16384; - const N: i64 = 16384; - const K: i64 = 16384; + // Increased matrix sizes for a more substantial workload. + const M: i64 = 8192; + const N: i64 = 8192; + const K: i64 = 8192; const shape_a = zml.Shape.init(.{ .m = M, .n = K }, .i32).withPartitioning(.{ .n = .x }); const shape_b = zml.Shape.init(.{ .m = M, .n = N }, .i32).withPartitioning(.{ .n = .x }); log.info("Compiling the OverlapModel...", .{}); + var compile_timer = try std.time.Timer.start(); const mod = try zml.compileFn(allocator, OverlapModel.forward, .{ shape_a, shape_b }, mesh, platform); defer mod.deinit(); - log.info("Compilation complete.", .{}); + log.info("Compilation complete in {d:.3}s.", .{@as(f32, @floatFromInt(compile_timer.read())) / 1e9}); log.info("Preparing input data...", .{}); const a_data_slice = try zml.slice.arange(allocator, shape_a, .{}); @@ -131,11 +141,12 @@ pub fn asyncMain() !void { log.info("Input data prepared and transferred to device.", .{}); log.info("Executing the model...", .{}); - var timer = try std.time.Timer.start(); + var exec_timer = try std.time.Timer.start(); const result_buffer = mod.call(.{ a_buffer, b_buffer }); defer result_buffer.deinit(); const result_data = try result_buffer.toHost(allocator); // Await completion for timing defer allocator.free(result_data); - log.info("Execution finished in {d:.3}ms.", .{@as(f32, @floatFromInt(timer.read())) / 1e6}); - log.info("Result (first 8 elements): {any}", .{result_data[0..@min(result_data.len, 8)]}); + log.info("✅ Execution finished in {d:.3}ms.", .{@as(f32, @floatFromInt(exec_timer.read())) / 1e6}); + + log.info("result: {}", .{zml.slice.pretty(result_buffer.shape(), result_data)}); } diff --git a/zml/module.zig b/zml/module.zig index 878331e5..4ac11fa7 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -922,6 +922,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); + setFlag(&options, "xla_gpu_nccl_async_execution", true); + setFlag(&options, "xla_gpu_nccl_blocking_communicators", false); + // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); From 77e68344d507d0d6170b6a520368fa7e4471aafa Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 15 Jul 2025 08:13:31 +0000 Subject: [PATCH 34/51] add flags for nccl --- zml/module.zig | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/zml/module.zig b/zml/module.zig index 4ac11fa7..9c974f25 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -924,6 +924,15 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); setFlag(&options, "xla_gpu_nccl_async_execution", true); setFlag(&options, "xla_gpu_nccl_blocking_communicators", false); + setFlag(&options, "xla_gpu_enable_nccl_user_buffers", true); + setFlag(&options, "xla_gpu_enable_pipelined_all_reduce", true); + setFlag(&options, "xla_gpu_enable_pipelined_reduce_scatter", true); + setFlag(&options, "xla_gpu_enable_pipelined_p2p", true); + setFlag(&options, "xla_gpu_enable_all_gather_combine_by_dim", true); + setFlag(&options, "xla_gpu_enable_reduce_scatter_combine_by_dim", true); + setFlag(&options, "xla_gpu_enable_approx_costly_collectives", true); + setFlag(&options, "xla_gpu_experimental_enable_nvshmem", true); + setFlag(&options, "xla_gpu_enable_nccl_comm_splitting", true); // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); From b3f84f3179191c9fb0832616bb4716daf3aa0b71 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 31 Jul 2025 09:20:33 +0000 Subject: [PATCH 35/51] back to something working --- examples/llama/BUILD.bazel | 2 +- examples/llama/llama.zig | 52 +++++++++++++++++++++----------------- zml/module.zig | 24 +++++++++--------- zml/platform.zig | 2 +- 4 files changed, 43 insertions(+), 37 deletions(-) diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 45b5b66b..d657f4a1 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -105,7 +105,7 @@ cc_binary( "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", "--prompt='Once upon a time, there was a little girl named Lily.'", "--no-llama3=1", # don't do template prompt encoding, I'm a simple model - "--sharding=true", # don't shard me, I'm so small + "--sharding=false", # don't shard me, I'm so small ], data = [ ":tinyllama_stories15M_json", diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 729f0b98..de68e042 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -93,11 +93,13 @@ pub const LlamaLM = struct { pub fn sampleTokens( self: LlamaLM, - lm_head_: ?Linear, // This should be the local `Linear` + lm_head_: ?Linear, out_: Tensor, rng: Tensor.Rng, opts: zml.nn.SamplingStrategy, ) struct { Tensor, Tensor.Rng } { + // The body of this function from the previous step is correct. + // We only need to fix the signature. const vocab_mesh = if (lm_head_) |lm_head| lm_head.weight.mesh() else self.model.embed_tokens.weight.mesh(); zml.pushMesh(vocab_mesh); defer zml.popMesh(); @@ -239,29 +241,18 @@ pub const TransformerLayer = struct { ) struct { Tensor, KvCache } { stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); - // --- Attention Block --- - const attn_input = zml.call(self.input_layernorm, .forward, .{x0}); - // self_attn will do the q,k,v projections and return the PARTIAL o_proj result. - const attn_partial_output, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ attn_input, token_index, kv_cache }); + const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - // Start the all_reduce for the attention block. XLA can schedule this to run in the background. - const attn_reduced_output = zml.ops.allReduce(attn_partial_output, .model, self.self_attn.o_proj.weight.mesh(), 1); + // self_attn now returns the final, reduced tensor. + const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); + const x1 = x0.add(delta0); - // --- MLP Block (starts while attn_reduced_output is being computed) --- - // The input to the MLP block depends on the result of the attention block. - const mlp_input = x0.add(attn_reduced_output); - const mlp_norm_input = zml.call(self.post_attention_layernorm, .forward, .{mlp_input}); + const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); + // mlp now returns the final, reduced tensor. + const delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); + const x2 = x1.add(delta1); - // The MLP forward pass computes the partial result. - const mlp_partial_output = zml.call(self.mlp, .forward, .{mlp_norm_input}); - - // Start the all_reduce for the MLP block. - const mlp_reduced_output = zml.ops.allReduce(mlp_partial_output, .model, self.mlp.down_proj.weight.mesh(), 2); - - // Final residual connection - const x_final = mlp_input.add(mlp_reduced_output); - - return .{ x_final.reuseBuffer(x0), updated_kv_cache }; + return .{ x2.reuseBuffer(x0), updated_kv_cache }; } }; @@ -299,7 +290,14 @@ pub const Mlp = struct { const gate_out = zml.call(self.gate_proj, .forward, .{x}); const up_out = zml.call(self.up_proj, .forward, .{x}); const output = gate_out.silu().mul(up_out); - return zml.call(self.down_proj, .forward, .{output}); + const down_proj_partial = zml.call(self.down_proj, .forward, .{output}); + const down_proj_reduced = zml.ops.allReduce(down_proj_partial, .model, self.down_proj.weight.mesh(), 1); + + // FIX: Re-shard the replicated output to match the input's sharding scheme for the residual connection. + return if (x.dim(.s) > 1) + down_proj_reduced.withSharding(.{ .s = .s }) + else + down_proj_reduced; } }; @@ -365,10 +363,18 @@ pub const SelfAttn = struct { } else null; const attn_output = zml.nn.sdpa(q, k_cached, v_cached, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); const o_proj_partial = zml.call(self.o_proj, .forward, .{attn}); - return .{ o_proj_partial, new_kv_cache }; + const sharded_output_reduced = zml.ops.allReduce(o_proj_partial, .model, self.o_proj.weight.mesh(), 2); + + const sharded_output = if (x.dim(.s) > 1) + sharded_output_reduced.withSharding(.{ .s = .s }) + else + sharded_output_reduced; + + return .{ sharded_output, new_kv_cache }; } }; diff --git a/zml/module.zig b/zml/module.zig index 9c974f25..86327c8a 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -600,7 +600,7 @@ pub const CompilationContext = struct { if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. const ctx = self.mlirCtx(); - if (!self._platform.compilation_options.sharding_enabled) return; + const default_layout = mlir.NamedAttribute.named(ctx, "mhlo.layout_mode", .string(ctx, "default")); for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { attr.appendAssumeCapacity(default_layout); @@ -922,17 +922,17 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); - setFlag(&options, "xla_gpu_nccl_async_execution", true); - setFlag(&options, "xla_gpu_nccl_blocking_communicators", false); - setFlag(&options, "xla_gpu_enable_nccl_user_buffers", true); - setFlag(&options, "xla_gpu_enable_pipelined_all_reduce", true); - setFlag(&options, "xla_gpu_enable_pipelined_reduce_scatter", true); - setFlag(&options, "xla_gpu_enable_pipelined_p2p", true); - setFlag(&options, "xla_gpu_enable_all_gather_combine_by_dim", true); - setFlag(&options, "xla_gpu_enable_reduce_scatter_combine_by_dim", true); - setFlag(&options, "xla_gpu_enable_approx_costly_collectives", true); - setFlag(&options, "xla_gpu_experimental_enable_nvshmem", true); - setFlag(&options, "xla_gpu_enable_nccl_comm_splitting", true); + // setFlag(&options, "xla_gpu_nccl_async_execution", true); + // setFlag(&options, "xla_gpu_nccl_blocking_communicators", false); + // setFlag(&options, "xla_gpu_enable_nccl_user_buffers", true); + // setFlag(&options, "xla_gpu_enable_pipelined_all_reduce", true); + // setFlag(&options, "xla_gpu_enable_pipelined_reduce_scatter", true); + // setFlag(&options, "xla_gpu_enable_pipelined_p2p", true); + // setFlag(&options, "xla_gpu_enable_all_gather_combine_by_dim", true); + // setFlag(&options, "xla_gpu_enable_reduce_scatter_combine_by_dim", true); + // setFlag(&options, "xla_gpu_enable_approx_costly_collectives", true); + // setFlag(&options, "xla_gpu_experimental_enable_nvshmem", true); + // setFlag(&options, "xla_gpu_enable_nccl_comm_splitting", true); // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); diff --git a/zml/platform.zig b/zml/platform.zig index 6008b0c5..ac2b2b24 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -14,7 +14,7 @@ pub const CompilationOptions = struct { xla_dump_to: ?[]const u8 = null, xla_dump_fusion_visualization: bool = false, xla_dump_hlo_pass_re: ?[]const u8 = null, - sharding_enabled: bool = false, + sharding_enabled: ?bool = false, // todo(hugo): remove this, it is only used in the old compilation path }; pub const Platform = struct { From 6e885435e6b0aa3db2c281cc5aafd2f9e37fe16a Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 31 Jul 2025 09:39:01 +0000 Subject: [PATCH 36/51] fix flags --- zml/module.zig | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 86327c8a..aed57869 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -922,17 +922,19 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); - // setFlag(&options, "xla_gpu_nccl_async_execution", true); - // setFlag(&options, "xla_gpu_nccl_blocking_communicators", false); - // setFlag(&options, "xla_gpu_enable_nccl_user_buffers", true); - // setFlag(&options, "xla_gpu_enable_pipelined_all_reduce", true); - // setFlag(&options, "xla_gpu_enable_pipelined_reduce_scatter", true); - // setFlag(&options, "xla_gpu_enable_pipelined_p2p", true); - // setFlag(&options, "xla_gpu_enable_all_gather_combine_by_dim", true); - // setFlag(&options, "xla_gpu_enable_reduce_scatter_combine_by_dim", true); - // setFlag(&options, "xla_gpu_enable_approx_costly_collectives", true); - // setFlag(&options, "xla_gpu_experimental_enable_nvshmem", true); - // setFlag(&options, "xla_gpu_enable_nccl_comm_splitting", true); + + // Collectives options + setFlag(&options, "xla_gpu_nccl_async_execution", true); + setFlag(&options, "xla_gpu_nccl_blocking_communicators", false); + // setFlag(&options, "xla_gpu_enable_nccl_user_buffers", true); // todo(hugo): check segfault + setFlag(&options, "xla_gpu_enable_pipelined_all_reduce", true); + setFlag(&options, "xla_gpu_enable_pipelined_reduce_scatter", true); + setFlag(&options, "xla_gpu_enable_pipelined_p2p", true); + setFlag(&options, "xla_gpu_enable_all_gather_combine_by_dim", true); + setFlag(&options, "xla_gpu_enable_reduce_scatter_combine_by_dim", true); + setFlag(&options, "xla_gpu_enable_approx_costly_collectives", true); + setFlag(&options, "xla_gpu_experimental_enable_nvshmem", true); + setFlag(&options, "xla_gpu_enable_nccl_comm_splitting", true); // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); From c6876002803351d0dc813221e9111e6d40804c6d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 31 Jul 2025 11:32:23 +0000 Subject: [PATCH 37/51] device description --- examples/partitioning/main.zig | 41 ++++++++++++++++++++++++++++++++++ pjrt/pjrt.zig | 12 ++++++++++ 2 files changed, 53 insertions(+) diff --git a/examples/partitioning/main.zig b/examples/partitioning/main.zig index 129865f6..b751a697 100644 --- a/examples/partitioning/main.zig +++ b/examples/partitioning/main.zig @@ -71,6 +71,47 @@ pub fn asyncMain() !void { }); context.printAvailablePlatforms(platform); + const devices = platform.getDevices(); + + for (devices, 0..) |device, i| { + const desc = device.getDescription(platform.pjrt_api); + log.info(" Device #{d}:", .{i}); + log.info(" - ID: {d}", .{desc.getId(platform.pjrt_api)}); + log.info(" - Process Index: {d}", .{desc.getProcessIndex(platform.pjrt_api)}); + log.info(" - Kind: {s}", .{desc.getKind(platform.pjrt_api)}); + log.info(" - Debug String: {s}", .{desc.debugString(platform.pjrt_api)}); + log.info(" - To String: {s}", .{desc.toString(platform.pjrt_api)}); + + const attributes = desc.getAttributes(platform.pjrt_api); + if (attributes.len > 0) { + log.info(" - Attributes:", .{}); + for (attributes) |attr| { + log.info(" - {s}:", .{attr.name()}); + switch (attr.kind()) { + .string => log.info(" (string): {s}", .{attr.inner.unnamed_0.string_value[0..attr.inner.value_size]}), + .int64 => { + log.info(" (int64): {d}", .{attr.inner.unnamed_0.int64_value}); + if (std.mem.eql(u8, attr.name(), "memory_bandwidth")) { + const bandwidth_gbps: f32 = @as(f32, @floatFromInt(attr.inner.unnamed_0.int64_value)) / (1024.0 * 1024.0 * 1024.0); + log.info(" (Computed Bandwidth: {:.2} GB/s)", .{bandwidth_gbps}); + } + if (std.mem.eql(u8, attr.name(), "core_count")) { + log.info(" (Core Count: {d})", .{attr.inner.unnamed_0.int64_value}); + } + }, + .int64list => { + const list_slice = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + log.info(" (int64list): {d}", .{list_slice}); + }, + .float => log.info(" (float): {d}", .{attr.inner.unnamed_0.float_value}), + .bool => log.info(" (bool): {any}", .{attr.inner.unnamed_0.bool_value}), + } + } + } else { + log.info(" - No specific attributes.", .{}); + } + } + // const mesh: zml.Mesh = .init(.{ .x = 4, .y = 3 }); const mesh: zml.Mesh = .auto(platform); // const mesh: zml.Mesh = .single(); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 2bd5a8fc..315b1482 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -625,6 +625,18 @@ pub const DeviceDescription = opaque { return @intCast(ret.process_index); } + pub fn getAttributes(self: *const DeviceDescription, api: *const Api) []const NamedValue { + const ret = api.call(.PJRT_DeviceDescription_Attributes, .{ + .device_description = self.inner(), + }) catch unreachable; + + if (ret.attributes) |attrs| { + return @ptrCast(attrs[0..ret.num_attributes]); + } + + return &.{}; + } + pub fn getKind(self: *const DeviceDescription, api: *const Api) []const u8 { const ret = api.call(.PJRT_DeviceDescription_Kind, .{ .device_description = self.inner(), From acfea5c6ae6083d31c5a91266f95f9e0445d9578 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 31 Jul 2025 13:54:32 +0000 Subject: [PATCH 38/51] fixes for tpu --- examples/llama/llama.zig | 4 ++-- runtimes/tpu/tpu.bzl | 4 ++-- zml/buffer.zig | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index de68e042..652bccfd 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -397,8 +397,8 @@ pub const KvCache = struct { const layer_index_sharding = zml.Sharding.init(mesh, zml.Shape.init(.{}, .u32)); return .{ - .k = try zml.Buffer.uninitialized(platform, kv_sharding, .{}), - .v = try zml.Buffer.uninitialized(platform, kv_sharding, .{}), + .k = try zml.Buffer.constant(platform, kv_sharding, 1), // try zml.Buffer.uninitialized(platform, kv_sharding, .{}), + .v = try zml.Buffer.constant(platform, kv_sharding, 1), // try zml.Buffer.uninitialized(platform, kv_sharding, .{}), .layer_index = try zml.Buffer.constant(platform, layer_index_sharding, 0), }; } diff --git a/runtimes/tpu/tpu.bzl b/runtimes/tpu/tpu.bzl index d8384152..e8a7e6a2 100644 --- a/runtimes/tpu/tpu.bzl +++ b/runtimes/tpu/tpu.bzl @@ -4,9 +4,9 @@ def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( name = "libpjrt_tpu", - url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu-nightly/libtpu_nightly-0.1.dev20250102+nightly-py3-none-linux_x86_64.whl", + url = "https://storage.googleapis.com/libtpu-lts-releases/wheels/libtpu/libtpu-0.0.19.1-py3-none-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "df8339b4f852bd19ad4ed380facc08f28c04e214e9dabb88863e70907b08817e", + sha256 = "373368791794e2ffa20d48ae522232c3146918e138825fe8f7c29b9eae8544e5", build_file = "libpjrt_tpu.BUILD.bazel", ) return mctx.extension_metadata( diff --git a/zml/buffer.zig b/zml/buffer.zig index 0b240ff6..bc8ba742 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -295,8 +295,8 @@ pub const Buffer = struct { }, else => unreachable, } - const data: []const u8 = bytes[0..max_bytes]; - return try from(platform, sharding, data, .{ .wait = true }); + const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, ._data = &bytes }; + return try from(platform, sharding, host_buffer.bytes(), .{ .wait = true }); } test constant { From adc28fa3253925837efd4b9b79d58e78529d135f Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 31 Jul 2025 13:57:26 +0000 Subject: [PATCH 39/51] remove check --- zml/hostbuffer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 2459110a..3e44e168 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -182,7 +182,7 @@ pub const HostBuffer = struct { } pub fn bytes(self: HostBuffer) []const u8 { - stdx.debug.assert(self.isContiguous(), "{} isn't contiguous, can't interpret as []const u8", .{self}); + // stdx.debug.assert(self.isContiguous(), "{} isn't contiguous, can't interpret as []const u8", .{self}); return self._data[0..self._shape.byteSize()]; } From 0acedf2ba34722af077b80f372939647b480d209 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 31 Jul 2025 14:44:35 +0000 Subject: [PATCH 40/51] restore uninitialized --- examples/llama/llama.zig | 4 ++-- zml/hostbuffer.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 652bccfd..de68e042 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -397,8 +397,8 @@ pub const KvCache = struct { const layer_index_sharding = zml.Sharding.init(mesh, zml.Shape.init(.{}, .u32)); return .{ - .k = try zml.Buffer.constant(platform, kv_sharding, 1), // try zml.Buffer.uninitialized(platform, kv_sharding, .{}), - .v = try zml.Buffer.constant(platform, kv_sharding, 1), // try zml.Buffer.uninitialized(platform, kv_sharding, .{}), + .k = try zml.Buffer.uninitialized(platform, kv_sharding, .{}), + .v = try zml.Buffer.uninitialized(platform, kv_sharding, .{}), .layer_index = try zml.Buffer.constant(platform, layer_index_sharding, 0), }; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 3e44e168..2459110a 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -182,7 +182,7 @@ pub const HostBuffer = struct { } pub fn bytes(self: HostBuffer) []const u8 { - // stdx.debug.assert(self.isContiguous(), "{} isn't contiguous, can't interpret as []const u8", .{self}); + stdx.debug.assert(self.isContiguous(), "{} isn't contiguous, can't interpret as []const u8", .{self}); return self._data[0..self._shape.byteSize()]; } From 5294d0b77c8d7ec46c951e521e14f84c04dc50c5 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 4 Aug 2025 15:38:38 +0000 Subject: [PATCH 41/51] cleanup --- examples/partitioning/collectives.zig | 4 ++-- examples/partitioning/main.zig | 10 ++++------ examples/partitioning/overlap.zig | 3 --- zml/ops.zig | 1 - zml/partitioning.zig | 6 +++--- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/examples/partitioning/collectives.zig b/examples/partitioning/collectives.zig index 1b94f2c5..52b981d1 100644 --- a/examples/partitioning/collectives.zig +++ b/examples/partitioning/collectives.zig @@ -15,7 +15,7 @@ const log = std.log.scoped(.@"examples/collectives"); const AllReduceModel = struct { pub fn forward(x: zml.Tensor) zml.Tensor { const mesh = x.mesh(); - const reduced = zml.ops.allReduce(x, .x, mesh); + const reduced = zml.ops.allReduce(x, .x, mesh, 1); const replicated = reduced.withSharding(.{}); return replicated.add(zml.Tensor.constant(replicated.shape(), replicated.dtype().zero())); @@ -25,7 +25,7 @@ const AllReduceModel = struct { const AllGatherModel = struct { pub fn forward(x_sharded: zml.Tensor) zml.Tensor { const mesh = x_sharded.mesh(); - const gathered = zml.ops.allGather(x_sharded, .x, mesh); + const gathered = zml.ops.allGather(x_sharded, .m, mesh); const replicated = gathered.withSharding(.{}); return replicated.add(zml.Tensor.constant(replicated.shape(), replicated.dtype().zero())); } diff --git a/examples/partitioning/main.zig b/examples/partitioning/main.zig index b751a697..c764f910 100644 --- a/examples/partitioning/main.zig +++ b/examples/partitioning/main.zig @@ -3,7 +3,6 @@ const zml = @import("zml"); const stdx = @import("stdx"); const asynk = @import("async"); -// set log level to debug to print the generated IR pub const std_options: std.Options = .{ .log_level = .debug, .logFn = asynk.logFn(std.log.defaultLog), @@ -18,14 +17,14 @@ const ThirdPartyModule = struct { weight: zml.Tensor, pub fn init(self: *ThirdPartyModule, mesh: zml.Mesh) void { - self.weight = self.weight.withTags(.{ .rows, .cols }).withMesh(mesh).withSharding(.{ .rows = .x }); + self.weight = self.weight.withTags(.{ .rows, .cols }).withMesh(mesh).withSharding(.{ .rows = .xoxo }); } pub fn forward(self: ThirdPartyModule, a: zml.Tensor) zml.Tensor { log.debug("Forwarding a: {} and weight: {}", .{ a, self.weight }); - zml.pushMesh(a.mesh().flatten(.x)); - const x = a.withSharding(.{ .m = .x }).add(self.weight.withSharding(.{ .rows = .x })); + zml.pushMesh(a.mesh().flatten(.xoxo)); + const x = a.withSharding(.{ .m = .xoxo }).add(self.weight.withSharding(.{ .rows = .x })); zml.popMesh(); return x.withSharding(.{ .m = .x, .n = .y }); @@ -53,7 +52,6 @@ pub fn main() !void { } pub fn asyncMain() !void { - // Short lived allocations var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); const allocator = gpa.allocator(); @@ -119,7 +117,7 @@ pub fn asyncMain() !void { const shape = zml.Shape.init(.{ .m = 12, .n = 6 }, .i32); - const a_shape = shape.withTags(.{ .m, .n }).withPartitioning(.{ .m = .x, .n = .y }); + const a_shape = shape.withPartitioning(.{ .m = .x, .n = .y }); const b_shape = shape.withTags(.{ .n, .m }).withPartitioning(.{ .n = .x, .m = .y }); var buffers: zml.aio.BufferStore.Buffers = .{}; diff --git a/examples/partitioning/overlap.zig b/examples/partitioning/overlap.zig index ab741247..118270db 100644 --- a/examples/partitioning/overlap.zig +++ b/examples/partitioning/overlap.zig @@ -54,7 +54,6 @@ const OverlapModel = struct { // It starts as the original `b` input tensor. var b_result = b.convert(.f32); - // --- Pipelined Execution Loop --- for (0..OverlapModel.num_layers) |i| { // Stage 1: Communication on the 'a' data path. // The result of the previous layer's communication is reduced. @@ -73,7 +72,6 @@ const OverlapModel = struct { // Use the result of the all-reduce. This forces a wait on the communication. const a_scalar = a_reduced.sum(0).sum(1).asScalar(); - // --- Prepare inputs for the next iteration --- // The result of the communication is the input for the next layer's communication. a_intermediate = a_reduced; // The result of this layer's computation is the input for the next layer's computation. @@ -114,7 +112,6 @@ pub fn asyncMain() !void { const mesh = zml.Mesh.init(.{ .x = num_devices }); log.info("Using mesh: {}", .{mesh}); - // Increased matrix sizes for a more substantial workload. const M: i64 = 8192; const N: i64 = 8192; const K: i64 = 8192; diff --git a/zml/ops.zig b/zml/ops.zig index 6b893265..e41c47c6 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -630,7 +630,6 @@ pub fn allReduce(input: Tensor, reduce_axis_name: anytype, mesh: Mesh, channel_i const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlirCtx(); - // FIX: Use the optional channel ID or get a new one. const actual_channel_id = if (channel_id) |id| id else ctx.getNextChannelId(); const body_block, _ = ctx.makeBlock(.hermetic, BlockSignNoCtx(Tensor.add), &Tensor.add, {}, .{ diff --git a/zml/partitioning.zig b/zml/partitioning.zig index c3f34fa7..b5c9e33c 100644 --- a/zml/partitioning.zig +++ b/zml/partitioning.zig @@ -741,7 +741,7 @@ fn testShardingCase(allocator: std.mem.Allocator, mesh: Mesh, shape: Shape, part std.debug.print("Original full slice of data: {any}\n\n", .{Shaped(i32, shape_partitioned, original_data)}); } - // --- SCATTER (H2D) --- + // Scatter var pjrt_buffers = std.ArrayList(*pjrtx.Buffer).init(allocator); defer pjrt_buffers.deinit(); @@ -772,7 +772,7 @@ fn testShardingCase(allocator: std.mem.Allocator, mesh: Mesh, shape: Shape, part } } - // --- GATHER (D2H) --- + // Gather if (verbose) std.debug.print("--- Reassembling from shards ---\n", .{}); const sharded_buffer = Buffer.fromPjrtBuffers(platform, sharding, pjrt_buffers.items); defer sharded_buffer.deinit(); // This will deinit all the underlying PJRT buffers. @@ -782,7 +782,7 @@ fn testShardingCase(allocator: std.mem.Allocator, mesh: Mesh, shape: Shape, part if (verbose) std.debug.print("Reassembled full slice of data: {any}\n\n", .{Shaped(i32, sharding.global_shape, reassembled_data)}); - // --- VERIFY --- + // Verify try std.testing.expectEqualSlices(u8, original_data, reassembled_data); std.debug.print("✅ Verification successful for: {}\n\n", .{sharding}); } From f0f11930e28e5c5d3ad94e9ae3b3b0e4dc0bf6d0 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 5 Aug 2025 18:20:48 +0000 Subject: [PATCH 42/51] 8b okay 70b ko --- examples/llama/llama.zig | 49 +++++------ mlir/dialects/stablehlo.zig | 8 ++ zml/buffer.zig | 2 +- zml/ops.zig | 159 ++++++++++++++++++++++++++++++++++++ zml/tensor.zig | 8 +- 5 files changed, 196 insertions(+), 30 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index de68e042..20a2fc24 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -67,8 +67,7 @@ pub const LlamaLM = struct { if (self.lm_head) |*lm_head| { lm_head.weight = lm_head.weight .withTags(.{ .voc, .d }) - .withMesh(vocab_mesh) - .withSharding(.{ .voc = .model }); + .withMesh(vocab_mesh); } } @@ -98,8 +97,6 @@ pub const LlamaLM = struct { rng: Tensor.Rng, opts: zml.nn.SamplingStrategy, ) struct { Tensor, Tensor.Rng } { - // The body of this function from the previous step is correct. - // We only need to fix the signature. const vocab_mesh = if (lm_head_) |lm_head| lm_head.weight.mesh() else self.model.embed_tokens.weight.mesh(); zml.pushMesh(vocab_mesh); defer zml.popMesh(); @@ -109,26 +106,26 @@ pub const LlamaLM = struct { var logits = blk: { if (lm_head_) |lm_head| { var partial_logits = zml.call(lm_head, .forward, .{out}); - if (partial_logits.shape().hasTag(.voc) == null) { partial_logits = partial_logits.rename(.{ .d = .voc }); } - - break :blk zml.ops.allGather(partial_logits, .voc, vocab_mesh); + break :blk partial_logits; } else { const embed_table = self.model.embed_tokens.weight.withTags(.{ .voc, .d }); const partial_logits = out.dotGeneral( - embed_table.transpose(.{ .d, .voc }), + embed_table.transpose(.{ .d = .voc, .voc = .d }), &.{.{ 1, 0 }}, &.{}, ); - break :blk zml.ops.allReduce(partial_logits, .model, vocab_mesh, 3); + break :blk partial_logits; } }; if (logits.shape().hasTag(.voc) == null) logits = logits.rename(.{ .d = .voc }); + // With replicated logits, sampleTokens becomes a simple local operation + // with no need for collectives. const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); return .{ next_tokens, new_rng }; } @@ -182,13 +179,13 @@ pub const Llama = struct { const replicated_sharding = .{ .d = .replicated }; self.norm.init(config, compute_mesh, replicated_sharding); + // Replicate the embedding table across all devices in the vocab_mesh. self.embed_tokens.weight = self.embed_tokens.weight .withTags(.{ .voc, .d }) - .withMesh(vocab_mesh) - .withSharding(.{ .d = .replicated, .voc = .model }); + .withMesh(vocab_mesh); for (self.layers) |*layer| { - layer.init(config, vocab_mesh); + layer.init(config, compute_mesh); } } @@ -289,15 +286,14 @@ pub const Mlp = struct { pub fn forward(self: Mlp, x: Tensor) Tensor { const gate_out = zml.call(self.gate_proj, .forward, .{x}); const up_out = zml.call(self.up_proj, .forward, .{x}); - const output = gate_out.silu().mul(up_out); - const down_proj_partial = zml.call(self.down_proj, .forward, .{output}); - const down_proj_reduced = zml.ops.allReduce(down_proj_partial, .model, self.down_proj.weight.mesh(), 1); + const silu_out = gate_out.silu().mul(up_out); + const down_proj_out = zml.call(self.down_proj, .forward, .{silu_out}); - // FIX: Re-shard the replicated output to match the input's sharding scheme for the residual connection. - return if (x.dim(.s) > 1) - down_proj_reduced.withSharding(.{ .s = .s }) - else - down_proj_reduced; + // The output `down_proj_out` is sharded on the `.hidden` dimension along the + // `.model` mesh axis. The subsequent residual connection requires this tensor + // to be replicated. GSPMD will automatically insert an AllReduce(sum) here + // to satisfy the sharding constraint. + return down_proj_out; } }; @@ -366,15 +362,12 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); - const o_proj_partial = zml.call(self.o_proj, .forward, .{attn}); - const sharded_output_reduced = zml.ops.allReduce(o_proj_partial, .model, self.o_proj.weight.mesh(), 2); + const o_proj_out = zml.call(self.o_proj, .forward, .{attn}); - const sharded_output = if (x.dim(.s) > 1) - sharded_output_reduced.withSharding(.{ .s = .s }) - else - sharded_output_reduced; - - return .{ sharded_output, new_kv_cache }; + // `o_proj_out` is sharded on the `.d` dimension (from `.o_hidden`) along + // the `.model` mesh axis. The subsequent residual connection requires this + // tensor to be replicated. GSPMD will automatically insert an AllReduce(sum) here. + return .{ o_proj_out, new_kv_cache }; } }; diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index 8010985d..3a07a764 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -70,6 +70,14 @@ const functors = struct { } }; +pub fn partition_id(ctx: mlir.Context, location: mlir.Location) mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ + .operands = &.{}, + .results = &.{mlir.Type.tensor(&.{}, mlir.Type.int(ctx, .u32))}, + .location = location, + }); +} + pub fn return_(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.return", .{ .variadic_operands = &.{&.{value}}, diff --git a/zml/buffer.zig b/zml/buffer.zig index bc8ba742..9cbd56d3 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -47,7 +47,7 @@ pub const Buffer = struct { _shape: Shape, _api: *const pjrt.Api, _shards: Shards, - _mesh: ?Mesh = null, + _mesh: ?Mesh = null, // ? todo pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); diff --git a/zml/ops.zig b/zml/ops.zig index e41c47c6..104abc8d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -662,6 +662,165 @@ pub fn allReduce(input: Tensor, reduce_axis_name: anytype, mesh: Mesh, channel_i return Tensor._result(result_shape, op.result(0)); } +fn allReduceArgMax(inputs: Tensor.ArgMaxRes, mesh: Mesh, channel_id: ?u64) Tensor.ArgMaxRes { + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const actual_channel_id = if (channel_id) |id| id else ctx.getNextChannelId(); + + // Create a reduction body that compares (value, index) pairs + // We'll encode this as: if lhs.value > rhs.value OR (lhs.value == rhs.value AND lhs.index < rhs.index) + // then return lhs, else return rhs + + const element_type_values = inputs.values.value().getType().as(mlir.RankedTensorType).?.getElementType(); + const element_type_indices = inputs.indices.value().getType().as(mlir.RankedTensorType).?.getElementType(); + + const scalar_type_values = mlir.Type.tensor(&.{}, element_type_values); + _ = scalar_type_values; // autofix + const scalar_type_indices = mlir.Type.tensor(&.{}, element_type_indices); + _ = scalar_type_indices; // autofix + + const loc = ctx.location(@src(), "argmax_reducer", .{}); + + // Create a 2-parameter reduction by interleaving values and indices + // Pack them into a 2-element vector: [value, index_as_float] + const vector_type = mlir.Type.tensor(&.{2}, element_type_values); + const locs = [_]mlir.Location{loc} ** 2; + var body_block = mlir.Block.init(&.{ vector_type, vector_type }, &locs) catch unreachable; + + const lhs_vec = Tensor.fromMlirValue(body_block.argument(0)); + const rhs_vec = Tensor.fromMlirValue(body_block.argument(1)); + + ctx.activate(); + // Extract components from vectors - use axis 0 for rank-1 tensors + const lhs_value = lhs_vec.slice1d(0, .{ .start = 0, .end = 1 }).squeeze(0); + const lhs_index_float = lhs_vec.slice1d(0, .{ .start = 1, .end = 2 }).squeeze(0); + const rhs_value = rhs_vec.slice1d(0, .{ .start = 0, .end = 1 }).squeeze(0); + const rhs_index_float = rhs_vec.slice1d(0, .{ .start = 1, .end = 2 }).squeeze(0); + + // Implement argmax comparison logic + const lhs_greater = lhs_value.cmp(.GT, rhs_value); + const values_equal = lhs_value.cmp(.EQ, rhs_value); + const lhs_index_smaller = lhs_index_float.cmp(.LT, rhs_index_float); + + const should_take_lhs = lhs_greater.logical(.OR, values_equal.logical(.AND, lhs_index_smaller)); + + const result_value = should_take_lhs.select(lhs_value, rhs_value); + const result_index_float = should_take_lhs.select(lhs_index_float, rhs_index_float); + + // Pack result back into vector - use axis 0 and add a new axis for stacking + const result_vec = Tensor.stack(&.{ result_value, result_index_float }, 0, ._); + ctx.deactivate(); + + const return_op = dialect.stablehlo.returns_(mlir_ctx, &.{result_vec.value()}, loc); + body_block.appendOperationRecursive(return_op, .hermetic); + + // Pack input into vectors - use axis 0 and add a new axis for stacking + const indices_as_float = inputs.indices.convert(inputs.values.dtype()); + const input_vec = Tensor.stack(&.{ inputs.values, indices_as_float }, 0, ._); + + const replicated_sharding = Sharding.init(mesh, Shape.init(.{}, .i32).withReplicatedPartitioning()); + const sharding_attr = ctx.getShardingAttr(replicated_sharding); + + const all_reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.all_reduce", .{ + .operands = &.{input_vec.value()}, + .results = &.{input_vec.value().getType()}, + .blocks = &.{body_block}, + .attributes = &.{ + .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, + .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(actual_channel_id), 1)) }, + .{ "use_global_device_ids", .unit(mlir_ctx) }, + .{ "mhlo.sharding", sharding_attr }, + }, + .location = ctx.location(@src(), "allReduceTuple", .{}), + }); + + // Unpack result - use axis 0 for rank-1 tensors + const result_vec2 = Tensor.fromMlirValue(all_reduce_op.result(0)); + const result_values = result_vec2.slice1d(0, .{ .start = 0, .end = 1 }).squeeze(0); + const result_indices_float = result_vec2.slice1d(0, .{ .start = 1, .end = 2 }).squeeze(0); + const result_indices = result_indices_float.convert(inputs.indices.dtype()); + + return Tensor.ArgMaxRes{ + .values = result_values, + .indices = result_indices, + }; +} + +pub fn allReduceTuple( + inputs: anytype, + init_values: @TypeOf(inputs), + comptime reduce_fn: anytype, + mesh: Mesh, + channel_id: ?u64, +) @TypeOf(inputs) { + _ = reduce_fn; // autofix + _ = init_values; // Not needed by the op, but useful for the caller to remember. + if (mesh.isSinglePartition()) { + return inputs; + } + + // Special case for ArgMaxRes to handle the distributed argmax pattern + if (@TypeOf(inputs) == Tensor.ArgMaxRes) { + return allReduceArgMax(inputs, mesh, channel_id); + } + + // Generic implementation for other types would go here + @compileError("allReduceTuple currently only supports ArgMaxRes"); +} + +// fn allReduceArgMax(inputs: Tensor.ArgMaxRes, mesh: Mesh, channel_id: ?u64) Tensor.ArgMaxRes { +// const ctx = CompilationContext.current(); +// const mlir_ctx = ctx.mlirCtx(); +// const actual_channel_id = if (channel_id) |id| id else ctx.getNextChannelId(); + +// // Create the reduction body for argmax +// const element_type_values = inputs.values.value().getType().as(mlir.RankedTensorType).?.getElementType(); +// const element_type_indices = inputs.indices.value().getType().as(mlir.RankedTensorType).?.getElementType(); + +// const scalar_type_values = mlir.Type.tensor(&.{}, element_type_values); +// const scalar_type_indices = mlir.Type.tensor(&.{}, element_type_indices); + +// const loc = ctx.location(@src(), "argmax_reducer", .{}); +// const locs = [_]mlir.Location{loc} ** 4; +// var body_block = mlir.Block.init(&.{ scalar_type_values, scalar_type_indices, scalar_type_values, scalar_type_indices }, &locs) catch unreachable; + +// // Extract block arguments +// const lhs_values = Tensor.fromMlirValue(body_block.argument(0)); +// const lhs_indices = Tensor.fromMlirValue(body_block.argument(1)); +// const rhs_values = Tensor.fromMlirValue(body_block.argument(2)); +// const rhs_indices = Tensor.fromMlirValue(body_block.argument(3)); + +// ctx.activate(); +// const lhs_struct = Tensor.ArgMaxRes{ .values = lhs_values, .indices = lhs_indices }; +// const rhs_struct = Tensor.ArgMaxRes{ .values = rhs_values, .indices = rhs_indices }; +// const result_struct = Tensor.ArgMaxRes.cmp(lhs_struct, rhs_struct); +// ctx.deactivate(); + +// const return_op = dialect.stablehlo.returns_(mlir_ctx, &.{ result_struct.values.value(), result_struct.indices.value() }, loc); +// body_block.appendOperationRecursive(return_op, .hermetic); + +// const replicated_sharding = Sharding.init(mesh, Shape.init(.{}, .i32).withReplicatedPartitioning()); +// const sharding_attr = ctx.getShardingAttr(replicated_sharding); + +// const all_reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.all_reduce", .{ +// .operands = &.{ inputs.values.value(), inputs.indices.value() }, +// .results = &.{ inputs.values.value().getType(), inputs.indices.value().getType() }, +// .blocks = &.{body_block}, +// .attributes = &.{ +// .{ "replica_groups", getSpmdReplicaGroups(mlir_ctx, mesh) }, +// .{ "channel_handle", mlir.Attribute.wrap(c.stablehloChannelHandleGet(mlir_ctx._inner, @intCast(actual_channel_id), 1)) }, +// .{ "use_global_device_ids", .unit(mlir_ctx) }, +// .{ "mhlo.sharding", sharding_attr }, +// }, +// .location = ctx.location(@src(), "allReduceTuple", .{}), +// }); + +// return Tensor.ArgMaxRes{ +// .values = Tensor.fromMlirValue(all_reduce_op.result(0)), +// .indices = Tensor.fromMlirValue(all_reduce_op.result(1)), +// }; +// } + pub fn allGather(input: Tensor, gather_dim: anytype, mesh: Mesh) Tensor { const gather_ax = input.axis(gather_dim); const partition_spec = input.shape().partition(gather_ax); diff --git a/zml/tensor.zig b/zml/tensor.zig index 2058fec7..8f8c5727 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -262,6 +262,12 @@ pub const Tensor = struct { }; } + pub fn partitionId() Tensor { + const ctx = CompilationContext.current(); + const op = dialect.stablehlo.partition_id(ctx.mlirCtx(), ctx.mlirCtx().location(@src())); + return _result(Shape.init(.{}, .u32), op.result(0)); + } + /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { var res = self; @@ -2819,7 +2825,7 @@ pub const Tensor = struct { values: Tensor, indices: Tensor, - fn cmp(left: ArgMaxRes, right: ArgMaxRes) ArgMaxRes { + pub fn cmp(left: ArgMaxRes, right: ArgMaxRes) ArgMaxRes { const left_val = left.values; const right_val = right.values; const left_idx = left.indices; From 717b0c405e73d2cb62115aca9824011e6399b463 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 5 Aug 2025 18:57:31 +0000 Subject: [PATCH 43/51] 100 toks/s --- examples/llama/llama.zig | 26 ++++----- examples/llama/main.zig | 30 ++++++----- zml/partitioning.zig | 110 +++++++++++++++++++++++---------------- 3 files changed, 91 insertions(+), 75 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 20a2fc24..5cc20f5f 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -57,17 +57,18 @@ pub const LlamaLM = struct { gen_opts: zml.nn.SamplingStrategy = .{}, config: Config, - pub fn init(self: *LlamaLM, compute_mesh: zml.Mesh, vocab_mesh: zml.Mesh, config: Config, options: Options) void { + pub fn init(self: *LlamaLM, mesh: zml.Mesh, config: Config, options: Options) void { self.config = config; self.gen_opts = options.sampling_strategy orelse .{}; - self.model.init(config, options.max_seq_len, compute_mesh, vocab_mesh); + self.model.init(config, options.max_seq_len, mesh); // Shard the final output layer across all devices if (self.lm_head) |*lm_head| { lm_head.weight = lm_head.weight .withTags(.{ .voc, .d }) - .withMesh(vocab_mesh); + .withMesh(mesh) + .withSharding(.{ .voc = .model }); } } @@ -166,7 +167,7 @@ pub const Llama = struct { }; } - pub fn init(self: *Llama, config: LlamaLM.Config, max_seq_len: usize, compute_mesh: zml.Mesh, vocab_mesh: zml.Mesh) void { + pub fn init(self: *Llama, config: LlamaLM.Config, max_seq_len: usize, mesh: zml.Mesh) void { self.num_heads = @intCast(config.num_attention_heads); self.num_kv_heads = @intCast(config.num_key_value_heads); self.max_seq_len = @intCast(max_seq_len); @@ -177,15 +178,15 @@ pub const Llama = struct { }; const replicated_sharding = .{ .d = .replicated }; - self.norm.init(config, compute_mesh, replicated_sharding); + self.norm.init(config, mesh, replicated_sharding); - // Replicate the embedding table across all devices in the vocab_mesh. + // Replicate the embedding table across all devices to avoid gather-related slowdowns. self.embed_tokens.weight = self.embed_tokens.weight .withTags(.{ .voc, .d }) - .withMesh(vocab_mesh); + .withMesh(mesh); for (self.layers) |*layer| { - layer.init(config, compute_mesh); + layer.init(config, mesh); } } @@ -206,14 +207,7 @@ pub const Llama = struct { const embeddings = zml.call(embed_tokens_, .forward, .{tokens_}); var tagged_embeddings = embeddings.withPartialTags(.{.d}); - // Sequence parallelism shards this dimension. - // Tensor parallelism replicates it. - // We are doing Tensor Parallelism, so it should be replicated. - if (tokens_.dim(.s) > 1) { // Prefill - return tagged_embeddings.withSharding(.{ .s = .replicated }); - } else { // Decode - return tagged_embeddings.replicated(); - } + return tagged_embeddings.replicated(); } }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 886bec6b..c4fc2237 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -246,17 +246,15 @@ pub fn asyncMain() !void { } const num_devices = platform.getDevices().len; - const compute_mesh = zml.Mesh.init(.{ .s = num_devices }); - const vocab_mesh = zml.Mesh.init(.{ .model = num_devices }); - log.info("Using Sequence Parallelism across {} devices.", .{num_devices}); - log.info("Compute Mesh (for sequence): {}", .{compute_mesh}); - log.info("Vocab/Weight Mesh: {}", .{vocab_mesh}); + const mesh = zml.Mesh.init(.{ .model = num_devices }); + log.info("Using Tensor Parallelism across {} devices.", .{num_devices}); + log.info("Mesh: {}", .{mesh}); var ts = try zml.aio.detectFormatAndOpen(allocator, res.args.weights.?); defer ts.deinit(); var model_arena = std.heap.ArenaAllocator.init(allocator); - var model_instance = try zml.aio.populateModel(llama.LlamaLM, model_arena.allocator(), ts); + var model_instance = try zml.aio.populateModel(LlamaLM, model_arena.allocator(), ts); const llama_options: llama.LlamaLM.Options = .{ .max_seq_len = @intCast(res.args.@"seq-len" orelse 256), @@ -265,13 +263,17 @@ pub fn asyncMain() !void { .temperature = 1.0, }, }; - model_instance.init(compute_mesh, vocab_mesh, config, llama_options); + // Pass the single, correct mesh to the model initialization. + model_instance.init(mesh, config, llama_options); const dims = model_instance.model.shape(); const dtype = model_instance.model.embed_tokens.weight.dtype(); - const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32).withPartitioning(.{ .s = .s }); - const tokens_shape_decode = zml.Shape.init(.{ .s = 1 }, .u32).withPartitioning(.{ .s = .replicated }); + // The prefill stage can be sharded by sequence length if desired, but for simplicity + // and consistency, we'll keep it replicated. GSPMD will handle it. + const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); + // Decode tokens are replicated across all devices. + const tokens_shape_decode = zml.Shape.init(.{ .s = 1 }, .u32); const token_idx_shape = zml.Shape.init(.{}, .u32); const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype); const kv_cache_shape = llama.KvCache.initShape(kv_shape); @@ -302,16 +304,16 @@ pub fn asyncMain() !void { token_idx_shape, kv_cache_shape, rng_shape, - }, compute_mesh, platform }); + }, mesh, platform }); var fut_mod_decode = try asynk.asyncc(zml.compileModel, .{ allocator, llama.LlamaLM.forward, model_instance, .{ tokens_shape_decode, token_idx_shape, kv_cache_shape, rng_shape, - }, compute_mesh, platform }); + }, mesh, platform }); log.info("\tLoading Llama weights from {?s}...", .{res.args.weights}); - var llama_weights = try zml.aio.loadModelBuffers(LlamaLM, model_instance, ts, model_arena.allocator(), platform, compute_mesh); + var llama_weights = try zml.aio.loadModelBuffers(LlamaLM, model_instance, ts, model_arena.allocator(), platform, mesh); defer zml.aio.unloadBuffers(&llama_weights); log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); @@ -325,7 +327,7 @@ pub fn asyncMain() !void { log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); log.info("Creating KvCache", .{}); - const kv_cache = try llama.KvCache.initBuffer(kv_shape, compute_mesh, platform); + const kv_cache = try llama.KvCache.initBuffer(kv_shape, mesh, platform); log.info("✅\tPrompt: {s}", .{prompt}); @@ -336,7 +338,7 @@ pub fn asyncMain() !void { profiler.start(); } - const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module_decode, kv_cache, tokenizer, allocator, compute_mesh, seed, prompt, skip_llama3_encoding); + const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module_decode, kv_cache, tokenizer, allocator, mesh, seed, prompt, skip_llama3_encoding); defer allocator.free(generated_text); if (use_pjrt_profiler) { diff --git a/zml/partitioning.zig b/zml/partitioning.zig index b5c9e33c..a077aee8 100644 --- a/zml/partitioning.zig +++ b/zml/partitioning.zig @@ -621,65 +621,85 @@ pub const Sharding = struct { } pub fn writeShardingRepresentation(self: Sharding, writer: anytype) !void { - if (self.getType() == .replicated) { + if (self.mesh.isSinglePartition()) { try writer.writeAll("{replicated}"); return; } - var product_of_tensor_tiles: i64 = 1; - var num_partitioned_dims: u32 = 0; + var partitioned_mesh_axis_indices: std.BoundedArray(u3, MaxMeshAxes) = .{}; + var is_fully_replicated = true; - // Calculate the product of explicitly defined tile dimensions for (0..self.global_shape.rank()) |i| { const part_spec = self.global_shape.partition(i); if (part_spec == .axis) { const mesh_axis_tag = part_spec.toTag(); - if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { - product_of_tensor_tiles *= self.mesh.topology.dim(mesh_axis_tag); - num_partitioned_dims += 1; + if (self.mesh.topology.hasTag(mesh_axis_tag)) |mesh_axis_idx| { + is_fully_replicated = false; + if (std.mem.indexOfScalar(u3, partitioned_mesh_axis_indices.constSlice(), mesh_axis_idx) == null) { + partitioned_mesh_axis_indices.appendAssumeCapacity(mesh_axis_idx); + } } } } - const is_fully_sharded_on_mesh = (product_of_tensor_tiles == self.mesh.numPartitions()); - - if (is_fully_sharded_on_mesh) { - // This is the easy case. The tensor is partitioned across all mesh devices. - // The tile array must have the same rank as the tensor. - try writer.writeAll("{devices=["); - for (0..self.global_shape.rank()) |i| { - const part_spec = self.global_shape.partition(i); - var tile_dim: i64 = 1; - if (part_spec == .axis) { - if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { - tile_dim = self.mesh.topology.dim(part_spec.toTag()); - } - } - try writer.print("{d}", .{tile_dim}); - if (i < self.global_shape.rank() - 1) { - try writer.writeByte(','); - } - } - try writer.print("]<=[{d}]}}", .{self.mesh.numPartitions()}); - } else { - // This is partial sharding. This is more complex. - // The `devices` array lists the tiling only for the *partitioned* dimensions. - // The remaining devices are handled by `last_tile_dim_replicate`. - try writer.writeAll("{devices=["); - var first = true; - for (0..self.global_shape.rank()) |i| { - const part_spec = self.global_shape.partition(i); - if (part_spec == .axis) { - const mesh_axis_tag = part_spec.toTag(); - if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { - if (!first) try writer.writeByte(','); - try writer.print("{d}", .{self.mesh.topology.dim(mesh_axis_tag)}); - first = false; - } - } - } - try writer.print("]<=[{d}] last_tile_dim_replicate}}", .{self.mesh.numPartitions()}); + if (is_fully_replicated) { + try writer.writeAll("{replicated}"); + return; } + + var tile_assignment_dims: Shape.DimsArray = .{}; + var unassigned_mesh_partitions: i64 = 1; + + var used_mesh_axes_mask: [MaxMeshAxes]bool = [_]bool{false} ** MaxMeshAxes; + for (partitioned_mesh_axis_indices.constSlice()) |idx| { + used_mesh_axes_mask[idx] = true; + } + for (0..@intCast(self.mesh.rank())) |i| { + if (!used_mesh_axes_mask[i]) { + unassigned_mesh_partitions *= self.mesh.topology.dim(i); + } + } + + var assigned_replication = false; + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + var tile_dim: i64 = 1; + + if (part_spec == .axis) { + if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + tile_dim = self.mesh.topology.dim(part_spec.toTag()); + } + } else { + if (!assigned_replication and unassigned_mesh_partitions > 1) { + tile_dim = unassigned_mesh_partitions; + assigned_replication = true; + } + } + tile_assignment_dims.appendAssumeCapacity(tile_dim); + } + + if (!assigned_replication and unassigned_mesh_partitions > 1) { + std.log.warn("Tensor shape {} on mesh {} is fully partitioned but does not use all mesh axes. Falling back to replicated.", .{ self.global_shape, self.mesh }); + try writer.writeAll("{replicated}"); + return; + } + + var product: u64 = 1; + for (tile_assignment_dims.constSlice()) |d| product *= @intCast(d); + stdx.debug.assert( + product == self.mesh.numDevices(), + "Internal error: tiling product {d} does not match mesh size {d} for shape {s} on mesh {s}", + .{ product, self.mesh.numDevices(), self.global_shape, self.mesh }, + ); + + try writer.writeAll("{devices=["); + for (tile_assignment_dims.constSlice(), 0..) |dim, i| { + try writer.print("{d}", .{dim}); + if (i < tile_assignment_dims.len - 1) { + try writer.writeByte(','); + } + } + try writer.print("]<=[{d}]}}", .{self.mesh.numDevices()}); } test "Sharding MLIR representation" { From 8c7afd6b0839966a129aef098159c6f82b5cf487 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 6 Aug 2025 12:34:02 +0000 Subject: [PATCH 44/51] physical mesh --- examples/llama/main.zig | 6 +- zml/module.zig | 20 ++- zml/platform.zig | 285 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 303 insertions(+), 8 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index c4fc2237..6233f56d 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -231,7 +231,7 @@ pub fn asyncMain() !void { // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' const create_opts_json = res.args.@"create-options" orelse "{}"; const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); - const platform = context.autoPlatform(.{ .cpu = .{ .cpu_device_count = 8 } }).withCompilationOptions(compilation_options); + const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); create_opts.deinit(); context.printAvailablePlatforms(platform); @@ -246,9 +246,9 @@ pub fn asyncMain() !void { } const num_devices = platform.getDevices().len; + // Define the logical mesh for tensor parallelism. const mesh = zml.Mesh.init(.{ .model = num_devices }); - log.info("Using Tensor Parallelism across {} devices.", .{num_devices}); - log.info("Mesh: {}", .{mesh}); + log.info("Using logical mesh for Tensor Parallelism: {}", .{mesh}); var ts = try zml.aio.detectFormatAndOpen(allocator, res.args.weights.?); defer ts.deinit(); diff --git a/zml/module.zig b/zml/module.zig index aed57869..75ea0ffe 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -13,6 +13,7 @@ const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const createDeviceMesh = @import("platform.zig").createDeviceMesh; const partitioning = @import("partitioning.zig"); const Mesh = partitioning.Mesh; const Partition = partitioning.Partition; @@ -894,14 +895,25 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m }, }; + // Create a device mesh that respects the physical hardware topology. + // This ordered list of devices is used to map logical partition IDs to physical device IDs. + const ordered_devices = try createDeviceMesh(arena, mesh, platform); + defer arena.free(ordered_devices); + stdx.debug.assert(ordered_devices.len == mesh.numPartitions(), "createDeviceMesh returned {d} devices, but mesh requires {d}", .{ ordered_devices.len, mesh.numPartitions() }); + const computation_devices = &options.executable_build_options.?.device_assignment.?.computation_devices; try computation_devices.ensureTotalCapacity(arena, @intCast(mesh.numPartitions())); - const replica_device_ids = try arena.alloc(i64, @intCast(mesh.numPartitions())); - for (0..@intCast(mesh.numPartitions())) |i| { - replica_device_ids[i] = @intCast(i); - computation_devices.appendAssumeCapacity(.{ .replica_device_ids = .fromOwnedSlice(replica_device_ids[i .. i + 1]) }); + + for (ordered_devices) |device| { + const device_id = @as(i64, @intCast(device.getDescription(platform.pjrt_api).getId(platform.pjrt_api))); + const replica_device_ids = try arena.alloc(i64, 1); + replica_device_ids[0] = device_id; + + computation_devices.appendAssumeCapacity(.{ .replica_device_ids = .fromOwnedSlice(replica_device_ids) }); } + stdx.debug.assert(computation_devices.items.len == mesh.numPartitions(), "computation_devices should have {d} items to match {d} partitions", .{ computation_devices.items.len, mesh.numPartitions() }); + // Let the arena deinit, zig-protobuf deinit is very slow. try options.env_option_overrides.ensureUnusedCapacity(arena, 16); if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { diff --git a/zml/platform.zig b/zml/platform.zig index ac2b2b24..3d9ab4ec 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,13 +3,296 @@ const std = @import("std"); const runtimes = @import("runtimes"); pub const Target = runtimes.Platform; const stdx = @import("stdx"); +const Mesh = @import("partitioning.zig").Mesh; +const Shape = @import("shape.zig").Shape; const pjrt = @import("pjrtx.zig"); -const log = std.log.scoped(.zml); +const log = std.log.scoped(.@"zml/platform"); pub const available_targets = std.enums.values(Target); +/// Represents the physical arrangement of hardware devices. +/// It contains the devices ordered according to their physical topology and the shape of that topology. +pub const PhysicalMesh = struct { + allocator: std.mem.Allocator, + devices: []const *const pjrt.Device, + shape: Shape, + + pub fn deinit(self: *PhysicalMesh) void { + self.allocator.free(self.devices); + self.* = undefined; + } +}; + +// Optimal orderings for TPU collectives, from JAX. +const tpu_v5_lite_2x2_ring_order = [_]u8{ 0, 1, 3, 2 }; +const tpu_v5_lite_2x2x2_iota_order = [_]u8{ 0, 4, 2, 6, 1, 5, 3, 7 }; +// Same ring order for v5e tray, v6e tray, and v7x 2x4 tray +const tpu_tray_ring_order_8 = [_]u8{ 0, 1, 2, 3, 7, 6, 5, 4 }; + +/// Sorts devices primarily by their coordinates (lexicographically, reversed) and then by core_on_chip. +/// This matches JAX's z,y,x,core sorting for creating a physical mesh. +/// This function is allocation-free. +/// This is a `lessThan` function for `std.sort.sort`. +fn sortDevicesForPhysicalMesh(context: *const pjrt.Api, lhs_dev: *const pjrt.Device, rhs_dev: *const pjrt.Device) bool { + const api: *const pjrt.Api = context; + + const lhs_desc = lhs_dev.getDescription(api); + const rhs_desc = rhs_dev.getDescription(api); + + var lhs_coords: ?[]const i64 = null; + var lhs_core: i64 = 0; + for (lhs_desc.getAttributes(api)) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + lhs_coords = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + lhs_core = attr.inner.unnamed_0.int64_value; + } + } + + var rhs_coords: ?[]const i64 = null; + var rhs_core: i64 = 0; + for (rhs_desc.getAttributes(api)) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + rhs_coords = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + rhs_core = attr.inner.unnamed_0.int64_value; + } + } + + // TPUs are expected to have coordinates. + // If not, we can't sort them topologically, so we treat them as equal and let the original order prevail. + if (lhs_coords == null or rhs_coords == null) { + return false; + } + + const lhs_coords_slice = lhs_coords.?; + const rhs_coords_slice = rhs_coords.?; + + stdx.debug.assert(lhs_coords_slice.len == rhs_coords_slice.len, "Devices have different coordinate ranks", .{}); + var i = lhs_coords_slice.len; + while (i > 0) { + i -= 1; + if (lhs_coords_slice[i] != rhs_coords_slice[i]) { + return lhs_coords_slice[i] < rhs_coords_slice[i]; + } + } + + return lhs_core < rhs_core; +} + +/// Creates a device mesh that arranges devices for optimal collective performance, +/// similar to `jax.experimental.mesh_utils.create_device_mesh`. +pub fn createDeviceMesh( + allocator: std.mem.Allocator, + logical_mesh: Mesh, + platform: Platform, +) ![]const *const pjrt.Device { + const api = platform.pjrt_api; + const devices = platform.getDevices(); + + const num_logical_devices: u64 = logical_mesh.topology.count(); + stdx.debug.assert(num_logical_devices <= devices.len, "Number of devices {d} must be >= the product of logical_mesh_shape {d}", .{ devices.len, logical_mesh.topology }); + const active_devices = devices[0..@intCast(num_logical_devices)]; + + if (active_devices.len == 0) return &.{}; + + switch (platform.target) { + .tpu => { + const first_device_kind = active_devices[0].getDescription(api).getKind(api); + + // The kind string is a reliable way to dispatch for TPUs. + if (std.mem.eql(u8, first_device_kind, "TPU v5 lite") or std.mem.eql(u8, first_device_kind, "TPU v6 lite")) { + if (try createTpuV5LiteMesh(allocator, logical_mesh, active_devices, api)) |ordered_devices| { + return ordered_devices; + } + } else if (std.mem.eql(u8, first_device_kind, "TPU7x")) { + if (try createTpu7xMesh(allocator, logical_mesh, active_devices, api)) |ordered_devices| { + return ordered_devices; + } + } + + // Fallback for other TPU kinds (e.g., v2, v3, v4, v5p). + log.warn("Using generic TPU physical mesh creation for {s}. For optimal performance, a specific handler may be needed.", .{first_device_kind}); + var physical_mesh = try getPhysicalTpuMesh(allocator, active_devices, api); + defer physical_mesh.deinit(); + return try createDeviceMeshForNdTorus(allocator, physical_mesh, logical_mesh); + }, + .cuda, .rocm, .cpu, .neuron => { + // Default for CPU, GPU, and Neuron: simple duplication of the device list. + return allocator.dupe(*const pjrt.Device, active_devices); + }, + } +} + +/// Creates a device mesh for TPU v5 lite and v6 lite, applying optimal orderings. +fn createTpuV5LiteMesh( + allocator: std.mem.Allocator, + logical_mesh: Mesh, + devices: []const *const pjrt.Device, + api: *const pjrt.Api, +) !?[]const *const pjrt.Device { + const sequential_devices = try allocator.dupe(*const pjrt.Device, devices); + defer allocator.free(sequential_devices); + std.sort.pdq(*const pjrt.Device, sequential_devices, api, sortDevicesForPhysicalMesh); + + const reordered_devices = try allocator.alloc(*const pjrt.Device, devices.len); + + const order: ?[]const u8 = switch (devices.len) { + 4 => &tpu_v5_lite_2x2_ring_order, + 8 => &tpu_v5_lite_2x2x2_iota_order, // JAX uses iota order for v5e 2x2x2 cubes + else => null, + }; + + if (order) |o| { + for (o, 0..) |old_idx, new_idx| { + reordered_devices[new_idx] = sequential_devices[old_idx]; + } + // Reshaping is conceptual; the flat ordered array is what matters for XLA's device_assignment. + _ = logical_mesh; + return reordered_devices; + } + + allocator.free(reordered_devices); + return null; +} + +/// Creates a device mesh for TPU v7x. +fn createTpu7xMesh( + allocator: std.mem.Allocator, + logical_mesh: Mesh, + devices: []const *const pjrt.Device, + api: *const pjrt.Api, +) !?[]const *const pjrt.Device { + const sequential_devices = try allocator.dupe(*const pjrt.Device, devices); + defer allocator.free(sequential_devices); + std.sort.pdq(*const pjrt.Device, sequential_devices, api, sortDevicesForPhysicalMesh); + + if (devices.len == 8) { + const reordered_devices = try allocator.alloc(*const pjrt.Device, devices.len); + for (tpu_tray_ring_order_8, 0..) |old_idx, new_idx| { + reordered_devices[new_idx] = sequential_devices[old_idx]; + } + _ = logical_mesh; + return reordered_devices; + } + + return null; +} + +/// Constructs a PhysicalMesh representing the N-D torus of a TPU slice. +fn getPhysicalTpuMesh(allocator: std.mem.Allocator, devices: []const *const pjrt.Device, api: *const pjrt.Api) !PhysicalMesh { + stdx.debug.assert(devices.len > 0, "Cannot create physical mesh from zero devices", .{}); + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + _ = arena; // autofix + + var min_coords: [3]i64 = .{ std.math.maxInt(i64), std.math.maxInt(i64), std.math.maxInt(i64) }; + var max_coords: [3]i64 = .{ std.math.minInt(i64), std.math.minInt(i64), std.math.minInt(i64) }; + var min_core: i64 = std.math.maxInt(i64); + var max_core: i64 = std.math.minInt(i64); + + for (devices) |device| { + const desc = device.getDescription(api); + const attrs = desc.getAttributes(api); + for (attrs) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + const coords = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + stdx.debug.assert(coords.len == 3, "Expected 3D coordinates for TPU device, got {any}", .{coords}); + for (0..3) |i| { + min_coords[i] = @min(min_coords[i], coords[i]); + max_coords[i] = @max(max_coords[i], coords[i]); + } + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + min_core = @min(min_core, attr.inner.unnamed_0.int64_value); + max_core = @max(max_core, attr.inner.unnamed_0.int64_value); + } + } + } + + const dims: [3]u64 = .{ + @intCast(max_coords[0] - min_coords[0] + 1), + @intCast(max_coords[1] - min_coords[1] + 1), + @intCast(max_coords[2] - min_coords[2] + 1), + }; + const cores_per_chip: u64 = @intCast(max_core - min_core + 1); + stdx.debug.assert(devices.len == dims[0] * dims[1] * dims[2] * cores_per_chip, "Device list does not form a contiguous cuboid.", .{}); + + var physical_shape: Shape = undefined; + var physical_dims: Shape.DimsArray = .{}; + if (cores_per_chip > 1) { + try physical_dims.appendSlice(&.{ @intCast(dims[0]), @intCast(dims[1]), @intCast(cores_per_chip) }); + physical_shape = Shape.init(physical_dims.constSlice(), .u8).withTags(.{ .x, .y, .c }); + } else { + try physical_dims.appendSlice(&.{ @intCast(dims[0]), @intCast(dims[1]), @intCast(dims[2]) }); + physical_shape = Shape.init(physical_dims.constSlice(), .u8).withTags(.{ .x, .y, .z }); + } + + const ordered_devices = try allocator.alloc(*const pjrt.Device, devices.len); + // @memset(ordered_devices, null); + + for (devices) |device| { + const desc = device.getDescription(api); + const attrs = desc.getAttributes(api); + var coords: [3]i64 = undefined; + var core_on_chip: i64 = 0; + for (attrs) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + const coords_slice = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + @memcpy(coords[0..coords_slice.len], coords_slice); + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + core_on_chip = attr.inner.unnamed_0.int64_value; + } + } + const x = @as(usize, @intCast(coords[0] - min_coords[0])); + const y = @as(usize, @intCast(coords[1] - min_coords[1])); + const z = @as(usize, @intCast(coords[2] - min_coords[2])); + const c = @as(usize, @intCast(core_on_chip - min_core)); + + const index = if (cores_per_chip > 1) (x * @as(usize, @intCast(dims[1])) * @as(usize, @intCast(cores_per_chip))) + (y * @as(usize, @intCast(cores_per_chip))) + c else (x * @as(usize, @intCast(dims[1])) * @as(usize, @intCast(dims[2]))) + (y * @as(usize, @intCast(dims[2]))) + z; + stdx.debug.assert(index < ordered_devices.len, "Index out of bounds", .{}); + // stdx.debug.assert(ordered_devices[index] == null, "Duplicate device position in physical mesh", .{}); + ordered_devices[index] = device; + } + + return PhysicalMesh{ + .allocator = allocator, + .devices = ordered_devices, + .shape = physical_shape, + }; +} + +/// Maps a logical mesh onto a physical N-D torus mesh. +fn createDeviceMeshForNdTorus( + allocator: std.mem.Allocator, + physical_mesh: PhysicalMesh, + logical_mesh: Mesh, +) ![]const *const pjrt.Device { + // This is a complex combinatorial problem. For now, we implement a simplified greedy approach + // that assumes logical axes map to single physical axes. A full implementation would explore + // combinations and splitting physical axes. + // TODO: Implement the full JAX axis assignment algorithm. + log.warn("Using simplified logical to physical mesh mapping. Performance may not be optimal.", .{}); + + var physical_axes = physical_mesh.shape._dims; + var logical_axes = logical_mesh.topology._dims; + std.sort.pdq(i64, physical_axes.slice(), {}, std.sort.desc(i64)); + std.sort.pdq(i64, logical_axes.slice(), {}, std.sort.desc(i64)); + + if (!std.mem.eql(i64, physical_axes.constSlice(), logical_axes.constSlice())) { + log.err("Physical mesh shape {any} cannot be mapped to logical mesh shape {any} with the current simplified algorithm.", .{ + physical_mesh.shape, + logical_mesh.topology, + }); + return error.MeshMappingNotSupported; + } + + // Since shapes are identical after sorting, a simple copy is sufficient for this simplified version. + return allocator.dupe(*const pjrt.Device, physical_mesh.devices); +} + pub const CompilationOptions = struct { xla_dump_to: ?[]const u8 = null, xla_dump_fusion_visualization: bool = false, From ca98ce0beaf3692ae5e9d6e3f90ac3ba0b1ecb80 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 6 Aug 2025 13:22:44 +0000 Subject: [PATCH 45/51] llama tpu --- examples/llama/llama.zig | 2 +- examples/llama/main.zig | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 5cc20f5f..ab38ea16 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -207,7 +207,7 @@ pub const Llama = struct { const embeddings = zml.call(embed_tokens_, .forward, .{tokens_}); var tagged_embeddings = embeddings.withPartialTags(.{.d}); - return tagged_embeddings.replicated(); + return tagged_embeddings.withSharding(.{ .s = .data }); } }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6233f56d..4301f0b5 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -246,9 +246,13 @@ pub fn asyncMain() !void { } const num_devices = platform.getDevices().len; - // Define the logical mesh for tensor parallelism. - const mesh = zml.Mesh.init(.{ .model = num_devices }); - log.info("Using logical mesh for Tensor Parallelism: {}", .{mesh}); + const mesh: zml.Mesh = if (num_devices >= 4 and platform.target == .tpu) + // For 4 devices on TPU pod a 2x2 mesh is ideal + // for a hybrid data and tensor parallelism strategy. + zml.Mesh.init(.{ .data = 2, .model = num_devices / 2 }) + else + zml.Mesh.init(.{ .model = num_devices }); + log.info("Using logical mesh for Hybrid Parallelism: {}", .{mesh}); var ts = try zml.aio.detectFormatAndOpen(allocator, res.args.weights.?); defer ts.deinit(); @@ -269,11 +273,11 @@ pub fn asyncMain() !void { const dims = model_instance.model.shape(); const dtype = model_instance.model.embed_tokens.weight.dtype(); - // The prefill stage can be sharded by sequence length if desired, but for simplicity - // and consistency, we'll keep it replicated. GSPMD will handle it. - const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); - // Decode tokens are replicated across all devices. - const tokens_shape_decode = zml.Shape.init(.{ .s = 1 }, .u32); + const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32) + .withPartitioning(.{ .s = .data }); + // Decode tokens are also sharded by the batch-like sequence dimension. + const tokens_shape_decode = zml.Shape.init(.{ .s = 1 }, .u32) + .withPartitioning(.{ .s = .data }); const token_idx_shape = zml.Shape.init(.{}, .u32); const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype); const kv_cache_shape = llama.KvCache.initShape(kv_shape); From ac6fd43f08feb2d257a8b361b8af42f97be2cd16 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 6 Aug 2025 14:06:21 +0000 Subject: [PATCH 46/51] tpu sharding fix --- examples/llama/main.zig | 3 +-- zml/partitioning.zig | 52 +++++++++++++++++++---------------------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 4301f0b5..a42c31fd 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -276,8 +276,7 @@ pub fn asyncMain() !void { const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32) .withPartitioning(.{ .s = .data }); // Decode tokens are also sharded by the batch-like sequence dimension. - const tokens_shape_decode = zml.Shape.init(.{ .s = 1 }, .u32) - .withPartitioning(.{ .s = .data }); + const tokens_shape_decode = zml.Shape.init(.{ .s = 1 }, .u32); const token_idx_shape = zml.Shape.init(.{}, .u32); const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype); const kv_cache_shape = llama.KvCache.initShape(kv_shape); diff --git a/zml/partitioning.zig b/zml/partitioning.zig index a077aee8..20156d9f 100644 --- a/zml/partitioning.zig +++ b/zml/partitioning.zig @@ -626,7 +626,7 @@ pub const Sharding = struct { return; } - var partitioned_mesh_axis_indices: std.BoundedArray(u3, MaxMeshAxes) = .{}; + var used_mesh_axes: std.BoundedArray(bool, MaxMeshAxes) = .{ .buffer = [_]bool{false} ** MaxMeshAxes, .len = @intCast(self.mesh.rank()) }; var is_fully_replicated = true; for (0..self.global_shape.rank()) |i| { @@ -635,9 +635,7 @@ pub const Sharding = struct { const mesh_axis_tag = part_spec.toTag(); if (self.mesh.topology.hasTag(mesh_axis_tag)) |mesh_axis_idx| { is_fully_replicated = false; - if (std.mem.indexOfScalar(u3, partitioned_mesh_axis_indices.constSlice(), mesh_axis_idx) == null) { - partitioned_mesh_axis_indices.appendAssumeCapacity(mesh_axis_idx); - } + used_mesh_axes.buffer[mesh_axis_idx] = true; } } } @@ -648,40 +646,38 @@ pub const Sharding = struct { } var tile_assignment_dims: Shape.DimsArray = .{}; - var unassigned_mesh_partitions: i64 = 1; - - var used_mesh_axes_mask: [MaxMeshAxes]bool = [_]bool{false} ** MaxMeshAxes; - for (partitioned_mesh_axis_indices.constSlice()) |idx| { - used_mesh_axes_mask[idx] = true; - } - for (0..@intCast(self.mesh.rank())) |i| { - if (!used_mesh_axes_mask[i]) { - unassigned_mesh_partitions *= self.mesh.topology.dim(i); - } - } - - var assigned_replication = false; for (0..self.global_shape.rank()) |i| { const part_spec = self.global_shape.partition(i); var tile_dim: i64 = 1; if (part_spec == .axis) { - if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { - tile_dim = self.mesh.topology.dim(part_spec.toTag()); - } - } else { - if (!assigned_replication and unassigned_mesh_partitions > 1) { - tile_dim = unassigned_mesh_partitions; - assigned_replication = true; + if (self.mesh.topology.hasTag(part_spec.toTag())) |mesh_axis_idx| { + tile_dim = self.mesh.topology.dim(mesh_axis_idx); } } tile_assignment_dims.appendAssumeCapacity(tile_dim); } - if (!assigned_replication and unassigned_mesh_partitions > 1) { - std.log.warn("Tensor shape {} on mesh {} is fully partitioned but does not use all mesh axes. Falling back to replicated.", .{ self.global_shape, self.mesh }); - try writer.writeAll("{replicated}"); - return; + var replication_factor: i64 = 1; + for (used_mesh_axes.constSlice(), 0..) |used, i| { + if (!used) { + replication_factor *= self.mesh.topology.dim(i); + } + } + + // Find a dimension to apply the replication factor to. + // GSPMD prefers applying replication to an existing replicated dimension if possible. + var applied_replication = false; + for (0..self.global_shape.rank()) |i| { + if (self.global_shape.partition(i) == .replicated) { + tile_assignment_dims.buffer[i] *= replication_factor; + applied_replication = true; + break; + } + } + // If no dimension was marked as replicated, we add a new virtual tiling dimension for replication. + if (!applied_replication and replication_factor > 1) { + tile_assignment_dims.appendAssumeCapacity(replication_factor); } var product: u64 = 1; From df2e6eb8845e7fae6a2b77b516d68fbb14d44aa4 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 6 Aug 2025 15:06:49 +0000 Subject: [PATCH 47/51] rollback some stuffs --- examples/llama/llama.zig | 58 +++++++++++++--------------------------- examples/llama/main.zig | 32 +++++++++++++++------- 2 files changed, 41 insertions(+), 49 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index ab38ea16..8fde0b6e 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -1,4 +1,3 @@ -// examples/llama/llama.zig const std = @import("std"); const testing = std.testing; @@ -63,12 +62,11 @@ pub const LlamaLM = struct { self.model.init(config, options.max_seq_len, mesh); - // Shard the final output layer across all devices if (self.lm_head) |*lm_head| { lm_head.weight = lm_head.weight .withTags(.{ .voc, .d }) .withMesh(mesh) - .withSharding(.{ .voc = .model }); + .withSharding(.{ .d = .model }); } } @@ -106,16 +104,12 @@ pub const LlamaLM = struct { var logits = blk: { if (lm_head_) |lm_head| { - var partial_logits = zml.call(lm_head, .forward, .{out}); - if (partial_logits.shape().hasTag(.voc) == null) { - partial_logits = partial_logits.rename(.{ .d = .voc }); - } - break :blk partial_logits; + break :blk zml.call(lm_head, .forward, .{out}); } else { const embed_table = self.model.embed_tokens.weight.withTags(.{ .voc, .d }); const partial_logits = out.dotGeneral( embed_table.transpose(.{ .d = .voc, .voc = .d }), - &.{.{ 1, 0 }}, + &.{.{ 1, -1 }}, &.{}, ); break :blk partial_logits; @@ -125,8 +119,6 @@ pub const LlamaLM = struct { if (logits.shape().hasTag(.voc) == null) logits = logits.rename(.{ .d = .voc }); - // With replicated logits, sampleTokens becomes a simple local operation - // with no need for collectives. const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); return .{ next_tokens, new_rng }; } @@ -177,13 +169,12 @@ pub const Llama = struct { .scaling = config.rope_scaling, }; - const replicated_sharding = .{ .d = .replicated }; - self.norm.init(config, mesh, replicated_sharding); + self.norm.init(config, mesh, .{ .d = .model }); - // Replicate the embedding table across all devices to avoid gather-related slowdowns. self.embed_tokens.weight = self.embed_tokens.weight .withTags(.{ .voc, .d }) - .withMesh(mesh); + .withMesh(mesh) + .withSharding(.{ .d = .model }); for (self.layers) |*layer| { layer.init(config, mesh); @@ -204,10 +195,12 @@ pub const Llama = struct { } pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { + // This function is now incredibly simple. It just does the lookup. + // The input `tokens_` has the correct sharding annotation (.data or .replicated) + // and the `embed_tokens_.weight` is sharded on `.d`. GSPMD propagates this + // to make the output `embeddings` sharded on both `.s` (by `.data`) and `.d` (by `.model`). const embeddings = zml.call(embed_tokens_, .forward, .{tokens_}); - var tagged_embeddings = embeddings.withPartialTags(.{.d}); - - return tagged_embeddings.withSharding(.{ .s = .data }); + return embeddings.withPartialTags(.{.d}); } }; @@ -218,10 +211,11 @@ pub const TransformerLayer = struct { mlp: Mlp, pub fn init(self: *TransformerLayer, config: LlamaLM.Config, mesh: zml.Mesh) void { + // RmsNorm weights are small and applied element-wise, so they are replicated. self.input_layernorm.init(config, mesh, .{ .d = .replicated }); self.self_attn.init(config, mesh); self.post_attention_layernorm.init(config, mesh, .{ .d = .replicated }); - self.mlp.init(mesh); + self.mlp.init(config, mesh); } pub fn forward( @@ -230,16 +224,12 @@ pub const TransformerLayer = struct { token_index: Tensor, kv_cache: KvCache, ) struct { Tensor, KvCache } { - stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); - + // This is now clean, high-level logic. All sharding is inferred. const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - - // self_attn now returns the final, reduced tensor. const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); const x1 = x0.add(delta0); const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - // mlp now returns the final, reduced tensor. const delta1 = zml.call(self.mlp, .forward, .{x1_normalized}); const x2 = x1.add(delta1); @@ -268,12 +258,13 @@ pub const Mlp = struct { gate_proj: Linear, down_proj: Linear, - pub fn init(self: *Mlp, mesh: zml.Mesh) void { - // Column-Parallel: Shard on the output/column dimension + pub fn init(self: *Mlp, config: LlamaLM.Config, mesh: zml.Mesh) void { + _ = config; + // Column-Parallel: Shard on the output/column dimension (.hidden) self.gate_proj.weight = self.gate_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(.{ .hidden = .model }); self.up_proj.weight = self.up_proj.weight.withTags(.{ .hidden, .d }).withMesh(mesh).withSharding(.{ .hidden = .model }); - // Row-Parallel: Shard on the INPUT/column dimension + // Row-Parallel: Shard on the INPUT/row dimension (.hidden) self.down_proj.weight = self.down_proj.weight.withTags(.{ .d, .hidden }).withMesh(mesh).withSharding(.{ .hidden = .model }); } @@ -281,13 +272,7 @@ pub const Mlp = struct { const gate_out = zml.call(self.gate_proj, .forward, .{x}); const up_out = zml.call(self.up_proj, .forward, .{x}); const silu_out = gate_out.silu().mul(up_out); - const down_proj_out = zml.call(self.down_proj, .forward, .{silu_out}); - - // The output `down_proj_out` is sharded on the `.hidden` dimension along the - // `.model` mesh axis. The subsequent residual connection requires this tensor - // to be replicated. GSPMD will automatically insert an AllReduce(sum) here - // to satisfy the sharding constraint. - return down_proj_out; + return zml.call(self.down_proj, .forward, .{silu_out}); } }; @@ -353,14 +338,9 @@ pub const SelfAttn = struct { } else null; const attn_output = zml.nn.sdpa(q, k_cached, v_cached, .{ .attn_mask = attn_mask, .allow_cudnn = true }); - const attn = attn_output.merge(.{ .o_hidden = .{ .h, .hd } }).rename(.{ .q = .s }); - const o_proj_out = zml.call(self.o_proj, .forward, .{attn}); - // `o_proj_out` is sharded on the `.d` dimension (from `.o_hidden`) along - // the `.model` mesh axis. The subsequent residual connection requires this - // tensor to be replicated. GSPMD will automatically insert an AllReduce(sum) here. return .{ o_proj_out, new_kv_cache }; } }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index a42c31fd..4691c9b7 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -246,13 +246,16 @@ pub fn asyncMain() !void { } const num_devices = platform.getDevices().len; - const mesh: zml.Mesh = if (num_devices >= 4 and platform.target == .tpu) - // For 4 devices on TPU pod a 2x2 mesh is ideal - // for a hybrid data and tensor parallelism strategy. - zml.Mesh.init(.{ .data = 2, .model = num_devices / 2 }) - else - zml.Mesh.init(.{ .model = num_devices }); - log.info("Using logical mesh for Hybrid Parallelism: {}", .{mesh}); + // Define the logical mesh for parallelism. + // Use 2D hybrid parallelism for 4+ devices, otherwise fallback to 1D tensor parallelism. + const mesh: zml.Mesh = if (num_devices >= 4 and @mod(num_devices, 2) == 0) blk: { + log.info("Using 2D mesh for Hybrid Data/Model Parallelism.", .{}); + break :blk zml.Mesh.init(.{ .data = num_devices / 2, .model = 2 }); + } else blk: { + log.info("Using 1D mesh for Tensor Parallelism.", .{}); + break :blk zml.Mesh.init(.{ .model = num_devices }); + }; + log.info("Using logical mesh: {}", .{mesh}); var ts = try zml.aio.detectFormatAndOpen(allocator, res.args.weights.?); defer ts.deinit(); @@ -273,9 +276,18 @@ pub fn asyncMain() !void { const dims = model_instance.model.shape(); const dtype = model_instance.model.embed_tokens.weight.dtype(); - const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32) - .withPartitioning(.{ .s = .data }); - // Decode tokens are also sharded by the batch-like sequence dimension. + // The prefill stage is sharded by sequence length along the `.data` axis. + const is_2d_mesh = mesh.rank() > 1 and mesh.topology.hasTag(.data) != null; + + // Prefill: Long sequence. Shard on `.data` if available, otherwise replicate. + var tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32) + .withPartitioning(.{ .s = .replicated }); + + if (is_2d_mesh) { + tokens_shape_prefill = tokens_shape_prefill.withPartitioning(.{ .s = .data }); + } + + // Decode: Single token. Always replicate the sequence dimension. const tokens_shape_decode = zml.Shape.init(.{ .s = 1 }, .u32); const token_idx_shape = zml.Shape.init(.{}, .u32); const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype); From 8fc2a7e9396ed7ad73be9c44696b72f423c5894d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 6 Aug 2025 15:14:33 +0000 Subject: [PATCH 48/51] another attempt --- examples/llama/llama.zig | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 8fde0b6e..216d2871 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -66,7 +66,7 @@ pub const LlamaLM = struct { lm_head.weight = lm_head.weight .withTags(.{ .voc, .d }) .withMesh(mesh) - .withSharding(.{ .d = .model }); + .withSharding(.{ .voc = .replicated, .d = .model }); } } @@ -174,7 +174,7 @@ pub const Llama = struct { self.embed_tokens.weight = self.embed_tokens.weight .withTags(.{ .voc, .d }) .withMesh(mesh) - .withSharding(.{ .d = .model }); + .withSharding(.{ .voc = .replicated, .d = .model }); for (self.layers) |*layer| { layer.init(config, mesh); @@ -195,12 +195,16 @@ pub const Llama = struct { } pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { - // This function is now incredibly simple. It just does the lookup. - // The input `tokens_` has the correct sharding annotation (.data or .replicated) - // and the `embed_tokens_.weight` is sharded on `.d`. GSPMD propagates this - // to make the output `embeddings` sharded on both `.s` (by `.data`) and `.d` (by `.model`). const embeddings = zml.call(embed_tokens_, .forward, .{tokens_}); - return embeddings.withPartialTags(.{.d}); + var tagged_embeddings = embeddings.withPartialTags(.{.d}); + + if (tokens_.shape().dim(.s) > 1) { + // Prefill phase: Input is sharded on `s` by `data`. Output is sharded on `s` by `data` and `d` by `model`. + return tagged_embeddings.withSharding(.{ .s = .data, .d = .model }); + } else { + // Decode phase: Input is replicated. Output is sharded on `d` by `model`. + return tagged_embeddings.withSharding(.{ .s = .replicated, .d = .model }); + } } }; @@ -211,7 +215,7 @@ pub const TransformerLayer = struct { mlp: Mlp, pub fn init(self: *TransformerLayer, config: LlamaLM.Config, mesh: zml.Mesh) void { - // RmsNorm weights are small and applied element-wise, so they are replicated. + // RMSNorm weights are small and applied element-wise. Replicate them everywhere. self.input_layernorm.init(config, mesh, .{ .d = .replicated }); self.self_attn.init(config, mesh); self.post_attention_layernorm.init(config, mesh, .{ .d = .replicated }); @@ -294,10 +298,13 @@ pub const SelfAttn = struct { .scaling = config.rope_scaling, }; - self.q_proj.weight = self.q_proj.weight.withTags(.{ .q_hidden, .d }).withMesh(mesh).withSharding(.{ .q_hidden = .model }); - self.k_proj.weight = self.k_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); - self.v_proj.weight = self.v_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model }); - self.o_proj.weight = self.o_proj.weight.withTags(.{ .d, .o_hidden }).withMesh(mesh).withSharding(.{ .o_hidden = .model }); + // Column-Parallel projections: shard output dim (.q_hidden, .kv_hidden) on model axis + self.q_proj.weight = self.q_proj.weight.withTags(.{ .q_hidden, .d }).withMesh(mesh).withSharding(.{ .q_hidden = .model, .d = .replicated }); + self.k_proj.weight = self.k_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model, .d = .replicated }); + self.v_proj.weight = self.v_proj.weight.withTags(.{ .kv_hidden, .d }).withMesh(mesh).withSharding(.{ .kv_hidden = .model, .d = .replicated }); + + // Row-Parallel projection: shard input dim (.o_hidden) on model axis + self.o_proj.weight = self.o_proj.weight.withTags(.{ .d, .o_hidden }).withMesh(mesh).withSharding(.{ .d = .replicated, .o_hidden = .model }); } pub fn forward( From bf9369abcc93c65d00dc6f644e1f8598d315a11e Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 6 Aug 2025 15:20:02 +0000 Subject: [PATCH 49/51] restore writeShardingRepresentation --- zml/partitioning.zig | 98 ++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 57 deletions(-) diff --git a/zml/partitioning.zig b/zml/partitioning.zig index 20156d9f..b5c9e33c 100644 --- a/zml/partitioning.zig +++ b/zml/partitioning.zig @@ -621,81 +621,65 @@ pub const Sharding = struct { } pub fn writeShardingRepresentation(self: Sharding, writer: anytype) !void { - if (self.mesh.isSinglePartition()) { + if (self.getType() == .replicated) { try writer.writeAll("{replicated}"); return; } - var used_mesh_axes: std.BoundedArray(bool, MaxMeshAxes) = .{ .buffer = [_]bool{false} ** MaxMeshAxes, .len = @intCast(self.mesh.rank()) }; - var is_fully_replicated = true; + var product_of_tensor_tiles: i64 = 1; + var num_partitioned_dims: u32 = 0; + // Calculate the product of explicitly defined tile dimensions for (0..self.global_shape.rank()) |i| { const part_spec = self.global_shape.partition(i); if (part_spec == .axis) { const mesh_axis_tag = part_spec.toTag(); - if (self.mesh.topology.hasTag(mesh_axis_tag)) |mesh_axis_idx| { - is_fully_replicated = false; - used_mesh_axes.buffer[mesh_axis_idx] = true; + if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + product_of_tensor_tiles *= self.mesh.topology.dim(mesh_axis_tag); + num_partitioned_dims += 1; } } } - if (is_fully_replicated) { - try writer.writeAll("{replicated}"); - return; - } + const is_fully_sharded_on_mesh = (product_of_tensor_tiles == self.mesh.numPartitions()); - var tile_assignment_dims: Shape.DimsArray = .{}; - for (0..self.global_shape.rank()) |i| { - const part_spec = self.global_shape.partition(i); - var tile_dim: i64 = 1; - - if (part_spec == .axis) { - if (self.mesh.topology.hasTag(part_spec.toTag())) |mesh_axis_idx| { - tile_dim = self.mesh.topology.dim(mesh_axis_idx); + if (is_fully_sharded_on_mesh) { + // This is the easy case. The tensor is partitioned across all mesh devices. + // The tile array must have the same rank as the tensor. + try writer.writeAll("{devices=["); + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + var tile_dim: i64 = 1; + if (part_spec == .axis) { + if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + tile_dim = self.mesh.topology.dim(part_spec.toTag()); + } + } + try writer.print("{d}", .{tile_dim}); + if (i < self.global_shape.rank() - 1) { + try writer.writeByte(','); } } - tile_assignment_dims.appendAssumeCapacity(tile_dim); - } - - var replication_factor: i64 = 1; - for (used_mesh_axes.constSlice(), 0..) |used, i| { - if (!used) { - replication_factor *= self.mesh.topology.dim(i); + try writer.print("]<=[{d}]}}", .{self.mesh.numPartitions()}); + } else { + // This is partial sharding. This is more complex. + // The `devices` array lists the tiling only for the *partitioned* dimensions. + // The remaining devices are handled by `last_tile_dim_replicate`. + try writer.writeAll("{devices=["); + var first = true; + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + if (!first) try writer.writeByte(','); + try writer.print("{d}", .{self.mesh.topology.dim(mesh_axis_tag)}); + first = false; + } + } } + try writer.print("]<=[{d}] last_tile_dim_replicate}}", .{self.mesh.numPartitions()}); } - - // Find a dimension to apply the replication factor to. - // GSPMD prefers applying replication to an existing replicated dimension if possible. - var applied_replication = false; - for (0..self.global_shape.rank()) |i| { - if (self.global_shape.partition(i) == .replicated) { - tile_assignment_dims.buffer[i] *= replication_factor; - applied_replication = true; - break; - } - } - // If no dimension was marked as replicated, we add a new virtual tiling dimension for replication. - if (!applied_replication and replication_factor > 1) { - tile_assignment_dims.appendAssumeCapacity(replication_factor); - } - - var product: u64 = 1; - for (tile_assignment_dims.constSlice()) |d| product *= @intCast(d); - stdx.debug.assert( - product == self.mesh.numDevices(), - "Internal error: tiling product {d} does not match mesh size {d} for shape {s} on mesh {s}", - .{ product, self.mesh.numDevices(), self.global_shape, self.mesh }, - ); - - try writer.writeAll("{devices=["); - for (tile_assignment_dims.constSlice(), 0..) |dim, i| { - try writer.print("{d}", .{dim}); - if (i < tile_assignment_dims.len - 1) { - try writer.writeByte(','); - } - } - try writer.print("]<=[{d}]}}", .{self.mesh.numDevices()}); } test "Sharding MLIR representation" { From fe0b99427382385a829681694f62c6697e23ae0a Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 6 Aug 2025 15:25:46 +0000 Subject: [PATCH 50/51] fix norm weight sharding --- examples/llama/llama.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 216d2871..49379747 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -179,6 +179,8 @@ pub const Llama = struct { for (self.layers) |*layer| { layer.init(config, mesh); } + + self.norm.init(config, mesh, .{ .d = .model }); } pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { From 6b6152a29179c5a985e10cf9b5d8db7f436d3d1d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 8 Aug 2025 12:42:00 +0000 Subject: [PATCH 51/51] cleanup from tpu --- examples/llama/main.zig | 7 +- examples/partitioning/main.zig | 26 ++++--- zml/buffer.zig | 7 +- zml/partitioning.zig | 132 ++++++++++++++++++++++++--------- zml/platform.zig | 32 ++++++-- zml/tensor.zig | 1 + 6 files changed, 147 insertions(+), 58 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 4691c9b7..42443e73 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -246,11 +246,10 @@ pub fn asyncMain() !void { } const num_devices = platform.getDevices().len; - // Define the logical mesh for parallelism. - // Use 2D hybrid parallelism for 4+ devices, otherwise fallback to 1D tensor parallelism. - const mesh: zml.Mesh = if (num_devices >= 4 and @mod(num_devices, 2) == 0) blk: { + const mesh: zml.Mesh = if (num_devices >= 8 and @mod(num_devices, 2) == 0 and platform.target == .tpu) blk: { log.info("Using 2D mesh for Hybrid Data/Model Parallelism.", .{}); - break :blk zml.Mesh.init(.{ .data = num_devices / 2, .model = 2 }); + // A common strategy: split batch over `data` and model layers over `model`. + break :blk zml.Mesh.init(.{ .data = @divExact(num_devices, 2), .model = @divExact(num_devices, 2) }); } else blk: { log.info("Using 1D mesh for Tensor Parallelism.", .{}); break :blk zml.Mesh.init(.{ .model = num_devices }); diff --git a/examples/partitioning/main.zig b/examples/partitioning/main.zig index c764f910..69dc90c3 100644 --- a/examples/partitioning/main.zig +++ b/examples/partitioning/main.zig @@ -17,14 +17,14 @@ const ThirdPartyModule = struct { weight: zml.Tensor, pub fn init(self: *ThirdPartyModule, mesh: zml.Mesh) void { - self.weight = self.weight.withTags(.{ .rows, .cols }).withMesh(mesh).withSharding(.{ .rows = .xoxo }); + self.weight = self.weight.withTags(.{ .rows, .cols }).withMesh(mesh).withSharding(.{ .rows = .x }); } pub fn forward(self: ThirdPartyModule, a: zml.Tensor) zml.Tensor { log.debug("Forwarding a: {} and weight: {}", .{ a, self.weight }); - zml.pushMesh(a.mesh().flatten(.xoxo)); - const x = a.withSharding(.{ .m = .xoxo }).add(self.weight.withSharding(.{ .rows = .x })); + zml.pushMesh(a.mesh().flatten(.x)); + const x = a.withSharding(.{ .m = .x }).add(self.weight.withSharding(.{ .rows = .x })); zml.popMesh(); return x.withSharding(.{ .m = .x, .n = .y }); @@ -110,15 +110,21 @@ pub fn asyncMain() !void { } } - // const mesh: zml.Mesh = .init(.{ .x = 4, .y = 3 }); - const mesh: zml.Mesh = .auto(platform); - // const mesh: zml.Mesh = .single(); - log.info("{}", .{mesh}); + const num_devices = devices.len; + const mesh: zml.Mesh = if (platform.target == .tpu and num_devices >= 4 and @mod(num_devices, 2) == 0) blk: { + const mesh2d = zml.Mesh.init(.{ .x = @divExact(num_devices, 2), .y = 2 }); + log.info("Using 2D mesh for TPU: {}", .{mesh2d}); + break :blk mesh2d; + } else blk: { + const mesh1d = zml.Mesh.auto(platform); + log.info("Using 1D mesh: {}", .{mesh1d}); + break :blk mesh1d; + }; - const shape = zml.Shape.init(.{ .m = 12, .n = 6 }, .i32); + const shape = zml.Shape.init(.{ .m = 24, .n = 12 }, .i32); const a_shape = shape.withPartitioning(.{ .m = .x, .n = .y }); - const b_shape = shape.withTags(.{ .n, .m }).withPartitioning(.{ .n = .x, .m = .y }); + const b_shape = shape.withPartitioning(.{ .m = .y, .n = .x }); var buffers: zml.aio.BufferStore.Buffers = .{}; const weight = try zml.slice.arange(allocator, shape, .{}); @@ -154,7 +160,7 @@ pub fn asyncMain() !void { const a_buffer = try zml.Buffer.from(platform, a_sharding, a_buffer_slice, .{}); defer a_buffer.deinit(); - const b_sharding: zml.Sharding = .init(mesh, a_shape); + const b_sharding: zml.Sharding = .init(mesh, b_shape); log.info("b_sharding: {}", .{b_sharding}); const b_buffer_slice = try zml.slice.arange(allocator, b_shape, .{}); diff --git a/zml/buffer.zig b/zml/buffer.zig index 9cbd56d3..2bb0f15b 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -8,6 +8,7 @@ const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const createDeviceMesh = @import("platform.zig").createDeviceMesh; const Shape = @import("shape.zig").Shape; const partitioning = @import("partitioning.zig"); const Mesh = partitioning.Mesh; @@ -59,7 +60,11 @@ pub const Buffer = struct { /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, sharding: Sharding, data: []const u8, opts: FromOptions) !Buffer { - var sharding_devices = sharding.iterator(platform.getDevices()); + var device_list_buffer: [1024]u8 = undefined; // todo remove allocator in createDeviceMesh + var fba = std.heap.FixedBufferAllocator.init(&device_list_buffer); + const ordered_devices = try createDeviceMesh(fba.allocator(), sharding.mesh, platform); + + var sharding_devices = sharding.iterator(ordered_devices); var res: Buffer = .{ ._api = platform.pjrt_api, diff --git a/zml/partitioning.zig b/zml/partitioning.zig index b5c9e33c..7b09efb6 100644 --- a/zml/partitioning.zig +++ b/zml/partitioning.zig @@ -627,7 +627,7 @@ pub const Sharding = struct { } var product_of_tensor_tiles: i64 = 1; - var num_partitioned_dims: u32 = 0; + var has_partitioned_axis = false; // Calculate the product of explicitly defined tile dimensions for (0..self.global_shape.rank()) |i| { @@ -636,52 +636,112 @@ pub const Sharding = struct { const mesh_axis_tag = part_spec.toTag(); if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { product_of_tensor_tiles *= self.mesh.topology.dim(mesh_axis_tag); - num_partitioned_dims += 1; + has_partitioned_axis = true; } } } + if (!has_partitioned_axis) { + try writer.writeAll("{replicated}"); + return; + } + const is_fully_sharded_on_mesh = (product_of_tensor_tiles == self.mesh.numPartitions()); + try writer.writeAll("{devices=["); + // The tile array must have the same rank as the tensor, for both partial and full sharding. + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + var tile_dim: i64 = 1; + if (part_spec == .axis) { + if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + tile_dim = self.mesh.topology.dim(part_spec.toTag()); + } + } + try writer.print("{d}", .{tile_dim}); + if (i < self.global_shape.rank() - 1) { + try writer.writeByte(','); + } + } + + try writer.writeAll("]<=["); + const mesh_dims = self.mesh.topology.dims(); + for (mesh_dims, 0..) |dim, i| { + try writer.print("{d}", .{dim}); + if (i < mesh_dims.len - 1) { + try writer.writeByte(','); + } + } + if (is_fully_sharded_on_mesh) { - // This is the easy case. The tensor is partitioned across all mesh devices. - // The tile array must have the same rank as the tensor. - try writer.writeAll("{devices=["); - for (0..self.global_shape.rank()) |i| { - const part_spec = self.global_shape.partition(i); - var tile_dim: i64 = 1; - if (part_spec == .axis) { - if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { - tile_dim = self.mesh.topology.dim(part_spec.toTag()); - } - } - try writer.print("{d}", .{tile_dim}); - if (i < self.global_shape.rank() - 1) { - try writer.writeByte(','); - } - } - try writer.print("]<=[{d}]}}", .{self.mesh.numPartitions()}); + try writer.writeAll("]}"); } else { - // This is partial sharding. This is more complex. - // The `devices` array lists the tiling only for the *partitioned* dimensions. - // The remaining devices are handled by `last_tile_dim_replicate`. - try writer.writeAll("{devices=["); - var first = true; - for (0..self.global_shape.rank()) |i| { - const part_spec = self.global_shape.partition(i); - if (part_spec == .axis) { - const mesh_axis_tag = part_spec.toTag(); - if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { - if (!first) try writer.writeByte(','); - try writer.print("{d}", .{self.mesh.topology.dim(mesh_axis_tag)}); - first = false; - } - } - } - try writer.print("]<=[{d}] last_tile_dim_replicate}}", .{self.mesh.numPartitions()}); + try writer.writeAll("] last_tile_dim_replicate}"); } } + // pub fn writeShardingRepresentation(self: Sharding, writer: anytype) !void { + // if (self.getType() == .replicated) { + // try writer.writeAll("{replicated}"); + // return; + // } + + // var product_of_tensor_tiles: i64 = 1; + // var num_partitioned_dims: u32 = 0; + + // // Calculate the product of explicitly defined tile dimensions + // for (0..self.global_shape.rank()) |i| { + // const part_spec = self.global_shape.partition(i); + // if (part_spec == .axis) { + // const mesh_axis_tag = part_spec.toTag(); + // if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + // product_of_tensor_tiles *= self.mesh.topology.dim(mesh_axis_tag); + // num_partitioned_dims += 1; + // } + // } + // } + + // const is_fully_sharded_on_mesh = (product_of_tensor_tiles == self.mesh.numPartitions()); + + // if (is_fully_sharded_on_mesh) { + // // This is the easy case. The tensor is partitioned across all mesh devices. + // // The tile array must have the same rank as the tensor. + // try writer.writeAll("{devices=["); + // for (0..self.global_shape.rank()) |i| { + // const part_spec = self.global_shape.partition(i); + // var tile_dim: i64 = 1; + // if (part_spec == .axis) { + // if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + // tile_dim = self.mesh.topology.dim(part_spec.toTag()); + // } + // } + // try writer.print("{d}", .{tile_dim}); + // if (i < self.global_shape.rank() - 1) { + // try writer.writeByte(','); + // } + // } + // try writer.print("]<=[{d}]}}", .{self.mesh.numPartitions()}); + // } else { + // // This is partial sharding. This is more complex. + // // The `devices` array lists the tiling only for the *partitioned* dimensions. + // // The remaining devices are handled by `last_tile_dim_replicate`. + // try writer.writeAll("{devices=["); + // var first = true; + // for (0..self.global_shape.rank()) |i| { + // const part_spec = self.global_shape.partition(i); + // if (part_spec == .axis) { + // const mesh_axis_tag = part_spec.toTag(); + // if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + // if (!first) try writer.writeByte(','); + // try writer.print("{d}", .{self.mesh.topology.dim(mesh_axis_tag)}); + // first = false; + // } + // } + // } + // try writer.print("]<=[{d}] last_tile_dim_replicate}}", .{self.mesh.numPartitions()}); + // } + // } + test "Sharding MLIR representation" { // Case 1: Fully Replicated const mesh_rep = Mesh.init(.{ .x = 4 }); diff --git a/zml/platform.zig b/zml/platform.zig index 3d9ab4ec..8f5ad6e1 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -34,7 +34,6 @@ const tpu_tray_ring_order_8 = [_]u8{ 0, 1, 2, 3, 7, 6, 5, 4 }; /// Sorts devices primarily by their coordinates (lexicographically, reversed) and then by core_on_chip. /// This matches JAX's z,y,x,core sorting for creating a physical mesh. /// This function is allocation-free. -/// This is a `lessThan` function for `std.sort.sort`. fn sortDevicesForPhysicalMesh(context: *const pjrt.Api, lhs_dev: *const pjrt.Device, rhs_dev: *const pjrt.Device) bool { const api: *const pjrt.Api = context; @@ -82,6 +81,17 @@ fn sortDevicesForPhysicalMesh(context: *const pjrt.Api, lhs_dev: *const pjrt.Dev return lhs_core < rhs_core; } +/// Sorts devices topologically for consistent ordering across platforms. +fn getTopologicallySortedDevices( + allocator: std.mem.Allocator, + devices: []const *const pjrt.Device, + api: *const pjrt.Api, +) ![]*const pjrt.Device { + const sorted_devices = try allocator.dupe(*const pjrt.Device, devices); + std.sort.pdq(*const pjrt.Device, sorted_devices, api, sortDevicesForPhysicalMesh); + return sorted_devices; +} + /// Creates a device mesh that arranges devices for optimal collective performance, /// similar to `jax.experimental.mesh_utils.create_device_mesh`. pub fn createDeviceMesh( @@ -98,30 +108,38 @@ pub fn createDeviceMesh( if (active_devices.len == 0) return &.{}; + // Always sort devices topologically first to ensure a consistent base order + // for mapping logical devices to physical devices across all platforms. + const sequential_devices = try getTopologicallySortedDevices(allocator, active_devices, api); + + // For TPUs, we might apply an additional reordering for performance. + // For other platforms, we will use the topologically sorted list directly. switch (platform.target) { .tpu => { + // Keep the TPU-specific reordering logic, but operate on the sorted list. const first_device_kind = active_devices[0].getDescription(api).getKind(api); - // The kind string is a reliable way to dispatch for TPUs. if (std.mem.eql(u8, first_device_kind, "TPU v5 lite") or std.mem.eql(u8, first_device_kind, "TPU v6 lite")) { - if (try createTpuV5LiteMesh(allocator, logical_mesh, active_devices, api)) |ordered_devices| { + if (try createTpuV5LiteMesh(allocator, logical_mesh, sequential_devices, api)) |ordered_devices| { + allocator.free(sequential_devices); // Free the intermediate sorted list return ordered_devices; } } else if (std.mem.eql(u8, first_device_kind, "TPU7x")) { - if (try createTpu7xMesh(allocator, logical_mesh, active_devices, api)) |ordered_devices| { + if (try createTpu7xMesh(allocator, logical_mesh, sequential_devices, api)) |ordered_devices| { + allocator.free(sequential_devices); // Free the intermediate sorted list return ordered_devices; } } // Fallback for other TPU kinds (e.g., v2, v3, v4, v5p). log.warn("Using generic TPU physical mesh creation for {s}. For optimal performance, a specific handler may be needed.", .{first_device_kind}); - var physical_mesh = try getPhysicalTpuMesh(allocator, active_devices, api); + var physical_mesh = try getPhysicalTpuMesh(allocator, sequential_devices, api); defer physical_mesh.deinit(); return try createDeviceMeshForNdTorus(allocator, physical_mesh, logical_mesh); }, .cuda, .rocm, .cpu, .neuron => { - // Default for CPU, GPU, and Neuron: simple duplication of the device list. - return allocator.dupe(*const pjrt.Device, active_devices); + // For other platforms, the topologically sorted list is the final list. + return sequential_devices; }, } } diff --git a/zml/tensor.zig b/zml/tensor.zig index 8f8c5727..11c8efcf 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -171,6 +171,7 @@ pub const Tensor = struct { } pub fn mesh(self: Tensor) Mesh { + if (self._mesh) |m| return m; const ctx = self.getContext(); return ctx.currentMesh(); }