From b2f0e03043f6471ce6458a85aeefbca0c3fb151e Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 6 Oct 2026 15:37:22 +0200 Subject: [PATCH] zml/Tensor: explicit sharding in Tensor code (#868) In Buffer code we already have explicit sharding, but not for tensor. I'm adding a `zml.Sharding` to `.withPartitioning` to make it explicit which mesh we are working with. To simplify the migration, `.withPartitioning` in Tensor code can use `.model` syntax to lookup a sharding by name. This replace the previous logic of `selectSharding(shape)` which would look up a sharding that matches the axes of the given partitioning which could yield surprising behavior. This also modifies how sharding are stored in zml.Shape. Before it was storing a list of axis names + a tag (2 * 8 * 64bits), now I'm storing a zml.Sharding (64 bits) and a compact PartitionArray (32 bits) that references the logical axis of the sharding by offset instead of by name. There is an impact on the `TensorRegistry` because it also needs to be aware of the possible shardings, instead of delegating the matching partition spec -> sharding to loading. This means error will be detected sooner. `store.createTensor` accepts `@enumLiteral()` to lookup the relevant sharding. This also simplifies `zml.Buffer` because the full sharding is now stored in the shape, so it doesn't need to be passed independently. It also means that it will propagate from the tensor who are graph inputs, to the buffers that are eg inputs. `Placement` logic also got a bit simpler. --- README.md | 8 +- bin/zml-smi/BUILD.bazel | 18 +- bin/zml-smi/platforms/amdsmi/BUILD.bazel | 5 +- bin/zml-smi/platforms/nvml/BUILD.bazel | 5 +- docs/howtos/howto_torch2zml.md | 16 +- examples/benchmark/main.zig | 25 +- examples/io/main.zig | 27 +- examples/llm/main.zig | 11 +- examples/llm/models.zig | 17 +- examples/llm/models/common.zig | 18 +- examples/llm/models/lfm2/inference.zig | 16 +- examples/llm/models/lfm2/model.zig | 56 +- examples/llm/models/lfm2/session.zig | 20 +- examples/llm/models/lfm2_tests.zig | 57 +- examples/llm/models/llama/inference.zig | 12 +- examples/llm/models/llama/model.zig | 55 +- examples/llm/models/llama/session.zig | 13 +- examples/llm/models/llama_tests.zig | 40 +- examples/llm/models/qwen3_5/inference.zig | 26 +- examples/llm/models/qwen3_5/model.zig | 163 ++- examples/llm/models/qwen3_5/session.zig | 11 +- examples/llm/models/qwen3_5_moe/inference.zig | 20 +- examples/llm/models/qwen3_5_moe/model.zig | 195 ++-- examples/llm/models/qwen3_5_moe/session.zig | 9 +- examples/mnist/mnist.zig | 23 +- examples/sharding/main.zig | 37 +- platforms/cuda/BUILD.bazel | 2 + stdx/debug.zig | 3 +- stdx/fmt.zig | 38 + tools/BUILD.bazel | 3 +- zml/Compiler.zig | 189 ++-- zml/Sharding.zig | 941 +++++++++++------- zml/attention.zig | 47 +- zml/attention/attnd.zig | 9 +- zml/attention/flashattn.zig | 161 +-- zml/attention/metal_attention.zig | 3 +- zml/attention/nki/attention.zig | 9 +- zml/attention/paged_attention.zig | 79 +- zml/attention/sparse_mla.zig | 2 +- zml/attention/tpu_attention.zig | 14 +- zml/buffer.zig | 83 +- zml/exe.zig | 37 +- zml/io.zig | 224 ++--- zml/kernel.zig | 4 +- zml/mem.zig | 1 - zml/meta.zig | 13 +- zml/moe/cute_mxfp4.zig | 2 +- zml/moe/cutlass_flashinfer.zig | 6 +- zml/moe/fused_experts.zig | 10 +- zml/moe/mosaic_tpu.zig | 2 +- zml/moe/triton_mxfp4.zig | 2 +- zml/nn.zig | 35 +- zml/ops.zig | 121 ++- zml/platform.zig | 37 +- zml/quantization.zig | 4 +- zml/shape.zig | 566 +++++------ zml/tensor.zig | 278 ++++-- zml/testing.zig | 10 +- zml/zml.zig | 1 + 59 files changed, 2113 insertions(+), 1726 deletions(-) diff --git a/README.md b/README.md index f35453ff..d9e68b81 100644 --- a/README.md +++ b/README.md @@ -145,8 +145,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createTensor("weight", .{ .d_out, .d }, null), - .bias = store.createTensor("bias", .{.d_out}, null), + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), }; } @@ -168,10 +168,10 @@ const Mnist = struct { io: std.Io, platform: *const zml.Platform, store: *const zml.io.TensorStore, - shardings: []const zml.Sharding, + meshes: []const *const zml.Mesh, ) !zml.Bufferized(Mnist) { return zml.io.load(Mnist, self, allocator, io, platform, store, .{ - .shardings = shardings, + .meshes = meshes, .parallelism = 1, .dma_chunks = 1, .dma_chunk_size = 16 * 1024 * 1024, diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index d940a801..077f5b34 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -22,6 +22,7 @@ run_binary( "$(location libnrt.so.1)", "$(location @aws-neuronx-runtime-lib//:libnrt.patchelf)", ], + target_compatible_with = ["@platforms//os:linux"], tool = "@llvm//tools:llvm-strip", ) @@ -35,6 +36,7 @@ run_binary( "$(location libncfw.so.2)", "$(location @aws-neuronx-runtime-lib//:libncfw.patchelf)", ], + target_compatible_with = ["@platforms//os:linux"], tool = "@llvm//tools:llvm-strip", ) @@ -76,18 +78,18 @@ run_binary( zig_binary( name = "zml-smi", - visibility = ["//visibility:public"], data = [":sandbox"], main = "main.zig", + visibility = ["//visibility:public"], deps = [ + "//bin/zml-smi/api", "//bin/zml-smi/collector", "//bin/zml-smi/csv", - "//bin/zml-smi/json", - "//bin/zml-smi/api", - "//bin/zml-smi/prometheus", - "//bin/zml-smi/utils:sysfs", "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/prometheus", "//bin/zml-smi/tui", + "//bin/zml-smi/utils:sysfs", "//stdx", "@libvaxis//:vaxis", "@rules_zig//zig/runfiles", @@ -97,9 +99,9 @@ zig_binary( }) + select({ "@llvm//platforms/config:linux_x86_64": [ "//bin/zml-smi/platforms/amdsmi", - "//bin/zml-smi/platforms/oneapi", "//bin/zml-smi/platforms/neuron", "//bin/zml-smi/platforms/nvml", + "//bin/zml-smi/platforms/oneapi", "//bin/zml-smi/platforms/tpu", "//platforms/neuron:libnrt_headers", ], @@ -168,8 +170,8 @@ _RELEASE_PLATFORMS = [ [platform_transition_filegroup( name = "zml-smi-" + os + "-" + arch, srcs = [":archive_" + os + "_" + arch], - target_platform = "//platforms:" + os + "_" + arch, tags = ["manual"], + target_platform = "//platforms:" + os + "_" + arch, ) for os, arch in _RELEASE_PLATFORMS] [hashes( @@ -195,6 +197,7 @@ oci_image( name = "image_", base = "@distroless_cc_debian12", entrypoint = ["/zml-smi/zml-smi"], + tags = ["manual"], target_compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", @@ -202,7 +205,6 @@ oci_image( tars = [ ":archive_linux_amd64", ], - tags = ["manual"], ) platform_transition_filegroup( diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel index e3ac355d..dac6b935 100644 --- a/bin/zml-smi/platforms/amdsmi/BUILD.bazel +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -2,12 +2,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "amdsmi", - import_name = "zml-smi/platforms/amdsmi", - main = "metrics.zig", srcs = [ "amdsmi.zig", "process.zig", ], + import_name = "zml-smi/platforms/amdsmi", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = [ "//bin/zml-smi/collector", diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel index 3a9e54ed..0ab0fae1 100644 --- a/bin/zml-smi/platforms/nvml/BUILD.bazel +++ b/bin/zml-smi/platforms/nvml/BUILD.bazel @@ -2,12 +2,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "nvml", - import_name = "zml-smi/platforms/nvml", - main = "metrics.zig", srcs = [ "nvml.zig", "process.zig", ], + import_name = "zml-smi/platforms/nvml", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = [ "//bin/zml-smi/collector", diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index d05e2b6f..3d246f11 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -190,7 +190,7 @@ convert it to a `TensorStore`: ```zig pub fn main(init: std.process.Init) !void { ... - var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); defer model_store.deinit(); } ``` @@ -205,18 +205,18 @@ pub fn main(init: std.process.Init) !void { const mlp: Mlp = .{ .up_proj = .init( - mlp_view.createTensor("up_proj.weight", .{ .dout, .d }, null), - mlp_view.maybeCreateTensor("up_proj.bias", .{.dout}, null), + mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), .d, ), .gate_proj = .init( - mlp_view.createTensor("gate_proj.weight", .{ .dout, .d }, null), - mlp_view.maybeCreateTensor("gate_proj.bias", .{.dout}, null), + mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), .d, ), .down_proj = .init( - mlp_view.createTensor("down_proj.weight", .{ .d, .dout }, null), - mlp_view.maybeCreateTensor("down_proj.bias", .{.d}, null), + mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), + mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), .dout, ), }; @@ -287,7 +287,7 @@ pub const Wrapper = struct { pub fn main(init: std.process.Init) !void { ... - var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry); + var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); defer activations_store.deinit(); try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 1bdaad7b..7ff70707 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -5,7 +5,7 @@ const zml = @import("zml"); const stdx = zml.stdx; pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.dot(b, .k).withPartitioning(.{ .m = .m, .n = .replicated }); + return a.dot(b, .k).withPartitioning(.benchmark_mesh, .{ .m = .m, .n = .replicated }); } pub fn main(init: std.process.Init) !void { @@ -26,34 +26,31 @@ pub fn main(init: std.process.Init) !void { log.info("\n{f}", .{platform.fmtVerbose()}); - const benchmark_sharding: zml.Sharding = try platform.registerSharding("benchmark_mesh", .mesh( + const mesh: *const zml.Mesh = try platform.registerMesh("mesh", .mesh( .{ .m = .low_bandwidth, .n = .high_bandwidth }, )); const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); - const a_shape = zml.Shape.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) - .withPartitioning(.{ .m = .m, .k = .replicated }); - const b_shape = zml.Shape.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) - .withPartitioning(.{ .k = .replicated, .n = .n }); - - const a: zml.Tensor = .fromShape(a_shape); - const b: zml.Tensor = .fromShape(b_shape); + const a = zml.Tensor.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .m = .m, .k = .replicated }); + const b = zml.Tensor.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .k = .replicated, .n = .n }); var exe = blk: { log.info("⏱️ Compiling benchmark...", .{}); const now: std.Io.Timestamp = .now(io, .awake); defer log.info("✅ Compiled benchmark [{f}]", .{now.untilNow(io, .awake)}); - break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .shardings = &.{benchmark_sharding} }); + break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .meshes = &.{mesh} }); }; defer exe.deinit(); var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); - var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), benchmark_sharding, random); + var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), random); defer a_buffer.deinit(); - var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), benchmark_sharding, random); + var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), random); defer b_buffer.deinit(); var exe_args = try exe.args(allocator); @@ -96,7 +93,7 @@ pub fn main(init: std.process.Init) !void { }); } -fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, sharding: zml.Sharding, random: std.Random) !zml.Buffer { +fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); defer slice.free(allocator); @@ -122,5 +119,5 @@ fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const }, } - return .fromSlice(io, platform, slice, sharding); + return .fromSlice(io, platform, slice); } diff --git a/examples/io/main.zig b/examples/io/main.zig index 29215866..1cec23b0 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -155,9 +155,9 @@ pub fn main(init: std.process.Init) !void { try stdout_writer.interface.flush(); }, .load => { - const ShardingType = enum { replicated, sharded }; + const MeshType = enum { replicated, sharded }; - const sharding_type: ShardingType = std.meta.stringToEnum(ShardingType, it.next() orelse "sharded") orelse return error.InvalidShardingKind; + const mesh_type: MeshType = std.meta.stringToEnum(MeshType, it.next() orelse "sharded") orelse return error.InvalidMesheKind; const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); @@ -165,7 +165,13 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + const tensor_parallel: *const zml.Mesh = try platform.registerMesh( + "tp", + .mesh(.{ .model = .high_bandwidth }), + ); + const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); defer store.deinit(); const AllTensorsModel = struct { @@ -180,22 +186,17 @@ pub fn main(init: std.process.Init) !void { var registry_it = registry.iterator(); var load_count: usize = 0; while (registry_it.next()) |entry| : (load_count += 1) { - tensors[load_count] = switch (sharding_type) { - .replicated => store.view().createTensor(entry.key_ptr.*, null, .replicated), + tensors[load_count] = switch (mesh_type) { + .replicated => store.view().createReplicatedTensor(entry.key_ptr.*, null), .sharded => if (entry.value_ptr.shape.rank() > 0) - store.view().createTensor(entry.key_ptr.*, null, .{ ._0 = .model }) + store.view().createTensor(entry.key_ptr.*, null, .tp, tp_partitioning) else - store.view().createTensor(entry.key_ptr.*, null, .replicated), + store.view().createReplicatedTensor(entry.key_ptr.*, null), }; } const model: AllTensorsModel = .{ .tensors = tensors }; - const sharded_sharding: zml.Sharding = try platform.registerSharding( - "playground_model", - .mesh(.{ .model = .high_bandwidth }), - ); - var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); progress.increaseEstimatedTotalItems(load_count); defer progress.end(); @@ -215,7 +216,7 @@ pub fn main(init: std.process.Init) !void { }); defer loader.deinit(); - try loader.load(io, AllTensorsModel, &model, &buffers, &store, &.{sharded_sharding}, .{ .progress = &progress }); + try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); try loader.await(io); const took = now.untilNow(io, .awake); diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 4407aadb..e10db570 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -109,7 +109,9 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + // Defines how the model's tensors are sharded across the available devices. + const meshes: models.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); const generation: models.GenerationOptions = .{ @@ -121,9 +123,6 @@ pub fn main(init: std.process.Init) !void { var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); defer model.deinit(allocator); - // Defines how the model's tensors are sharded across the available devices. - const shardings: models.Shardings = try .init(platform); - // // Load the model and compile it // @@ -135,11 +134,11 @@ pub fn main(init: std.process.Init) !void { var compiled_model = try allocator.create(models.CompiledModel); defer allocator.destroy(compiled_model); - compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, shardings, args.seqlen, &progress); + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); defer compiled_model.deinit(); // Load buffers after the model compilation to be sure to give enough room to the autotune. - var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress, shardings); + var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); defer model.unloadBuffers(&model_buffers, allocator); progress.end(); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index b88a855c..20aa60c7 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -3,7 +3,7 @@ const std = @import("std"); const zml = @import("zml"); const common = @import("models/common.zig"); -pub const Shardings = common.Shardings; +pub const Meshes = common.Meshes; pub const GenerationOptions = common.GenerationOptions; pub const parseConfig = common.parseConfig; pub const lfm2 = @import("models/lfm2.zig"); @@ -55,12 +55,19 @@ pub const LoadedModel = union(ModelType) { } } - pub fn loadBuffers(self: *LoadedModel, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !Buffers { + pub fn loadBuffers( + self: *LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { return switch (self.*) { inline else => |*m, t| @unionInit( Buffers, @tagName(t), - try m.loadBuffers(allocator, io, platform, store, progress, shardings), + try m.loadBuffers(allocator, io, platform, store, progress), ), }; } @@ -77,7 +84,7 @@ pub const LoadedModel = union(ModelType) { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: Shardings, + meshes: Meshes, seqlen: usize, progress: *std.Progress.Node, ) !CompiledModel { @@ -85,7 +92,7 @@ pub const LoadedModel = union(ModelType) { inline else => |*m, t| @unionInit( CompiledModel.Inner, @tagName(t), - try m.compile(allocator, io, platform, backend, shardings, seqlen, progress), + try m.compile(allocator, io, platform, backend, meshes, seqlen, progress), ), }; return .{ diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index debeff00..62fc7560 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -42,11 +42,11 @@ pub const Phase = enum { } }; -pub const Shardings = struct { - model: zml.Sharding, - experts: zml.Sharding, +pub const Meshes = struct { + model: *const zml.Mesh, + experts: *const zml.Mesh, - pub fn init(platform: *zml.Platform) !Shardings { + pub fn init(platform: *zml.Platform) !Meshes { switch (platform.target) { .tpu => { const has_link_y = platform.physical_mesh.hasAxis(.link_y); @@ -59,18 +59,18 @@ pub const Shardings = struct { } return .{ - .model = try platform.registerShardingWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), - .experts = try platform.registerShardingWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), + .model = try platform.registerMeshWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), + .experts = try platform.registerMeshWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), }; }, .cuda, .rocm, .oneapi, .neuron, .metal, .cpu => return .{ - .model = try platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })), - .experts = try platform.registerSharding("experts", .mesh(.{ .experts = .high_bandwidth })), + .model = try platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })), + .experts = try platform.registerMesh("experts", .mesh(.{ .experts = .high_bandwidth })), }, } } - pub fn all(self: Shardings) [2]zml.Sharding { + pub fn all(self: Meshes) [2]*const zml.Mesh { return .{ self.model, self.experts }; } }; diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 407df36e..92ab4462 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -17,9 +17,9 @@ pub const CompilationParameters = struct { attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, seqlen: u32, - shardings: common.Shardings, + meshes: common.Meshes, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); const cache: model.Cache = .{ .kv = .init(.init(.{ @@ -28,7 +28,7 @@ pub const CompilationParameters = struct { .k = seqlen, .h = config.num_key_value_heads, .hd = config.hidden_size / config.num_attention_heads, - }, mdl.embed_tokens.weight.dtype())), + }, mdl.embed_tokens.weight.dtype()), meshes.model), .conv = .init(.init(.{ .layer = mdl.num_conv_layers, .batch = 1, @@ -42,10 +42,10 @@ pub const CompilationParameters = struct { .batch_dim = 1, .rng = .init(), .cache = cache, - .attention_metadata = .init(.fromBackend(backend, seqlen, config.num_attention_heads)), + .attention_metadata = .init(.fromBackend(backend, seqlen, config.num_attention_heads), meshes.model), .attention_parameters = .init(.fromBackend(backend)), .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, }; } }; @@ -235,7 +235,7 @@ fn compileEmbed( defer phase.logCompileDone(log, "embed_tokens", io, from); return zml.FnExe(model.TokenEmbedding.forward).compile(allocator, io, platform, .{ - .shardings = &opts.shardings.all(), + .meshes = &opts.meshes.all(), .program_name = phase.programName("lfm2", "embed_tokens"), }, .{.{ .embedding = embed_tokens, @@ -273,7 +273,7 @@ fn compileLayer( } else unreachable; return zml.FnExe(model.DecoderLayer.forward).compile(allocator, io, platform, .{ - .shardings = &opts.shardings.all(), + .meshes = &opts.meshes.all(), .program_name = phase.programName("lfm2", if (kind == .conv) "conv_layer" else "attn_layer"), }, .{.{ .layer = layer, @@ -306,7 +306,7 @@ fn compileSample( defer phase.logCompileDone(log, "lm_head", io, from); return zml.FnExe(model.LmHead.forward).compile(allocator, io, platform, .{ - .shardings = &opts.shardings.all(), + .meshes = &opts.meshes.all(), .program_name = phase.programName("lfm2", "lm_head"), }, .{.{ .lm_head = mdl.lm_head, diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 53f1071c..1d627025 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -79,7 +79,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -94,8 +93,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -117,11 +115,11 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); return inference.CompiledModel.init(allocator, io, @constCast(platform), self, self.inner, params, progress); } }; @@ -173,7 +171,6 @@ pub const Model = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -188,8 +185,7 @@ pub const Model = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - loader.load(io, Model, self, &buffers, store, &all_shardings); + loader.load(io, Model, self, &buffers, store); try loader.await(io); const took = now.untilNow(io, .awake); @@ -278,7 +274,7 @@ pub const TokenEmbedding = struct { }; pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { - return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }; + return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }; } pub fn forward(input: Input) Output { @@ -448,16 +444,16 @@ pub const ShortConv = struct { pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); return .{ - .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), - .kernel = store.createTensor("conv.weight", .{ .out, .in, .kernel_size }, .replicated), + .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .kernel = store.createReplicatedTensor("conv.weight", .{ .out, .in, .kernel_size }), .config = config, }; } pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, actual_seq_len: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { var cache = cache_; - const BCx = self.in_proj.forward(input.withPartitioning(.{ .d = .replicated })); + const BCx = self.in_proj.forward(input.withPartitioning(.model, .{ .d = .replicated })); const B, const C, const x = BCx.chunkExact(.d, 3); const Bx = B.mul(x); @@ -523,10 +519,10 @@ pub const Attention = struct { const head_dim = config.hidden_size / config.num_attention_heads; const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; return .{ - .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), + .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), .q_layernorm = RmsNorm.init(store.withPrefix("q_layernorm"), config.norm_eps, .hd), .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), .head_dim = head_dim, @@ -544,7 +540,7 @@ pub const Attention = struct { attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, ) struct { zml.Tensor, KvCache } { - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); @@ -608,8 +604,8 @@ pub const Cache = struct { conv: ConvCache, kv: KvCache, - pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Cache) { - return .{ .conv = try self.conv.initBuffers(allocator, io, platform, sharding), .kv = try self.kv.initBuffers(io, platform, sharding) }; + pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Cache) { + return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; } pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { @@ -629,12 +625,12 @@ pub const ConvCache = struct { return .{ .state = .fromShape(shape) }; } - pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(ConvCache) { + pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvCache) { const sh = self.state.shape(); const host = try allocator.alloc(u8, sh.byteSize()); defer allocator.free(host); @memset(host, 0); - return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, sharding, host) }; + return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; } pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { @@ -650,13 +646,13 @@ pub const KvCache = struct { k: zml.Tensor, v: zml.Tensor, - pub fn init(kv_shape: zml.Shape) KvCache { - const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); return .{ .k = .fromShape(sharded_shape), .v = .fromShape(sharded_shape) }; } - pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { - return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}) }; + pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; } pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { @@ -717,9 +713,9 @@ const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ - .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), - .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), + .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), }; } @@ -740,7 +736,7 @@ const RmsNorm = struct { tag: zml.Shape.Tag, pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { - return .{ .weight = store.createTensor("weight", .{tag}, .replicated), .eps = eps, .tag = zml.Shape.toTag(tag) }; + return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 67287861..b59a0416 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -30,9 +30,9 @@ pub const Session = struct { model_buffers: *model.Buffers, ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.model); + var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); errdefer model.Cache.unloadBuffers(&cache_buffers); - var rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); errdefer generated_token_slice.free(allocator); @@ -120,21 +120,21 @@ pub const Session = struct { @memset(tokens, self.config.pad_token_id); @memcpy(tokens[0..all_tokens.len], all_tokens); - var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice, .replicated); + var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); defer tokens_buf.deinit(); const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, .replicated); + var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); defer tokens_pos_buf.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, .replicated); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); const params = self.compiled_model.params; var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - else => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + else => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); @@ -161,18 +161,18 @@ pub const Session = struct { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, .replicated); + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, .replicated); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); const params = self.compiled_model.params; - var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model); + var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); generation: while (true) { @@ -194,7 +194,7 @@ pub const Session = struct { if (all_tokens.items.len >= self.seqlen) break :generation; const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); - var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, .replicated); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); defer token_pos_buffer.deinit(); var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index cbc33811..2da991ec 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -40,16 +40,16 @@ pub fn main(init: std.process.Init) !void { defer repo.close(io); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const shardings: common.Shardings = try .init(platform); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); defer repo_model.unloadBuffers(&model_buffers, allocator); const backend = args.backend orelse zml.attention.Backend.auto(platform); @@ -58,11 +58,11 @@ pub fn main(init: std.process.Init) !void { repo_model.parsed_config.value, repo_model.parsed_config.value.max_position_embeddings, backend, - shardings, + meshes, ); progress.end(); - try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters); + try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters, meshes); } pub fn run( @@ -75,11 +75,12 @@ pub fn run( model_buffers: *lfm2.Buffers, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, + meshes: common.Meshes, ) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer activation_store.deinit(); var ctx = TestContext{ @@ -89,7 +90,7 @@ pub fn run( .activations_store = &activation_store, .attention_metadata = attention_metadata, .attention_parameters = attention_parameters, - .sharding = platform.replicated_sharding, + .mesh = platform.replicated_mesh, }; try ctx.testLayer("embed_tokens", .{ .batch, .seq }, mdl.embed_tokens, model_buffers.embed_tokens, .{}); @@ -134,7 +135,7 @@ const TestContext = struct { activations_store: *zml.io.TensorStore, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, - sharding: zml.Sharding, + mesh: *const zml.Mesh, fn testLayerPrint(self: *TestContext, comptime name_fmt: []const u8, name_args: anytype, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { const name = try std.fmt.allocPrint(self.allocator, name_fmt, name_args); @@ -147,19 +148,19 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(tagz); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); defer out_buffer_expected.deinit(); const exe = if (comptime @TypeOf(layer) == model.TokenEmbedding) - try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .shardings = &.{self.sharding} }) + try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .meshes = &.{self.mesh} }) else - try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .shardings = &.{self.sharding} }); + try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); var args = try exe.args(self.allocator); @@ -185,42 +186,42 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache", .{name}); defer self.allocator.free(cache_key); - var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key, self.sharding); + var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key); defer cache_buffer.deinit(); const cache_tensor = zml.Tensor.fromShape(cache_buffer.shape()).withTags(.{ .layer, .batch, .seq, .d }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); defer out_buffer_expected.deinit(); const actual_seq_len_tensor: zml.Tensor = .init(.{}, .u32); const actual_seq_len: u32 = @intCast(in_tensor.dim(.seq)); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .shardings = &.{self.sharding} }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, self.sharding); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); - var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice, self.sharding); + var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); defer cache_index_buf.deinit(); var runner = try exe.runner(self.allocator); @@ -243,41 +244,41 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const key_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.key", .{name}); defer self.allocator.free(key_cache_key); - var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key, self.sharding); + var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key); defer key_cache_buffer.deinit(); const key_cache_tensor = zml.Tensor.fromShape(key_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const value_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.value", .{name}); defer self.allocator.free(value_cache_key); - var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key, self.sharding); + var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key); defer value_cache_buffer.deinit(); const value_cache_tensor = zml.Tensor.fromShape(value_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); defer out_buffer_expected.deinit(); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .shardings = &.{self.sharding} }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; - var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform, self.sharding); + var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32); @@ -297,7 +298,7 @@ const TestContext = struct { } }; -fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8, sharding: zml.Sharding) !zml.Buffer { +fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { const shape = store.view().getShape(key) orelse return error.NotFound; const host_bytes = try allocator.alloc(u8, shape.byteSize()); @@ -309,5 +310,5 @@ fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml _ = try reader.interface.readSliceAll(host_bytes); - return zml.Buffer.fromBytes(io, platform, shape, sharding, host_bytes); + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); } diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 4eafdcea..331ee0dc 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -18,9 +18,9 @@ pub const CompilationParameters = struct { prefill_attention_parameters: zml.attention.Parameters, decode_attention_parameters: zml.attention.Parameters, seqlen: usize, - shardings: common.Shardings, + meshes: common.Meshes, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { const head_dim = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads); return .{ @@ -32,11 +32,11 @@ pub const CompilationParameters = struct { .k = seqlen, .h = config.num_key_value_heads, .hd = head_dim, - }, mdl.model.embed_tokens.weight.dtype())), + }, mdl.model.embed_tokens.weight.dtype()), meshes.model), .rng = .init(), .attention_metadata = switch (backend) { .attnd => .{ .attnd = .init() }, - else => .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads))), + else => .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads)), meshes.model), }, .prefill_attention_parameters = switch (backend) { .attnd => .{ .attnd = .init(.{ @@ -59,7 +59,7 @@ pub const CompilationParameters = struct { else => .init(.fromBackend(backend)), }, .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, }; } }; @@ -211,7 +211,7 @@ fn compileKernel( defer phase.logCompileDone(log, "forward", io, from); return KernelExe.compile(allocator, io, platform, .{ - .shardings = ¶meters.shardings.all(), + .meshes = ¶meters.meshes.all(), .program_name = phase.programName("llama", "forward"), }, .{.{ .weights = llama_model, diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index b86c6d6b..ae988b97 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -76,7 +76,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -91,8 +90,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -114,11 +112,11 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } @@ -144,6 +142,7 @@ pub const Model = struct { const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensor( "weight", .{ .dout, .d }, + .model, .{ .dout = .model, .d = .replicated }, )) |weight| .init(weight, null, .d) @@ -168,7 +167,6 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.Sharding, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -184,7 +182,7 @@ pub const Model = struct { }); defer loader.deinit(); - loader.load(io, Model, self, &buffers, store, shardings); + loader.load(io, Model, self, &buffers, store); try loader.await(io); const took = now.untilNow(io, .awake); @@ -256,10 +254,11 @@ const Llama = struct { .embed_tokens = .{ .weight = store.createTensor( "embed_tokens.weight", .{ .voc, .d }, + .model, .{ .voc = .replicated, .d = .model }, ) }, .norm = .{ - .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .{ .d = .replicated }), + .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = config.rms_norm_eps, }, .layers = layers, @@ -329,7 +328,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.{ .d = .replicated }) }; + .withPartitioning(.model, .{ .d = .replicated }) }; } }; @@ -426,7 +425,7 @@ pub const TransformerLayer = struct { stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); // Keep the residual stream replicated to avoid repeated gathers before q/k/v. - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const x0_normalized = self.input_layernorm.forward(x0_replicated); const delta0, const updated_kv_cache = self.self_attn.forward( x0_normalized, @@ -438,13 +437,13 @@ pub const TransformerLayer = struct { ); // Fully Connected - const x1 = x0_replicated.add(delta0).withPartitioning(.{ .d = .replicated }); + const x1 = x0_replicated.add(delta0).withPartitioning(.model, .{ .d = .replicated }); const x1_normalized = self.post_attention_layernorm.forward(x1); const x2 = self.mlp.forward(x1_normalized) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }) + .withPartitioning(.model, .{ .d = .replicated }) .add(x1) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(.model, .{ .d = .replicated }); return .{ .hidden = x2.reuseBuffer(x0), .kv_cache = updated_kv_cache }; } @@ -456,7 +455,7 @@ const RmsNorm = struct { pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { return .{ - .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), + .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps, }; } @@ -480,9 +479,9 @@ const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ - .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .{ .d = .model }), null, .d), + .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), }; } @@ -515,10 +514,10 @@ const SelfAttn = struct { var rope_scaling = config.rope_scaling; rope_scaling.setRopeTheta(config.rope_theta); return .{ - .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .{ .d = .model }), null, .d), + .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), // TODO(Corentin): fix that .q_norm = null, .k_norm = null, @@ -555,7 +554,7 @@ const SelfAttn = struct { // Make hidden state replicated once and reuse it across q/k/v projections. // This avoids paying gather-style collectives independently for each projection. - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); var q = self.q_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); var k = self.k_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); @@ -605,7 +604,7 @@ const SelfAttn = struct { const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); const delta = self.o_proj.forward(attn, attn.dtype()) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(.model, .{ .d = .replicated }); return .{ delta, new_kv_cache }; } }; @@ -616,8 +615,8 @@ pub const KvCache = struct { pub const Buffer = zml.Bufferized(KvCache); - pub fn init(kv_shape: zml.Shape) KvCache { - const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); return .{ .k = .fromShape(sharded_shape), @@ -625,10 +624,10 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !Buffer { + pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform) !Buffer { return .{ - .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), }; } diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 2269464f..6a04cf11 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -29,8 +29,7 @@ pub const Session = struct { compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - const shardings = &compiled_model.params.shardings; - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const token_index_buffers = try allocator.alloc(zml.Buffer, compiled_model.params.seqlen); @@ -49,7 +48,7 @@ pub const Session = struct { const conversation_id: u64 = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); @@ -138,7 +137,7 @@ pub const Session = struct { @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice, .replicated); + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice); defer prefill_tokens_buffer.deinit(); const params = self.compiled_model.params; @@ -149,7 +148,7 @@ pub const Session = struct { .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32), } }, .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + .vanilla, .cuda_fa2, .cuda_fa3, .nki => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); @@ -174,7 +173,7 @@ pub const Session = struct { defer self.allocator.free(decoder_out_buffer); var last_token_id: u32 = self.last_generated_token; - var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), .replicated, @ptrCast(&last_token_id)); + var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); const params = self.compiled_model.params; @@ -184,7 +183,7 @@ pub const Session = struct { .layer_id = try .scalar(self.io, self.platform, 0, .u16), .num_tokens = try .scalar(self.io, self.platform, 1, .u32), } }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 98798887..1b5894f1 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -38,20 +38,20 @@ pub fn main(init: std.process.Init) !void { defer repo.close(io); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const shardings: common.Shardings = try .init(platform); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); defer repo_model.unloadBuffers(&model_buffers, allocator); progress.end(); - try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, platform.replicated_sharding); + try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); } fn run( @@ -61,28 +61,28 @@ fn run( activations_path: []const u8, mdl: model.Model, model_buffers: *model.Buffers, - sharding: zml.Sharding, + meshes: []const *const zml.Mesh, ) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); defer activation_store.deinit(); - try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, sharding, .{ .absolute_tolerance = 1e-3 }); + try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); if (mdl.model.layers.len == 0) return; const layer = mdl.model.layers[0]; const layer_buffers = model_buffers.model.layers[0]; - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, sharding, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, sharding, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, sharding, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, sharding, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, sharding, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, sharding, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, sharding, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); } fn testLayer( @@ -93,19 +93,19 @@ fn testLayer( name: []const u8, layer: anytype, layer_weights: zml.Bufferized(@TypeOf(layer)), - sharding: zml.Sharding, + meshes: []const *const zml.Mesh, opts: zml.testing.CompareOpts, ) !void { const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); defer allocator.free(in_key); const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; - var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key, sharding); + var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_shape); const out_key = try std.fmt.allocPrint(allocator, "{s}.out", .{name}); defer allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key, sharding); + var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key); defer out_buffer_expected.deinit(); // `zml.nn.Linear.forward` takes an explicit output dtype; every other layer here @@ -116,7 +116,7 @@ fn testLayer( return if (Layer == zml.nn.Linear) l.forward(x, x.dtype()) else Layer.forward(l, x); } }; - const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .shardings = &.{sharding} }); + const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .meshes = meshes }); defer exe.deinit(); var args = try exe.args(allocator); @@ -133,7 +133,7 @@ fn testLayer( try zml.testing.expectClose(io, out_result, out_buffer_expected, opts); } -fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8, sharding: zml.Sharding) !zml.Buffer { +fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { const shape = store.getShape(key) orelse return error.NotFound; const host_bytes = try allocator.alloc(u8, shape.byteSize()); @@ -145,5 +145,5 @@ fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *cons _ = try reader.interface.readSliceAll(host_bytes); - return zml.Buffer.fromBytes(io, platform, shape, sharding, host_bytes); + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); } diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 33a41086..6bfb71c5 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -15,18 +15,18 @@ pub const CompilationParameters = struct { kv_cache: model.KvCache, rng: zml.Tensor.Rng, seqlen: u32, - shardings: common.Shardings, + meshes: common.Meshes, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, meshes: common.Meshes) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); return .{ .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), .decode_tokens = .init(.{ .b = 1, .s = 1 }, .u32), .token_index = .init(.{}, .u32), - .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), .rng = .init(), .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, }; } }; @@ -244,7 +244,7 @@ fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.P defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "embed tokens", io, from); - return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ + return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ .embedding = .{ .embed_tokens = mdl.text_model.embed_tokens }, .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), }}); @@ -256,9 +256,9 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "full attention layer", io, from); - return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ + return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .token_index = parameters.token_index, .cache = .{ .k = parameters.kv_cache.self_attn.k, @@ -274,9 +274,9 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "linear attention layer", io, from); - return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ + return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .active_length = zml.Tensor.init(.{}, .u32), .cache = .{ .conv_state = parameters.kv_cache.gated_delta_net.conv_state, @@ -292,19 +292,19 @@ fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml. defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "sampler", io, from); - return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ + return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ .sampler = mdl.sampler(), - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .rng = parameters.rng, .token_index = parameters.token_index, }}); } -fn hiddenTensor(mdl: model.Model, seqlen: usize) zml.Tensor { +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { return .fromShape(zml.Shape.init( .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, mdl.text_model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated })); + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); } fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 5a2a41ac..14edd022 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -81,7 +81,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -96,8 +95,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -120,12 +118,12 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { _ = backend; - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), meshes); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } }; @@ -161,7 +159,7 @@ pub const Model = struct { return .{ .text_model = try .init(allocator, store.withPrefix("model.language_model"), config), .lm_head = .init( - store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), + store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), null, .d, ), @@ -180,7 +178,7 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.Sharding, + meshes: []const *const zml.Mesh, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -196,7 +194,7 @@ pub const Model = struct { }); defer loader.deinit(); - loader.load(io, Model, self, &buffers, store, shardings); + loader.load(io, Model, self, &buffers, store, meshes); try loader.await(io); const took = now.untilNow(io, .awake); @@ -286,7 +284,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.{ .d = .replicated }) }; + .withPartitioning(.model, .{ .d = .replicated }) }; } }; @@ -308,7 +306,7 @@ pub const TextModel = struct { } return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, .layers = layers, .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), }; @@ -414,7 +412,7 @@ pub const TransformerLayer = struct { active_length: zml.Tensor, kv_cache: KvCache.LayerView, ) struct { zml.Tensor, KvCache } { - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; @@ -440,18 +438,18 @@ pub const TransformerLayer = struct { }, } - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); - return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + return .{ mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; } pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { const self = input.layer; const x0 = input.hidden; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const self_attn = switch (self.attn) { @@ -460,12 +458,12 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); return .{ - .hidden = mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -473,7 +471,7 @@ pub const TransformerLayer = struct { pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { const self = input.layer; const x0 = input.hidden; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const linear_attn = switch (self.attn) { @@ -482,12 +480,12 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); return .{ - .hidden = mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -501,18 +499,18 @@ pub const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ .up_proj = .init( - store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .gate_proj = .init( - store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .down_proj = .init( - store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .{ .d = .replicated, .dout = .model }), - store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .{ .d = .replicated }), + store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .model, .{ .d = .replicated, .dout = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), .dout, ), }; @@ -549,8 +547,8 @@ pub const SelfAttn = struct { fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { return .init( - store.createTensor("weight", .{ .dout, .d }, partitions), - store.maybeCreateTensor("bias", .{.dout}, bias_partitions), + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), .d, ); } @@ -594,26 +592,26 @@ pub const SelfAttn = struct { return .{ k, v }; } - fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - return switch (kv_head_sharding) { + fn partitionProjectedKv(kv: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { + return switch (kv_head_meshe) { .sharded => |heads| blk: { const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; - break :blk sharded_kv.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => kv.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), }; } - fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + fn partitionCachedKv(tensor: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { var cache_tensor = tensor.rename(.{ .s = .k }); - return switch (kv_head_sharding) { + return switch (kv_head_meshe) { .sharded => |heads| blk: { if (heads.factor != 1) { cache_tensor = cache_tensor.stutter1d(cache_tensor.axis(.h), heads.factor); } - break :blk cache_tensor.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + break :blk cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => cache_tensor.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), }; } @@ -623,18 +621,15 @@ pub const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = zml.Compiler.current().partitioning.shardableDim( - k.shape().withPartitioning(.{ .h = .model }), - .h, - q.dim(.h), - ) catch unreachable; + const kv_head_meshe = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -646,22 +641,22 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }); - k = partitionCachedKv(k, kv_head_sharding); - v = partitionCachedKv(v, kv_head_sharding); + k = partitionCachedKv(k, kv_head_meshe); + v = partitionCachedKv(v, kv_head_meshe); const attn_output = zml.attention.attention( q, k, v, token_index, - zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads), tp), zml.attention.Parameters.init(.fromBackend(.vanilla)), ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); @@ -669,7 +664,7 @@ pub const SelfAttn = struct { const projected_output = self.o_proj .forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(tp, .{ .d = .replicated }); return .{ projected_output, new_kv_cache }; } @@ -772,7 +767,7 @@ pub const GatedDeltaNet = struct { conv_kernel_size: i64, fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, partitions), null, .d); + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); } pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { @@ -784,9 +779,9 @@ pub const GatedDeltaNet = struct { .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), - .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), - .dt_bias = store.createTensor("dt_bias", .{.vh}, .{ .vh = .model }), - .aLog = store.createTensor("A_log", .{.vh}, .{ .vh = .model }), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), .num_k_heads = config.text_config.linear_num_key_heads, .num_v_heads = config.text_config.linear_num_value_heads, @@ -873,10 +868,10 @@ pub const GatedDeltaNet = struct { const conv_dim = 2 * key_dim + value_dim; const left_pad = self.conv_kernel_size - 1; - const x_in = x.withPartitioning(.{ .d = .replicated }); + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) .rename(.{ .dout = .mix }) - .withPartitioning(.{ .s = .replicated, .mix = .model }); + .withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; const conv_input = b: { if (use_cached_state) break :b zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s); @@ -906,7 +901,7 @@ pub const GatedDeltaNet = struct { if (use_cached_state) { mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); } - mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const z = self.in_proj_z.forward(x_in, x_in.dtype()) .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); @@ -959,7 +954,7 @@ pub const GatedDeltaNet = struct { const output = self.out_proj .forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(.model, .{ .d = .replicated }); const updated_cache = cache.update( if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), last_recurrent_state, @@ -973,7 +968,7 @@ pub const RmsNorm = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { @@ -994,7 +989,7 @@ pub const RmsNormGated = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { @@ -1026,7 +1021,7 @@ pub const KvCache = struct { pub const Buffers = zml.Bufferized(SelfAttnCache); - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_sharding: zml.Sharding) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -1035,10 +1030,10 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const kv_head_sharding = model_sharding.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_sharding) { - .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(.{ .h = .model }), - .replicated => kv_shape.withPartitioning(.{ .h = .replicated }), + const kv_head_meshe = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_meshe) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), }; return .{ .k = .fromShape(sharded_kv_shape), @@ -1047,10 +1042,10 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !SelfAttnCache.Buffers { + pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !SelfAttnCache.Buffers { return .{ - .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1127,7 +1122,7 @@ pub const KvCache = struct { pub const Buffers = zml.Bufferized(GatedDeltaNetCache); - pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + @@ -1145,19 +1140,19 @@ pub const KvCache = struct { .khd = config.text_config.linear_key_head_dim, .vhd = config.text_config.linear_value_head_dim, }, recurrent_dtype); - const sharded_conv_state_shape = conv_state_shape.withPartitioning(.{ .mix = .model }); - const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(.{ .vh = .model }); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); return .{ - .conv_state = .fromShape(sharded_conv_state_shape), - .recurrent_state = .fromShape(sharded_recurrent_state_shape), + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !GatedDeltaNetCache.Buffers { + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !GatedDeltaNetCache.Buffers { return .{ - .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), - .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1231,19 +1226,19 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, - model_sharding: zml.Sharding, + model_meshe: *const zml.Mesh, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_sharding), - .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype), + .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), + .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { return .{ - .self_attn = try self.self_attn.initBuffer(io, platform, sharding), - .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform, sharding), + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), }; } diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index a49940cc..84963773 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -31,11 +31,11 @@ pub const Session = struct { compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, compiled_model.params.shardings.model); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; @@ -111,9 +111,7 @@ pub const Session = struct { @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - const replicated_sharding: zml.Sharding = .replicated; - - var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice); defer prefill_tokens_buffer.deinit(); var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); @@ -141,9 +139,8 @@ pub const Session = struct { const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); - const replicated_sharding: zml.Sharding = .replicated; - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, replicated_sharding); + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 80c6be65..f29ecabd 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -12,17 +12,17 @@ pub const CompilationParameters = struct { rng: zml.Tensor.Rng, moe_backend: zml.moe.Backend, seqlen: u32, - shardings: common.Shardings, + meshes: common.Meshes, xla_dump_to: ?[]const u8, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, meshes: common.Meshes) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); return .{ - .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), .rng = .init(), .moe_backend = moe_backend, .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, .xla_dump_to = "/home/ubuntu/xla_dump", }; } @@ -274,7 +274,7 @@ fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.P fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardSelfAttn, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .token_index = zml.Tensor.init(.{}, .u32), .cache = .{ .k = parameters.kv_cache.self_attn.k, @@ -289,7 +289,7 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .active_length = zml.Tensor.init(.{}, .u32), .cache = .{ .conv_state = parameters.kv_cache.gated_delta_net.conv_state, @@ -304,7 +304,7 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { return compileExe(allocator, io, platform, model.Sampler.sampleTokens, .{.{ .sampler = mdl.text_model.sampler(), - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .rng = parameters.rng, .token_index = zml.Tensor.init(.{}, .u32), }}, parameters, progress, phase, "sampling"); @@ -329,16 +329,16 @@ fn compileExe( const now: std.Io.Timestamp = .now(io, .awake); defer log.info("Compiled {s} {s} [{f}]", .{ phase, component, now.untilNow(io, .awake) }); return zml.FnExe(function).compile(allocator, io, platform, .{ - .shardings = ¶meters.shardings.all(), + .meshes = ¶meters.meshes.all(), .xla_dump_to = parameters.xla_dump_to, }, args); } -fn hiddenTensor(mdl: model.Model, seqlen: usize) zml.Tensor { +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { return .fromShape(zml.Shape.init( .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, mdl.text_model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated })); + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); } fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 6b6f9bb0..59d52815 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -47,26 +47,26 @@ pub const RopeParameters = struct { rope_theta: f32, }; -fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - return switch (kv_head_sharding) { +fn partitionProjectedKv(kv: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { + return switch (kv_head_meshe) { .sharded => |heads| blk: { const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; - break :blk sharded_kv.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => kv.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), }; } -fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { +fn partitionCachedKv(tensor: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { var kv = tensor.rename(.{ .s = .k }); - return switch (kv_head_sharding) { + return switch (kv_head_meshe) { .sharded => |heads| blk: { if (heads.factor != 1) { kv = kv.stutter1d(kv.axis(.h), heads.factor); } - break :blk kv.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + break :blk kv.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => kv.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => kv.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), }; } @@ -107,7 +107,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -122,8 +121,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -145,7 +143,7 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { @@ -153,7 +151,7 @@ pub const LoadedModel = struct { const moe_gate_up = self.inner.text_model.layers[0].moe.gate_up_proj; log.info("Moe dtype : {}", .{moe_gate_up.weight.dtype()}); const moe_backend = try zml.moe.Backend.auto(platform, moe_gate_up.quantizationScheme(), moe_gate_up.weight.dtype()); - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, meshes); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } }; @@ -196,7 +194,7 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.Sharding, + meshes: []const *const zml.Mesh, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -212,7 +210,7 @@ pub const Model = struct { }); defer loader.deinit(); - loader.load(io, Model, self, &buffers, store, shardings); + loader.load(io, Model, self, &buffers, store, meshes); try loader.await(io); const took = now.untilNow(io, .awake); @@ -314,10 +312,10 @@ pub const TextModel = struct { } return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, .layers = layers, .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), - .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), null, .d), + .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), null, .d), .gen_options = gen_options, }; } @@ -447,7 +445,7 @@ pub const TransformerLayer = struct { const self = input.layer; const x0 = input.hidden; _ = input.config; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const self_attn = switch (self.attn) { @@ -456,13 +454,13 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); return .{ - .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -471,7 +469,7 @@ pub const TransformerLayer = struct { const self = input.layer; const x0 = input.hidden; _ = input.config; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const linear_attn = switch (self.attn) { @@ -480,13 +478,13 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); return .{ - .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -501,7 +499,7 @@ pub const TransformerLayer = struct { moe_backend: zml.moe.Backend, ) struct { zml.Tensor, KvCache } { _ = config; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; @@ -519,12 +517,12 @@ pub const TransformerLayer = struct { }, } - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, moe_backend); - return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }), updated_kv_cache }; + return .{ moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }), updated_kv_cache }; } }; @@ -550,8 +548,8 @@ pub const SelfAttn = struct { fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { return .init( - store.createTensor("weight", .{ .dout, .d }, partitions), - store.maybeCreateTensor("bias", .{.dout}, bias_partitions), + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), .d, ); } @@ -605,19 +603,16 @@ pub const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = zml.Compiler.current().partitioning.shardableDim( - k.shape().withPartitioning(.{ .h = .model }), - .h, - q.dim(.h), - ) catch unreachable; + const kv_head_meshe = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); - q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + gate = gate.withPartitioning(tp, .{ .s = .replicated, .d_out_proj = .model }); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -629,28 +624,28 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); - q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - k = partitionCachedKv(k, kv_head_sharding); - v = partitionCachedKv(v, kv_head_sharding); + q = q.rename(.{ .s = .q }).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }); + k = partitionCachedKv(k, kv_head_meshe); + v = partitionCachedKv(v, kv_head_meshe); const attn_output = zml.attention.attention( q, k, v, token_index, - zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads), tp), zml.attention.Parameters.init(.fromBackend(.vanilla)), - ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + ).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); - const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); return .{ projected_output, new_kv_cache }; } @@ -667,20 +662,20 @@ pub const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ .up_proj = .init( - store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .up_proj_scale = null, .gate_proj = .init( - store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .gate_proj_scale = null, .down_proj = .init( - store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .replicated, .d = .model }), - store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .{ .d = .replicated }), + store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .replicated, .d = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), .d, ), .down_proj_scale = null, @@ -708,8 +703,8 @@ const Router = struct { pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { return .{ .router = .init( - store.createTensor("weight", .{ .expert, .d }, .{ .expert = .replicated, .d = .replicated }), - store.maybeCreateTensor("bias", .{.expert}, .{ .expert = .replicated }), + store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), + store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), .d, ), .num_experts_per_tok = num_experts_per_tok, @@ -742,19 +737,21 @@ pub const Moe = struct { const gate_up_proj_tensor = experts_store.createTensor( "gate_up_proj", .{ .expert, .dout, .d }, + .experts, .{ .expert = .experts, .dout = .replicated, .d = .replicated }, ); const down_proj_tensor = experts_store.createTensor( "down_proj", .{ .expert, .d, .dout }, + .experts, .{ .expert = .experts, .d = .replicated, .dout = .replicated }, ); return .{ .shared_expert = Mlp.init(store.withPrefix("shared_expert")), .shared_expert_gate = .init( - store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .{ .dout = .replicated, .d = .replicated }), - store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .{ .dout = .replicated }), + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), .d, ), .gate_up_proj = .init(gate_up_proj_tensor, null, .d), @@ -786,7 +783,7 @@ pub const Moe = struct { ); const shared_gate = self.shared_expert_gate.forward(x, x.dtype()).sigmoid().broad(x.shape()); - const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.{ + const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.model, .{ .b = .replicated, .s = .replicated, .d = .replicated, @@ -896,7 +893,7 @@ pub const GatedDeltaNet = struct { conv_kernel_size: i64, fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, partitions), null, .d); + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); } pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { @@ -911,9 +908,9 @@ pub const GatedDeltaNet = struct { .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), .out_proj_scale = null, - .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), - .dt_bias = store.createTensor("dt_bias", .{.vh}, .{ .vh = .model }), - .aLog = store.createTensor("A_log", .{.vh}, .{ .vh = .model }), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), .num_k_heads = config.text_config.linear_num_key_heads, .num_v_heads = config.text_config.linear_num_value_heads, @@ -997,9 +994,9 @@ pub const GatedDeltaNet = struct { const conv_dim = 2 * key_dim + value_dim; const left_pad = self.conv_kernel_size - 1; - const x_in = x.withPartitioning(.{ .d = .replicated }); + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) - .rename(.{ .dout = .mix }).withPartitioning(.{ .s = .replicated, .mix = .model }); + .rename(.{ .dout = .mix }).withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; const conv_input = if (use_cached_state) zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) @@ -1029,26 +1026,26 @@ pub const GatedDeltaNet = struct { if (use_cached_state) { mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); } - mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const z = self.in_proj_z.forward(x_in, x_in.dtype()) .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); - const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); - const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); + const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); + const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); const query = mixed_qkv .slice(.mix, .{ .start = 0, .end = key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); const key = mixed_qkv .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); const value = mixed_qkv .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); const beta = b.sigmoid(); const aLog_type = self.aLog.dtype(); @@ -1073,10 +1070,10 @@ pub const GatedDeltaNet = struct { z.rename(.{ .vhd = .d }), ) .rename(.{ .d = .vhd }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) - .rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + .rename(.{ .dout = .d }).withPartitioning(.model, .{ .d = .replicated }); const updated_cache = cache.update( if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), last_recurrent_state, @@ -1090,7 +1087,7 @@ pub const RmsNorm = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { @@ -1111,7 +1108,7 @@ pub const RmsNormGated = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { @@ -1141,7 +1138,7 @@ pub const KvCache = struct { v: zml.Tensor, layer_index: zml.Tensor, - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_sharding: zml.Sharding) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -1150,22 +1147,22 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const kv_head_sharding = model_sharding.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_sharding) { - .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(.{ .h = .model }), - .replicated => kv_shape.withPartitioning(.{ .h = .replicated }), + const kv_head_meshe = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_meshe) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), }; return .{ - .k = .fromShape(sharded_kv_shape), - .v = .fromShape(sharded_kv_shape), + .k = zml.Tensor.fromShape(sharded_kv_shape), + .v = zml.Tensor.fromShape(sharded_kv_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(SelfAttnCache) { + pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(SelfAttnCache) { return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1238,7 +1235,7 @@ pub const KvCache = struct { recurrent_state: zml.Tensor, layer_index: zml.Tensor, - pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; const conv_state_shape = zml.Shape.init(.{ @@ -1254,19 +1251,19 @@ pub const KvCache = struct { .khd = config.text_config.linear_key_head_dim, .vhd = config.text_config.linear_value_head_dim, }, recurrent_dtype); - const sharded_conv_state_shape = conv_state_shape.withPartitioning(.{ .mix = .model }); - const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(.{ .vh = .model }); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); return .{ - .conv_state = .fromShape(sharded_conv_state_shape), - .recurrent_state = .fromShape(sharded_recurrent_state_shape), + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(GatedDeltaNetCache) { + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(GatedDeltaNetCache) { return .{ - .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), - .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1334,19 +1331,19 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, - model_sharding: zml.Sharding, + model_meshe: *const zml.Mesh, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_sharding), - .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype), + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), + .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { return .{ - .self_attn = try self.self_attn.initBuffer(io, platform, sharding), - .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform, sharding), + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), }; } diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index e1509df5..35998797 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -31,12 +31,11 @@ pub const Session = struct { compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - const shardings = compiled_model.params.shardings; - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; @@ -122,7 +121,7 @@ pub const Session = struct { @memset(tokens_slice.items(u32), 0); @memcpy(tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice, .replicated); + var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice); defer tokens_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); defer token_index_buffer.deinit(); @@ -149,7 +148,7 @@ pub const Session = struct { const out_tokens_buffer = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, .replicated); + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); defer token_index_buffer.deinit(); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 3b4349c4..d1291c1b 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -19,8 +19,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createTensor("weight", .{ .d_out, .d }, .replicated), - .bias = store.createTensor("bias", .{.d_out}, .replicated), + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), }; } @@ -49,7 +49,7 @@ const Mnist = struct { var loader: zml.io.Loader = try .init(allocator, platform, .default); errdefer loader.deinit(); - try loader.load(io, Mnist, self, &buffers, store, &.{}, .{}); + try loader.load(io, Mnist, self, &buffers, store, .{}); try loader.await(io); return buffers; @@ -66,7 +66,10 @@ const Mnist = struct { pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; - for (layers) |layer| { + const compiler = zml.Compiler.current(); + for (0.., layers) |i, layer| { + compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); + defer compiler.popLocation(); x = layer.forward(x); } return x.argMax(0).indices.convert(.u8); @@ -87,15 +90,15 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); defer registry.deinit(); - // Init model - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - const mnist_model: Mnist = .init(store.view()); - // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); + // Init model + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + defer store.deinit(); + const mnist_model: Mnist = .init(store.view()); + // // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); var exe = blk: { @@ -137,7 +140,7 @@ pub fn main(init: std.process.Init) !void { var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample), .replicated); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); defer input_buffer.deinit(); printDigit(sample); diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 61a16318..cc46a1bd 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -2,7 +2,7 @@ const std = @import("std"); const zml = @import("zml"); -const log = std.log.scoped(.sharding); +const log = std.log.scoped(.mesh); pub const std_options: std.Options = .{ .log_level = .info, @@ -34,10 +34,10 @@ const DemoModel = struct { var y = x.dot(self.w, .feature); y = y.add(self.b.broad(y.shape())); - y = y.withPartitioning(.{ .batch = .data, .hidden = .model }); + y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); y.print("dense_out"); - const gate = y.scale(0.01).sigmoid(); + const gate = y.scale(0.01).sigmoid().withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); return zml.ops.manualComputation( (struct { y: zml.Tensor, @@ -142,7 +142,6 @@ fn createSequenceBuffer( io: std.Io, platform: *const zml.Platform, shape: zml.Shape, - sharding: zml.Sharding, start: f32, ) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); @@ -152,7 +151,7 @@ fn createSequenceBuffer( e.* = start + @as(f32, @floatFromInt(i)); } - return zml.Buffer.fromSlice(io, platform, slice, sharding); + return zml.Buffer.fromSlice(io, platform, slice); } pub fn main(init: std.process.Init) !void { @@ -193,24 +192,24 @@ pub fn main(init: std.process.Init) !void { } log.info("{f}", .{platform.physical_mesh}); - const sharding: zml.Sharding = try platform.registerSharding( - "demo_mesh", + const mesh: *const zml.Mesh = try platform.registerMesh( + "dp_mp", .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), ); - log.info("{f}", .{sharding.data.logical}); - log.info("{f}", .{sharding}); + log.info("{f}", .{mesh.logical}); + log.info("{f}", .{mesh}); const input_shape = zml.Shape.init(.{ .batch = 16, .feature = 32 }, .f32) - .withPartitioning(.{ .batch = .data, .feature = .replicated }); + .withPartitioning(mesh, .{ .batch = .data, .feature = .replicated }); const w_shape = zml.Shape.init(.{ .feature = 32, .hidden = 64 }, .f32) - .withPartitioning(.{ .feature = .replicated, .hidden = .model }); + .withPartitioning(mesh, .{ .feature = .replicated, .hidden = .model }); const b_shape = zml.Shape.init(.{ .hidden = 64 }, .f32) - .withPartitioning(.{ .hidden = .model }); + .withPartitioning(mesh, .{ .hidden = .model }); - const input: zml.Tensor = .fromShape(input_shape); - const w: zml.Tensor = .fromShape(w_shape); - const b: zml.Tensor = .fromShape(b_shape); + const input: zml.Tensor = zml.Tensor.fromShape(input_shape); + const w: zml.Tensor = zml.Tensor.fromShape(w_shape); + const b: zml.Tensor = zml.Tensor.fromShape(b_shape); const model: DemoModel = .init(w, b); var exe = try platform.compile( @@ -221,16 +220,16 @@ pub fn main(init: std.process.Init) !void { .{input}, .{ .partitioner = args.partitioner, - .shardings = &.{sharding}, + .meshes = &.{mesh}, }, ); defer exe.deinit(); - var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, sharding, 0.0); + var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, 0.0); defer w_buf.deinit(); - var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, sharding, 100.0); + var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, 100.0); defer b_buf.deinit(); - var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, sharding, 1000.0); + var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, 1000.0); defer input_buf.deinit(); log.info("input placement: {f}", .{input_buf}); diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index b337cc9a..7f7aed55 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -20,6 +20,7 @@ zig_shared_library( cc_library( name = "libzml_cuda_repo", + target_compatible_with = ["@platforms//os:linux"], deps = select({ "@llvm//platforms/config:linux_x86_64": ["@libzml_cuda_linux_amd64//:libzml_cuda"], "@llvm//platforms/config:linux_aarch64": ["@libzml_cuda_linux_arm64//:libzml_cuda"], @@ -28,6 +29,7 @@ cc_library( cc_library( name = "cuda_nvtx_headers", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = select({ "@llvm//platforms/config:linux_x86_64": ["@cuda_nvtx_linux_x86_64//:headers"], diff --git a/stdx/debug.zig b/stdx/debug.zig index 4686b3b5..a217eca1 100644 --- a/stdx/debug.zig +++ b/stdx/debug.zig @@ -10,11 +10,12 @@ pub inline fn internalAssert(check: bool, comptime msg: []const u8, args: anytyp pub inline fn assert(check: bool, comptime msg: []const u8, args: anytype) void { if (!check) { + @branchHint(.cold); panic(msg, args); } } -pub inline fn panic(comptime format: []const u8, args: anytype) noreturn { +pub fn panic(comptime format: []const u8, args: anytype) noreturn { std.debug.panic(format, args); } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 47ec59aa..33351d93 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -5,6 +5,16 @@ pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { return .{ .slice = any_slice }; } +/// Properly format a slice of []const u8. +pub fn strings(strs: []const []const u8) FmtStrings { + return .{ .strings = strs }; +} + +/// Properly format a slice of [*:0]const u8. +pub fn stringsZ(strs: []const [*:0]const u8) FmtZeroStrings { + return .{ .strings = strs }; +} + fn FmtSlice(T: type) type { return struct { slice: []const T, @@ -31,6 +41,26 @@ fn FmtSlice(T: type) type { }; } +pub const FmtStrings = struct { + strings: []const []const u8, + + pub fn format(f: FmtStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStr, f.strings, .{}, 1, writer); + } + + pub fn formatNumber(f: FmtStrings, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + return try formatIntSlice(f.strings, n, 1, writer); + } +}; + +pub const FmtZeroStrings = struct { + strings: []const [*:0]const u8, + + pub fn format(f: FmtZeroStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStrZ, f.strings, .{}, 1, writer); + } +}; + pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), @@ -77,6 +107,14 @@ pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !vo try writer.alignBufferOptions(if (value) "1" else "0", .{ .alignment = spec.alignment, .fill = spec.fill }); } +pub fn formatStr(value: []const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(value); +} + +pub fn formatStrZ(value: [*:0]const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(std.mem.span(value)); +} + pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; const T = @TypeOf(value); diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index e8c8519d..844778f5 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -16,10 +16,11 @@ sh_binary( sh_binary( name = "rocprofv3", + srcs = ["rocprofv3.sh"], data = [ "@amdrocm-profiler-base//:rocprofv3_tool", "@libzml_rocm//:sandbox", ], - srcs = ["rocprofv3.sh"], + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], ) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 23259b1d..41f175ae 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -20,7 +20,6 @@ const Platform = @import("platform.zig").Platform; const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); -const Partitioning = Sharding.Partitioning; const Tensor = @import("tensor.zig").Tensor; const Compiler = @This(); @@ -35,7 +34,8 @@ mlir_ctx: *mlir.Context, mlir_pass_manager: *mlir.PassManager, module: *mlir.Module, platform: *const Platform, -partitioning: Sharding.Partitioning, +meshes: []const *const Sharding.Mesh, +partitioner: Sharding.Partitioner, mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), @@ -74,7 +74,7 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { pub const Options = struct { /// Incremented once after successful compilation. progress: std.Progress.Node = .none, - shardings: []const Sharding = &.{}, + meshes: []const *const Sharding.Mesh = &.{}, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, // Debugging options @@ -161,16 +161,13 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, } } - // Ensure replicated sharding is always included as a fallback option. - var shardings = std.ArrayList(Sharding).initCapacity(arena.allocator(), opts.shardings.len + 1) catch @panic("OOM"); - var needs_replicated: bool = true; - for (opts.shardings) |sharding| { - if (sharding.data == platform.replicated_sharding.data) needs_replicated = false; - shardings.appendAssumeCapacity(sharding.resolve(platform)); + var mesh_list: std.ArrayList(*const Sharding.Mesh) = .empty; + mesh_list.appendSlice(arena.allocator(), opts.meshes) catch @panic("OOM"); + if (std.mem.indexOfScalar(*const Sharding.Mesh, mesh_list.items, platform.replicated_mesh) == null) { + mesh_list.append(arena.allocator(), platform.replicated_mesh) catch @panic("OOM"); } - if (needs_replicated) shardings.appendAssumeCapacity(platform.replicated_sharding); - - const partitioning = Sharding.Partitioning.init(opts.partitioner orelse .fromTarget(platform.target), shardings.items) catch @panic("OOM"); + const meshes = mesh_list.items; + validateMeshes(meshes) catch |err| stdx.debug.panic("Incompatible meshes: {t}", .{err}); return .{ .allocator = allocator, @@ -182,7 +179,8 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, .mlir_known_types = mlir_known_types, .module = module, .platform = platform, - .partitioning = partitioning, + .partitioner = opts.partitioner orelse .fromTarget(platform.target), + .meshes = meshes, .location = unknown_location, .unknown_location = unknown_location, }; @@ -276,7 +274,8 @@ test pushLocation { .mlir_known_types = undefined, .module = undefined, .platform = undefined, - .partitioning = undefined, + .partitioner = undefined, + .meshes = undefined, .location = unknown_location, .unknown_location = unknown_location, }; @@ -328,6 +327,63 @@ pub fn allocPrint(self: *Compiler, comptime fmt: []const u8, args: anytype) []u8 return std.fmt.allocPrint(self.arena.allocator(), fmt, args) catch self.abortOOM(); } +/// Lookup a mesh by it's name +pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (compiler.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + if (name == .replicated) return compiler.platform.replicated_mesh; + std.debug.panic( + \\Found no meshes named {s}. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(compiler.meshes) }); +} + +/// Find a mesh that covers all the given axes +pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { + if (@TypeOf(logical_axes) != []const Shape.Tag) { + const comp_tags = comptime Shape.parseTags(logical_axes); + const parsed_tags: []const Shape.Tag = comptime comp_tags.constSlice(); + return compiler.resolveMesh(parsed_tags); + } + + var ok_meshe: ?*const Sharding.Mesh = null; + for (compiler.meshes) |shd| { + var covers_all: bool = true; + for (logical_axes) |ax| { + const input_axis = Shape.toTag(ax); + var covers_this: bool = false; + for (shd.logical.axes.slice()) |existing_axis| { + if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { + covers_this = true; + break; + } + } + covers_all = covers_all and covers_this; + } + if (covers_all) { + if (ok_meshe) |first_match| { + std.debug.panic( + \\Found two meshes covering axes: {any}, expected exacty one. + \\- First match: {f} + \\- Second match: {f} + , .{ logical_axes, first_match, shd }); + } + ok_meshe = shd; + } + } + + return ok_meshe orelse std.debug.panic( + \\Found no meshes covering axes: {any}, expected exacty one. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ logical_axes, stdx.fmt.slice(compiler.meshes) }); +} + pub fn Typed(comptime func: anytype) type { return struct { pub fn compile( @@ -402,9 +458,9 @@ pub fn compileInternal( _ = result.func.appendTo(compiler.module.body()); - const num_partitions = compiler.partitioning.numPartitions(); - const num_replicas = compiler.partitioning.numReplicas(); - const num_devices = compiler.partitioning.numDevices(); + const num_partitions = compiler.meshes[0].numPartitions(); + const num_replicas = compiler.meshes[0].numReplicas(); + const num_devices = num_partitions * num_replicas; compiler.module.operation().setAttributeByName( "mhlo.num_partitions", @@ -422,10 +478,7 @@ pub fn compileInternal( }, }; - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compiler.module, compiler.partitioning, opts) catch |err| { + const loaded_executable = compiler.compileModuleToPjrtExecutable(opts) catch |err| { compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Pjrt failed to compile the following MLIR"); return err; }; @@ -441,8 +494,6 @@ pub fn compileInternal( // This will get copied into exe result.input_info.items(.shape), result.output_info.items(.shape), - result.input_info.items(.sharding), - result.output_info.items(.sharding), result.input_info.items(.aliasing_output), ); errdefer exe.deinit(); @@ -455,16 +506,15 @@ fn addPartitionerOperations(ctx: *Compiler) !void { const allocator = ctx.arena.allocator(); const mlir_ctx = ctx.mlir_ctx; const module = ctx.module; - const partitioning = ctx.partitioning; - switch (partitioning.partitioner) { + switch (ctx.partitioner) { .gspmd => {}, .shardy => { - for (partitioning.shardings) |sharding| { - const attr_str = try sharding.data.sdyMeshAttr(allocator); + for (ctx.meshes) |shd| { + const attr_str = try shd.sdyMeshAttr(allocator); defer allocator.free(attr_str); - const name = sharding.data.name; + const name = shd.name; const mesh_attr = try mlir.Attribute.parse(mlir_ctx, attr_str); const mesh_op = mlir.Operation.make(mlir_ctx, "sdy.mesh", .{ @@ -491,7 +541,6 @@ const EmitMlirResult = struct { pub const TensorInfo = struct { id: Tensor.Id, shape: Shape, - sharding: Sharding, value: *const mlir.Value, // Only used for input tensors, stores which output tensor ends up with their buffer @@ -501,12 +550,12 @@ pub const TensorInfo = struct { var attrs: AttributeList = .empty; const mlir_ctx = compiler.mlir_ctx; - const sharding_attr = try compiler.partitioning.tensorShardingAttr(arena, mlir_ctx, info.shape, info.sharding); - const name = switch (compiler.partitioning.partitioner) { + const meshe_attr = try compiler.tensorShardingAttr(arena, mlir_ctx, info.shape); + const name = switch (compiler.partitioner) { .gspmd => "mhlo.sharding", .shardy => "sdy.sharding", }; - attrs.appendAssumeCapacity(.named(mlir_ctx, name, sharding_attr)); + attrs.appendAssumeCapacity(.named(mlir_ctx, name, meshe_attr)); const memory = scope.id_to_memory.get(info.id) orelse .device; if (memory != .device) { @@ -583,16 +632,9 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou defer ctx.current_argument_id += 1; - const input_sharding = ctx.compiler.partitioning.selectSharding(packed_shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "Failed to resolve sharding for input {f}({d}) because it's using unknown sharding. Pass more shardings to `platform.compile`. Known shardings: {f}", - .{ packed_shape, ctx.current_argument_id, stdx.fmt.slice(ctx.compiler.partitioning.shardings) }, - ), - }; try ctx.infos.append(ctx.compiler.allocator, .{ .id = tensor.id, .shape = og_shape, - .sharding = input_sharding, .value = value, }); } @@ -615,7 +657,6 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf fn cb(ctx: *@This(), tensor: *const Tensor) !void { const og_shape = tensor.shape(); - const packed_shape = og_shape.packedShape(); var value = ctx.scope.id_to_argument.get(tensor.id) orelse tensor._value orelse @panic("no value found for output tensor"); @@ -626,9 +667,9 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf try ctx.infos.append(ctx.compiler.allocator, .{ .id = tensor.id, + // const packed_shape = og_shape.packedShape(); + // TODO: clarify why this og_shape and not packedShape() .shape = og_shape, - // Note: the panic should have been triggered during createBlockArguments or emitMlir - .sharding = ctx.compiler.partitioning.selectSharding(packed_shape) catch @panic("failed to resolve output sharding"), .value = value, }); } @@ -666,14 +707,14 @@ fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.Multi } } - // Input sharding/memory/aliasing attributes + // Input mesh/memory/aliasing attributes const input_attributes = try arena.alloc(*const mlir.Attribute, input_info.len); for (0.., input_attributes) |i, *input_attrs| { const attrs_list = try input_info.get(i).attributes(arena, compiler, fn_scope); input_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); } - // Output sharding/memory attributes + // Output mesh/memory attributes const output_attributes = try arena.alloc(*const mlir.Attribute, output_info.len); for (0.., output_attributes) |i, *output_attrs| { const attrs_list = try output_info.get(i).attributes(arena, compiler, fn_scope); @@ -748,6 +789,38 @@ fn repack(compiler: *Compiler, scope: *Scope, og_shape: Shape, value: *const mli return bit_cast_op.result(0); } +fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { + stdx.debug.assert(meshes.len >= 1, "Waiting at leat 1 mesh strategy to be implemented", .{}); + + const first = meshes[0]; + const partitions = first.numPartitions(); + const replicas = first.numReplicas(); + + for (meshes[1..]) |s| { + if (s.numPartitions() != partitions or s.numReplicas() != replicas) { + // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers + return error.InconsistentMesheCardinality; + } + } +} + +pub fn tensorShardingAttr( + compiler: *const Compiler, + allocator: std.mem.Allocator, + mlir_ctx: *mlir.Context, + shape: Shape, +) error{OutOfMemory}!*const mlir.Attribute { + const mesh_ = shape._sharding.mesh orelse compiler.platform.replicated_mesh; + return switch (compiler.partitioner) { + .shardy => (try mesh_.sdyShardingAttrForShape(allocator, mlir_ctx, shape)).asAttr(), + .gspmd => mesh_.gspmdShardingAttrForShape(allocator, mlir_ctx, shape) catch |err| switch (err) { + error.WriteFailed => error.OutOfMemory, // We're writing to memory + error.OutOfMemory => error.OutOfMemory, + error.IncompatibleSharding => @panic("Shape dimensions must be divisible by their mesh partitions"), + }, + }; +} + fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { const result = c.upb_Map_Set( map, @@ -770,21 +843,25 @@ fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_are } } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, partitioning: Partitioning, opts: Options) !*pjrt.LoadedExecutable { - var upb_alloc: upb.Allocator = .init(arena); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - defer c.upb_Arena_Free(upb_arena); - - const use_shardy_partitioner = switch (partitioning.partitioner) { +fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.LoadedExecutable { + const use_shardy_partitioner = switch (compiler.partitioner) { .shardy => true, .gspmd => false, }; - const num_partitions = partitioning.numPartitions(); - const num_replicas = partitioning.numReplicas(); + const platform = compiler.platform; + const main_mesh = compiler.meshes[0]; + const num_partitions = main_mesh.numPartitions(); + const num_replicas = main_mesh.numReplicas(); - const device_assignment = try partitioning.deviceAssignment(arena); + const device_assignment = try main_mesh.deviceAssignment(compiler.allocator); + defer compiler.allocator.free(device_assignment); + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); const options = blk: { const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { @@ -918,9 +995,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: const loaded_executable = try pjrtx.Client.compile( platform.pjrt_client, platform.pjrt_api, - arena, - io, - module, + compiler.allocator, + compiler.io, + compiler.module, try upb.serialize(options, upb_arena), ); errdefer loaded_executable.deinit(); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index b510a9c4..761721fa 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -16,33 +16,152 @@ const Target = @import("platform.zig").Target; const Sharding = @This(); -data: *const Data, - -pub const Error = error{ MissingLogicalBinding, IncompatibleSharding }; +mesh: ?*const Mesh, +partition: Partitioning, pub const MAX_MESH_RANK = 4; +pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; -var _replicated: [11]u8 align(@alignOf(Data)) = "_replicated".*; - -// special value to make public apis more fluent. -pub const replicated: Sharding = .{ .data = @ptrCast(&_replicated) }; - -pub fn resolve(sharding: Sharding, platform: *const Platform) Sharding { - return if (sharding.data == replicated.data) platform.replicated_sharding else sharding; +pub fn get(sharding: Sharding, ax: usize) PartitionSpec { + return if (sharding.mesh == null) .replicated else sharding.partition.get(ax); } -pub fn numPartitionsForLogicalAxis(sharding: Sharding, logical_axis: anytype) i64 { - return sharding.data.numPartitionsForLogicalAxis(logical_axis); +pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.set(ax, spec); + return res; } -pub fn devicesInCanonicalOrder(sharding: Sharding) []const Device { - return sharding.data.physical.devices_in_canonical_order; +pub fn insert(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.insert(ax, spec); + return res; } -pub fn format(sharding: Sharding, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return sharding.data.format(writer); +pub fn orderedRemove(self: Sharding, ax: usize) Sharding { + var res = self; + res.partition = res.partition.orderedRemove(ax); + return res; } +pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { + var result: [Shape.MAX_RANK]PartitionSpec = undefined; + for (&result, 0..) |*spec, ax| spec.* = self.get(ax); + return result; +} + +pub fn isFullyReplicated(self: Sharding) bool { + if (self.mesh == null) return true; + for (self.partition.toArray()) |spec| { + if (spec != .replicated and spec != .out_of_bound) return false; + } + return true; +} + +/// An unspecified mesh is compatible with any sharding. On the same mesh, +/// open axes accept any in-bounds partition, while closed axes must match. +pub fn isCompatible(self: Sharding, other: Sharding) bool { + if (self.mesh == null or other.mesh == null) return true; + if (self.mesh != other.mesh) return false; + for (self.partition.toArray(), other.partition.toArray()) |lhs, rhs| { + if (lhs == rhs) continue; + if (lhs == .out_of_bound or rhs == .out_of_bound) return false; + if (lhs != .open and rhs != .open) return false; + } + return true; +} + +test isCompatible { + const tp: Mesh = .initTest(.tp, .{.model}); + const dp: Mesh = .initTest(.dp, .{ .data, .model }); + + const sharded: Sharding = .{ .mesh = &tp, .partition = .init(&.{ .mesh_axis_0, .replicated }) }; + const cases = [_]struct { lhs: Sharding, rhs: Sharding, compatible: bool }{ + .{ .lhs = .replicated, .rhs = .replicated, .compatible = true }, + .{ .lhs = .replicated, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = .{ .mesh = &tp, .partition = .open(2) }, .compatible = true }, + .{ .lhs = sharded.set(1, .open), .rhs = sharded.set(0, .open), .compatible = true }, + .{ .lhs = sharded, .rhs = sharded.set(0, .mesh_axis_1), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(0, .replicated), .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = sharded.partition }, .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = .open(2) }, .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(1, .out_of_bound), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(2, .open), .compatible = false }, + }; + for (cases) |case| { + errdefer std.log.err("{f}.isCompatible({f}) expected to be {}, got {}", .{ case.lhs, case.rhs, case.compatible, !case.compatible }); + + try std.testing.expectEqual(case.compatible, case.lhs.isCompatible(case.rhs)); + try std.testing.expectEqual(case.compatible, case.rhs.isCompatible(case.lhs)); + } +} + +pub fn format(self: Sharding, writer: *std.Io.Writer) !void { + const mesh = self.mesh orelse return writer.writeAll("{replicated}"); + const specs = self.partition.toArray(); + var rank = specs.len; + while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; + + try writer.writeByte('{'); + for (specs[0..rank], 0..) |spec, i| { + if (i > 0) try writer.writeByte(','); + switch (spec) { + .replicated => try writer.writeAll("replicated"), + .open => try writer.writeByte('?'), + .out_of_bound => try writer.writeByte('!'), + else => try writer.print("{s}", .{mesh.logical.axes.get(spec.meshAxis().?)}), + } + } + try writer.print("}}@{s}", .{mesh.name}); +} + +/// Describes how a given Shape axis behaves inside a mesh. +/// Is it replicated ? sharded along a specific logical axis ? or open to replication ? +pub const PartitionSpec = enum(u4) { + mesh_axis_0 = 0, + mesh_axis_1 = 1, + mesh_axis_2 = 2, + mesh_axis_3 = 3, + mesh_axis_4 = 4, + mesh_axis_5 = 5, + mesh_axis_6 = 6, + mesh_axis_7 = 7, + // an 8D mesh seems already a lot, the max we know about is 3D. + replicated = 8, + + // Partitioning don't encode the rank of corresponding tensor, so this is used to mark invalid positions. + out_of_bound = 0xa, + open = 15, + + pub fn sharded(mesh_axis: u3) PartitionSpec { + return @enumFromInt(mesh_axis); + } + + /// Extract the mesh axis along which we are sharded. Null if not sharded. + pub fn meshAxis(self: PartitionSpec) ?u3 { + const ax = @intFromEnum(self); + return if (ax < @intFromEnum(PartitionSpec.replicated)) @intCast(ax) else null; + } + + pub fn isSharded(self: PartitionSpec) bool { + return @intFromEnum(self) < @intFromEnum(PartitionSpec.replicated); + } + + pub fn isClosed(self: PartitionSpec) bool { + return @intFromEnum(self) <= @intFromEnum(PartitionSpec.replicated); + } + + test isClosed { + try std.testing.expect(PartitionSpec.mesh_axis_2.isClosed()); + try std.testing.expect(!PartitionSpec.open.isClosed()); + + try std.testing.expect(PartitionSpec.replicated.isClosed()); + + try std.testing.expect(!PartitionSpec.out_of_bound.isClosed()); + } +}; + pub const Partitioner = union(enum) { shardy, gspmd, @@ -54,121 +173,14 @@ pub const Partitioner = union(enum) { } }; -pub const Partitioning = struct { - partitioner: Partitioner, - shardings: []const Sharding, - - pub fn init(partitioner: Partitioner, shardings: []const Sharding) !Partitioning { - stdx.debug.assert(shardings.len >= 1, "Waiting at leat 1 sharding strategy to be implemented", .{}); - var plan: Partitioning = .{ .partitioner = partitioner, .shardings = shardings }; - - const first = plan.primarySharding(); - const partitions = first.data.numPartitions(); - const replicas = first.data.numReplicas(); - - for (shardings[1..]) |s| { - if (s.data.numPartitions() != partitions or s.data.numReplicas() != replicas) { - // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers - return error.InconsistentShardingCardinality; - } - } - - return plan; - } - - pub fn numPartitions(self: Partitioning) i32 { - const primary = self.primarySharding(); - return primary.data.numPartitions(); - } - - pub fn numReplicas(self: Partitioning) i32 { - const primary = self.primarySharding(); - return primary.data.numReplicas(); - } - - pub fn numDevices(self: Partitioning) i32 { - return self.numPartitions() * self.numReplicas(); - } - - pub fn deviceAssignment(self: Partitioning, allocator: std.mem.Allocator) ![]usize { - return switch (self.partitioner) { - .shardy, .gspmd => blk: { - const sharding = self.primarySharding(); - break :blk sharding.data.deviceAssignment(allocator); - }, - }; - } - - pub fn tensorShardingAttr( - self: Partitioning, - allocator: std.mem.Allocator, - ctx: *mlir.Context, - shape: Shape, - sharding: Sharding, - ) error{OutOfMemory}!*const mlir.Attribute { - return switch (self.partitioner) { - .shardy => (try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape)).asAttr(), - .gspmd => sharding.data.gspmdShardingAttrForShape(allocator, ctx, shape) catch |err| switch (err) { - error.WriteFailed => error.OutOfMemory, // We're writing to memory - error.OutOfMemory => error.OutOfMemory, - // TODO(hugomano): clarify what can trigger this and consider moving the check to the Sharding creation - error.MissingDeviceInTile => @panic("MissingDeviceInTile"), - }, - }; - } - - pub fn localShapeForShape(self: Partitioning, shape: Shape) !Shape { - const sharding = try self.selectSharding(shape); - const ordered_devices = sharding.devicesInCanonicalOrder(); - if (ordered_devices.len == 0) return error.InvalidPhysicalMesh; - return sharding.shardedShape(shape); - } - - pub fn numPartitionsForLogicalAxis(self: Partitioning, shape: Shape, logical_axis: anytype) !i64 { - const sharding = try self.selectSharding(shape); - return sharding.data.numPartitionsForLogicalAxis(logical_axis); - } - - pub fn shardableDim(self: Partitioning, shape: Shape, axis: anytype, must_divide: i64) !DimSharding { - const ax = shape.axis(axis); - const spec = shape.partition(ax); - if (spec != .axis) return .replicated; - - const sharding = try self.selectSharding(shape); - return sharding.shardableDim(shape.dim(ax), spec.axis, must_divide); - } - - pub fn selectSharding(self: Partitioning, shape: Shape) error{NoSuitableSharding}!Sharding { - return pickSharding(self.shardings, shape, .any_covering) orelse error.NoSuitableSharding; - } - - fn primarySharding(self: Partitioning) Sharding { - return self.shardings[0]; - } -}; - -pub const SelectShardingMode = enum { - any_covering, - explicit_axis_binding, -}; - -pub fn pickSharding(shardings: []const Sharding, shape: Shape, mode: SelectShardingMode) ?Sharding { - if (mode == .explicit_axis_binding and !shapeHasAxisPartition(shape)) return null; - - for (shardings) |sharding| { - if (sharding.data.covers(shape)) return sharding; - } - return null; -} - fn shapeHasAxisPartition(shape: Shape) bool { for (0..shape.rank()) |ax| { - if (shape.partition(ax) == .axis) return true; + if (shape.partition(ax).meshAxis()) |_| return true; } return false; } -pub const DimSharding = union(enum) { +pub const DimMeshe = union(enum) { sharded: struct { dim: i64, factor: u32, @@ -176,43 +188,25 @@ pub const DimSharding = union(enum) { replicated, }; -pub fn shardableDim(sharding: Sharding, dim: i64, logical_axis: anytype, must_divide: i64) DimSharding { - const partitions = sharding.numPartitionsForLogicalAxis(logical_axis); - - if (@mod(dim, partitions) == 0) { - return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; - } - - const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); - const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); - const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; - if (@mod(must_divide, materialized_dim) == 0) { - return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; - } else { - return .replicated; - } -} - pub fn sdyPerValueShardingAttr( allocator: std.mem.Allocator, ctx: *mlir.Context, shapes: []const Shape, - shardings: []const Sharding, + mesh: *const Mesh, ) error{OutOfMemory}!*const mlir.Attribute { - const sharding_attrs = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); - for (sharding_attrs, shapes, shardings) |*attr, shape, sharding| { - attr.* = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + const meshe_attrs = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); + for (meshe_attrs, shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); } - return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, sharding_attrs).asAttr(); + return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, meshe_attrs).asAttr(); } pub fn sdyManualAxesAttr( allocator: std.mem.Allocator, ctx: *mlir.Context, in_shapes: []const Shape, - in_shardings: []const Sharding, out_shapes: []const Shape, - out_shardings: []const Sharding, + mesh: *const Mesh, ) error{OutOfMemory}!*const mlir.Attribute { var axis_names = std.ArrayList([]const u8).empty; defer axis_names.deinit(allocator); @@ -226,8 +220,8 @@ pub fn sdyManualAxesAttr( } }; - for (in_shapes, in_shardings) |shape, sharding| { - const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (in_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); for (0..attr.numReplicatedAxes()) |i| { Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); } @@ -238,8 +232,8 @@ pub fn sdyManualAxesAttr( } } } - for (out_shapes, out_shardings) |shape, sharding| { - const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (out_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); for (0..attr.numReplicatedAxes()) |i| { Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); } @@ -259,6 +253,192 @@ pub fn sdyManualAxesAttr( return dialects.shardy.ManualAxesAttribute.init(ctx, axes).asAttr(); } +pub const Partitioning = packed struct { + _0: PartitionSpec, + _1: PartitionSpec, + _2: PartitionSpec, + _3: PartitionSpec, + _4: PartitionSpec, + _5: PartitionSpec, + _6: PartitionSpec, + _7: PartitionSpec, + + pub const out_of_bound: Partitioning = splat(.out_of_bound); + + const MAX_RANK = Shape.MAX_RANK; + const Vec = @Vector(MAX_RANK, u4); + + pub fn init(specs: []const PartitionSpec) Partitioning { + var res: Partitioning = out_of_bound; + for (0.., specs) |i, spec| { + res = res.set(i, spec); + } + return res; + } + + pub fn replicated(rank_: usize) Partitioning { + return repeat(.replicated, rank_); + } + + pub fn open(rank_: usize) Partitioning { + return repeat(.open, rank_); + } + + pub fn splat(spec: PartitionSpec) Partitioning { + const vec: Vec = @splat(@intFromEnum(spec)); + return @bitCast(vec); + } + + pub fn repeat(spec: PartitionSpec, rank_: usize) Partitioning { + std.debug.assert(rank_ <= MAX_RANK); + const splatted: Vec = @splat(@intFromEnum(spec)); + const mask = std.simd.iota(u4, 8) < @as(Vec, @splat(@truncate(rank_))); + return @bitCast(@select(u4, mask, splatted, @as(Vec, @bitCast(out_of_bound)))); + } + + pub fn get(p: Partitioning, ax: usize) PartitionSpec { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + return @enumFromInt(@as(u4, @truncate(pack >> shift))); + } + + pub fn set(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const mask = @as(u32, 0xf) << shift; + return @bitCast((pack & ~mask) | (@as(u32, @intFromEnum(spec)) << shift)); + } + + pub fn rank(p: Partitioning) u8 { + for (0..MAX_RANK) |i_usize| { + const i: u8 = @truncate(i_usize); + if (p.get(i) == .out_of_bound) return i; + } + return MAX_RANK; + } + + /// Inserts a spec, shifting subsequent slots right and discarding the last slot. + pub fn insert(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @intFromEnum(spec)) << shift)); + } + + /// Removes a spec, shifting subsequent slots left and filling the last with unknown. + pub fn orderedRemove(p: Partitioning, ax: usize) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @intFromEnum(PartitionSpec.out_of_bound)) << 28)); + } + + pub fn toArray(p: Partitioning) [MAX_RANK]PartitionSpec { + var res: [MAX_RANK]PartitionSpec = undefined; + for (&res, 0..) |*spec, ax| spec.* = p.get(ax); + return res; + } + + pub fn hasUniqueAxes(p: Partitioning) bool { + var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .initEmpty(); + for (0..MAX_RANK) |shape_ax| { + const spec = p.get(shape_ax); + if (spec.meshAxis()) |mesh_axis| { + if (used_mesh_axes.isSet(mesh_axis)) return false; + used_mesh_axes.set(mesh_axis); + } + } + return true; + } + + test hasUniqueAxes { + try std.testing.expect(hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_2 }))); + try std.testing.expect(!hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_0 }))); + } + + test "packed access and updates" { + try std.testing.expectEqual(32, @bitSizeOf(Partitioning)); + try std.testing.expectEqual(4, @sizeOf(Partitioning)); + inline for (std.meta.tags(PartitionSpec)) |spec| { + const filled = Partitioning.splat(spec); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(spec, filled.get(ax)); + const updated = Partitioning.out_of_bound.set(ax, spec); + for (0..MAX_RANK) |i| { + try std.testing.expectEqual(if (i == ax) spec else .out_of_bound, updated.get(i)); + } + } + } + for (0..MAX_RANK + 1) |rank_| { + const parts = Partitioning.replicated(rank_); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(if (ax < rank_) PartitionSpec.replicated else .out_of_bound, parts.get(ax)); + } + } + } + + test "insertion and removal at every slot" { + var parts: Partitioning = .out_of_bound; + for (0..MAX_RANK) |ax| parts = parts.set(ax, @enumFromInt(ax)); + const original = parts.toArray(); + for (0..MAX_RANK) |ax| { + var inserted = original; + std.mem.copyBackwards(PartitionSpec, inserted[ax + 1 ..], original[ax .. MAX_RANK - 1]); + inserted[ax] = .open; + try std.testing.expectEqualSlices(PartitionSpec, &inserted, &parts.insert(ax, .open).toArray()); + + var removed = original; + std.mem.copyForwards(PartitionSpec, removed[ax .. MAX_RANK - 1], original[ax + 1 ..]); + removed[MAX_RANK - 1] = .out_of_bound; + try std.testing.expectEqualSlices(PartitionSpec, &removed, &parts.orderedRemove(ax).toArray()); + try std.testing.expectEqual(parts, parts.orderedRemove(ax).insert(ax, parts.get(ax))); + } + try std.testing.expectEqual(Partitioning.out_of_bound, Partitioning.out_of_bound.orderedRemove(0)); + const comptime_parts = comptime Partitioning.out_of_bound.insert(7, .open).orderedRemove(0).set(0, .replicated); + try std.testing.expectEqual(PartitionSpec.replicated, comptime_parts.get(0)); + try std.testing.expectEqual(PartitionSpec.open, comptime_parts.get(6)); + try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); + } + + /// Parse a struct literal into a partitioning object. + /// The struct passed to partitioning must only have fields names present in `tags`. + /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. + /// The default value is important. + /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. + pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { + var partition_: Partitioning = .repeat(default_value, tags.len); + + const T = @TypeOf(partitioning); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "parsePartitioning expected a struct of enum literals eg {{ .b = .data, .d = .model }}, got: {any}", .{T}); + inline for (std.meta.fields(T)) |field| { + const shape_tag = Shape.toTag(field); + const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field.name }); + }; + + const value = @field(partitioning, field.name); + const spec: PartitionSpec = if (@TypeOf(value) == PartitionSpec) value else switch (value) { + .replicated => .replicated, + .out_of_bound => std.debug.panic("value out_of_bound not allowed", .{}), + .open => .open, + else => spec: { + const mesh_tag = Shape.toTag(value); + const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { + stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); + }; + break :spec .sharded(@intCast(mesh_ax)); + }, + }; + partition_ = partition_.set(shape_ax, spec); + } + return partition_; + } +}; + /// Device as part of a PhysicalMesh. pub const Device = struct { /// Unique device identifier in the mesh @@ -1127,7 +1307,7 @@ pub const PhysicalMesh = struct { }; pub const LogicalAxisIntent = enum { - /// Needs the fastest possible path (e.g. Weight sharding / All-Reduce). + /// Needs the fastest possible path (e.g. Weight mesh / All-Reduce). /// Requires high-bandwidth, low-latency interconnects (Cores/NVLink). high_bandwidth, @@ -1135,12 +1315,12 @@ pub const LogicalAxisIntent = enum { /// Suitable for chip-to-chip interconnects (NeuronLink/ICI). balanced, - /// Can tolerate lower bandwidth and higher latency (e.g. Batch sharding). + /// Can tolerate lower bandwidth and higher latency (e.g. Batch mesh). /// Suitable for system bus or network (PCIe/Ethernet). low_bandwidth, }; -// Defines the logical mesh used for sharding the model and data. +// Defines the logical mesh used for mesh the model and data. // E.g., batch, model, context, experts, etc. // Each axis represents a dimension along which the workload can be partitioned. The value is the intent of the axis. /// LogicalMesh defines *semantic* axes only (batch/model/context/...). @@ -1214,16 +1394,11 @@ pub const LogicalMesh = struct { } }; -pub const AxisList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); -pub const Binding = struct { - logical: Shape.Tag, - physical: AxisList, -}; -pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); +pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); pub const Fold = struct { target: PhysicalAxisTag, - sources: AxisList, + sources: PhysicalList, }; pub const Folds = stdx.BoundedArray(Fold, MAX_MESH_RANK); @@ -1260,38 +1435,73 @@ pub const PhysicalView = struct { } }; -pub const Data = struct { +pub const Mesh = struct { + pub const zml_no_meta_visit: void = {}; + name: []const u8, physical: *const PhysicalMesh, logical: LogicalMesh, /// Compact binding table: logical axis -> physical axes - bindings: Bindings, + bindings: stdx.BoundedArray(PhysicalList, MAX_MESH_RANK), /// Explicit folding rules: kept axis -> ordered source axes. folds: Folds, folds_consumed: std.EnumSet(PhysicalAxisTag), - pub fn binding(self: *const Data, tag: Shape.Tag) ?[]const PhysicalAxisTag { - for (self.bindings.slice()) |*b| { - if (b.logical == tag) return b.physical.slice(); + pub fn shardableDim(mesh: *const Mesh, dim: i64, logical_axis: anytype, must_divide: i64) DimMeshe { + const partitions = mesh.numPartitionsForLogicalAxis(logical_axis); + + if (@mod(dim, partitions) == 0) { + return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; + } + + const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); + const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); + const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; + if (@mod(must_divide, materialized_dim) == 0) { + return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; + } else { + return .replicated; + } + } + + pub fn binding(self: *const Mesh, mesh_axis: usize) []const PhysicalAxisTag { + return self.bindings.constSlice()[mesh_axis].constSlice(); + } + + pub fn resolveLogicalAxis(self: *const Mesh, tag: Shape.Tag) ?u8 { + for (0.., self.logical.axes.slice()) |i, logical_axis| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(tag))) return @truncate(i); } return null; } pub fn init( - name: []const u8, + name_: []const u8, physical: *const PhysicalMesh, logical: LogicalMesh, strategy: Strategy, - ) !Data { + ) !Mesh { const axis_order = physical.axisOrder().slice(); if (axis_order.len == 0) return error.InvalidPhysicalMesh; if (strategy.bindings.len == 0) return error.InvalidStrategy; - var bindings: Bindings = .empty; - for (strategy.bindings.slice()) |bind| { - bindings.appendAssumeCapacity(.{ .logical = bind.logical, .physical = bind.physical }); + // Create one binding per logical axis + var bindings: @FieldType(Mesh, "bindings") = .empty; + for (logical.axes.slice()) |_| { + bindings.appendAssumeCapacity(.empty); + } + + for (strategy.bindings.slice()) |strat_binding| { + for (logical.axes.slice(), bindings.slice()) |logical_axis, *binding_| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(strat_binding.logical))) { + binding_.* = strat_binding.physical; + break; + } + } else { + std.debug.panic("{f} expects strategy to only list known logical axes, got unknown {s}", .{ logical, strat_binding.logical }); + } } var folds: Folds = .empty; @@ -1305,7 +1515,7 @@ pub const Data = struct { } return .{ - .name = name, + .name = name_, .logical = logical, .physical = physical, .bindings = bindings, @@ -1314,14 +1524,34 @@ pub const Data = struct { }; } - fn foldSources(self: *const Data, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { + pub fn initTest(name_: @EnumLiteral(), tagz: anytype) Mesh { + if (!builtin.is_test) @compileError("Cannot use zml.Sharding.Mesh.initTest outside of a test block"); + const axes_tags = Shape.parseTags(tagz); + return .{ + .name = @tagName(name_), + .logical = .{ + .axes = .init(axes_tags.slice()), + .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, + }, + .physical = undefined, + .bindings = .empty, + .folds = .empty, + .folds_consumed = .empty, + }; + } + + fn foldSources(self: *const Mesh, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { for (self.folds.slice()) |*f| { if (f.target == tag) return f.sources.slice(); } return null; } - pub fn physicalView(self: *const Data) PhysicalView { + pub fn devicesInCanonicalOrder(mesh: *const Mesh) []const Device { + return mesh.physical.devices_in_canonical_order; + } + + pub fn physicalView(self: *const Mesh) PhysicalView { const physical = self.physical; var view: PhysicalView = .{ .axes = .empty, @@ -1359,17 +1589,18 @@ pub const Data = struct { return view; } - pub fn logicalIndexFromCoords(self: *const Data, coords: []const u8) usize { + pub fn logicalIndexFromCoords(self: *const Mesh, coords: []const u8) usize { return self.physical.linearIndexFromCoords(coords); } - pub fn numPartitions(self: *const Data) i32 { + pub fn numPartitions(self: *const Mesh) i32 { return @intCast(self.physicalView().total_devices); } - pub fn numPartitionsForLogicalAxis(self: *const Data, logical_axis: anytype) i64 { + pub fn numPartitionsForLogicalAxis(self: *const Mesh, logical_axis: anytype) i64 { const logical_tag = Shape.toTag(logical_axis); - const bound_axes = self.binding(logical_tag) orelse return 1; + const logical_ax = self.resolveLogicalAxis(logical_tag) orelse std.debug.panic("mesh {f} has no axis {s}", .{ self, logical_tag }); + const bound_axes = self.binding(logical_ax); var physical_axes: std.EnumSet(PhysicalAxisTag) = .empty; for (bound_axes) |bound_axis| { @@ -1389,25 +1620,15 @@ pub const Data = struct { return partitions; } - pub fn numReplicas(_: *const Data) i32 { + pub fn numReplicas(_: *const Mesh) i32 { return 1; } - pub fn numDevices(self: *const Data) i32 { + pub fn numDevices(self: *const Mesh) i32 { return self.numPartitions() * self.numReplicas(); } - pub fn covers(sharding: *const Data, shape: Shape) bool { - for (shape._partitioning.slice()) |partitioning| { - switch (partitioning) { - .axis => |tag| if (sharding.binding(tag) == null) return false, - else => {}, - } - } - return true; - } - - pub fn sdyMeshAttr(self: *const Data, allocator: std.mem.Allocator) ![]const u8 { + pub fn sdyMeshAttr(self: *const Mesh, allocator: std.mem.Allocator) ![]const u8 { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); @@ -1422,7 +1643,7 @@ pub const Data = struct { } const DimMapping = struct { - /// For each tensor dimension, the indices of physical axes (in PhysicalView) sharding it. + /// For each tensor dimension, the indices of physical axes (in PhysicalView) mesh it. axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK), /// Indices of physical axes used for replication. replicated_axes: stdx.BoundedArray(usize, Shape.MAX_RANK), @@ -1431,7 +1652,7 @@ pub const Data = struct { }; /// Common logic to map tensor dimensions to physical mesh indices. - fn getDimMapping(self: *const Data, shape: Shape) DimMapping { + fn getDimMapping(self: *const Mesh, shape: Shape) DimMapping { const view = self.physicalView(); var axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK) = .empty; var used_mask: [Shape.MAX_RANK]bool = @splat(false); @@ -1441,16 +1662,15 @@ pub const Data = struct { var dim_axes: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; const spec = shape.partition(ax); - if (spec == .axis) { - if (self.binding(spec.axis)) |binding_| { - for (binding_) |p_tag| { - for (view.axes.slice(), 0..) |v_ax, i| { - // Only use the axis if it's bound and hasn't been consumed by a previous dimension - if (v_ax.contains(p_tag) and !globally_used.contains(v_ax.tag)) { - dim_axes.appendAssumeCapacity(i); - globally_used.insert(v_ax.tag); - used_mask[i] = true; - } + if (spec.meshAxis()) |mesh_ax| { + const binding_ = self.bindings.get(mesh_ax); + for (binding_.slice()) |p_tag| { + for (view.axes.slice(), 0..) |v_ax, i| { + // Only use the axis if it's bound and hasn't been consumed by a previous dimension + if (v_ax.contains(p_tag) and !globally_used.contains(v_ax.tag)) { + dim_axes.appendAssumeCapacity(i); + globally_used.insert(v_ax.tag); + used_mask[i] = true; } } } @@ -1471,7 +1691,7 @@ pub const Data = struct { } pub fn sdyShardingAttrForShape( - data: *const Data, + data: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape, @@ -1479,39 +1699,28 @@ pub const Data = struct { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); - var any_explicit = false; - for (0..shape.rank()) |ax| { - if (shape.partition(ax) != .unknown) { - any_explicit = true; - break; - } - } - const all_replicated = !any_explicit; - const mapping = data.getDimMapping(shape); const dimensions = try allocator.alloc(*const dialects.shardy.DimensionShardingAttribute, shape.rank()); + // Note: here we map from the logical axis to physical. Is this a good idea ? + // why not stay in logical realm ? for (0.., dimensions) |ax, *d| { - const spec = shape.partition(ax); - d.* = switch (spec) { - .axis => |logical_tag| d: { - if (data.binding(logical_tag)) |_| { - const dim_phys_indices = mapping.axes_per_dim.get(ax); - if (dim_phys_indices.len == 0) { - break :d .replicated(ctx); - } else { - const axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, dim_phys_indices.len); - for (dim_phys_indices.slice(), 0..) |p_idx, i| { - axes[i] = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); - } - break :d .closed(ctx, axes); - } + d.* = switch (shape.partition(ax)) { + .out_of_bound => @panic("undefined"), + .replicated => .replicated(ctx), + .open => .open(ctx, &.{}), + else => d: { + const dim_phys_indices = mapping.axes_per_dim.get(ax); + if (dim_phys_indices.len == 0) { + break :d .replicated(ctx); } else { - break :d .open(ctx, &.{}); + const axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, dim_phys_indices.len); + for (dim_phys_indices.slice(), 0..) |p_idx, i| { + axes[i] = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); + } + break :d .closed(ctx, axes); } }, - .replicated => .replicated(ctx), - .open, .unknown => if (all_replicated) .replicated(ctx) else .open(ctx, &.{}), }; } @@ -1523,14 +1732,14 @@ pub const Data = struct { return .init(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes }); } - pub fn gspmdShardingAttrForShape(self: *const Data, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { + pub fn gspmdShardingAttrForShape(self: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); var has_sharding = false; for (0..shape.rank()) |ax| { - if (shape.partition(ax) == .axis) { + if (shape.partition(ax).isSharded()) { has_sharding = true; break; } @@ -1562,9 +1771,7 @@ pub const Data = struct { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); - // Emit explicit device assignment list to avoid XLA requiring - // a sharding attribute on the custom-call instruction. - const ids = try self.deviceAssignment(allocator); + const ids = try self.gspmdTileAssignment(allocator, shape); defer allocator.free(ids); try out.writer.writeAll("{devices=["); @@ -1591,12 +1798,36 @@ pub const Data = struct { return .string(ctx, try out.toOwnedSlice()); } - pub fn deviceAssignment(self: *const Data, allocator: std.mem.Allocator) ![]usize { - const view = self.physicalView(); - const count: usize = @intCast(view.total_devices); + fn gspmdTileAssignment(self: *const Mesh, allocator: std.mem.Allocator, shape: Shape) ![]u32 { + const placement = try Placement.init(shape); + const devices = self.devicesInCanonicalOrder(); + var tile_count: usize = 1; + for (placement.axis_plans.constSlice()) |plan| tile_count *= @intCast(plan.num_devices); + const replicas_per_tile = @divExact(devices.len, tile_count); + const replica_indices = try allocator.alloc(usize, tile_count); + defer allocator.free(replica_indices); + @memset(replica_indices, 0); + const ids = try allocator.alloc(u32, devices.len); - var ids = try allocator.alloc(usize, count); - @memset(ids, std.math.maxInt(usize)); + for (devices, 0..) |device, partition_index| { + var tile_index: usize = 0; + for (placement.axis_plans.constSlice()) |plan| { + tile_index = tile_index * @as(usize, @intCast(plan.num_devices)) + @as(usize, @intCast(plan.linearIndex(device.coords))); + } + // HLO shardings refer to logical partitions; executable device assignment + // separately maps those partitions to physical PJRT device IDs. + ids[tile_index * replicas_per_tile + replica_indices[tile_index]] = @intCast(partition_index); + replica_indices[tile_index] += 1; + } + return ids; + } + + pub fn deviceAssignment(self: *const Mesh, allocator: std.mem.Allocator) ![]u32 { + const view = self.physicalView(); + const count: u32 = @intCast(view.total_devices); + + var ids = try allocator.alloc(u32, count); + @memset(ids, std.math.maxInt(u32)); for (self.physical.devices_in_canonical_order) |d| { const coords = d.coords; @@ -1605,31 +1836,35 @@ pub const Data = struct { } for (ids) |id| { - if (id == std.math.maxInt(usize)) return error.MissingDeviceInTile; + if (id == std.math.maxInt(u32)) return error.MissingDeviceInTile; } return ids; } - pub fn format(self: Data, writer: *std.Io.Writer) !void { - try writer.print("Sharding(name={s})\n", .{self.name}); + /// Extract coordinate data from the physical mesh for a specific axis. + fn addPhysicalToSplit(mesh: *const Mesh, plan: *AxisSplit, tag: PhysicalAxisTag) void { + const physical = mesh.physical; + const info = physical.axisInfo(tag) orelse return; + const depth = physical.axis_traversal.depth(tag) orelse return; + plan.add(@intCast(info.size), depth); + } + + pub fn format(self: Mesh, writer: *std.Io.Writer) !void { + try writer.print("Mesh(name={s})\n", .{self.name}); try writer.writeAll("Bindings:\n"); - for (self.logical.axes.slice(), self.logical.intents.slice()) |l_tag, l_intent| { + for (self.logical.axes.slice(), self.logical.intents.slice(), self.bindings.slice()) |l_tag, l_intent, bind| { try writer.print(" - {s} ({s}) -> ", .{ l_tag, @tagName(l_intent) }); - if (self.binding(l_tag)) |axes| { - if (axes.len == 0) { - try writer.writeAll("replicated\n"); - } else { - for (axes, 0..) |p, i| { - if (i > 0) try writer.writeAll(", "); - try writer.writeAll(@tagName(p)); - } - try writer.writeAll("\n"); - } - } else { + if (bind.len == 0) { try writer.writeAll("unbound\n"); + } else { + for (bind.slice(), 0..) |physical, i| { + if (i > 0) try writer.writeAll(", "); + try writer.writeAll(@tagName(physical)); + } + try writer.writeAll("\n"); } } @@ -1665,19 +1900,19 @@ pub const Data = struct { }; pub const Strategy = struct { - pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); - pub const Binding = struct { - logical: Shape.Tag, - physical: PhysicalList, - }; - pub const Bindings = stdx.BoundedArray(Strategy.Binding, MAX_MESH_RANK); pub const Fold = struct { target: PhysicalAxisTag, sources: PhysicalList, }; + pub const Binding = struct { + logical: Shape.Tag, + physical: PhysicalList, + }; + + pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); pub const Folding = stdx.BoundedArray(Strategy.Fold, MAX_MESH_RANK); - bindings: Strategy.Bindings, + bindings: Bindings, folding: Folding, // TODO: remove .init, promote .parseBindings to .init @@ -1784,16 +2019,19 @@ pub const Strategy = struct { }; /// For a given shape, compute the shape of the slice each shard will receive. -pub fn shardedShape(sharding: Sharding, shape: Shape) Error!Shape { - const pl = try sharding.placement(shape); - return pl.shape; +pub fn shardedShape(shape: Shape) error{IncompatibleSharding}!Shape { + if (shape._sharding.mesh != null) { + const pl: Placement = try .init(shape); + var local_shape = pl.shape; + local_shape._sharding = .replicated; + return local_shape; + } else { + // fully replicated + return shape; + } } -pub fn placement(sharding: Sharding, shape: Shape) Error!Placement { - return .init(sharding, shape); -} - -/// Precompute sharding information common to all devices for a given shape. +/// Precompute mesh information common to all devices for a given shape. pub const Placement = struct { pub const Slices = stdx.BoundedArray(Slice1d, Shape.MAX_RANK); pub const Slice1d = struct { @@ -1801,26 +2039,26 @@ pub const Placement = struct { size: i64, }; - sharding: Sharding, // Note this shape is mainly use to indicate the dims of the sharded buffer. // We may want to only store that. shape: Shape, global_shape: if (builtin.mode == .Debug) Shape else void, - axis_plans: stdx.BoundedArray(AxisSplit, Shape.MAX_RANK), - pub fn init(sharding: Sharding, shape: Shape) Error!Placement { + pub fn init(shape: Shape) error{IncompatibleSharding}!Placement { var pl: Placement = .{ - .sharding = sharding, .shape = shape, // modified below .global_shape = if (builtin.mode == .Debug) shape else {}, .axis_plans = .empty, // set below }; + // mesh == null => replicated => no axis plan + if (shape._sharding.mesh == null) return pl; + const mesh_ = shape._sharding.mesh.?; var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; for (0..shape.rank()) |ax| { const axis_index: u8 = @intCast(ax); - pl.axis_plans.appendAssumeCapacity(try axisSplit(sharding, shape, &used_axes, axis_index)); + pl.axis_plans.appendAssumeCapacity(try axisSplit(mesh_, shape, &used_axes, axis_index)); } for (pl.shape._dims.slice(), shape.dims(), pl.axis_plans.slice()) |*shard_dim, dim, plan| { @@ -1829,8 +2067,16 @@ pub const Placement = struct { return pl; } + pub fn mesh(pl: *const Placement) ?*const Mesh { + return pl.shape._sharding.mesh; + } + pub fn slices(pl: *const Placement, device: Device.Coords) Slices { var res: Slices = .{ .buffer = undefined, .len = pl.shape.rank() }; + if (pl.mesh() == null) { + for (res.slice(), pl.shape.dims()) |*slice, dim| slice.* = .{ .start = 0, .size = dim }; + return res; + } for (res.slice(), pl.shape.dims(), pl.axis_plans.slice()) |*r, size, plan| { const start = plan.linearIndex(device) * size; r.* = .{ @@ -1848,6 +2094,8 @@ pub const Placement = struct { } var ptr: [*]const u8 = slice.constData().ptr; + // mesh = null => implies replicated => each device read from the start. + if (pl.shape._sharding.mesh == null) return ptr; for (pl.shape.dims(), pl.axis_plans.slice(), slice.byte_strides.slice()) |size, plan, stride| { const start = plan.linearIndex(device) * size; ptr = ptr[@intCast(start * stride)..]; @@ -1870,7 +2118,9 @@ pub const Placement = struct { } pub fn format(pl: Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const ordered_devices = pl.sharding.devicesInCanonicalOrder(); + const ordered_devices = if (pl.mesh()) |m| m.devicesInCanonicalOrder() else { + return try writer.print("Placement(shape={f}, replicated)\n", .{pl.shape}); + }; try pl.formatPlacementSummary(pl.shape, ordered_devices.len, writer); try writer.writeAll("Per-shard placement:\n"); @@ -1945,7 +2195,7 @@ const AxisSplit = struct { }; fn axisSplit( - sharding: Sharding, + mesh: *const Mesh, shape: Shape, used_axes: *std.EnumSet(PhysicalAxisTag), axis_index: u8, @@ -1953,37 +2203,34 @@ fn axisSplit( const dim = shape.dim(axis_index); const spec = shape.partition(axis_index); - switch (spec) { - .axis => |logical_tag| { - const binding = sharding.data.binding(logical_tag) orelse return error.MissingLogicalBinding; + return if (spec.meshAxis()) |mesh_axis| { + const binding = mesh.binding(mesh_axis); - // Calculate the split based on the physical coordinates of the current device. - return try calculateSplit(sharding, dim, binding, used_axes); - }, - else => return .empty, - } + // Calculate the split based on the physical coordinates of the current device. + return try calculateSplit(mesh, dim, binding, used_axes); + } else .empty; } fn calculateSplit( - sharding: Sharding, + mesh: *const Mesh, dim: i64, binding: []const PhysicalAxisTag, used_axes: *std.EnumSet(PhysicalAxisTag), -) !AxisSplit { +) error{IncompatibleSharding}!AxisSplit { var plan: AxisSplit = .empty; for (binding) |p_tag| { // Expand the physical tag: check if it's a target for folded source axes. - if (sharding.data.foldSources(p_tag)) |sources| { + if (mesh.foldSources(p_tag)) |sources| { for (sources) |src| { if (used_axes.contains(src)) continue; - addPhysicalToSplit(sharding, &plan, src); + mesh.addPhysicalToSplit(&plan, src); used_axes.insert(src); } } else { // Standard case: directly bound, not part of an explicit fold. if (used_axes.contains(p_tag)) continue; - addPhysicalToSplit(sharding, &plan, p_tag); + mesh.addPhysicalToSplit(&plan, p_tag); used_axes.insert(p_tag); } } @@ -1994,14 +2241,6 @@ fn calculateSplit( return plan; } -/// Extract coordinate data from the physical mesh for a specific axis. -fn addPhysicalToSplit(sharding: Sharding, plan: *AxisSplit, tag: PhysicalAxisTag) void { - const physical = sharding.data.physical; - const info = physical.axisInfo(tag) orelse return; - const depth = physical.axis_traversal.depth(tag) orelse return; - plan.add(@intCast(info.size), depth); -} - const ShardingTest = struct { pub const ExpectedShard = struct { // TODO: remove device_id from here, we always have ids from 0 to N @@ -2010,7 +2249,7 @@ const ShardingTest = struct { }; pub const Scenario = struct { - sharding: Sharding.Data, + mesh: *const Mesh, shape: Shape, expect_error: ?anyerror = null, @@ -2064,8 +2303,8 @@ const ShardingTest = struct { } pub fn run(self: ShardingTest, s: Scenario) !void { - const sharding = s.sharding; - errdefer std.log.warn("Failed sharding {f}", .{s}); + const mesh = s.mesh; + errdefer std.log.warn("Failed mesh {f}", .{s}); // Verify MLIR String if (s.expected_sdy) |expected_attr| { @@ -2079,7 +2318,7 @@ const ShardingTest = struct { defer ctx.deinit(); ctx.loadAllAvailableDialects(); - const actual_attr = try sharding.sdyShardingAttrForShape(self.allocator, ctx, s.shape); + const actual_attr = try mesh.sdyShardingAttrForShape(self.allocator, ctx, s.shape); var writer: std.Io.Writer.Allocating = .init(self.allocator); defer writer.deinit(); try actual_attr.asAttr().format(&writer.writer); @@ -2087,18 +2326,17 @@ const ShardingTest = struct { } // Verify Placement logic / Error - const sharding_ref: Sharding = .{ .data = &sharding }; - const ordered_devices = sharding_ref.devicesInCanonicalOrder(); + const ordered_devices = mesh.devicesInCanonicalOrder(); if (s.expect_error) |err| { errdefer std.log.warn("Expected error failed", .{}); try std.testing.expect(ordered_devices.len > 0); - try std.testing.expectError(err, sharding_ref.placement(s.shape)); + try std.testing.expectError(err, Sharding.Placement.init(s.shape)); return; } // Verify Shard Slices (Math) if (s.expected_shards.len > 0) { - const pl = try sharding_ref.placement(s.shape); + const pl: Placement = try .init(s.shape); errdefer std.log.warn("Expected shards failed. Got {f}", .{pl}); try std.testing.expectEqual(s.expected_shards.len, ordered_devices.len); for (s.expected_shards, ordered_devices) |expected, device| { @@ -2132,7 +2370,7 @@ const ShardingTest = struct { } }; -test "sharding: unknown partitioning implies replication" { +test "mesh: unknown partitioning implies replication" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2141,9 +2379,10 @@ test "sharding: unknown partitioning implies replication" { const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth }); const strategy: Strategy = .parseBindings(.{ .batch = .link_x }); + const mesh: Sharding.Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), + .mesh = &mesh, // No partitioning on shape .shape = Shape.init(.{ .batch = 8 }, .f32), .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", @@ -2156,7 +2395,7 @@ test "sharding: unknown partitioning implies replication" { }); } -test "sharding: suggest strategy realization" { +test "mesh: suggest strategy realization" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2169,11 +2408,12 @@ test "sharding: suggest strategy realization" { }); const strategy: Strategy = .suggest(logical, &physical); + const mesh: Mesh = try .init("suggested_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("suggested_mesh", &physical, logical, strategy), + .mesh = &mesh, .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) - .withPartitioning(.{ .batch = .batch, .model = .model }), + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model }), .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y0, z0 @@ -2188,7 +2428,7 @@ test "sharding: suggest strategy realization" { }); } -test "sharding: suggest folds logical axes with same intent" { +test "mesh: suggest folds logical axes with same intent" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2202,10 +2442,11 @@ test "sharding: suggest folds logical axes with same intent" { const strategy: Strategy = .suggest(logical, &physical); + const mesh: Mesh = try .init("fold_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("fold_mesh", &physical, logical, strategy), + .mesh = &mesh, .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) - .withPartitioning(.{ .model = .model, .experts = .experts }), + .withPartitioning(&mesh, .{ .model = .model, .experts = .experts }), .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 1 }, .{ 0, 4 } } }, @@ -2216,7 +2457,7 @@ test "sharding: suggest folds logical axes with same intent" { }); } -test "sharding: multiple physical axes on one logical dimension (folding)" { +test "mesh: multiple physical axes on one logical dimension (folding)" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2226,9 +2467,10 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { { const strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_y } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, @@ -2243,9 +2485,10 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { // Swap link_x and link_y order: // -> the roles of devices 1 (0,1) and 2 (1,0) are swapped const strategy: Strategy = .parseBindings(.{ .model = .{ .link_y, .link_x } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_y\", \"link_x\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, @@ -2257,7 +2500,7 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { } } -test "sharding: explicit strategy folding" { +test "mesh: explicit strategy folding" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2268,9 +2511,10 @@ test "sharding: explicit strategy folding" { var strategy: Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_z }); + const mesh: Mesh = try .init("strategy_fold", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("strategy_fold", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, @@ -2285,7 +2529,7 @@ test "sharding: explicit strategy folding" { }); } -test "sharding: open and replicated dimension mix" { +test "mesh: open and replicated dimension mix" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2295,9 +2539,10 @@ test "sharding: open and replicated dimension mix" { const strategy: Strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }); + const mesh: Mesh = try .init("mix_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("mix_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(.{ .batch = .open, .model = .replicated }), + .mesh = &mesh, + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, @@ -2308,7 +2553,7 @@ test "sharding: open and replicated dimension mix" { }); } -test "sharding: full 3D cluster sharding" { +test "mesh: full 3D cluster mesh" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2322,11 +2567,12 @@ test "sharding: full 3D cluster sharding" { .context = .link_z, }); + const mesh: Mesh = try .init("3d_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("3d_mesh", &physical, logical, strategy), + .mesh = &mesh, // Note: .context and .model are swapped compared to .link_x and .link_z .shape = Shape.init(.{ .batch = 4, .context = 4, .model = 4 }, .f32) - .withPartitioning(.{ .batch = .batch, .model = .model, .context = .context }), + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model, .context = .context }), .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_z\"}, {\"link_y\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, @@ -2341,7 +2587,7 @@ test "sharding: full 3D cluster sharding" { }); } -test "sharding: num partitions for logical axis" { +test "mesh: num partitions for logical axis" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2355,8 +2601,35 @@ test "sharding: num partitions for logical axis" { var strategy: Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_z }); - const sharding: Sharding.Data = try .init("axis_parts_mesh", &physical, logical, strategy); + const mesh: Mesh = try .init("axis_parts_mesh", &physical, logical, strategy); - try std.testing.expectEqual(4, sharding.numPartitionsForLogicalAxis(.model)); - try std.testing.expectEqual(1, sharding.numPartitionsForLogicalAxis(.batch)); + try std.testing.expectEqual(4, mesh.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(1, mesh.numPartitionsForLogicalAxis(.batch)); +} + +test "localized shapes do not retain global sharding" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{2}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("local", &physical, .mesh(.{ .batch = .balanced }), .parseBindings(.{ .batch = .link_x })); + try std.testing.expectEqual(mesh_.bindings.constSlice()[0].constSlice().ptr, mesh_.binding(0).ptr); + const global = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }); + const local = try shardedShape(global); + try std.testing.expectEqual(4, local.dim(.batch)); + try std.testing.expectEqual(null, local._sharding.mesh); + try std.testing.expectEqual(8, global.dim(.batch)); + try std.testing.expectEqual(&mesh_, global._sharding.mesh); +} + +test "GSPMD tiles use tensor axis order and logical partition indices" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); + const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + const shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }); + const ids = try mesh_.gspmdTileAssignment(arena.allocator(), shape); + try std.testing.expectEqualSlices(u32, &.{ 0, 2, 4, 6, 1, 3, 5, 7 }, ids); } diff --git a/zml/attention.zig b/zml/attention.zig index b3701b48..f5399c5e 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -116,22 +116,22 @@ pub const Metadata = union(Backend) { } }; - pub fn init(opts: InitOptions) Metadata { + pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { return switch (opts) { .vanilla => .{ .vanilla = {} }, .attnd => @panic("Must be initialized manually"), .nki => .{ .nki = {} }, - .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, - .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, + .cuda_fa2 => |o| .{ .cuda_fa2 = .init(o, mesh) }, + .cuda_fa3 => |o| .{ .cuda_fa3 = .init(o, mesh) }, .metal_fa => .{ .metal_fa = .init() }, }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return switch (self) { .vanilla => .{ .vanilla = {} }, .nki => .{ .nki = {} }, - inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform, sharding)), + inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform)), }; } @@ -179,6 +179,31 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T }; } +test "FlashAttention metadata initializes with explicit sharding outside compilation" { + try std.testing.expect(zml.Compiler.currentOrNull() == null); + const mesh: zml.Sharding.Mesh = .{ + .name = "metadata_mesh", + .physical = undefined, + .logical = .mesh(.{ .model = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .folds = .empty, + .folds_consumed = .empty, + }; + inline for (.{ Backend.cuda_fa2, Backend.cuda_fa3 }) |backend| { + const metadata: Metadata = .init(.fromBackend(backend, 16, 8), &mesh); + const cuda_metadata = @field(metadata, @tagName(backend)); + inline for (std.meta.fields(@TypeOf(cuda_metadata))) |field| { + const shape = @field(cuda_metadata, field.name).shape(); + try std.testing.expectEqual(&mesh, shape._sharding.mesh); + if (shape.hasTag(.h)) |axis| { + try std.testing.expectEqual(zml.Sharding.PartitionSpec.sharded(0), shape.partition(axis)); + } else { + try std.testing.expectEqual(zml.Sharding.PartitionSpec.replicated, shape.partition(.meta)); + } + } + } +} + test "attention: q=1,qh=64,kh=8" { try testAttention( .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), @@ -270,13 +295,13 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co defer k.deinit(); var v = try rng_k.eval(allocator, io, {}); defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, .replicated, @ptrCast(token_index_h)); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); defer token_index.deinit(); - const shardings = platform.shardings.values(); + const meshes = platform.meshes.values(); const vanilla_exe = try platform.compileFn(allocator, io, attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .vanilla, .vanilla }, .{ .program_name = "attention_vanilla", - .shardings = shardings, + .meshes = meshes, }); defer vanilla_exe.deinit(); @@ -292,7 +317,7 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co else => if (!backend.isAvailable(platform)) continue, } - const metadata: Metadata = .init(.fromBackend(backend, tensors.k.dim(.k), tensors.q.dim(.h))); + const metadata: Metadata = .init(.fromBackend(backend, tensors.k.dim(.k), tensors.q.dim(.h)), platform.meshes.get("model").?); const parameters: Parameters = .init(.fromBackend(backend)); const exe = try platform.compileFn( allocator, @@ -301,12 +326,12 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, .{ .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .shardings = shardings, + .meshes = meshes, }, ); defer exe.deinit(); - var metadata_d = try metadata.initBuffer(io, platform, platform.shardings.get("model").?); + var metadata_d = try metadata.initBuffer(io, platform); defer Metadata.deinitBuffer(&metadata_d); var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); diff --git a/zml/attention/attnd.zig b/zml/attention/attnd.zig index b1c39cd1..a986ab7b 100644 --- a/zml/attention/attnd.zig +++ b/zml/attention/attnd.zig @@ -74,12 +74,11 @@ pub const Metadata = struct { self: Metadata, io: std.Io, platform: *const zml.Platform, - sharding: zml.Sharding, ) !zml.Bufferized(Metadata) { return .{ .conversation_id = try zml.Buffer.scalar(io, platform, 749, .u64), - .layer_id = try zml.Buffer.uninitialized(io, platform, self.layer_id.shape(), sharding, .{}), - .num_tokens = try zml.Buffer.uninitialized(io, platform, self.num_tokens.shape(), sharding, .{}), + .layer_id = try zml.Buffer.uninitialized(io, platform, self.layer_id.shape(), .{}), + .num_tokens = try zml.Buffer.uninitialized(io, platform, self.num_tokens.shape(), .{}), }; } @@ -140,8 +139,8 @@ const Context = struct { }; pub fn causalAttention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_offset: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - const ctx = zml.Compiler.current(); - const num_partitions = ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable; + const tp = zml.Compiler.current().mesh(.model); + const num_partitions = tp.numPartitionsForLogicalAxis(.model); const actual_k, const actual_v = if (parameters.is_prefill) .{ k, v } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index f533b008..dd010217 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -236,27 +236,24 @@ pub const fa2 = struct { num_heads: i64, }; - pub fn init(opts: InitOptions) Metadata { + pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { return .{ - .softmax_lse = .fromShape(zml.Shape.init(.{ opts.seqlen, opts.num_heads, 1 }, .f32) - .withTags(.{ .s, .h, .dummy }) - .withPartitioning(.{ .h = .model })), + .softmax_lse = zml.Tensor.init(.{ .s = opts.seqlen, .h = opts.num_heads, .dummy = 1 }, .f32) + .withPartitioning(mesh, .{ .h = .model }), - .softmax_lse_accum = .fromShape(zml.Shape.init(.{ 1, opts.num_heads, 128 }, .f32) - .withTags(.{ .dummy, .h, .hd }) - .withPartitioning(.{ .h = .model })), + .softmax_lse_accum = zml.Tensor.init(.{ .dummy = 1, .h = opts.num_heads, .hd = 128 }, .f32) + .withPartitioning(mesh, .{ .h = .model }), - .out_accum = .fromShape(zml.Shape.init(.{ opts.seqlen, opts.num_heads, 128 }, .f32) - .withTags(.{ .s, .h, .hd }) - .withPartitioning(.{ .h = .model })), + .out_accum = zml.Tensor.init(.{ .s = opts.seqlen, .h = opts.num_heads, .hd = 128 }, .f32) + .withPartitioning(mesh, .{ .h = .model }), }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), sharding, .{}), + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), }; } @@ -313,20 +310,21 @@ pub const fa2 = struct { .merge(.{ .tot = .{ .tot, .ngroups } }); } - const q_sharded = q.withPartitioning(.{ .h = .model }); - const model_partitions: i32 = @intCast(ctx.partitioning.numPartitionsForLogicalAxis(q_sharded.shape(), .model) catch std.debug.panic("cu_fa2 attention backend requires a .model sharding", .{})); + const attn_sdy = ctx.resolveMesh(.{.model}); + const q_sharded = q.withPartitioning(attn_sdy, .{ .h = .model }); + const model_partitions: i32 = @intCast(attn_sdy.numPartitionsForLogicalAxis(.model)); const output = fa2_mha_varlen_fwd.call( .{ .q = q_sharded, - .k = k.withPartitioning(.{ .h = .model }), - .v = v.withPartitioning(.{ .h = .model }), + .k = k.withPartitioning(attn_sdy, .{ .h = .model }), + .v = v.withPartitioning(attn_sdy, .{ .h = .model }), .cu_seqlens_q = cu_seqlens_q, .cu_seqlens_k = cu_seqlens_k, .seqused_k = seqused_k, - .softmax_lse = metadata.softmax_lse.withPartitioning(.{ .h = .model }), - .softmax_lse_accum = metadata.softmax_lse_accum.withPartitioning(.{ .h = .model }), - .out_accum = metadata.out_accum.withPartitioning(.{ .h = .model }), + .softmax_lse = metadata.softmax_lse.withPartitioning(attn_sdy, .{ .h = .model }), + .softmax_lse_accum = metadata.softmax_lse_accum.withPartitioning(attn_sdy, .{ .h = .model }), + .out_accum = metadata.out_accum.withPartitioning(attn_sdy, .{ .h = .model }), }, .{ .o = q_sharded.shape(), @@ -451,25 +449,25 @@ pub const fa3 = struct { num_heads: i64, }; - pub fn init(opts: InitOptions) Metadata { + pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { return .{ - .softmax_lse = .fromShape(zml.Shape.init(.{opts.num_heads * opts.seqlen * 4}, .i8) - .withTags(.{.h}).withPartitioning(.{ .h = .model })), - .softmax_lse_accum = .fromShape(zml.Shape.init(.{opts.num_heads * 128 * 4}, .i8) - .withTags(.{.h}).withPartitioning(.{ .h = .model })), - .out_accum = .fromShape(zml.Shape.init(.{opts.num_heads * opts.seqlen * 128 * 4}, .i8) - .withTags(.{.h}).withPartitioning(.{ .h = .model })), - .scheduler_metadata = .fromShape(zml.Shape.init(.{2}, .i32) - .withTags(.{.meta}).withPartitioning(.{ .meta = .replicated })), + .softmax_lse = zml.Tensor.init(.{ .h = opts.num_heads * opts.seqlen * 4 }, .i8) + .withPartitioning(mesh, .{ .h = .model }), + .softmax_lse_accum = zml.Tensor.init(.{ .h = opts.num_heads * 128 * 4 }, .i8) + .withPartitioning(mesh, .{ .h = .model }), + .out_accum = zml.Tensor.init(.{ .h = opts.num_heads * opts.seqlen * 128 * 4 }, .i8) + .withPartitioning(mesh, .{ .h = .model }), + .scheduler_metadata = zml.Tensor.init(.{ .meta = 2 }, .i32) + .withPartitioning(mesh, .{ .meta = .replicated }), }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), sharding, .{}), - .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), sharding, .{}), + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), }; } @@ -496,7 +494,7 @@ pub const fa3 = struct { const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); // TODO(Corendos): replace with cumsum const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })) - .withPartitioning(.{ ._0 = .replicated }); + .withPartitioning(.model, .{ ._0 = .replicated }); var o = zml.ops.customCall( custom_call_name, @@ -909,15 +907,18 @@ pub const paged_fa2 = struct { const head_dim = q.dim(.hd); const num_heads = num_head_groups * num_kv_heads; // FIXME: remove unreachable and propagate error correctly. - const num_heads_per_shard = @divExact(num_heads, ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable); + + const attn_sdy = ctx.resolveMesh(.{.model}); + const model_partitions: i32 = @intCast(attn_sdy.numPartitionsForLogicalAxis(.model)); + const num_heads_per_shard = @divExact(num_heads, model_partitions); const o = switch (parameters) { .decode => |decode_parameters| b: { const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; - const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); - const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); + const block_table = decode_parameters.block_table.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k = decode_parameters.seqused_k.withPartitioning(.model, .{ .b = .replicated }); const out_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -925,27 +926,27 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const dummy_cu_seqlens_k: zml.Tensor = .zeroes(cu_seqlens_q.shape()); const batch_dim = q.dim(.b); var q2 = q; if (seqlenq_ngroups_swapped) { - q2 = q2.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); + q2 = q2.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.model, .{ .hkv = .model }); } else { - q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); } const output_shape = q2.shape(); @@ -1003,12 +1004,12 @@ pub const paged_fa2 = struct { .mixed => |mixed_parameters| b: { const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; - const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.{ .b = .replicated }); - const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.{ .b = .replicated }); - const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); - const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); + const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.model, .{ .b = .replicated }); + const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.model, .{ .b = .replicated }); const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -1016,22 +1017,22 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const dummy_cu_seqlens_k_prefill: zml.Tensor = .zeroes(cu_seqlens_q_prefill.shape()); var q2 = q; - q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const output_shape = q2.shape(); var o = zml.ops.manualComputation( @@ -1088,25 +1089,25 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = batch_dim_decode, .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = batch_dim_decode, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum_decode = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = batch_dim_decode, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const dummy_cu_seqlens_k_decode = zml.Tensor.zeroes(cu_seqlens_q_decode.shape()); var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_dim_decode)); if (seqlenq_ngroups_swapped) { - q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); + q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.model, .{ .hkv = .model }); } else { - q_decode = q_decode.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q_decode = q_decode.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); } const output_shape_decode = q_decode.shape(); @@ -1563,9 +1564,9 @@ pub const paged_fa3 = struct { .decode => |decode_parameters| b: { const batch_size = decode_parameters.block_table.dim(0); - const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); - const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); + const block_table = decode_parameters.block_table.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k = decode_parameters.seqused_k.withPartitioning(.model, .{ .b = .replicated }); const out_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -1573,21 +1574,21 @@ pub const paged_fa3 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .q = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .q = q.dim(.b), .hkv = num_kv_heads, .hg = num_head_groups, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const scheduler_metadata = zml.Tensor.zeroes(.init(.{ .b = batch_size + 1 }, .i32)); - var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const output_shape = q2.shape(); var o = zml.ops.manualComputation( @@ -1636,12 +1637,12 @@ pub const paged_fa3 = struct { .mixed => |mixed_parameters| b: { const batch_size_prefill = mixed_parameters.block_table_prefill.dim(0); - const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.{ .b = .replicated }); - const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.{ .b = .replicated }); - const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); - const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); + const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.model, .{ .b = .replicated }); + const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.model, .{ .b = .replicated }); const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -1649,21 +1650,21 @@ pub const paged_fa3 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .q = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .q = q.dim(.b), .hkv = num_kv_heads, .hg = num_head_groups, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const scheduler_metadata_prefill = zml.Tensor.zeroes(.init(.{ .b = batch_size_prefill + 1 }, .i32)); - var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const output_shape = q2.shape(); var o = zml.ops.manualComputation( @@ -1714,7 +1715,7 @@ pub const paged_fa3 = struct { .hg = num_head_groups, .b = batch_size_decode, .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, @@ -1727,9 +1728,9 @@ pub const paged_fa3 = struct { .hg = num_head_groups, }, .f32)); const scheduler_metadata_decode = zml.Tensor.zeroes(.init(.{ .b = batch_size_decode + 1 }, .i32)); - var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_size_decode)).withPartitioning(.{ .hkv = .model }); + var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_size_decode)).withPartitioning(.model, .{ .hkv = .model }); - q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const decode_output_shape = q_decode.shape(); var o_decode = zml.ops.manualComputation( diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig index 11942afe..42821a1f 100644 --- a/zml/attention/metal_attention.zig +++ b/zml/attention/metal_attention.zig @@ -14,9 +14,8 @@ pub const Metadata = struct { return .{ .num_tokens = .fromShape(.scalar(.u32)) }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { _ = self; - _ = sharding; return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; } diff --git a/zml/attention/nki/attention.zig b/zml/attention/nki/attention.zig index 6553b463..cdbad308 100644 --- a/zml/attention/nki/attention.zig +++ b/zml/attention/nki/attention.zig @@ -43,13 +43,14 @@ pub const Parameters = struct { /// Attention for Neuron. Both prefill and decode lower to NKI custom calls. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - const q_sharded = q.withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - const k_sharded = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - const v_sharded = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + const tp_mesh = zml.Compiler.current().mesh(.model); + const q_sharded = q.withPartitioning(tp_mesh, .{ .q = .replicated, .h = .model, .hd = .replicated }); + const k_sharded = k.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); + const v_sharded = v.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); if (q.dim(.q) == 1) { const token_index_2d = token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) - .withPartitioning(.{ .row = .replicated, .col = .replicated }); + .withPartitioning(tp_mesh, .{ .row = .replicated, .col = .replicated }); return zml.ops.manualComputation( (struct { diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 97960240..403b5a97 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -339,8 +339,8 @@ test "Backend.auto selects mosaic_tpu on TPU" { .devices = &.{}, .memories = &.{}, .physical_mesh = undefined, - .replicated_sharding = undefined, - .shardings = .empty, + .replicated_mesh = undefined, + .meshes = .empty, .io_impl = .threaded, }; @@ -357,8 +357,8 @@ test "Backend.auto selects triton on oneAPI" { .devices = &.{}, .memories = &.{}, .physical_mesh = undefined, - .replicated_sharding = undefined, - .shardings = .empty, + .replicated_mesh = undefined, + .meshes = .empty, .io_impl = .threaded, }; @@ -383,25 +383,26 @@ test pagedAttention { const batch_size = active_batch_size + 1; const query_token_count = prefill_token_count + num_decode; const dt: zml.DataType = .bf16; + const model_meshe = platform.meshes.get("model").?; const partition = .{ .hkv = .model }; const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt), partition), - .k = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt), partition), - .v = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt), partition), + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), .kv_cache = .{ .split = .{ - .k = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt), partition), - .v = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt), partition), + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), }, }, }; - const shardings: []const zml.Sharding = &.{ platform.replicated_sharding, platform.shardings.get("model").? }; - const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .shardings = shardings }); + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, platform.meshes.get("model").? }; + const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); defer rng_q.deinit(); - const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .shardings = shardings }); + const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); defer rng_k.deinit(); - const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .shardings = shardings }); + const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .meshes = meshes }); defer rng_kv_cache.deinit(); const triton_options_args: Options.Args = .{ @@ -458,9 +459,9 @@ test pagedAttention { const query_start_len: [batch_size + 1]i32 = .{ 0, 32, 33, 34, 35, 36, 37, 38, 38 }; var triton_parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ - .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), .replicated, @ptrCast(&block_table)), - .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), .replicated, @ptrCast(&seq_lens)), - .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), .replicated, @ptrCast(&query_start_len)), + .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), @ptrCast(&block_table)), + .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), @ptrCast(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), @ptrCast(&query_start_len)), } }; defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); @@ -514,7 +515,7 @@ test pagedAttention { io, pagedAttention, .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, - .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .shardings = shardings }, + .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, ); defer exe.deinit(); @@ -540,13 +541,13 @@ test pagedAttention { @memcpy(&seqused_k_decode, seq_lens[num_prefill .. num_prefill + num_decode]); break :cuda_fa2 .{ .cuda_fa2 = .{ .mixed = .{ - .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), .replicated, @ptrCast(&block_table_prefill)), - .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), .replicated, @ptrCast(&cu_seqlens_q_prefill)), - .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), .replicated, @ptrCast(&seqused_k_prefill)), + .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), @ptrCast(&block_table_prefill)), + .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), @ptrCast(&cu_seqlens_q_prefill)), + .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), @ptrCast(&seqused_k_prefill)), - .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), .replicated, @ptrCast(&block_table_decode)), - .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), .replicated, @ptrCast(&cu_seqlens_q_decode)), - .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), .replicated, @ptrCast(&seqused_k_decode)), + .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), @ptrCast(&block_table_decode)), + .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), @ptrCast(&cu_seqlens_q_decode)), + .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), @ptrCast(&seqused_k_decode)), .metadata = .{ .decode_offset = try .scalar(io, platform, prefill_token_count, .i32) }, } } }; @@ -1060,20 +1061,20 @@ test "Triton sparse MLA value ranks and padded queries" { const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); var parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ - .block_table = try .fromBytes(std.testing.io, platform, parameters.triton.block_table.shape(), .replicated, std.mem.sliceAsBytes(&block_table)), - .seq_lens = try .fromBytes(std.testing.io, platform, parameters.triton.seq_lens.shape(), .replicated, std.mem.sliceAsBytes(&seq_lens)), - .query_start_len = try .fromBytes(std.testing.io, platform, parameters.triton.query_start_len.shape(), .replicated, std.mem.sliceAsBytes(&query_start_len)), + .block_table = try .fromBytes(std.testing.io, platform, parameters.triton.block_table.shape(), std.mem.sliceAsBytes(&block_table)), + .seq_lens = try .fromBytes(std.testing.io, platform, parameters.triton.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(std.testing.io, platform, parameters.triton.query_start_len.shape(), std.mem.sliceAsBytes(&query_start_len)), } }; defer zml.Buffer.deinitAll(Parameters, ¶meters_d); - var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); + var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, std.mem.sliceAsBytes(&q_data)); defer q_d.deinit(); - var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)); + var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, std.mem.sliceAsBytes(&kv_data)); defer kv_d.deinit(); - var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); + var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, std.mem.sliceAsBytes(&sink_data)); defer sink_d.deinit(); - var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); + var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, std.mem.sliceAsBytes(&topk_data)); defer topk_d.deinit(); - var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, .replicated, std.mem.sliceAsBytes(&tokens_pos_data)); + var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, std.mem.sliceAsBytes(&tokens_pos_data)); defer tokens_pos_d.deinit(); const TestCase = struct { @@ -1174,20 +1175,20 @@ test "execute stablehlo mla kernel" { const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); var parameters_d: zml.Bufferized(Parameters) = .{ .stablehlo = .{ - .block_table = try .fromBytes(std.testing.io, platform, parameters.stablehlo.block_table.shape(), .replicated, std.mem.sliceAsBytes(&block_table)), - .seq_lens = try .fromBytes(std.testing.io, platform, parameters.stablehlo.seq_lens.shape(), .replicated, std.mem.sliceAsBytes(&seq_lens)), - .query_start_len = try .fromBytes(std.testing.io, platform, parameters.stablehlo.query_start_len.shape(), .replicated, std.mem.sliceAsBytes(&query_start_len)), + .block_table = try .fromBytes(std.testing.io, platform, parameters.stablehlo.block_table.shape(), std.mem.sliceAsBytes(&block_table)), + .seq_lens = try .fromBytes(std.testing.io, platform, parameters.stablehlo.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(std.testing.io, platform, parameters.stablehlo.query_start_len.shape(), std.mem.sliceAsBytes(&query_start_len)), } }; defer zml.Buffer.deinitAll(Parameters, ¶meters_d); - var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); + var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, std.mem.sliceAsBytes(&q_data)); defer q_d.deinit(); - var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)); + var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, std.mem.sliceAsBytes(&kv_data)); defer kv_d.deinit(); - var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); + var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, std.mem.sliceAsBytes(&sink_data)); defer sink_d.deinit(); - var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); + var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, std.mem.sliceAsBytes(&topk_data)); defer topk_d.deinit(); - var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, .replicated, std.mem.sliceAsBytes(&tokens_pos_data)); + var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, std.mem.sliceAsBytes(&tokens_pos_data)); defer tokens_pos_d.deinit(); const exe = try platform.compileFn( diff --git a/zml/attention/sparse_mla.zig b/zml/attention/sparse_mla.zig index fbb9573b..3edbbe67 100644 --- a/zml/attention/sparse_mla.zig +++ b/zml/attention/sparse_mla.zig @@ -103,7 +103,7 @@ pub fn launchConfig(paged_opts: Options, topk_count: usize, cu_count_: usize) Co } // Full 256-query prefill chunks already expose enough query parallelism. // Retain BLOCK_M=16 and the generic split model, but process wider sparse - // tiles. The predicate uses the post-sharding flattened query shape. + // tiles. The predicate uses the post-mesh flattened query shape. if (is_sm103 and !paged_opts.all_decode and paged_opts.total_q_blocks >= 256 and diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index d1806730..82b39006 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -155,12 +155,12 @@ pub const mosaic_tpu = struct { .{ q.dim(.hd), target_head_dim }, ); return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }) - .withPartitioning(.{ .h = .model }); + .withPartitioning(.model, .{ .h = .model }); } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const kv_pages_partitioned = kv_pages.withPartitioning(.{ .hkv = .model }); - const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + const kv_pages_partitioned = kv_pages.withPartitioning(.model, .{ .hkv = .model }); + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); const cache_head_dim: usize = @intCast(kv_pages_partitioned.dim(.hd)); stdx.debug.assert( @@ -169,15 +169,15 @@ pub const mosaic_tpu = struct { .{ cache_head_dim, logical_head_dim }, ); const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); - const query_start_len = parameters.query_start_len.withPartitioning(.{ .b = .replicated }); + const query_start_len = parameters.query_start_len.withPartitioning(.model, .{ .b = .replicated }); return .{ .q = q_ragged, .kv_pages = kv_pages_partitioned, - .seq_lens = parameters.seq_lens.withPartitioning(.{ ._0 = .replicated }), - .block_table = parameters.block_table.withPartitioning(.{ ._0 = .replicated, ._1 = .replicated }), + .seq_lens = parameters.seq_lens.withPartitioning(.model, .{ ._0 = .replicated }), + .block_table = parameters.block_table.withPartitioning(.model, .{ ._0 = .replicated, ._1 = .replicated }), .query_start_len = query_start_len, - .num_seqs = activeSequenceCount(query_start_len).withPartitioning(.{ ._0 = .replicated }), + .num_seqs = activeSequenceCount(query_start_len).withPartitioning(.model, .{ ._0 = .replicated }), }; } diff --git a/zml/buffer.zig b/zml/buffer.zig index 5b86cae3..6f7acd8c 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -31,7 +31,6 @@ test { pub const Buffer = struct { _platform: *const Platform, _shape: Shape, - _sharding: Sharding, _shards: Shards, pub const MAX_NUM_SHARDS: u16 = Platform.MAX_NUM_DEVICES; @@ -115,10 +114,10 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - const placement = self._sharding.placement(self._shape) catch { - return try writer.print("sharding error {} vs {}", .{ self._sharding, self._shape }); + const pl = Sharding.Placement.init(self._shape) catch |err| { + return try writer.print("Buffer({f}) with invalid sharding {} ({})", .{ self._shape, self._shape._sharding, err }); }; - try writer.print("{f}", .{placement}); + try writer.print("{f}", .{pl}); } /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -126,7 +125,6 @@ pub const Buffer = struct { io: std.Io, platform: *const Platform, shape_: Shape, - sharding: Sharding, data_: []const u8, opts: FromOptions, ) !Buffer { @@ -135,7 +133,6 @@ pub const Buffer = struct { var res: Buffer = .{ ._platform = platform, ._shape = shape_, - ._sharding = sharding.resolve(platform), ._shards = .empty, }; errdefer for (res._shards.slice()) |shard| { @@ -146,7 +143,7 @@ pub const Buffer = struct { const slice = Slice.init(sh, data_); const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - const placement = placementOrPanic(res._sharding, sh); + const placement = placementOrPanic(platform, sh); const shard_dims: []const i64 = placement.shape.dims(); const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); @@ -179,28 +176,28 @@ pub const Buffer = struct { /// Copies the given Zig bytes to the accelerator memory and /// return a Buffer with the given dimensions. - pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, data: []const u8) !Buffer { - return fromBytesOpts(io, platform, sh, sharding, data, .{}); + pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8) !Buffer { + return fromBytesOpts(io, platform, sh, data, .{}); } - pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, data: []const u8, opts: FromOptions) !Buffer { - return from(io, platform, sh, sharding, data, opts); + pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { + return from(io, platform, sh, data, opts); } /// Copies the given zml.Slice to the accelerator memory and /// return a Buffer. - pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice, sharding: Sharding) !Buffer { - return fromSliceOpts(io, platform, slice, sharding, .{}); + pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice) !Buffer { + return fromSliceOpts(io, platform, slice, .{}); } - pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, sharding: Sharding, opts: FromOptions) !Buffer { - return from(io, platform, slice.shape, sharding, std.mem.sliceAsBytes(slice.constData()), opts); + pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, opts: FromOptions) !Buffer { + return from(io, platform, slice.shape, std.mem.sliceAsBytes(slice.constData()), opts); } /// Creates a Buffer with a single element. pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); - return fromBytes(io, platform, .scalar(dtype_), .replicated, x.asBytes()); + return fromBytes(io, platform, .scalar(dtype_), x.asBytes()); } pub fn await(self: Buffer, io: std.Io) !void { @@ -217,7 +214,6 @@ pub const Buffer = struct { _: std.Io, platform: *const Platform, shape_: Shape, - sharding: Sharding, opts: UnitializedOptions, ) !Buffer { std.log.debug("uninitialized {f}", .{shape_}); @@ -225,7 +221,6 @@ pub const Buffer = struct { var res: Buffer = .{ ._platform = platform, ._shape = shape_, - ._sharding = sharding.resolve(platform), ._shards = .empty, }; errdefer for (res._shards.slice()) |shard| { @@ -234,7 +229,7 @@ pub const Buffer = struct { stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); const element_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - const placement = placementOrPanic(res._sharding, sh); + const placement = placementOrPanic(platform, sh); const shard_dims: []const i64 = placement.shape.dims(); const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); @@ -257,14 +252,13 @@ pub const Buffer = struct { } /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, sharding: Sharding, pjrt_buffers: []const *pjrt.Buffer) Buffer { + pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); return .{ ._platform = platform, ._shape = sh, - ._sharding = sharding, ._shards = Shards.fromSlice(pjrt_buffers) catch unreachable, }; } @@ -283,8 +277,8 @@ pub const Buffer = struct { pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); - const placement = placementOrPanic(self._sharding, self._shape); - for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { + const placement = placementOrPanic(self._platform, self._shape); + for (self._platform.physical_mesh.devices_in_canonical_order, 0..) |device, shard_index| { // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed const sub_slice = placement.shardSlice(device.coords, slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; @@ -306,12 +300,12 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - const placement = placementOrPanic(self._sharding, self._shape); + const placement = placementOrPanic(self._platform, self._shape); var shard_slice = try Slice.alloc(allocator, placement.shape); defer shard_slice.free(allocator); - for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { + for (self._platform.physical_mesh.devices_in_canonical_order, 0..) |device, shard_index| { const sub_slice = placement.shardSlice(device.coords, slice); const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { @@ -331,8 +325,8 @@ pub const Buffer = struct { /// ie: `num_devices * shard_byte_size` /// `shard_byte_size` can be up to `self.shape().byteSize()` when the buffer is fully replicated. pub fn byteSize(self: Buffer) usize { - const placement = placementOrPanic(self._sharding, self._shape); - return placement.shape.byteSize() * self._sharding.devicesInCanonicalOrder().len; + const placement = placementOrPanic(self._platform, self._shape); + return placement.shape.byteSize() * self._shards.len; } pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { @@ -360,7 +354,7 @@ pub const Buffer = struct { view_shard_list.appendAssumeCapacity(view_shard); } - return fromPjrtBuffers(platform, self._shape, self._sharding, view_shard_list.items); + return fromPjrtBuffers(platform, self._shape, view_shard_list.items); } }; @@ -381,13 +375,14 @@ test "device round-trip" { .{ 56, 57, 58, 59, 60, 61, 62, 63 }, }; + const mp: *const zml.Mesh = platform.meshes.get("model").?; const x_h: zml.Slice = .init(.withPartitioning( .init(.{ .b = 8, .d = 8 }, .u32), + mp, .{ .b = .model }, ), std.mem.asBytes(&x)); // no free: x_h is stack allocated - const model_sharding: zml.Sharding = platform.shardings.get("model").?; - const x_d: zml.Buffer = try .fromSlice(io, platform, x_h, model_sharding); + const x_d: zml.Buffer = try .fromSlice(io, platform, x_h); try std.testing.expectEqual(platform.devices.len, x_d.numShards()); { @@ -409,29 +404,11 @@ test "device round-trip" { } } -fn placementOrPanic(sharding: Sharding, shape: Shape) Sharding.Placement { - return sharding.placement(shape) catch |err| { +fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { + var sh = shape; + if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; + return Sharding.Placement.init(sh) catch |err| { @branchHint(.cold); - switch (err) { - error.MissingLogicalBinding => { - log.err( - \\Failed to shard Buffer of shape {f}, with sharding: - \\{f} - \\ - \\The Buffer is probably inheriting a partitionned shape from a Tensor, - \\So Buffer creation must pass a Sharding, that maps the logical sharding of the Tensor to the physical mesh. - , .{ shape, sharding }); - @panic("Buffer shape and sharding should be consistent"); - }, - error.IncompatibleSharding => { - log.err( - \\Failed to shard Buffer of shape {f}, with sharding: - \\{f} - \\ - \\The Buffer dimension isn't properly divisible by the number of devices along the sharded axis. - , .{ shape, sharding }); - @panic("Buffer shape should be divisible by the number of devices along the sharded axis."); - }, - } + std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); }; } diff --git a/zml/exe.zig b/zml/exe.zig index f97b8f7d..1db04837 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -23,9 +23,6 @@ pub const Exe = struct { input_shapes: []const Shape, output_shapes: []const Shape, - input_shardings: []const Sharding, - output_shardings: []const Sharding, - /// Inputs whose buffers are donated to an output (`reuseBuffer`). /// Runners destroy their PJRT handles after the raw call returns. donated_input_indices: []const usize, @@ -43,8 +40,6 @@ pub const Exe = struct { num_partitions: i32, input_shapes: []const Shape, output_shapes: []const Shape, - input_shardings: []const Sharding, - output_shardings: []const Sharding, input_aliasing: []const ?u32, ) !Exe { var arena = std.heap.ArenaAllocator.init(allocator); @@ -53,10 +48,6 @@ pub const Exe = struct { const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); - // Re-home sharding pointers into arena-owned values so exe doesn't depend on caller lifetimes. - const input_shardings_copy = try arena.allocator().dupe(Sharding, input_shardings); - const output_shardings_copy = try arena.allocator().dupe(Sharding, output_shardings); - var donated_count: usize = 0; for (input_aliasing) |aliasing| donated_count += @intFromBool(aliasing != null); @@ -74,8 +65,6 @@ pub const Exe = struct { .exe = exe, .input_shapes = input_shapes_copy, .output_shapes = output_shapes_copy, - .input_shardings = input_shardings_copy, - .output_shardings = output_shardings_copy, .donated_input_indices = donated_input_indices, .num_devices = num_devices, .num_partitions = num_partitions, @@ -90,11 +79,11 @@ pub const Exe = struct { } pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { - return Arguments.init(allocator, self.input_shapes, self.input_shardings, self.num_devices); + return Arguments.init(allocator, self.input_shapes, self.num_devices); } pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { - return Results.init(allocator, self.output_shapes, self.output_shardings, self.platform, self.num_devices); + return Results.init(allocator, self.output_shapes, self.platform, self.num_devices); } pub const FlatBuffers = struct { @@ -131,9 +120,8 @@ pub const Exe = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, baked_count: usize = 0, - shardings: []const Sharding, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, num_devices: usize) error{OutOfMemory}!Arguments { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) error{OutOfMemory}!Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -143,7 +131,6 @@ pub const Exe = struct { return .{ .flat_buffers = flat_buffers, .expected_shapes = expected_shapes, - .shardings = shardings, }; } @@ -219,9 +206,8 @@ pub const Exe = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, - shardings: []const Sharding, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, platform: *const Platform, num_devices: usize) !Results { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: *const Platform, num_devices: usize) !Results { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -232,7 +218,6 @@ pub const Exe = struct { .platform = platform, .flat_buffers = flat_buffers, .expected_shapes = expected_shapes, - .shardings = shardings, }; } @@ -254,7 +239,7 @@ pub const Exe = struct { for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], context_.self.shardings[context_.current_index], shards.constSlice()); + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); context_.current_index += 1; } }.cb, &context, &result); @@ -274,7 +259,7 @@ pub const Exe = struct { for (0..ctx.results.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(ctx.results.flat_buffers.buffers[device_index][ctx.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], ctx.results.shardings[ctx.current_index], shards.constSlice()); + buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], shards.constSlice()); ctx.current_index += 1; } }.cb, &context, &v); @@ -639,7 +624,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { defer exe.deinit(); try std.testing.expectEqualSlices(usize, &.{0}, exe.donated_input_indices); - var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), .replicated, std.mem.asBytes(&[2]i32{ 10, 20 })); + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); defer weight_buffer.deinit(); var args = try exe.args(allocator); defer args.deinit(allocator); @@ -647,7 +632,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { defer results.deinit(allocator); for ([_]bool{ false, true }) |wait| { - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); args.set(.{ value_buffer, weight_buffer }); exe.call(io, args, &results, .{ .wait = wait }); @@ -679,9 +664,9 @@ test "FnExe runner replaces donated buffers with and without waiting" { const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .weight = weight, .value = value }}); defer exe.deinit(); - var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), .replicated, std.mem.asBytes(&[2]i32{ 10, 20 })); + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); defer weight_buffer.deinit(); - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); var runner = try Model.Runner(.{.weight}).init(&exe, allocator, .{ .weight = weight_buffer }); defer runner.deinit(allocator); @@ -714,7 +699,7 @@ test "FnExe runner replaces unchanged donated buffers" { defer exe.deinit(); try std.testing.expectEqualSlices(usize, &.{0}, exe.raw.donated_input_indices); - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); var runner = try exe.runner(allocator); defer runner.deinit(allocator); diff --git a/zml/io.zig b/zml/io.zig index 589abd47..5fce0793 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -31,14 +31,16 @@ pub const TensorStore = struct { }; registry: *safetensors.TensorRegistry, + meshes: []const *const Sharding.Mesh, id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, - pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry) TensorStore { - const arena: std.heap.ArenaAllocator = .init(allocator); + pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { + var arena: std.heap.ArenaAllocator = .init(allocator); return .{ .registry = registry, + .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), .id_to_sources = .empty, .allocator = allocator, .arena = arena, @@ -104,6 +106,20 @@ pub const TensorStore = struct { return .{ .store = self }; } + pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (store.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + std.debug.panic( + \\Found no meshes named {s} in TensorStore. + \\Try passing more meshes to `zml.TensorStore.fromRegistry`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(store.meshes) }); + } + pub const View = struct { store: *TensorStore, @@ -162,19 +178,50 @@ pub const TensorStore = struct { } else false; } - pub const CreateTensorOpts = struct { - memory: Memory.Kind = .default, - }; - - pub fn maybeCreateTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{}); + /// Creates a zml.Tensor from a specific entry in the store. + pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{ .memory = .host_pinned }); + pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); + const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + if (mesh_ == .replicated) { + return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .replicated); + } + const resolved_mesh = self.store.mesh(mesh_); + // Unspecified tensor axes are treated as replicated + const partition: Sharding.Partitioning = if (has_tags) + .parse(parsed_tags.constSlice(), resolved_mesh, .replicated, partitioning) + else + partitioning; + return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .{ .mesh = resolved_mesh, .partition = partition }); } - fn maybeCreateTensorInternal(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype, opts: CreateTensorOpts) ?Tensor { + pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { + const tensor = self.maybeCreateTensor(subkey, tags, mesh_, partitioning); + if (tensor) |t| { + const storage = self.store.id_to_sources.getPtr(t.id); + storage.?.memory = .host_pinned; + } + return tensor; + } + + pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { + const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); + const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + + return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.slice() else null, .replicated) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { var buffer: [256]u8 = undefined; const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); const source = self.store.dupeSource(key) orelse return null; @@ -184,57 +231,19 @@ pub const TensorStore = struct { sources[0] = source; var shape = source.shape; - shape = applyTags(shape, tagz); - shape = applyPartitioning(shape, partitioning); + if (tags) |user_tags| { + stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); + @memcpy(shape._tags.slice(), user_tags); + } + shape._sharding = sharding; + if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = opts.memory }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); return tensor; } - pub fn createTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{}) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - pub fn createHostPinnedTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{ .memory = .host_pinned }) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - fn applyTags(shape_: Shape, tagz: anytype) Shape { - var shape = shape_; - if (@TypeOf(tagz) != @TypeOf(null)) { - switch (@typeInfo(@TypeOf(tagz))) { - .optional => if (tagz) |t| { - shape = shape.withTags(t); - }, - else => shape = shape.withTags(tagz), - } - } - return shape; - } - - fn applyPartitioning(shape_: Shape, partitioning: anytype) Shape { - var shape = shape_; - - if (@TypeOf(partitioning) == @TypeOf(null)) { - @compileError("TensorStore.View.createTensor partitioning cannot be null; pass .replicated or an explicit partitioning"); - } - - switch (@typeInfo(@TypeOf(partitioning))) { - .optional => @compileError("TensorStore.View.createTensor partitioning cannot be optional; pass .replicated or an explicit partitioning"), - .enum_literal => switch (partitioning) { - .replicated => shape = shape.withReplicatedPartitioning(), - else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), - }, - else => shape = shape.withPartitioning(partitioning), - } - - return shape; - } - pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { const arena = self.store.arena.allocator(); @@ -365,7 +374,7 @@ pub const Loader = struct { pub const LoadError = error{TransformedTensorNotDelivered}; - pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) LoadError!void { + pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { const tensor_count = meta.count(Tensor, model); var arena: std.heap.ArenaAllocator = .init(self.allocator); @@ -406,7 +415,7 @@ pub const Loader = struct { if (sources.transformed) continue; } - self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, shardings, opts }); + self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); } } @@ -424,25 +433,25 @@ pub const Loader = struct { log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); } - pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: Tensor, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { - loader.group.async(io, defaultCallback, .{ loader, io, &tensor, buffer, store, shardings, opts }); + pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { + loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); } - fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { + fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { const sources = store.getSourcesById(tensor.id) orelse { std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); return; }; stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); - self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, shardings, opts) catch |e| { + self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); unreachable; }; } - fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, shardings: []const Sharding, opts: LoadOpts) void { - self.loadSingleInner(io, source, shape, buffer, .device, shardings, opts) catch |e| { + fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { + self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { log.err("Failed to load tensor {s}: {}", .{ source.name, e }); loaded.* = false; return; @@ -457,17 +466,11 @@ pub const Loader = struct { shape: Shape, buffer: *Buffer, memory: Memory.Kind, - shardings: []const Sharding, opts: LoadOpts, ) !void { var reader = try source.reader(io, &.{}, .{}); defer reader.deinit(); - const sharding = Sharding.pickSharding(shardings, shape, .explicit_axis_binding) orelse blk: { - log.debug("No sharding strategy found for tensor {s} with shape {f}, using replicated sharding", .{ reader.tensor.name, shape }); - break :blk self.platform.replicated_sharding; - }; - var writer = try MemoryWriter.init( self.allocator, io, @@ -476,7 +479,6 @@ pub const Loader = struct { self.dma_allocators, self.dma_chunk_size, shape, - sharding, buffer, memory, ); @@ -507,7 +509,6 @@ pub const Loader = struct { tensor: Tensor, buffer: *Buffer, store: *const TensorStore, - shardings: []const Sharding, exe: *const Exe, opts: LoadOpts, ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { @@ -530,7 +531,7 @@ pub const Loader = struct { defer if (node) |*n| n.end(); for (sources, 0..) |source, i| { - self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], shardings, .{} }); + self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); } self.group.await(io) catch |err| switch (err) { error.Canceled => return error.Canceled, @@ -566,7 +567,7 @@ test "Loader.loadExecute cleans up donated and non-donated inputs" { defer registry.deinit(); try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: TensorStore = .fromRegistry(allocator, ®istry); + var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); defer store.deinit(); var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); defer loader.deinit(); @@ -586,7 +587,7 @@ test "Loader.loadExecute cleans up donated and non-donated inputs" { try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); var output: Buffer = undefined; - try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &.{}, &exe, .{}); + try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); defer output.deinit(); try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); try std.testing.expect(loader.delivered.contains(tensor.id)); @@ -681,18 +682,17 @@ pub const MemoryWriter = union(enum) { dma_allocators: []const mem.DmaAllocator, dma_chunk_size: usize, shape: Shape, - sharding: Sharding, buffer: *Buffer, memory: Memory.Kind, ) !MemoryWriter { return switch (platform.target) { .cuda, .rocm, .oneapi => .{ - .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer, memory), + .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), }, .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) else - .{ .buffered = try .init(allocator, io, platform, shape, sharding, buffer) }, + .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, }; } @@ -722,16 +722,14 @@ pub const BufferedMemoryWriter = struct { io: std.Io, platform: *const Platform, shape: Shape, - sharding: Sharding, buffer: *Buffer, interface: std.Io.Writer, - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, sharding: Sharding, buffer: *Buffer) !BufferedMemoryWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { return .{ .io = io, .platform = platform, .shape = shape, - .sharding = sharding, .buffer = buffer, .interface = .{ .buffer = try allocator.alloc(u8, shape.byteSize()), @@ -757,7 +755,6 @@ pub const BufferedMemoryWriter = struct { self.io, self.platform, self.shape, - self.sharding, @ptrCast(self.interface.buffer), .{ .wait = true }, ) catch return std.Io.Writer.Error.WriteFailed; @@ -982,9 +979,9 @@ const DispatchSpans = struct { spans: []DispatchSpan, mirror_writers: []usize, - fn init(allocator: std.mem.Allocator, shape: Shape, sharding: Sharding) !DispatchSpans { - const placement = try sharding.placement(shape); - const ordered_devices = sharding.devicesInCanonicalOrder(); + fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { + const placement = try Sharding.Placement.init(shape); + const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); var placement_span_count: usize = 0; for (ordered_devices) |device| { @@ -1197,11 +1194,14 @@ pub const DirectMemoryWriter = struct { dma_allocators: []const mem.DmaAllocator, dma_chunk_size: usize, shape: Shape, - sharding: Sharding, buffer: *Buffer, memory: Memory.Kind, ) !DirectMemoryWriter { - const ordered_devices = sharding.devicesInCanonicalOrder(); + var placed_shape = shape; + if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; + const placement = try Sharding.Placement.init(placed_shape); + const mesh = placement.mesh().?; + const ordered_devices = mesh.devicesInCanonicalOrder(); var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); errdefer allocator.free(shard_writers); @@ -1211,7 +1211,6 @@ pub const DirectMemoryWriter = struct { }; var pjrt_buffers: Buffer.Shards = .empty; - const placement = try sharding.placement(shape); for (ordered_devices, 0..) |device, i| { const pool = &pools[device.id]; const shard_dma_allocator = dma_allocators[device.id].allocator(); @@ -1223,9 +1222,9 @@ pub const DirectMemoryWriter = struct { pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); } - buffer.* = .fromPjrtBuffers(platform, shape, sharding, pjrt_buffers.constSlice()); + buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); - const dispatch_spans: DispatchSpans = try .init(allocator, shape, sharding); + const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); errdefer dispatch_spans.deinit(allocator); const first_span = dispatch_spans.spans[0]; @@ -1535,15 +1534,14 @@ const DirectMemoryWriterDeviceTest = struct { allocator: std.mem.Allocator, io: std.Io, - fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario) !void { + fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; defer platform.deinit(self.allocator, self.io); - const sharding: Sharding.Data = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); try self.runDirectMemoryWriter( platform, - scenario.shape, - .{ .data = &sharding }, + scenario.shape.withPartitioning(&mesh, partitioning), scenario.write_mode, scenario.writable_slice_min_len, scenario.pool_chunks, @@ -1556,7 +1554,6 @@ const DirectMemoryWriterDeviceTest = struct { self: DirectMemoryWriterDeviceTest, platform: *const Platform, shape: Shape, - sharding: Sharding, write_mode: WriteMode, writable_slice_min_len: usize, pool_chunks: usize, @@ -1595,7 +1592,6 @@ const DirectMemoryWriterDeviceTest = struct { dma_allocators, pool_chunk_size, shape, - sharding, &written_buffer, memory, ); @@ -1643,11 +1639,10 @@ test "DirectMemoryWriter: replicated with auto topology" { .physical_mesh = .auto, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32) - .withPartitioning(.{ .rows = .replicated, .cols = .replicated }), + .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), .logical_mesh = .mesh(.{ .x = .high_bandwidth }), .strategy = .parseBindings(.{ .x = .link_x }), - }); + }, .{ .rows = .replicated, .cols = .replicated }); } test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { @@ -1662,11 +1657,10 @@ test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) - .withPartitioning(.{ .rows = .replicated, .cols = .model }), + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = .parseBindings(.{ .model = .link_x }), - }); + }, .{ .rows = .replicated, .cols = .model }); } test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { @@ -1681,17 +1675,16 @@ test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32) - .withPartitioning(.{ .batch = .batch, .model = .model }), + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, }), .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), - }); + }, .{ .batch = .batch, .model = .model }); } -test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { +test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { const case: DirectMemoryWriterDeviceTest = .{ .allocator = std.testing.allocator, .io = std.testing.io, @@ -1703,14 +1696,14 @@ test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .model = 4096 }, .f32).withPartitioning(.{ .model = .model }), + .shape = Shape.init(.{ .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = blk: { var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_y }); break :blk strategy; }, - }); + }, .{ .model = .model }); } test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { @@ -1725,14 +1718,13 @@ test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) - .withPartitioning(.{ .rows = .replicated, .cols = .model }), + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = .parseBindings(.{ .model = .link_x }), .write_mode = .writable_slice_greedy, .writable_slice_min_len = 64, .pool_chunk_size = 1024, - }); + }, .{ .rows = .replicated, .cols = .model }); } test "DirectMemoryWriter: 3D topology folded model + replicated batch" { @@ -1747,8 +1739,7 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { .physical_mesh = .{ .custom = buildMesh2x2x2 }, .cpu = .{ .device_count = 8 }, }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) - .withPartitioning(.{ .batch = .replicated, .model = .model }), + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, @@ -1758,7 +1749,7 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { strategy.addFold(.link_x, &.{ .link_x, .link_z }); break :blk strategy; }, - }); + }, .{ .batch = .replicated, .model = .model }); } test "MemoryWriter can produce a host pinned buffer" { @@ -1772,10 +1763,9 @@ test "MemoryWriter can produce a host pinned buffer" { .create_options = .{ .physical_mesh = .{ .custom = buildMesh2 }, }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) - .withPartitioning(.{ .batch = .replicated, .model = .model }), + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = .parseBindings(.{ .model = .link_x }), .memory = .host_pinned, - }); + }, .{ .batch = .replicated, .model = .model }); } diff --git a/zml/kernel.zig b/zml/kernel.zig index c2114001..2d8945e4 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -955,9 +955,9 @@ test "fly kernels run on rocm" { fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), .replicated, std.mem.sliceAsBytes(ha)); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), .replicated, std.mem.sliceAsBytes(hb)); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); defer bb.deinit(); var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); defer out.deinit(); diff --git a/zml/mem.zig b/zml/mem.zig index 5ba82c69..f471a83a 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -333,7 +333,6 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu // I'm not sure I like that. I'd rather set all fields but _shards than leaving most of the m undefined. ._shape = model._shape, ._shards = .empty, - ._sharding = undefined, ._platform = undefined, }; return; diff --git a/zml/meta.zig b/zml/meta.zig index 9ff4d39a..08738e11 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -456,6 +456,7 @@ const Visitor = struct { pub const Action = enum { callback, recurse, + exit, }; pub fn determineAction(comptime callback: anytype, comptime PtrTypeOfV: type) Action { @@ -473,6 +474,13 @@ const Visitor = struct { return .callback; } + switch (@typeInfo(ptr_info.child)) { + .@"struct", .@"union" => { + if (@hasDecl(ptr_info.child, "zml_no_meta_visit")) return .exit; + }, + else => {}, + } + return .recurse; } }; @@ -487,6 +495,7 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) const can_error = stdx.meta.FnReturnErrorSet(callback) != null; return switch (action) { + .exit => {}, .callback => callback(ctx, v), .recurse => { const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { @@ -628,8 +637,8 @@ pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: const err_args = .{ChildTypeV}; switch (action) { + .exit => {}, .callback => return callback(ctx, v), - .recurse => { if (comptime !Contains(ChildTypeV, TargetType)) return; @@ -929,6 +938,7 @@ pub fn Contains(Haystack: type, T: type) bool { return switch (@typeInfo(Haystack)) { .@"struct" => |info| { + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; inline for (info.fields) |field| { if (!field.is_comptime and Contains(field.type, T)) { return true; @@ -937,6 +947,7 @@ pub fn Contains(Haystack: type, T: type) bool { return false; }, .@"union" => |info| { + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; inline for (info.fields) |field| { if (Contains(field.type, T)) return true; diff --git a/zml/moe/cute_mxfp4.zig b/zml/moe/cute_mxfp4.zig index 7cb235d0..9928aeb4 100644 --- a/zml/moe/cute_mxfp4.zig +++ b/zml/moe/cute_mxfp4.zig @@ -47,7 +47,7 @@ pub fn fusedExperts( stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); + const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); const hidden = down.weight.dim(1); const intermediate = down.weight.dim(2) * 2; diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index 18527910..f90a7882 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -252,7 +252,7 @@ fn ffiCallNvfp4( const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ .name = "flashinfer_cutlass_nvfp4_routed_moe", - // Expert sharding is owned by forwardMoe outer manual computation. + // Expert mesh is owned by forwardMoe outer manual computation. .sharding_aware = false, .has_side_effect = false, }); @@ -694,7 +694,7 @@ pub fn fusedExperts( // TODO(Corentin): Do error checking on nvfp4 // Also, maybe pass `zml.nn.Linear` directly - if (expert_partition.eql(.init(.experts))) { + if (expert_partition.isSharded()) { return zml.ops.manualComputation( (struct { input: zml.Tensor, @@ -799,7 +799,7 @@ pub fn fusedExperts( ); } - if (expert_partition.eql(.init(.experts))) { + if (expert_partition.isSharded()) { return zml.ops.manualComputation( (struct { input: zml.Tensor, diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index 2c58baa8..b59e892b 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -46,7 +46,7 @@ pub fn fusedExperts( }; const expert_partition = gate_up.weight.shape().partition(.expert); - if (!expert_partition.eql(.init(.experts))) { + if (!expert_partition.isSharded()) { return fusedExpertsImpl(args, backend); } @@ -238,7 +238,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); values[if (layout == .concatenated) i + 3 else 2 * i + 1] = .fromF32(up); } - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; @@ -270,7 +270,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { // }; // const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); // const values = [_]f32{ -3, 0, 1, 2, 4 }; -// var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); +// var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); // defer input.deinit(); // for ([_]?f32{ null, 2 }) |threshold| { // var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); @@ -412,7 +412,7 @@ test "FP8 routed GEMM with bias matches dequantized weights" { @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), } } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); initialized += 1; } var output: zml.Bufferized(Local.Outputs) = undefined; @@ -491,7 +491,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const host = try zml.Slice.alloc(allocator, x.shape()); defer host.free(allocator); @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); - var input = try zml.Buffer.fromSlice(io, platform, host, .replicated); + var input = try zml.Buffer.fromSlice(io, platform, host); defer input.deinit(); for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { diff --git a/zml/moe/mosaic_tpu.zig b/zml/moe/mosaic_tpu.zig index 6f6a7bc5..f43d1b84 100644 --- a/zml/moe/mosaic_tpu.zig +++ b/zml/moe/mosaic_tpu.zig @@ -238,7 +238,7 @@ pub fn fusedExperts( const expert_partition = gate_up.weight.shape().partition(.expert); - if (expert_partition.eql(.init(.experts))) { + if (expert_partition.isSharded()) { const global_num_experts = down.weight.dim(.expert); const partial_output = zml.ops.manualComputation( (struct { diff --git a/zml/moe/triton_mxfp4.zig b/zml/moe/triton_mxfp4.zig index bf6bbdf5..d20ffe8f 100644 --- a/zml/moe/triton_mxfp4.zig +++ b/zml/moe/triton_mxfp4.zig @@ -39,7 +39,7 @@ pub fn fusedExperts( stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); + const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); const context: Context = .{ .input = input, diff --git a/zml/nn.zig b/zml/nn.zig index a49865fa..1403788f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -141,7 +141,7 @@ pub fn scaledDot( const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - var res_shape: Shape = .{ ._dtype = out_dtype }; + var res_shape: Shape = .scalar(out_dtype); var lhs_batching_axes: Axes = .empty; var rhs_batching_axes: Axes = .empty; for (dot_axes.batching.constSlice()) |b_axes| { @@ -290,7 +290,7 @@ test "block128 scaled dot layouts" { slice.items(zml.floats.BFloat16)[j] = .fromF32(value); } } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); initialized += 1; } var output: zml.Bufferized(Local.Outputs) = undefined; @@ -443,7 +443,7 @@ test normalizeL2 { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, .{}); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); defer input_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -1041,7 +1041,7 @@ test rope { defer exe_sequential.deinit(); const x_values: [5][4]f32 = @splat(.{ 1.0, 0.1, -1.0, -0.5 }); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_values)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_values)); defer x_buffer.deinit(); var res1 = try exe_interleaved.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -1075,7 +1075,7 @@ test "rope: Proportional" { defer exe.deinit(); const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); - var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), .replicated, @ptrCast(&x_h)); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); defer x_buffer.deinit(); const expected_h: [5][16]f32 = .{ @@ -1127,7 +1127,7 @@ test "rope: Yarn with partial_rotary_factor" { defer exe.deinit(); const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); - var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), .replicated, @ptrCast(&x_h)); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); defer x_buffer.deinit(); const expected_h: [5][16]f32 = .{ @@ -1223,7 +1223,6 @@ test nearest { std.testing.io, platform, input_3d_basic.shape(), - .replicated, std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), ); defer input_3d_basic_buffer.deinit(); @@ -1248,7 +1247,7 @@ test nearest { ); defer exe.deinit(); - var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), .replicated, std.mem.sliceAsBytes(&[2][3][4]i32{ + var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, })); @@ -1285,7 +1284,7 @@ test nearest { ); defer exe.deinit(); - var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_basic_buffer}); @@ -1314,7 +1313,7 @@ test nearest { ); defer exe.deinit(); - var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), .replicated, std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ + var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ .{ .{ 1, 2 }, .{ 3, 4 } }, .{ .{ 5, 6 }, .{ 7, 8 } }, }, .{ @@ -1371,7 +1370,7 @@ test nearest { ); defer exe.deinit(); - var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_5d_buffer}); @@ -1821,7 +1820,6 @@ test "gated delta net" { std.testing.io, platform, queries.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, .{ .{ 1.0, 1.0 }, .{ 1.0, -1.0 } }, @@ -1832,7 +1830,6 @@ test "gated delta net" { std.testing.io, platform, keys.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 2.0 }, .{ 0.0, 1.0 } }, .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, @@ -1843,7 +1840,6 @@ test "gated delta net" { std.testing.io, platform, values.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 3.0, 1.0 }, .{ 2.0, 4.0 } }, .{ .{ 1.0, 5.0 }, .{ 3.0, 0.0 } }, @@ -1854,7 +1850,6 @@ test "gated delta net" { std.testing.io, platform, alphas.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2]f32{ .{ 0.5, 0.25 }, .{ 0.8, 0.6 }, @@ -1865,7 +1860,6 @@ test "gated delta net" { std.testing.io, platform, betas.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2]f32{ .{ 1.0, 0.5 }, .{ 0.75, 1.0 }, @@ -1876,7 +1870,6 @@ test "gated delta net" { std.testing.io, platform, initial_s.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, @@ -2004,7 +1997,7 @@ test sampleTokens { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, .{}); defer exe.deinit(); - var rng_buffer = try zml.Tensor.Rng.initBuffer(std.testing.io, platform, .replicated, 0xdeadbeef); + var rng_buffer = try rng.initBuffer(std.testing.io, platform, 0xdeadbeef); defer rng_buffer._state.deinit(); const inf = std.math.inf(f32); @@ -2016,7 +2009,7 @@ test sampleTokens { }) |logits_expected| { const logits, const expected: i32 = logits_expected; - var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), .replicated, std.mem.sliceAsBytes(&logits)); + var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); var sampled: zml.Buffer = undefined; @@ -2149,7 +2142,7 @@ test sampleTokensDynamic { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, .{}); defer exe.deinit(); - var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), .replicated, std.mem.sliceAsBytes(&logits_data)); + var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); defer logits_buffer.deinit(); const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; @@ -2193,7 +2186,7 @@ test sampleTokensDynamic { const boost = bf16.inf; const nerf = bf16.minus_inf; - var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), .replicated, std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); + var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); defer logits_bf16_buffer.deinit(); var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); diff --git a/zml/ops.zig b/zml/ops.zig index fd97ceb1..186805ea 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -21,6 +21,7 @@ const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; +const log = std.log.scoped(.@"zml/Compiler"); pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); @@ -66,7 +67,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), }; - const num_devices = ctx.partitioning.numPartitions(); + const num_devices = ctx.meshes[0].numPartitions(); if (num_devices <= 1) return inputs; const reducer_block = b: { @@ -635,9 +636,9 @@ test @"while" { ); defer exe.deinit(); - var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{1})); + var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), std.mem.sliceAsBytes(&[1]i64{1})); defer i_buffer.deinit(); - var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{0})); + var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), std.mem.sliceAsBytes(&[1]i64{0})); defer sum_buffer.deinit(); var results: zml.Bufferized(While.State) = undefined; @@ -865,10 +866,10 @@ fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(b fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); - // copy tags and sharding info over + // copy tags and mesh info over // some ops can change dims eg reduceWindow, so we trust mlir here. new._shape._tags = tensor._shape._tags; - new._shape._partitioning = tensor._shape._partitioning; + new._shape._sharding = tensor._shape._sharding; tensor.* = new; inner_ctx.index += 1; } @@ -1055,7 +1056,7 @@ test "fly custom call, both entry ABIs" { fn f(comptime forward: anytype, p: *const zml.Platform, t: Tensor, h: []const f32) !zml.Slice { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{t}, p, .{}); defer exe.deinit(); - var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), .replicated, std.mem.sliceAsBytes(h)); + var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), std.mem.sliceAsBytes(h)); defer buf.deinit(); var out = try exe.eval(std.testing.allocator, std.testing.io, .{buf}); defer out.deinit(); @@ -1274,9 +1275,9 @@ test "triton" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{1})); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); defer a_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{3})); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); var results: [2]zml.Buffer = undefined; @@ -1497,7 +1498,7 @@ test "cute" { var input: [128]f32 = undefined; for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -1556,7 +1557,7 @@ test "cuda_tile" { var input: [128]f32 = undefined; for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -1623,7 +1624,7 @@ test "cuda_tile grid" { defer exe.deinit(); const input = [_]f32{0} ** n; - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -1683,7 +1684,7 @@ test "cuda_tile zeroed_outputs" { var input: [128]f32 = undefined; for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -2109,7 +2110,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, if (indices_shape.count() == 1 and idx_axes.len == 1) { return self .slice(idx_axes[0], .dyn(indices_per_axis.get(0).asScalar(), 1)) - // Keep downstream resharding after the slice. SPMD engines like Neuron otherwise + // Keep downstream remeshe after the slice. SPMD engines like Neuron otherwise // may hoist an all-gather before the slice and materialize the full tensor. .optimizationBarrier() .reshape(res_shape); @@ -2522,6 +2523,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; } +/// Runs a per-shard body using one mesh for all input and output partition specs. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), @@ -2576,6 +2578,47 @@ fn manualComputationLocalizeInputs(allocator: std.mem.Allocator, inputs: anytype return local_inputs; } +fn manualComputationMesh(inputs: []const Shape, outputs: []const Shape, fallback: *const Sharding.Mesh) error{IncompatibleMeshes}!*const Sharding.Mesh { + var mesh: ?*const Sharding.Mesh = null; + for ([_][]const Shape{ inputs, outputs }) |shapes| { + for (shapes) |shape| { + if (shape._sharding.mesh) |shape_mesh| { + if (mesh) |m| { + if (m != shape_mesh) return error.IncompatibleMeshes; + } else mesh = shape_mesh; + } + } + } + return mesh orelse fallback; +} + +test "manualComputation infers one mesh from inputs and outputs" { + const fallback: Sharding.Mesh = .{ + .name = "fallback", + .physical = undefined, + .logical = .mesh(.{ .batch = .balanced }), + .bindings = .empty, + .folds = .empty, + .folds_consumed = .empty, + }; + var first = fallback; + first.name = "first"; + var second = fallback; + second.name = "second"; + const replicated: Shape = .init(.{8}, .f32); + var sharded = replicated; + sharded._sharding = .{ .mesh = &first, .partition = .init(&.{.mesh_axis_0}) }; + var other = sharded; + other._sharding.mesh = &second; + + try std.testing.expectEqual(&fallback, try manualComputationMesh(&.{replicated}, &.{replicated}, &fallback)); + try std.testing.expectEqual(&first, try manualComputationMesh(&.{ replicated, sharded, sharded }, &.{sharded}, &fallback)); + try std.testing.expectEqual(&first, try manualComputationMesh(&.{replicated}, &.{sharded}, &fallback)); + try std.testing.expectEqual(&first, try manualComputationMesh(&.{}, &.{sharded}, &fallback)); + try std.testing.expectError(error.IncompatibleMeshes, manualComputationMesh(&.{ sharded, other }, &.{}, &fallback)); + try std.testing.expectError(error.IncompatibleMeshes, manualComputationMesh(&.{sharded}, &.{other}, &fallback)); +} + fn manualComputationInternal( inputs: anytype, outputs: []const Shape, @@ -2596,35 +2639,21 @@ fn manualComputationInternal( const local_input_shapes = try arena.alloc(Shape, input_shapes.len); const local_output_shapes = try arena.alloc(Shape, outputs.len); - const input_shardings = try arena.alloc(Sharding, input_shapes.len); - const output_shardings = try arena.alloc(Sharding, outputs.len); + const manual_mesh = manualComputationMesh(input_shapes, outputs, ctx.platform.replicated_mesh) catch + @panic("manualComputation expects all inputs and outputs to use the same mesh"); for (input_shapes, 0..) |shape, i| { - const sharding = ctx.partitioning.selectSharding(shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "failed to shard manualComputation input {f}({d}) because it's using unknown sharding. Pass more shardings to `.compile`. Known shardings: {f}", - .{ shape, i, stdx.fmt.slice(ctx.partitioning.shardings) }, - ), - }; - input_shardings[i] = sharding; - local_input_shapes[i] = sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, sharding }); + local_input_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); } for (outputs, 0..) |shape, i| { - const sharding = ctx.partitioning.selectSharding(shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "failed to shard manualComputation output {f}({d}) because it's using unknown sharding. Pass more shardings to `.compile`. Known shardings: {f}", - .{ shape, i, stdx.fmt.slice(ctx.partitioning.shardings) }, - ), - }; - output_shardings[i] = sharding; - local_output_shapes[i] = sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, sharding }); + local_output_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); } - return switch (ctx.partitioning.partitioner) { + return switch (ctx.partitioner) { .shardy => { - const in_shardings_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, input_shapes, input_shardings); - const out_shardings_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, outputs, output_shardings); - const manual_axes_attr = try Sharding.sdyManualAxesAttr(arena, ctx.mlir_ctx, input_shapes, input_shardings, outputs, output_shardings); + const in_meshes_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, input_shapes, manual_mesh); + const out_meshes_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, outputs, manual_mesh); + const manual_axes_attr = try Sharding.sdyManualAxesAttr(arena, ctx.mlir_ctx, input_shapes, outputs, manual_mesh); const block_types = try arena.alloc(*const mlir.Type, input_shapes.len); for (local_input_shapes, 0..) |input_shape, i| { @@ -2675,8 +2704,8 @@ fn manualComputationInternal( .results = .{ .flat = global_result_types }, .blocks = &.{manual_block}, .attributes = &.{ - .named(ctx.mlir_ctx, "in_shardings", in_shardings_attr), - .named(ctx.mlir_ctx, "out_shardings", out_shardings_attr), + .named(ctx.mlir_ctx, "in_shardings", in_meshes_attr), + .named(ctx.mlir_ctx, "out_shardings", out_meshes_attr), .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr), }, .verify = true, @@ -2686,7 +2715,7 @@ fn manualComputationInternal( // Use the compiler allocator to return memory to the parent const sharded_outputs = ctx.alloc(Tensor, outputs.len); for (outputs, 0..) |output, i| { - sharded_outputs[i] = Tensor._result(output, op.result(i)); + sharded_outputs[i] = Tensor._resultPropagateSharding(output, op.result(i)); } return sharded_outputs; }, @@ -2732,8 +2761,8 @@ fn manualComputationInternal( const global_values = try arena.alloc(*const mlir.Value, outputs.len); const global_types = try arena.alloc(*const mlir.Type, outputs.len); - for (outputs, output_shardings, 0..) |output_shape, output_sharding, i| { - const gspmd_attr = try ctx.partitioning.tensorShardingAttr(arena, ctx.mlir_ctx, output_shape, output_sharding); + for (outputs, 0..) |output_shape, i| { + const gspmd_attr = try ctx.tensorShardingAttr(arena, ctx.mlir_ctx, output_shape); global_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, output_shape); const shard_to_full = dialects.stablehlo.custom_call( @@ -2762,7 +2791,7 @@ fn manualComputationInternal( const sharded_outputs = ctx.alloc(Tensor, outputs.len); for (outputs, 0..) |output_shape, i| { - sharded_outputs[i] = Tensor._result(output_shape, barrier.result(i)); + sharded_outputs[i] = Tensor._resultPropagateSharding(output_shape, barrier.result(i)); } return sharded_outputs; }, @@ -3153,7 +3182,7 @@ pub fn typedCustomCall( } break :b &input_tensors; }, - // Extra case to support []const Tensor from shardingAwareTypedCustomCall + // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -3169,7 +3198,7 @@ pub fn typedCustomCall( } break :b &output_shapes; }, - // Extra case to support []const Shape from shardingAwareTypedCustomCall + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -3234,7 +3263,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioning.partitioner == .gspmd) { + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -3246,7 +3275,7 @@ pub fn typedCustomCall( } return out; }, - // Extra case to support []const Shape from shardingAwareTypedCustomCall + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -3307,7 +3336,7 @@ test customCall { const scope = comp.pushBlock(block); defer scope.pop(); - const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(.{ ._0 = .x }); + const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(platform.meshes.get("model").?, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ .has_side_effect = false, diff --git a/zml/platform.zig b/zml/platform.zig index 826eeaee..7a9ee710 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -296,8 +296,8 @@ pub const Platform = struct { devices: []const Device, memories: []const Memory, physical_mesh: zml.Sharding.PhysicalMesh, - replicated_sharding: zml.Sharding, - shardings: std.StringArrayHashMapUnmanaged(zml.Sharding), + meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), + replicated_mesh: *const zml.Mesh, io_impl: IoImpl, pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; @@ -337,12 +337,12 @@ pub const Platform = struct { .pjrt_api = api, .pjrt_client = pjrt_client, .state = State.init(target), - .shardings = .empty, + .meshes = .empty, + .replicated_mesh = undefined, // set below .devices = undefined, .memories = undefined, .physical_mesh = undefined, - .replicated_sharding = undefined, .io_impl = options.io_impl, }; break :platform platform; @@ -350,7 +350,7 @@ pub const Platform = struct { const arena = platform.arena.allocator(); errdefer platform.arena.deinit(); - try platform.shardings.ensureTotalCapacity(arena, 8); + try platform.meshes.ensureTotalCapacity(arena, 8); { const devices = try arena.alloc(Device, pjrt_devices.len); @@ -375,7 +375,7 @@ pub const Platform = struct { .auto => zml.Sharding.PhysicalMesh.auto(arena, target, devices), .custom => |builder| builder(arena, target, devices), }; - platform.replicated_sharding = try platform.registerSharding("replicated", .mesh(.{ .x = .high_bandwidth })); + platform.replicated_mesh = try platform.registerMesh("replicated", .mesh(.{ .all = .high_bandwidth })); } switch (target) { @@ -639,38 +639,37 @@ pub const Platform = struct { return try profiler_.profiler(self.pjrt_api, allocator, io, options); } - /// Create a Sharding based on the given logical mesh and the default strategy. + /// Create a Mesh based on the given logical mesh and the default strategy. /// Memory is owned by the platform, making it safe to copy around. - pub fn registerSharding(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!Sharding { - return platform.registerShardingWithStrategy( + pub fn registerMesh(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!*const Sharding.Mesh { + return platform.registerMeshWithStrategy( name, logical, .suggest(logical, &platform.physical_mesh), ) catch |err| switch (err) { error.InvalidPhysicalMesh, error.InvalidStrategy, error.InvalidPhysicalAxis => { - std.debug.panic("ZML failed to create a valid sharding for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); + std.debug.panic("ZML failed to create a valid mesh for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); }, error.OutOfMemory => |e| return e, }; } - /// Create a Sharding based on the given logical mesh and a strategy. + /// Create a Mesh based on the given logical mesh and a strategy. /// Memory is owned by the platform, making it safe to copy around. - pub fn registerShardingWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !Sharding { + pub fn registerMeshWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !*const Sharding.Mesh { const arena = platform.arena.allocator(); - const entry = try platform.shardings.getOrPut(arena, name); + const entry = try platform.meshes.getOrPut(arena, name); if (entry.found_existing) { - std.debug.panic("Another sharding already exists with this name: {s}", .{name}); + std.debug.panic("Another mesh already exists with this name: {s}", .{name}); } const owned_name = try arena.dupe(u8, name); - const owned_data = try arena.create(Sharding.Data); - owned_data.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); - const sharding: Sharding = .{ .data = owned_data }; + const owned_mesh = try arena.create(Sharding.Mesh); + owned_mesh.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); entry.key_ptr.* = owned_name; - entry.value_ptr.* = sharding; + entry.value_ptr.* = owned_mesh; - return sharding; + return owned_mesh; } fn memoryFromPjrt(self: *const Platform, pjrt_memory: *const pjrt.Memory) *const Memory { diff --git a/zml/quantization.zig b/zml/quantization.zig index 489162fb..b125513a 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -436,7 +436,7 @@ test "block FP8 quantization preserves axes and reconstructs constant blocks in else => -magnitude, }); } - var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); defer buffer.deinit(); var output: zml.Bufferized(Local.Outputs) = undefined; var runner = try exe.runner(allocator); @@ -495,7 +495,7 @@ test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" const exponent = (bits >> 23) + @intFromBool(bits & 0x7fffff != 0); scale.* = @bitCast(exponent << 23); } - var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); defer buffer.deinit(); var output: zml.Bufferized(Local.Outputs) = undefined; var runner = try exe.runner(allocator); diff --git a/zml/shape.zig b/zml/shape.zig index 8afae990..11a3efac 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -6,141 +6,26 @@ const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const Sharding = @import("Sharding.zig"); const log = std.log.scoped(.shape); +test { + std.testing.refAllDecls(Shape); +} + /// Represent the shape of a tensor. +/// Contains also the sharding information, +/// so it describes exactly how the corresponding tensor/buffer will be layed out +/// across all devices of the current Platform. +/// +/// By default all created Shape are replicated, meaning the corresponding tensor / buffer will be copied on each device. +/// `.withPartitioning` allows to specify how it should be split. pub const Shape = struct { - pub const MAX_RANK: u8 = 8; - - pub const PartitionSpec = union(enum) { - axis: Tag, - open, - replicated, - unknown, - - pub fn init(comptime value: anytype) PartitionSpec { - const T = @TypeOf(value); - - if (T == PartitionSpec) { - return value; - } - - if (comptime T == @EnumLiteral()) { - const name = @tagName(value); - inline for (std.meta.fields(PartitionSpec)) |field| { - if (field.type == void and std.mem.eql(u8, field.name, name)) { - return @field(PartitionSpec, field.name); - } - } - } - - if (comptime isTagConvertible(T)) { - const tag_ = Shape.toTag(value); - - return .{ .axis = tag_ }; - } - - stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); - } - - test "PartitionSpec.init" { - const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; - try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); - - const spec_axis_open: PartitionSpec = .open; - try testing.expect(spec_axis_open.eql(.init(.open))); - } - - pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { - if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { - return false; - } - - switch (self) { - .axis => |t1| { - const t2 = other.axis; - return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); - }, - else => return true, - } - } - - test "PartitionSpec.eql" { - const spec_axis: PartitionSpec = .init(.a); - const spec_axis_b: PartitionSpec = .init(.b); - const spec_open: PartitionSpec = .open; - const spec_replicated: PartitionSpec = .replicated; - const spec_unknown: PartitionSpec = .unknown; - - try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); - try testing.expect(!spec_axis.eql(spec_axis_b)); - try testing.expect(!spec_axis.eql(.open)); - try testing.expect(!spec_axis.eql(.replicated)); - try testing.expect(!spec_axis.eql(.unknown)); - - try testing.expect(spec_open.eql(.open)); - try testing.expect(!spec_open.eql(spec_axis)); - try testing.expect(!spec_open.eql(.replicated)); - try testing.expect(!spec_open.eql(.unknown)); - - try testing.expect(spec_replicated.eql(.replicated)); - try testing.expect(!spec_replicated.eql(spec_axis)); - try testing.expect(!spec_replicated.eql(.open)); - try testing.expect(!spec_replicated.eql(.unknown)); - - try testing.expect(spec_unknown.eql(.unknown)); - try testing.expect(!spec_unknown.eql(spec_axis)); - try testing.expect(!spec_unknown.eql(.open)); - try testing.expect(!spec_unknown.eql(.replicated)); - } - - pub fn isClosed(self: PartitionSpec) bool { - return switch (self) { - .axis => true, - .open => false, - .replicated => true, - .unknown => false, - }; - } - - test isClosed { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.isClosed()); - - const spec_open: PartitionSpec = .open; - try testing.expect(!spec_open.isClosed()); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.isClosed()); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(!spec_unknown.isClosed()); - } - - pub fn toTag(self: PartitionSpec) Tag { - return switch (self) { - .axis => |t| return t, - .open => TagOpen, - .replicated => TagReplicated, - .unknown => TagUnknown, - }; - } - - test "PartitionSpec.toTag" { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); - - const spec_open: PartitionSpec = .open; - try testing.expect(spec_open.toTag() == TagOpen); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.toTag() == TagReplicated); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(spec_unknown.toTag() == TagUnknown); - } - }; + _dtype: DataType, + _dims: DimsArray, + _tags: TagsArray, + _sharding: Sharding, pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; @@ -148,17 +33,10 @@ pub const Shape = struct { const TagReplicated = "replicated".ptr; const TagOpen = "open".ptr; + pub const MAX_RANK: u8 = constants.MAX_RANK; pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); - pub const PartitionArray = stdx.BoundedArray(PartitionSpec, constants.MAX_RANK); - - const UnknownTags: TagsArray = .{ .len = 0, .buffer = @splat(TagUnknown) }; - - _dtype: DataType, - _dims: DimsArray = .empty, - _tags: TagsArray = UnknownTags, - _partitioning: PartitionArray = .empty, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -252,28 +130,39 @@ pub const Shape = struct { /// Create a shape from a struct literal, eg: /// Shape.init(.{ .h = 1024, .w = 512, .c = 3 }); /// Shape.init(.{ 1024, 512, 3 }); + /// + /// The created shape is replicated. pub fn init(dimz: anytype, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; - res._dims, res._tags = parseDimensions(dimz); - res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); - - return res; + const _dims, const _tags = parseDimensions(dimz); + return .{ + ._dtype = dt, + ._dims = _dims, + ._tags = _tags, + ._sharding = .replicated, + }; } pub fn scalar(dt: DataType) Shape { - return .{ ._dtype = dt }; + return .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .len = 0, .buffer = @splat(TagUnknown) }, + ._sharding = .replicated, + }; } /// Creates a Shape with dims set to `.{0, 1, 2, ..., rank-1}`. pub fn range(rank_: usize, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; + stdx.debug.assert(rank_ <= MAX_RANK, "zml.Shape.range expects a maximum rank of {d}, got {}", .{ MAX_RANK, rank_ }); + var res: Shape = .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .buffer = @splat(TagUnknown), .len = rank_ }, + ._sharding = .replicated, + }; for (0..rank_) |i| { - res._dims.append(@intCast(i)) catch { - stdx.debug.panic("Too many dimensions! Max: {d}, passed: {d}", .{ res._dims.capacity(), rank_ }); - }; - res._tags.append(TagUnknown) catch unreachable; + res._dims.appendAssumeCapacity(@intCast(i)); } - res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -319,7 +208,7 @@ pub const Shape = struct { fn ensureAttributesAreSync(self: Shape) void { if (builtin.mode == .Debug) { - stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); + stdx.debug.assert(self._dims.len == self._tags.len, "Tags, dims and partitioning have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); } } @@ -349,7 +238,7 @@ pub const Shape = struct { } pub fn hasTag(self: Shape, tag_: anytype) ?u3 { - return self.axisFromTagMaybe(toTag(tag_)); + return axisFromTagMaybe(self.tags(), toTag(tag_)); } pub fn hasTags(self: Shape, tagz: anytype) bool { @@ -443,7 +332,7 @@ pub const Shape = struct { @intCast(a); } - fn axisFromTagMaybe(self: Shape, t: Tag) ?u3 { + pub fn axisFromTagMaybe(tagz: []const Tag, t: Tag) ?u3 { if (t == TagUnknown) return null; if (axisFromLiteralInt(t)) |ax| return ax; @@ -451,7 +340,7 @@ pub const Shape = struct { if (@inComptime()) { // At comptime two duplicated strings may have two different representations const t_bytes: []const u8 = std.mem.span(t); - for (self.tags(), 0..) |self_tag, ax| { + for (tagz, 0..) |self_tag, ax| { if (std.mem.eql(u8, t_bytes, std.mem.span(self_tag))) { return @truncate(ax); } @@ -460,7 +349,7 @@ pub const Shape = struct { } // But at runtime the comptime strings have been deduplicated and ptr match is enough. - if (std.mem.indexOfScalar(Tag, self.tags(), t)) |ax| { + if (std.mem.indexOfScalar(Tag, tagz, t)) |ax| { return @truncate(ax); } return null; @@ -477,7 +366,7 @@ pub const Shape = struct { fn axisFromTag(self: Shape, d: Tag) u3 { stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); - return self.axisFromTagMaybe(d) orelse { + return axisFromTagMaybe(self.tags(), d) orelse { stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); }; } @@ -519,8 +408,7 @@ pub const Shape = struct { } /// Format the shape. - /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" - /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" + /// "Shape({.a=10, .b=20}, dtype=.f32)" pub fn format(self: Shape, writer: *std.Io.Writer) !void { _ = try writer.writeByte('{'); @@ -534,9 +422,16 @@ pub const Shape = struct { try writer.print("{d}", .{d}); } - const part = self._partitioning.get(i); - if (part.toTag() != TagUnknown) { - try writer.print("/{s}", .{part.toTag()}); + const partitioning = self._sharding.get(i); + switch (partitioning) { + .replicated => {}, + .open => try writer.writeAll("/?"), + // this state is invalid, but don't panic cause we may be called from a panic + .out_of_bound => try writer.writeAll("/!"), + else => { + const mesh_ax = partitioning.meshAxis().?; + try writer.print("/{s}", .{self._sharding.mesh.?.logical.axes.get(mesh_ax)}); + }, } need_comma = true; @@ -544,6 +439,10 @@ pub const Shape = struct { if (need_comma) try writer.writeByte(','); _ = try writer.write(@tagName(self.dtype())); _ = try writer.writeByte('}'); + if (self._sharding.mesh) |m| { + try writer.writeByte('@'); + try writer.writeAll(m.name); + } } /// Broadcasts a Tensor to the given shape, extending dimensions if needed. @@ -569,10 +468,17 @@ pub const Shape = struct { return true; } + /// Reshape a given shape but keep the same total number of elements. + /// Partitioning information is generally NOT preserved, + /// unless input was fully replicated. pub fn reshape(self: Shape, new_shape_: anytype) Shape { - var new_shape: Shape = .{ ._dtype = self.dtype() }; - new_shape._dims, new_shape._tags = parseDimensions(new_shape_); - new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); + const _dims, const _tags = parseDimensions(new_shape_); + var new_shape: Shape = .{ + ._dtype = self._dtype, + ._dims = _dims, + ._tags = _tags, + ._sharding = .{ .mesh = self._sharding.mesh, .partition = if (self.isFullyReplicated()) .replicated(_dims.len) else .open(_dims.len) }, + }; new_shape.inferMissingAxis(self.count()) catch |err| { std.debug.panic("Can't reshape {any} to {any}: {t}", .{ self.dims(), new_shape.dims(), err }); }; @@ -655,7 +561,7 @@ pub const Shape = struct { res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; for (0..dims_.len) |_| { - res._partitioning.insert(ax, .unknown) catch unreachable; + res._sharding = res._sharding.insert(ax, .replicated); } return res; @@ -678,7 +584,7 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; - res._partitioning.insert(ax, .unknown) catch unreachable; + res._sharding = res._sharding.insert(ax, .replicated); return res; } @@ -688,7 +594,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); - res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); + for (self.rank()..res.rank()) |ax| res._sharding = res._sharding.set(ax, .replicated); return res; } @@ -710,7 +616,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); - res._partitioning.appendAssumeCapacity(.unknown); + res._sharding = res._sharding.set(self.rank(), .replicated); return res; } @@ -719,7 +625,7 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); - _ = res._partitioning.orderedRemove(a); + res._sharding = res._sharding.orderedRemove(a); return res; } @@ -736,6 +642,7 @@ pub const Shape = struct { std.mem.sort(u3, to_remove.slice(), {}, std.sort.asc(u3)); var sh: Shape = self; + sh._sharding.partition = .out_of_bound; const rk = self.rank(); var res_ax: u32 = 0; for (0..rk) |ax| { @@ -745,12 +652,11 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; - sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; + sh._sharding = sh._sharding.set(res_ax, self._sharding.get(ax)); res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; - sh._partitioning.len = rk - to_remove.len; return sh; } @@ -773,12 +679,12 @@ pub const Shape = struct { var res = self; const original_dims = self._dims; const original_tags = self._tags; - const original_parts = self._partitioning; + const original_parts = self._sharding; for (permutations_.constSlice(), 0..) |permutation, i| { res._dims.set(i, original_dims.get(permutation)); res._tags.set(i, original_tags.get(permutation)); - res._partitioning.set(i, original_parts.get(permutation)); + res._sharding = res._sharding.set(i, original_parts.get(permutation)); } return res; @@ -797,9 +703,9 @@ pub const Shape = struct { ), ); - const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); + try testing.expectEqualDeep(Sharding.Partitioning.init(&.{ .mesh_axis_0, .open, .mesh_axis_2 }), shape._sharding.partition); } /// Tag each ax of this shape with tags from a tuple. @@ -908,154 +814,131 @@ pub const Shape = struct { return res; } - pub fn withPartitioning(self: Shape, specs: anytype) Shape { - const T = @TypeOf(specs); + // These tests only resolve logical names; placement does not use this fixture. + const test_mesh: Sharding.Mesh = .{ + .name = "test_mesh", + .physical = undefined, + .logical = .mesh(.{ .batch = .balanced, .feature = .balanced, .colors = .balanced }), + .bindings = .init(&.{ + .init(&.{.link_x}), + .init(&.{.link_y}), + .init(&.{.link_z}), + }), + .folds = .empty, + .folds_consumed = .empty, + }; - var res = self.withDefaultPartitioning(); // todo add test for this new change - - if (stdx.meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - const partition_axis = @field(specs, field.name); - const axis_ = res.axisFromTagMaybe(toTag(field)); - - if (axis_) |ax| { - res._partitioning.set(ax, PartitionSpec.init(partition_axis)); - } else { - stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); - } - } - } else { - stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { + if (@TypeOf(partitioning) != Sharding.Partitioning) { + return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); } - - // Check that no mesh axis is used to partition multiple tensor dimensions. - var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .empty; - - for (res._partitioning.constSlice()) |spec| { - const axis_tag = spec.toTag(); - - if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { - stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); - } - - switch (spec) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), - } + var res = self; + res._sharding = .{ .mesh = mesh, .partition = partitioning }; + for (0..self.rank()) |ax| { + if (res._sharding.get(ax) == .out_of_bound) res._sharding = res._sharding.set(ax, .replicated); } + for (self.rank()..MAX_RANK) |ax| res._sharding.partition = res._sharding.partition.set(ax, .out_of_bound); + + stdx.debug.assert(res._sharding.partition.hasUniqueAxes(), "{f}.withPartitioning({s}, ...) expects mesh axes to be used a most once, got {any}", .{ self, mesh.name, partitioning.toArray()[0..self.rank()] }); return res; } test withPartitioning { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .replicated, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withDefaultPartitioning(); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .open, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); } - pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { + pub fn mapPartitioningAxes(self: Shape, mesh: *const Sharding.Mesh, mapping: anytype) Shape { const T = @TypeOf(mapping); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); - var res = self; - for (res._partitioning.slice()) |*spec| { - switch (spec.*) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => |*old_axis_tag| { - inline for (std.meta.fields(T)) |field| { - const logical_axis_in_map = toTag(field); - - if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { - const physical_axis_from_map = @field(mapping, field.name); - spec.* = PartitionSpec.init(physical_axis_from_map); - break; - } - } - }, + for (0..self.rank()) |ax| { + const mesh_axis = self.partition(ax).meshAxis() orelse continue; + inline for (std.meta.fields(T)) |field| { + const old_axis = mesh.resolveLogicalAxis(toTag(field)) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field.name }); + if (mesh_axis == old_axis) { + const new_tag = toTag(@field(mapping, field.name)); + const new_axis = mesh.resolveLogicalAxis(new_tag) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, new_tag }); + res._sharding = res._sharding.set(ax, .sharded(@intCast(new_axis))); + break; + } } } return res; } test mapPartitioningAxes { - const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ - .batch = .data, - .seq = .replicated, - .features = .model, - }); - - const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); - - try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); - try testing.expect(physical_shape.partition(.features).eql(.init(.y))); - try testing.expect(physical_shape.partition(.seq).eql(.replicated)); - - const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); - try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); - try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); + const shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32) + .withPartitioning(&test_mesh, .{ .batch = .batch, .seq = .replicated, .features = .feature }); + const mapped = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .feature, .feature = .batch }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), mapped.partition(.batch)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), mapped.partition(.features)); + try testing.expectEqual(Sharding.PartitionSpec.replicated, mapped.partition(.seq)); + const partial = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .colors }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(2), partial.partition(.batch)); + try testing.expectEqual(shape.partition(.features), partial.partition(.features)); } pub fn withDefaultPartitioning(self: Shape) Shape { var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + res._sharding.partition = .replicated(self.rank()); return res; } test withDefaultPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withDefaultPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); } pub fn withReplicatedPartitioning(self: Shape) Shape { var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + res._sharding.partition = .replicated(self._dims.len); return res; } test withReplicatedPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withReplicatedPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); } pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec == .axis) return true; + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (spec.isSharded()) return true; } return false; } test hasAtLeastOnePartitionedAxis { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(shape.hasAtLeastOnePartitionedAxis()); - shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); shape = shape.withDefaultPartitioning(); @@ -1063,20 +946,18 @@ pub const Shape = struct { } pub fn isFullyPartitioned(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - switch (spec) { - .replicated, .open, .unknown => return false, - .axis => {}, - } + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (!spec.isSharded()) return false; } return true; } test isFullyPartitioned { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .feature }); try testing.expect(shape.isFullyPartitioned()); - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(!shape.isFullyPartitioned()); shape = shape.withDefaultPartitioning(); @@ -1084,54 +965,47 @@ pub const Shape = struct { } pub fn isFullyReplicated(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec != .replicated and spec != .unknown) { - return false; - } - } - return true; + return self._sharding.isFullyReplicated(); } test isFullyReplicated { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); try testing.expect(shape.isFullyReplicated()); - shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); - try testing.expect(shape.isFullyReplicated()); - - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(!shape.isFullyReplicated()); } - pub fn partition(self: Shape, ax: anytype) PartitionSpec { - return self._partitioning.get(self.axis(ax)); + pub fn partition(self: Shape, ax: anytype) Sharding.PartitionSpec { + return self._sharding.get(self.axis(ax)); } test partition { - const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), shape.partition(.a)); try testing.expectEqual(.open, shape.partition(.b)); - try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), shape.partition(.c)); } - pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { - for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { - if (dim_part.eql(part)) return true; + pub fn containsPartitionSpec(self: Shape, part: Sharding.PartitionSpec) bool { + for (0..self.rank()) |ax| { + const dim_part = self._sharding.get(ax); + if (dim_part == part) return true; } return false; } test containsPartitionSpec { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expect(shape.containsPartitionSpec(.init(.batch))); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.sharded(0))); try testing.expect(shape.containsPartitionSpec(.open)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); - try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); shape = shape.withDefaultPartitioning(); - try testing.expect(!shape.containsPartitionSpec(.init(.batch))); - try testing.expect(shape.containsPartitionSpec(.unknown)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.sharded(0))); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); + try testing.expect(shape.containsPartitionSpec(.replicated)); try testing.expect(!shape.containsPartitionSpec(.open)); } @@ -1329,10 +1203,10 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; - _ = new_shape._partitioning.orderedRemove(ax); + new_shape._sharding = new_shape._sharding.orderedRemove(ax); for (0..dims_.len) |_| { - new_shape._partitioning.insert(ax, .unknown) catch unreachable; + new_shape._sharding = new_shape._sharding.insert(ax, .replicated); } new_shape.inferMissingAxis(self.count()) catch |err| { @@ -1383,39 +1257,34 @@ pub const Shape = struct { const axes_to_merge = self.axes(axes_); stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); - var new_dim: i64 = 1; // Merging partitioned axes is complex. If ANY axis being merged is partitioned, - // the resulting sharding is ambiguous. A propagation pass must resolve this. + // the resulting mesh is ambiguous. A propagation pass must resolve this. // Marking it 'open' is the correct signal. If all are replicated/unknown, the // result is also replicated/unknown. - // rule: .replicated wins over .unknown. .open wins over everything. - var resulting_partition_spec: PartitionSpec = .unknown; - - for (0..axes_to_merge.len) |i| { - const current_axis = axes_to_merge.get(i); - new_dim *= self.dim(current_axis); - - if (i > 0) { - stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); - } - - switch (self.partition(current_axis)) { - .axis, .open => resulting_partition_spec = .open, - .replicated => if (resulting_partition_spec != .open) { - resulting_partition_spec = .replicated; - }, - .unknown => {}, - } - } + // rule: .replicated wins over .out_of_bound. .open wins over everything. const first_axis_to_merge = axes_to_merge.get(0); const num_axes_to_merge = axes_to_merge.len; - var new_shape = self; + var merged_dim: i64 = 1; + var merged_spec: Sharding.PartitionSpec = .out_of_bound; + for (axes_to_merge.constSlice()) |ax| { + merged_dim *= self.dim(ax); + const spec = self.partition(ax); + if (spec.isSharded() or spec == .open) { + merged_spec = .open; + } else if (spec == .replicated and merged_spec != .open) { + merged_spec = .replicated; + } + } - new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); + var new_shape = self; + for (1..num_axes_to_merge) |_| { + new_shape._sharding = new_shape._sharding.orderedRemove(first_axis_to_merge); + } + new_shape._sharding = new_shape._sharding.set(first_axis_to_merge, merged_spec); + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{merged_dim}) catch @panic("mergeAxis failed on dims"); new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); - new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); return new_shape; } @@ -1451,23 +1320,23 @@ pub const Shape = struct { test "mergeAxis partitioning" { // Merging a partitioned axis with a replicated one -> open const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .batch, .c = .replicated }); const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged1.partition(.ab).eql(.open)); + try testing.expect(merged1.partition(.ab) == .open); try testing.expect(merged1.dim(.ab) == 10); try testing.expect(merged1.rank() == 2); // Merging two replicated axes -> replicated const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated, .c = .feature }); const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged2.partition(.ab).eql(.replicated)); - try testing.expect(merged2.partition(.c).eql(.init(.y))); + try testing.expect(merged2.partition(.ab) == .replicated); + try testing.expect(merged2.partition(.c) == Sharding.PartitionSpec.sharded(1)); // Merging two unknown axes -> unknown - const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32).withDefaultPartitioning(); const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged3.partition(.ab).eql(.unknown)); + try testing.expect(merged3.partition(.ab) == .replicated); } pub fn mergeAxes(self: Shape, axes_: anytype) Shape { @@ -1641,6 +1510,37 @@ pub const Shape = struct { const x: Shape = .init(.{ 4, 8 }, .u2); try expectEqualShapes(.init(.{ 4, 2 }, .u8), x.packedShape()); } + + test "shape transformations preserve partition specs and clear unused slots" { + var shape = Shape.init(.{ 2, 3, 4 }, .f32); + shape._sharding = .{ .mesh = &test_mesh, .partition = Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .open).set(2, .mesh_axis_1) }; + for ([_]Shape{ + shape.insert(1, .{5}), + shape.append(.{5}), + shape.remove(1), + shape.removeMany(.{ 0, 1 }), + shape.transpose(.{ 2, 0, 1 }), + shape.reshape(.{24}), + shape.splitAxis(1, .{ 1, 3 }), + shape.mergeAxis(.merged, .{ 0, 1 }), + shape.withDefaultPartitioning(), + }) |transformed| { + try testing.expectEqual(&test_mesh, transformed._sharding.mesh); + } + const parts = shape._sharding.partition; + try testing.expectEqual(parts.insert(1, .replicated), shape.insert(1, .{5})._sharding.partition); + try testing.expectEqual(parts.insert(1, .replicated), shape.insertTag(1, 5, .new)._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.append(.{5})._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.appendDim(5, null)._sharding.partition); + try testing.expectEqual(parts.orderedRemove(1), shape.remove(1)._sharding.partition); + const removed = shape.removeMany(.{ 0, 1 }); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1), removed._sharding.partition); + try testing.expectEqual(Sharding.PartitionSpec.replicated, removed.append(.{5}).partition(1)); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1).set(1, .mesh_axis_0).set(2, .open), shape.transpose(.{ 2, 0, 1 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .replicated).set(2, .replicated).set(3, .mesh_axis_1), shape.splitAxis(1, .{ 1, 3 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withDefaultPartitioning()._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withReplicatedPartitioning()._sharding.partition); + } }; /// Iterates over all coordinates of a shape, yielding the multi-dimensional diff --git a/zml/tensor.zig b/zml/tensor.zig index 83d4beba..e630af1f 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -77,8 +77,11 @@ pub const Tensor = struct { /// Internal use /// /// Creates a tensor from a Shape and an mlir.Value. + /// Note: sharding information is discarded, the output tensor is considered open for replication. + /// Use _resultPropagateSharding when the shape sharding should be propagated. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { - const res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; + var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; + res._shape._sharding.partition = .open(sh.rank()); if (builtin.mode == .Debug) { // Check that the MLIR value actually have the same shape. @@ -91,9 +94,16 @@ pub const Tensor = struct { return res; } + pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { + var res = _result(sh, val); + res._shape._sharding = sh._sharding; + return res; + } + /// Creates a Tensor from a mlir.Value /// /// The shape is derived from the type of the mlir.Value. + /// mesh is `unknown` pub fn fromMlirValue(val: *const mlir.Value) Tensor { const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; @@ -101,13 +111,15 @@ pub const Tensor = struct { stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); - var sh: Shape = .{ ._dtype = ctx.dtype(ranked_tensor.elementType()) }; + var sh: Shape = .{ + ._dtype = ctx.dtype(ranked_tensor.elementType()), + ._dims = .empty, + ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, + ._sharding = .replicated, + }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } - sh._tags.appendNTimes(Shape.TagUnknown, n) catch unreachable; - sh._partitioning.appendNTimes(.unknown, n) catch unreachable; - return .{ ._shape = sh, ._value = val, .id = nextTensorId() }; } @@ -153,8 +165,24 @@ pub const Tensor = struct { return res; } - pub fn withPartitioning(self: Tensor, axes_: anytype) Tensor { - const partitioned_shape = self._shape.withPartitioning(axes_); + /// Specify the mesh of the input tensor. + /// * mesh: *const zml.Mesh, but during compilation the `.meshe_name` syntax can used to get + /// a known mesh from the compilation options. + /// * partition spec: a struct where the field names match the axis of the given mesh + /// + /// eg `x.withPartitioning(tp, .{ .h = .model }))` or `x.withPartitioning(.tp, .{ .h = .model }))` + pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { + if (@TypeOf(mesh_) == @EnumLiteral()) { + const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); + return self.withPartitioning(compiler.mesh(mesh_), partitioning); + } + + if (@TypeOf(partitioning) != Sharding.Partitioning) { + const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, .open, partitioning); + return self.withPartitioning(mesh_, parsed); + } + + const partitioned_shape = self._shape.withPartitioning(mesh_, partitioning); const ctx = Compiler.currentOrNull() orelse { var res = self; @@ -162,15 +190,9 @@ pub const Tensor = struct { return res; }; - const sharding = ctx.partitioning.selectSharding(partitioned_shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "{f}.withPartitioning({f}) failed to resolve because it's using unknown sharding. Pass more shardings to `zml.compile`. Known shardings: {f}", - .{ self, partitioned_shape, stdx.fmt.slice(ctx.partitioning.shardings) }, - ), - }; - const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape, sharding) catch @panic("OOM"); + const attr = ctx.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape) catch @panic("OOM"); - const op_result = switch (ctx.partitioning.partitioner) { + const op_result = switch (ctx.partitioner) { .shardy => blk: { const op = mlir.Operation.make(ctx.mlir_ctx, "sdy.sharding_constraint", .{ .operands = .{ .flat = &.{self.value()} }, @@ -201,7 +223,52 @@ pub const Tensor = struct { }, }; - return _result(partitioned_shape, op_result); + std.log.warn("{f}.withPartitioning({f}) -> {f}", .{ self, partitioned_shape, attr }); + return _resultPropagateSharding(partitioned_shape, op_result); + } + + /// Force the input tensor to be fully replicated. + /// This can be detrimental for performance. + pub fn replicate(self: Tensor) Tensor { + const compiler = Compiler.current(); + const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); + return self.withPartitioning(compiler.platform.replicated_mesh, partitioning); + } + + test withPartitioning { + // Create two similar meshes, + // Then check that tensor.withPartitioning don't confuse the meshes + const dp_mp: Sharding.Mesh = .{ + .name = "dp_mp", + .physical = undefined, + .logical = .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .folds = .empty, + .folds_consumed = .empty, + }; + + const mp_dp: Sharding.Mesh = .{ + .name = "mp_dp", + .physical = undefined, + .logical = .mesh(.{ .model = .low_bandwidth, .data = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .folds = .empty, + .folds_consumed = .empty, + }; + + const x = Tensor.init(.{ .h = 8 }, .f32).withPartitioning(&dp_mp, .{ .h = .model }); + try std.testing.expectEqual(&dp_mp, x.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(1), x.shape().partition(.h)); + try std.testing.expectEqual(&dp_mp, x.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2 = x.withPartitioning(&mp_dp, .{ .h = .model }); + try std.testing.expectEqual(&mp_dp, x2.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(0), x2.shape().partition(.h)); + try std.testing.expectEqual(&mp_dp, x2.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2_replicated = x.withPartitioning(&mp_dp, .{ .h = .replicated }); + try std.testing.expectEqual(&mp_dp, x2_replicated.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); } /// Copy the given tensor to the specified memory. @@ -257,7 +324,7 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcpyH2D, .{x_t}, platform, .{}); defer exe.deinit(); - var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), .replicated, @ptrCast(&inputs), .{ .memory = .host_pinned }); + var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), @ptrCast(&inputs), .{ .memory = .host_pinned }); defer x_h.deinit(); const x_h_ptr: [*]f32 = @ptrCast(@alignCast(x_h.opaqueDevicePtr(0))); @@ -359,6 +426,8 @@ pub const Tensor = struct { /// cannot use the donated input's contents after execution. Runners destroy its /// handle automatically; raw `Exe.call` leaves it for the caller to deinit. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. + /// + /// The sharding of the origin tensor is propagated to this one. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { const compilation_context = Compiler.current(); const scope = compilation_context.currentScope(); @@ -369,8 +438,30 @@ pub const Tensor = struct { .explicit => og_donation, }; scope.id_to_donation.put(scope.arena.allocator(), self.id, donation) catch @panic("OOM"); + + // The origin is mapped to a physical buffer either because it's an input or through `reuseBuffer` + // We can now look at its sharding. + if (origin._shape.isFullyReplicated()) { + stdx.debug.assert(self.byteSize() == origin.byteSize(), ".reuseBuffer expects two tensors with same byteSize, got {f} ({B}) and {f} ({B})", .{ self, self.byteSize(), origin, origin.byteSize() }); + } else { + // Origin is sharded, we can't just use the global byte size, we actually need the same byte size per shard. + // We are stricter here and require the same shape. + stdx.debug.assert(self._shape.eql(origin._shape), ".reuseBuffer expects two sharded tensors with same shape, got {f} and {f}", .{ self, origin }); + + // Normally self sharding is unspecified and we can just propagate origin sharding, + // but if the user called `.withPartitioning` explicitly it would be weird to override it silently, hence the check. + stdx.debug.assert(self._shape._sharding.isCompatible(origin._shape._sharding), ".reuseBuffer expects compatible shardings, got {f} and {f}", .{ self, origin }); + } + + var res = self; + res._shape._sharding = origin._shape._sharding; + return res; + } else { + // Origin tensor doesn't has any physical buffer associated so we can move on. + // This is allowed to allow a function like KVCache.update to call `reuseBuffer`, + // even though the caller may have constructed the inputs internally. + return self; } - return self; } test reuseBuffer { @@ -393,13 +484,13 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcopy, .{ x_t, y_t, z_t }, platform, .{}); defer exe.deinit(); - var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), .replicated, @ptrCast(&inputs)); + var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), @ptrCast(&inputs)); defer x_d.deinit(); - var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .replicated, .{}); + var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .{}); defer y_d.deinit(); - var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .replicated, .{}); + var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .{}); defer z_d.deinit(); var y_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; @@ -532,7 +623,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&i)); + var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -729,9 +820,9 @@ pub const Tensor = struct { return .{ ._state = .init(.{2}, .u64) }; } - pub fn initBuffer(io: std.Io, platform: *const Platform, sharding: Sharding, seed: u128) !Buffer { + pub fn initBuffer(rng: Rng, io: std.Io, platform: *const Platform, seed: u128) !Buffer { return .{ - ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), sharding, std.mem.asBytes(&seed)), + ._state = try .fromBytes(io, platform, rng._state._shape, std.mem.asBytes(&seed)), }; } @@ -753,12 +844,12 @@ pub const Tensor = struct { mlirx.Type.rankedTensor(mlirCtx(), sh), currentLoc(), ).appendTo(currentBlock()); - return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; + return .{ self.update(op.result(0)), _resultPropagateSharding(sh, op.result(1)) }; } fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ - ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), + ._state = _resultPropagateSharding(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, }; } @@ -799,7 +890,7 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - return .{ rng, floats.convert(shape_.dtype()) }; + return .{ rng, _resultPropagateSharding(shape_, floats.convert(shape_.dtype()).value()) }; } test uniform { @@ -834,7 +925,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); defer rng_buffer._state.deinit(); var stats: zml.Bufferized(Stats) = undefined; @@ -878,7 +969,7 @@ pub const Tensor = struct { const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, currentLoc()).appendTo(currentBlock()); - return _result(sh, op.result(0)); + return _resultPropagateSharding(sh, op.result(0)); } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. @@ -896,7 +987,8 @@ pub const Tensor = struct { // We don't want 0 to be sampled otherwise `log` will return -inf. .{ .min = std.math.floatEps(f32), .max = 1 }, ); - return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; + const result = u.log().scale(-1).log().scale(-1).convert(shape_.dtype()); + return .{ rand, _resultPropagateSharding(shape_, result.value()) }; } test gumbel { @@ -944,9 +1036,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); defer rng_buffer._state.deinit(); - var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), .replicated, @ptrCast(&tgt_dist_data)); + var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), @ptrCast(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); var stats: zml.Bufferized(Stats) = undefined; @@ -1294,7 +1386,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, @ptrCast(&x_f32)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), @ptrCast(&x_f32)); defer x_d_buffer.deinit(); var x_f4_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); @@ -1326,7 +1418,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, std.mem.sliceAsBytes(&x)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); var x_f8e3_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); @@ -1356,7 +1448,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), .replicated, @ptrCast(&x_u2)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), @ptrCast(&x_u2)); defer x_d_buffer.deinit(); var x_u8_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); @@ -1384,7 +1476,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), .replicated, @ptrCast(&x_f4_packed)); + var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), @ptrCast(&x_f4_packed)); defer x_f4_d.deinit(); var x_f32_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_f4_d}); @@ -1526,7 +1618,7 @@ pub const Tensor = struct { const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + var res_shape: Shape = .scalar(lhs.dtype()); // Validate batching axes var lhs_batching_axes: Axes = .empty; var rhs_batching_axes: Axes = .empty; @@ -1626,7 +1718,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -1670,27 +1762,27 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); defer input_buffer.deinit(); - var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), .replicated, std.mem.sliceAsBytes(&[3][4]f32{ + var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), std.mem.sliceAsBytes(&[3][4]f32{ .{ 1, 0, 2, 0 }, .{ 2, 1, 3, 3 }, .{ 1, 1, 0, 0 }, })); defer w_buffer.deinit(); - var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), .replicated, std.mem.sliceAsBytes(&[3][4]f32{ + var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), std.mem.sliceAsBytes(&[3][4]f32{ .{ 0, 0, 2, 1 }, .{ 1, 2, 1, 3 }, .{ 0, 0, 2, 0 }, })); defer v_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), .replicated, std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); defer b_buffer.deinit(); - var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), .replicated, std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); + var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); defer c_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ input_buffer, w_buffer, v_buffer, b_buffer, c_buffer }); @@ -1948,7 +2040,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -2236,7 +2328,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 2, 5 }, .f32); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); // Wrap slice to hide the anytype in the signature. @@ -2291,9 +2383,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.dynSliceResolved, .{ x, ax, .dyn(z, 2) }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), .replicated, std.mem.asBytes(&z_value)); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); defer z_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .dyn_start = z_buffer } }); @@ -2417,7 +2509,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -2481,7 +2573,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -3173,7 +3265,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), .replicated, std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); + var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); defer indices_buffer.deinit(); var operand_buffer: zml.Buffer = b: { const temp_func = struct { @@ -3391,11 +3483,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), .replicated, @ptrCast(&[2]i32{ 0, 2 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), @ptrCast(&[2]i32{ 0, 2 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); @@ -3414,11 +3506,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), .replicated, std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); defer updates_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); @@ -3436,9 +3528,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform, .{}); defer exe.deinit(); - var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), .replicated, std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); + var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); defer operand_buffer.deinit(); - var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), .replicated, std.mem.sliceAsBytes(&[2][2][3][2]i32{ + var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ .{ .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, @@ -3449,7 +3541,7 @@ pub const Tensor = struct { }, })); defer start_indices_buffer.deinit(); - var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), .replicated, std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); + var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); defer values_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, start_indices_buffer, values_buffer }); defer result.deinit(); @@ -3554,7 +3646,7 @@ pub const Tensor = struct { defer exe.deinit(); { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; @@ -3571,7 +3663,7 @@ pub const Tensor = struct { } { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; @@ -3628,7 +3720,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); defer x_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -3647,7 +3739,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f16{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, @@ -3678,7 +3770,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 89, 31, 22, 42, 64, 39, 0, 30, 64, 71, 46, 31, @@ -4015,7 +4107,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._split, .{ x, ax, &args }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); const res = try std.testing.allocator.alloc(zml.Buffer, expectation.len); @@ -4143,11 +4235,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{4})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); @@ -4169,11 +4261,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); @@ -4198,11 +4290,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); @@ -4228,13 +4320,13 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{-1})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); @@ -4260,13 +4352,13 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); @@ -4306,12 +4398,12 @@ pub const Tensor = struct { stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); - const p = self.shape()._partitioning.get(a); + const p = self.shape()._sharding.partition.get(a); var res_shape = self._shape; res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; - // TODO(Corentin): Not sure about that - res_shape._partitioning.replaceRange(a, 1, &.{ p, p }) catch unreachable; + + res_shape._sharding.partition = res_shape._sharding.partition.insert(a, p); const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); @@ -4340,7 +4432,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -4388,7 +4480,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 3, 3 }, .u8); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[3][3]u8{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ .{ 1, 1, 1 }, .{ 1, 1, 1 }, .{ 1, 1, 1 }, @@ -4570,9 +4662,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var xs: zml.Buffer = undefined; @@ -4647,9 +4739,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var z = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, y_buffer }); @@ -4799,7 +4891,7 @@ test "Tensor.maxPool1d" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; @@ -4841,7 +4933,7 @@ test "Tensor.maxPool2d" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; diff --git a/zml/testing.zig b/zml/testing.zig index e2e84502..366351d7 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -26,8 +26,8 @@ pub fn env() *const Platform { }, ) catch @panic("Pjrt not available"); - // Register shardings required by flash attention. - _ = platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; + // Register meshes required by flash attention. + _ = platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; _platform = platform; } @@ -394,7 +394,7 @@ pub fn testLayer( activation_store: zml.io.TensorStore.View, name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), - shardings: []const *const zml.Sharding, + meshes: []const *const zml.Mesh, opts: CompareOpts, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); @@ -424,12 +424,12 @@ pub fn testLayer( var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; - tensor.* = ctx_.activation_store.createTensor(subkey, null, .replicated); + tensor.* = ctx_.activation_store.createReplicatedTensor(subkey, null); ctx_.index += 1; } }.cb, &ctx, &args); - const exe = try platform.compile(allocator, io, layer, func, args, .{ .shardings = if (shardings.len != 0) shardings else &.{} }); + const exe = try platform.compile(allocator, io, layer, func, args, .{ .meshes = if (meshes.len != 0) meshes else &.{} }); defer exe.deinit(); const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); diff --git a/zml/zml.zig b/zml/zml.zig index 995318eb..2eb84a17 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -45,6 +45,7 @@ pub const tracer = @import("profiling/tracer.zig"); pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; +pub const Mesh = Sharding.Mesh; pub const Sharding = @import("Sharding.zig"); pub const slice = @import("slice.zig"); pub const Slice = slice.Slice;