diff --git a/README.md b/README.md index f35453ff..d9e68b81 100644 --- a/README.md +++ b/README.md @@ -145,8 +145,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createTensor("weight", .{ .d_out, .d }, null), - .bias = store.createTensor("bias", .{.d_out}, null), + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), }; } @@ -168,10 +168,10 @@ const Mnist = struct { io: std.Io, platform: *const zml.Platform, store: *const zml.io.TensorStore, - shardings: []const zml.Sharding, + meshes: []const *const zml.Mesh, ) !zml.Bufferized(Mnist) { return zml.io.load(Mnist, self, allocator, io, platform, store, .{ - .shardings = shardings, + .meshes = meshes, .parallelism = 1, .dma_chunks = 1, .dma_chunk_size = 16 * 1024 * 1024, diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index d940a801..077f5b34 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -22,6 +22,7 @@ run_binary( "$(location libnrt.so.1)", "$(location @aws-neuronx-runtime-lib//:libnrt.patchelf)", ], + target_compatible_with = ["@platforms//os:linux"], tool = "@llvm//tools:llvm-strip", ) @@ -35,6 +36,7 @@ run_binary( "$(location libncfw.so.2)", "$(location @aws-neuronx-runtime-lib//:libncfw.patchelf)", ], + target_compatible_with = ["@platforms//os:linux"], tool = "@llvm//tools:llvm-strip", ) @@ -76,18 +78,18 @@ run_binary( zig_binary( name = "zml-smi", - visibility = ["//visibility:public"], data = [":sandbox"], main = "main.zig", + visibility = ["//visibility:public"], deps = [ + "//bin/zml-smi/api", "//bin/zml-smi/collector", "//bin/zml-smi/csv", - "//bin/zml-smi/json", - "//bin/zml-smi/api", - "//bin/zml-smi/prometheus", - "//bin/zml-smi/utils:sysfs", "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/prometheus", "//bin/zml-smi/tui", + "//bin/zml-smi/utils:sysfs", "//stdx", "@libvaxis//:vaxis", "@rules_zig//zig/runfiles", @@ -97,9 +99,9 @@ zig_binary( }) + select({ "@llvm//platforms/config:linux_x86_64": [ "//bin/zml-smi/platforms/amdsmi", - "//bin/zml-smi/platforms/oneapi", "//bin/zml-smi/platforms/neuron", "//bin/zml-smi/platforms/nvml", + "//bin/zml-smi/platforms/oneapi", "//bin/zml-smi/platforms/tpu", "//platforms/neuron:libnrt_headers", ], @@ -168,8 +170,8 @@ _RELEASE_PLATFORMS = [ [platform_transition_filegroup( name = "zml-smi-" + os + "-" + arch, srcs = [":archive_" + os + "_" + arch], - target_platform = "//platforms:" + os + "_" + arch, tags = ["manual"], + target_platform = "//platforms:" + os + "_" + arch, ) for os, arch in _RELEASE_PLATFORMS] [hashes( @@ -195,6 +197,7 @@ oci_image( name = "image_", base = "@distroless_cc_debian12", entrypoint = ["/zml-smi/zml-smi"], + tags = ["manual"], target_compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", @@ -202,7 +205,6 @@ oci_image( tars = [ ":archive_linux_amd64", ], - tags = ["manual"], ) platform_transition_filegroup( diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel index e3ac355d..dac6b935 100644 --- a/bin/zml-smi/platforms/amdsmi/BUILD.bazel +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -2,12 +2,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "amdsmi", - import_name = "zml-smi/platforms/amdsmi", - main = "metrics.zig", srcs = [ "amdsmi.zig", "process.zig", ], + import_name = "zml-smi/platforms/amdsmi", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = [ "//bin/zml-smi/collector", diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel index 3a9e54ed..0ab0fae1 100644 --- a/bin/zml-smi/platforms/nvml/BUILD.bazel +++ b/bin/zml-smi/platforms/nvml/BUILD.bazel @@ -2,12 +2,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "nvml", - import_name = "zml-smi/platforms/nvml", - main = "metrics.zig", srcs = [ "nvml.zig", "process.zig", ], + import_name = "zml-smi/platforms/nvml", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = [ "//bin/zml-smi/collector", diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index d05e2b6f..3d246f11 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -190,7 +190,7 @@ convert it to a `TensorStore`: ```zig pub fn main(init: std.process.Init) !void { ... - var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); defer model_store.deinit(); } ``` @@ -205,18 +205,18 @@ pub fn main(init: std.process.Init) !void { const mlp: Mlp = .{ .up_proj = .init( - mlp_view.createTensor("up_proj.weight", .{ .dout, .d }, null), - mlp_view.maybeCreateTensor("up_proj.bias", .{.dout}, null), + mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), .d, ), .gate_proj = .init( - mlp_view.createTensor("gate_proj.weight", .{ .dout, .d }, null), - mlp_view.maybeCreateTensor("gate_proj.bias", .{.dout}, null), + mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), .d, ), .down_proj = .init( - mlp_view.createTensor("down_proj.weight", .{ .d, .dout }, null), - mlp_view.maybeCreateTensor("down_proj.bias", .{.d}, null), + mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), + mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), .dout, ), }; @@ -287,7 +287,7 @@ pub const Wrapper = struct { pub fn main(init: std.process.Init) !void { ... - var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry); + var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); defer activations_store.deinit(); try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 1bdaad7b..7ff70707 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -5,7 +5,7 @@ const zml = @import("zml"); const stdx = zml.stdx; pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.dot(b, .k).withPartitioning(.{ .m = .m, .n = .replicated }); + return a.dot(b, .k).withPartitioning(.benchmark_mesh, .{ .m = .m, .n = .replicated }); } pub fn main(init: std.process.Init) !void { @@ -26,34 +26,31 @@ pub fn main(init: std.process.Init) !void { log.info("\n{f}", .{platform.fmtVerbose()}); - const benchmark_sharding: zml.Sharding = try platform.registerSharding("benchmark_mesh", .mesh( + const mesh: *const zml.Mesh = try platform.registerMesh("mesh", .mesh( .{ .m = .low_bandwidth, .n = .high_bandwidth }, )); const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); - const a_shape = zml.Shape.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) - .withPartitioning(.{ .m = .m, .k = .replicated }); - const b_shape = zml.Shape.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) - .withPartitioning(.{ .k = .replicated, .n = .n }); - - const a: zml.Tensor = .fromShape(a_shape); - const b: zml.Tensor = .fromShape(b_shape); + const a = zml.Tensor.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .m = .m, .k = .replicated }); + const b = zml.Tensor.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .k = .replicated, .n = .n }); var exe = blk: { log.info("⏱️ Compiling benchmark...", .{}); const now: std.Io.Timestamp = .now(io, .awake); defer log.info("✅ Compiled benchmark [{f}]", .{now.untilNow(io, .awake)}); - break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .shardings = &.{benchmark_sharding} }); + break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .meshes = &.{mesh} }); }; defer exe.deinit(); var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); - var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), benchmark_sharding, random); + var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), random); defer a_buffer.deinit(); - var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), benchmark_sharding, random); + var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), random); defer b_buffer.deinit(); var exe_args = try exe.args(allocator); @@ -96,7 +93,7 @@ pub fn main(init: std.process.Init) !void { }); } -fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, sharding: zml.Sharding, random: std.Random) !zml.Buffer { +fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); defer slice.free(allocator); @@ -122,5 +119,5 @@ fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const }, } - return .fromSlice(io, platform, slice, sharding); + return .fromSlice(io, platform, slice); } diff --git a/examples/io/main.zig b/examples/io/main.zig index 29215866..1cec23b0 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -155,9 +155,9 @@ pub fn main(init: std.process.Init) !void { try stdout_writer.interface.flush(); }, .load => { - const ShardingType = enum { replicated, sharded }; + const MeshType = enum { replicated, sharded }; - const sharding_type: ShardingType = std.meta.stringToEnum(ShardingType, it.next() orelse "sharded") orelse return error.InvalidShardingKind; + const mesh_type: MeshType = std.meta.stringToEnum(MeshType, it.next() orelse "sharded") orelse return error.InvalidMesheKind; const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); @@ -165,7 +165,13 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + const tensor_parallel: *const zml.Mesh = try platform.registerMesh( + "tp", + .mesh(.{ .model = .high_bandwidth }), + ); + const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); defer store.deinit(); const AllTensorsModel = struct { @@ -180,22 +186,17 @@ pub fn main(init: std.process.Init) !void { var registry_it = registry.iterator(); var load_count: usize = 0; while (registry_it.next()) |entry| : (load_count += 1) { - tensors[load_count] = switch (sharding_type) { - .replicated => store.view().createTensor(entry.key_ptr.*, null, .replicated), + tensors[load_count] = switch (mesh_type) { + .replicated => store.view().createReplicatedTensor(entry.key_ptr.*, null), .sharded => if (entry.value_ptr.shape.rank() > 0) - store.view().createTensor(entry.key_ptr.*, null, .{ ._0 = .model }) + store.view().createTensor(entry.key_ptr.*, null, .tp, tp_partitioning) else - store.view().createTensor(entry.key_ptr.*, null, .replicated), + store.view().createReplicatedTensor(entry.key_ptr.*, null), }; } const model: AllTensorsModel = .{ .tensors = tensors }; - const sharded_sharding: zml.Sharding = try platform.registerSharding( - "playground_model", - .mesh(.{ .model = .high_bandwidth }), - ); - var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); progress.increaseEstimatedTotalItems(load_count); defer progress.end(); @@ -215,7 +216,7 @@ pub fn main(init: std.process.Init) !void { }); defer loader.deinit(); - try loader.load(io, AllTensorsModel, &model, &buffers, &store, &.{sharded_sharding}, .{ .progress = &progress }); + try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); try loader.await(io); const took = now.untilNow(io, .awake); diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 4407aadb..e10db570 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -109,7 +109,9 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + // Defines how the model's tensors are sharded across the available devices. + const meshes: models.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); const generation: models.GenerationOptions = .{ @@ -121,9 +123,6 @@ pub fn main(init: std.process.Init) !void { var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); defer model.deinit(allocator); - // Defines how the model's tensors are sharded across the available devices. - const shardings: models.Shardings = try .init(platform); - // // Load the model and compile it // @@ -135,11 +134,11 @@ pub fn main(init: std.process.Init) !void { var compiled_model = try allocator.create(models.CompiledModel); defer allocator.destroy(compiled_model); - compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, shardings, args.seqlen, &progress); + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); defer compiled_model.deinit(); // Load buffers after the model compilation to be sure to give enough room to the autotune. - var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress, shardings); + var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); defer model.unloadBuffers(&model_buffers, allocator); progress.end(); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index b88a855c..20aa60c7 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -3,7 +3,7 @@ const std = @import("std"); const zml = @import("zml"); const common = @import("models/common.zig"); -pub const Shardings = common.Shardings; +pub const Meshes = common.Meshes; pub const GenerationOptions = common.GenerationOptions; pub const parseConfig = common.parseConfig; pub const lfm2 = @import("models/lfm2.zig"); @@ -55,12 +55,19 @@ pub const LoadedModel = union(ModelType) { } } - pub fn loadBuffers(self: *LoadedModel, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !Buffers { + pub fn loadBuffers( + self: *LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { return switch (self.*) { inline else => |*m, t| @unionInit( Buffers, @tagName(t), - try m.loadBuffers(allocator, io, platform, store, progress, shardings), + try m.loadBuffers(allocator, io, platform, store, progress), ), }; } @@ -77,7 +84,7 @@ pub const LoadedModel = union(ModelType) { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: Shardings, + meshes: Meshes, seqlen: usize, progress: *std.Progress.Node, ) !CompiledModel { @@ -85,7 +92,7 @@ pub const LoadedModel = union(ModelType) { inline else => |*m, t| @unionInit( CompiledModel.Inner, @tagName(t), - try m.compile(allocator, io, platform, backend, shardings, seqlen, progress), + try m.compile(allocator, io, platform, backend, meshes, seqlen, progress), ), }; return .{ diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index debeff00..62fc7560 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -42,11 +42,11 @@ pub const Phase = enum { } }; -pub const Shardings = struct { - model: zml.Sharding, - experts: zml.Sharding, +pub const Meshes = struct { + model: *const zml.Mesh, + experts: *const zml.Mesh, - pub fn init(platform: *zml.Platform) !Shardings { + pub fn init(platform: *zml.Platform) !Meshes { switch (platform.target) { .tpu => { const has_link_y = platform.physical_mesh.hasAxis(.link_y); @@ -59,18 +59,18 @@ pub const Shardings = struct { } return .{ - .model = try platform.registerShardingWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), - .experts = try platform.registerShardingWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), + .model = try platform.registerMeshWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), + .experts = try platform.registerMeshWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), }; }, .cuda, .rocm, .oneapi, .neuron, .metal, .cpu => return .{ - .model = try platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })), - .experts = try platform.registerSharding("experts", .mesh(.{ .experts = .high_bandwidth })), + .model = try platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })), + .experts = try platform.registerMesh("experts", .mesh(.{ .experts = .high_bandwidth })), }, } } - pub fn all(self: Shardings) [2]zml.Sharding { + pub fn all(self: Meshes) [2]*const zml.Mesh { return .{ self.model, self.experts }; } }; diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 407df36e..92ab4462 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -17,9 +17,9 @@ pub const CompilationParameters = struct { attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, seqlen: u32, - shardings: common.Shardings, + meshes: common.Meshes, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); const cache: model.Cache = .{ .kv = .init(.init(.{ @@ -28,7 +28,7 @@ pub const CompilationParameters = struct { .k = seqlen, .h = config.num_key_value_heads, .hd = config.hidden_size / config.num_attention_heads, - }, mdl.embed_tokens.weight.dtype())), + }, mdl.embed_tokens.weight.dtype()), meshes.model), .conv = .init(.init(.{ .layer = mdl.num_conv_layers, .batch = 1, @@ -42,10 +42,10 @@ pub const CompilationParameters = struct { .batch_dim = 1, .rng = .init(), .cache = cache, - .attention_metadata = .init(.fromBackend(backend, seqlen, config.num_attention_heads)), + .attention_metadata = .init(.fromBackend(backend, seqlen, config.num_attention_heads), meshes.model), .attention_parameters = .init(.fromBackend(backend)), .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, }; } }; @@ -235,7 +235,7 @@ fn compileEmbed( defer phase.logCompileDone(log, "embed_tokens", io, from); return zml.FnExe(model.TokenEmbedding.forward).compile(allocator, io, platform, .{ - .shardings = &opts.shardings.all(), + .meshes = &opts.meshes.all(), .program_name = phase.programName("lfm2", "embed_tokens"), }, .{.{ .embedding = embed_tokens, @@ -273,7 +273,7 @@ fn compileLayer( } else unreachable; return zml.FnExe(model.DecoderLayer.forward).compile(allocator, io, platform, .{ - .shardings = &opts.shardings.all(), + .meshes = &opts.meshes.all(), .program_name = phase.programName("lfm2", if (kind == .conv) "conv_layer" else "attn_layer"), }, .{.{ .layer = layer, @@ -306,7 +306,7 @@ fn compileSample( defer phase.logCompileDone(log, "lm_head", io, from); return zml.FnExe(model.LmHead.forward).compile(allocator, io, platform, .{ - .shardings = &opts.shardings.all(), + .meshes = &opts.meshes.all(), .program_name = phase.programName("lfm2", "lm_head"), }, .{.{ .lm_head = mdl.lm_head, diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 53f1071c..1d627025 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -79,7 +79,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -94,8 +93,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -117,11 +115,11 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); return inference.CompiledModel.init(allocator, io, @constCast(platform), self, self.inner, params, progress); } }; @@ -173,7 +171,6 @@ pub const Model = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -188,8 +185,7 @@ pub const Model = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - loader.load(io, Model, self, &buffers, store, &all_shardings); + loader.load(io, Model, self, &buffers, store); try loader.await(io); const took = now.untilNow(io, .awake); @@ -278,7 +274,7 @@ pub const TokenEmbedding = struct { }; pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { - return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }; + return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }; } pub fn forward(input: Input) Output { @@ -448,16 +444,16 @@ pub const ShortConv = struct { pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); return .{ - .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), - .kernel = store.createTensor("conv.weight", .{ .out, .in, .kernel_size }, .replicated), + .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .kernel = store.createReplicatedTensor("conv.weight", .{ .out, .in, .kernel_size }), .config = config, }; } pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, actual_seq_len: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { var cache = cache_; - const BCx = self.in_proj.forward(input.withPartitioning(.{ .d = .replicated })); + const BCx = self.in_proj.forward(input.withPartitioning(.model, .{ .d = .replicated })); const B, const C, const x = BCx.chunkExact(.d, 3); const Bx = B.mul(x); @@ -523,10 +519,10 @@ pub const Attention = struct { const head_dim = config.hidden_size / config.num_attention_heads; const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; return .{ - .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), + .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), .q_layernorm = RmsNorm.init(store.withPrefix("q_layernorm"), config.norm_eps, .hd), .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), .head_dim = head_dim, @@ -544,7 +540,7 @@ pub const Attention = struct { attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, ) struct { zml.Tensor, KvCache } { - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); @@ -608,8 +604,8 @@ pub const Cache = struct { conv: ConvCache, kv: KvCache, - pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Cache) { - return .{ .conv = try self.conv.initBuffers(allocator, io, platform, sharding), .kv = try self.kv.initBuffers(io, platform, sharding) }; + pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Cache) { + return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; } pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { @@ -629,12 +625,12 @@ pub const ConvCache = struct { return .{ .state = .fromShape(shape) }; } - pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(ConvCache) { + pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvCache) { const sh = self.state.shape(); const host = try allocator.alloc(u8, sh.byteSize()); defer allocator.free(host); @memset(host, 0); - return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, sharding, host) }; + return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; } pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { @@ -650,13 +646,13 @@ pub const KvCache = struct { k: zml.Tensor, v: zml.Tensor, - pub fn init(kv_shape: zml.Shape) KvCache { - const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); return .{ .k = .fromShape(sharded_shape), .v = .fromShape(sharded_shape) }; } - pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { - return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}) }; + pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; } pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { @@ -717,9 +713,9 @@ const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ - .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), - .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), + .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), }; } @@ -740,7 +736,7 @@ const RmsNorm = struct { tag: zml.Shape.Tag, pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { - return .{ .weight = store.createTensor("weight", .{tag}, .replicated), .eps = eps, .tag = zml.Shape.toTag(tag) }; + return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 67287861..b59a0416 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -30,9 +30,9 @@ pub const Session = struct { model_buffers: *model.Buffers, ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.model); + var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); errdefer model.Cache.unloadBuffers(&cache_buffers); - var rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); errdefer generated_token_slice.free(allocator); @@ -120,21 +120,21 @@ pub const Session = struct { @memset(tokens, self.config.pad_token_id); @memcpy(tokens[0..all_tokens.len], all_tokens); - var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice, .replicated); + var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); defer tokens_buf.deinit(); const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, .replicated); + var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); defer tokens_pos_buf.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, .replicated); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); const params = self.compiled_model.params; var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - else => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + else => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); @@ -161,18 +161,18 @@ pub const Session = struct { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, .replicated); + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, .replicated); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); const params = self.compiled_model.params; - var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model); + var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); generation: while (true) { @@ -194,7 +194,7 @@ pub const Session = struct { if (all_tokens.items.len >= self.seqlen) break :generation; const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); - var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, .replicated); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); defer token_pos_buffer.deinit(); var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index cbc33811..2da991ec 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -40,16 +40,16 @@ pub fn main(init: std.process.Init) !void { defer repo.close(io); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const shardings: common.Shardings = try .init(platform); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); defer repo_model.unloadBuffers(&model_buffers, allocator); const backend = args.backend orelse zml.attention.Backend.auto(platform); @@ -58,11 +58,11 @@ pub fn main(init: std.process.Init) !void { repo_model.parsed_config.value, repo_model.parsed_config.value.max_position_embeddings, backend, - shardings, + meshes, ); progress.end(); - try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters); + try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters, meshes); } pub fn run( @@ -75,11 +75,12 @@ pub fn run( model_buffers: *lfm2.Buffers, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, + meshes: common.Meshes, ) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer activation_store.deinit(); var ctx = TestContext{ @@ -89,7 +90,7 @@ pub fn run( .activations_store = &activation_store, .attention_metadata = attention_metadata, .attention_parameters = attention_parameters, - .sharding = platform.replicated_sharding, + .mesh = platform.replicated_mesh, }; try ctx.testLayer("embed_tokens", .{ .batch, .seq }, mdl.embed_tokens, model_buffers.embed_tokens, .{}); @@ -134,7 +135,7 @@ const TestContext = struct { activations_store: *zml.io.TensorStore, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, - sharding: zml.Sharding, + mesh: *const zml.Mesh, fn testLayerPrint(self: *TestContext, comptime name_fmt: []const u8, name_args: anytype, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { const name = try std.fmt.allocPrint(self.allocator, name_fmt, name_args); @@ -147,19 +148,19 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(tagz); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); defer out_buffer_expected.deinit(); const exe = if (comptime @TypeOf(layer) == model.TokenEmbedding) - try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .shardings = &.{self.sharding} }) + try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .meshes = &.{self.mesh} }) else - try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .shardings = &.{self.sharding} }); + try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); var args = try exe.args(self.allocator); @@ -185,42 +186,42 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache", .{name}); defer self.allocator.free(cache_key); - var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key, self.sharding); + var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key); defer cache_buffer.deinit(); const cache_tensor = zml.Tensor.fromShape(cache_buffer.shape()).withTags(.{ .layer, .batch, .seq, .d }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); defer out_buffer_expected.deinit(); const actual_seq_len_tensor: zml.Tensor = .init(.{}, .u32); const actual_seq_len: u32 = @intCast(in_tensor.dim(.seq)); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .shardings = &.{self.sharding} }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, self.sharding); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); - var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice, self.sharding); + var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); defer cache_index_buf.deinit(); var runner = try exe.runner(self.allocator); @@ -243,41 +244,41 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const key_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.key", .{name}); defer self.allocator.free(key_cache_key); - var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key, self.sharding); + var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key); defer key_cache_buffer.deinit(); const key_cache_tensor = zml.Tensor.fromShape(key_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const value_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.value", .{name}); defer self.allocator.free(value_cache_key); - var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key, self.sharding); + var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key); defer value_cache_buffer.deinit(); const value_cache_tensor = zml.Tensor.fromShape(value_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); defer out_buffer_expected.deinit(); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .shardings = &.{self.sharding} }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; - var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform, self.sharding); + var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32); @@ -297,7 +298,7 @@ const TestContext = struct { } }; -fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8, sharding: zml.Sharding) !zml.Buffer { +fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { const shape = store.view().getShape(key) orelse return error.NotFound; const host_bytes = try allocator.alloc(u8, shape.byteSize()); @@ -309,5 +310,5 @@ fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml _ = try reader.interface.readSliceAll(host_bytes); - return zml.Buffer.fromBytes(io, platform, shape, sharding, host_bytes); + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); } diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 4eafdcea..331ee0dc 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -18,9 +18,9 @@ pub const CompilationParameters = struct { prefill_attention_parameters: zml.attention.Parameters, decode_attention_parameters: zml.attention.Parameters, seqlen: usize, - shardings: common.Shardings, + meshes: common.Meshes, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { const head_dim = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads); return .{ @@ -32,11 +32,11 @@ pub const CompilationParameters = struct { .k = seqlen, .h = config.num_key_value_heads, .hd = head_dim, - }, mdl.model.embed_tokens.weight.dtype())), + }, mdl.model.embed_tokens.weight.dtype()), meshes.model), .rng = .init(), .attention_metadata = switch (backend) { .attnd => .{ .attnd = .init() }, - else => .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads))), + else => .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads)), meshes.model), }, .prefill_attention_parameters = switch (backend) { .attnd => .{ .attnd = .init(.{ @@ -59,7 +59,7 @@ pub const CompilationParameters = struct { else => .init(.fromBackend(backend)), }, .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, }; } }; @@ -211,7 +211,7 @@ fn compileKernel( defer phase.logCompileDone(log, "forward", io, from); return KernelExe.compile(allocator, io, platform, .{ - .shardings = ¶meters.shardings.all(), + .meshes = ¶meters.meshes.all(), .program_name = phase.programName("llama", "forward"), }, .{.{ .weights = llama_model, diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index b86c6d6b..ae988b97 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -76,7 +76,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -91,8 +90,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -114,11 +112,11 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } @@ -144,6 +142,7 @@ pub const Model = struct { const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensor( "weight", .{ .dout, .d }, + .model, .{ .dout = .model, .d = .replicated }, )) |weight| .init(weight, null, .d) @@ -168,7 +167,6 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.Sharding, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -184,7 +182,7 @@ pub const Model = struct { }); defer loader.deinit(); - loader.load(io, Model, self, &buffers, store, shardings); + loader.load(io, Model, self, &buffers, store); try loader.await(io); const took = now.untilNow(io, .awake); @@ -256,10 +254,11 @@ const Llama = struct { .embed_tokens = .{ .weight = store.createTensor( "embed_tokens.weight", .{ .voc, .d }, + .model, .{ .voc = .replicated, .d = .model }, ) }, .norm = .{ - .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .{ .d = .replicated }), + .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = config.rms_norm_eps, }, .layers = layers, @@ -329,7 +328,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.{ .d = .replicated }) }; + .withPartitioning(.model, .{ .d = .replicated }) }; } }; @@ -426,7 +425,7 @@ pub const TransformerLayer = struct { stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); // Keep the residual stream replicated to avoid repeated gathers before q/k/v. - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const x0_normalized = self.input_layernorm.forward(x0_replicated); const delta0, const updated_kv_cache = self.self_attn.forward( x0_normalized, @@ -438,13 +437,13 @@ pub const TransformerLayer = struct { ); // Fully Connected - const x1 = x0_replicated.add(delta0).withPartitioning(.{ .d = .replicated }); + const x1 = x0_replicated.add(delta0).withPartitioning(.model, .{ .d = .replicated }); const x1_normalized = self.post_attention_layernorm.forward(x1); const x2 = self.mlp.forward(x1_normalized) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }) + .withPartitioning(.model, .{ .d = .replicated }) .add(x1) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(.model, .{ .d = .replicated }); return .{ .hidden = x2.reuseBuffer(x0), .kv_cache = updated_kv_cache }; } @@ -456,7 +455,7 @@ const RmsNorm = struct { pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { return .{ - .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), + .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps, }; } @@ -480,9 +479,9 @@ const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ - .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .{ .d = .model }), null, .d), + .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), }; } @@ -515,10 +514,10 @@ const SelfAttn = struct { var rope_scaling = config.rope_scaling; rope_scaling.setRopeTheta(config.rope_theta); return .{ - .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .{ .d = .model }), null, .d), + .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), // TODO(Corentin): fix that .q_norm = null, .k_norm = null, @@ -555,7 +554,7 @@ const SelfAttn = struct { // Make hidden state replicated once and reuse it across q/k/v projections. // This avoids paying gather-style collectives independently for each projection. - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); var q = self.q_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); var k = self.k_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); @@ -605,7 +604,7 @@ const SelfAttn = struct { const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); const delta = self.o_proj.forward(attn, attn.dtype()) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(.model, .{ .d = .replicated }); return .{ delta, new_kv_cache }; } }; @@ -616,8 +615,8 @@ pub const KvCache = struct { pub const Buffer = zml.Bufferized(KvCache); - pub fn init(kv_shape: zml.Shape) KvCache { - const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); return .{ .k = .fromShape(sharded_shape), @@ -625,10 +624,10 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !Buffer { + pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform) !Buffer { return .{ - .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), }; } diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 2269464f..6a04cf11 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -29,8 +29,7 @@ pub const Session = struct { compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - const shardings = &compiled_model.params.shardings; - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const token_index_buffers = try allocator.alloc(zml.Buffer, compiled_model.params.seqlen); @@ -49,7 +48,7 @@ pub const Session = struct { const conversation_id: u64 = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); @@ -138,7 +137,7 @@ pub const Session = struct { @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice, .replicated); + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice); defer prefill_tokens_buffer.deinit(); const params = self.compiled_model.params; @@ -149,7 +148,7 @@ pub const Session = struct { .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32), } }, .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + .vanilla, .cuda_fa2, .cuda_fa3, .nki => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); @@ -174,7 +173,7 @@ pub const Session = struct { defer self.allocator.free(decoder_out_buffer); var last_token_id: u32 = self.last_generated_token; - var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), .replicated, @ptrCast(&last_token_id)); + var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); const params = self.compiled_model.params; @@ -184,7 +183,7 @@ pub const Session = struct { .layer_id = try .scalar(self.io, self.platform, 0, .u16), .num_tokens = try .scalar(self.io, self.platform, 1, .u32), } }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 98798887..1b5894f1 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -38,20 +38,20 @@ pub fn main(init: std.process.Init) !void { defer repo.close(io); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const shardings: common.Shardings = try .init(platform); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); defer repo_model.unloadBuffers(&model_buffers, allocator); progress.end(); - try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, platform.replicated_sharding); + try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); } fn run( @@ -61,28 +61,28 @@ fn run( activations_path: []const u8, mdl: model.Model, model_buffers: *model.Buffers, - sharding: zml.Sharding, + meshes: []const *const zml.Mesh, ) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); defer activation_store.deinit(); - try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, sharding, .{ .absolute_tolerance = 1e-3 }); + try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); if (mdl.model.layers.len == 0) return; const layer = mdl.model.layers[0]; const layer_buffers = model_buffers.model.layers[0]; - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, sharding, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, sharding, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, sharding, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, sharding, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, sharding, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, sharding, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, sharding, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); } fn testLayer( @@ -93,19 +93,19 @@ fn testLayer( name: []const u8, layer: anytype, layer_weights: zml.Bufferized(@TypeOf(layer)), - sharding: zml.Sharding, + meshes: []const *const zml.Mesh, opts: zml.testing.CompareOpts, ) !void { const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); defer allocator.free(in_key); const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; - var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key, sharding); + var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_shape); const out_key = try std.fmt.allocPrint(allocator, "{s}.out", .{name}); defer allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key, sharding); + var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key); defer out_buffer_expected.deinit(); // `zml.nn.Linear.forward` takes an explicit output dtype; every other layer here @@ -116,7 +116,7 @@ fn testLayer( return if (Layer == zml.nn.Linear) l.forward(x, x.dtype()) else Layer.forward(l, x); } }; - const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .shardings = &.{sharding} }); + const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .meshes = meshes }); defer exe.deinit(); var args = try exe.args(allocator); @@ -133,7 +133,7 @@ fn testLayer( try zml.testing.expectClose(io, out_result, out_buffer_expected, opts); } -fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8, sharding: zml.Sharding) !zml.Buffer { +fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { const shape = store.getShape(key) orelse return error.NotFound; const host_bytes = try allocator.alloc(u8, shape.byteSize()); @@ -145,5 +145,5 @@ fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *cons _ = try reader.interface.readSliceAll(host_bytes); - return zml.Buffer.fromBytes(io, platform, shape, sharding, host_bytes); + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); } diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 33a41086..6bfb71c5 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -15,18 +15,18 @@ pub const CompilationParameters = struct { kv_cache: model.KvCache, rng: zml.Tensor.Rng, seqlen: u32, - shardings: common.Shardings, + meshes: common.Meshes, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, meshes: common.Meshes) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); return .{ .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), .decode_tokens = .init(.{ .b = 1, .s = 1 }, .u32), .token_index = .init(.{}, .u32), - .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), .rng = .init(), .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, }; } }; @@ -244,7 +244,7 @@ fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.P defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "embed tokens", io, from); - return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ + return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ .embedding = .{ .embed_tokens = mdl.text_model.embed_tokens }, .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), }}); @@ -256,9 +256,9 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "full attention layer", io, from); - return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ + return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .token_index = parameters.token_index, .cache = .{ .k = parameters.kv_cache.self_attn.k, @@ -274,9 +274,9 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "linear attention layer", io, from); - return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ + return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .active_length = zml.Tensor.init(.{}, .u32), .cache = .{ .conv_state = parameters.kv_cache.gated_delta_net.conv_state, @@ -292,19 +292,19 @@ fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml. defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "sampler", io, from); - return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ + return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ .sampler = mdl.sampler(), - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .rng = parameters.rng, .token_index = parameters.token_index, }}); } -fn hiddenTensor(mdl: model.Model, seqlen: usize) zml.Tensor { +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { return .fromShape(zml.Shape.init( .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, mdl.text_model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated })); + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); } fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 5a2a41ac..14edd022 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -81,7 +81,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -96,8 +95,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -120,12 +118,12 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { _ = backend; - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), meshes); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } }; @@ -161,7 +159,7 @@ pub const Model = struct { return .{ .text_model = try .init(allocator, store.withPrefix("model.language_model"), config), .lm_head = .init( - store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), + store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), null, .d, ), @@ -180,7 +178,7 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.Sharding, + meshes: []const *const zml.Mesh, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -196,7 +194,7 @@ pub const Model = struct { }); defer loader.deinit(); - loader.load(io, Model, self, &buffers, store, shardings); + loader.load(io, Model, self, &buffers, store, meshes); try loader.await(io); const took = now.untilNow(io, .awake); @@ -286,7 +284,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.{ .d = .replicated }) }; + .withPartitioning(.model, .{ .d = .replicated }) }; } }; @@ -308,7 +306,7 @@ pub const TextModel = struct { } return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, .layers = layers, .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), }; @@ -414,7 +412,7 @@ pub const TransformerLayer = struct { active_length: zml.Tensor, kv_cache: KvCache.LayerView, ) struct { zml.Tensor, KvCache } { - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; @@ -440,18 +438,18 @@ pub const TransformerLayer = struct { }, } - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); - return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + return .{ mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; } pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { const self = input.layer; const x0 = input.hidden; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const self_attn = switch (self.attn) { @@ -460,12 +458,12 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); return .{ - .hidden = mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -473,7 +471,7 @@ pub const TransformerLayer = struct { pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { const self = input.layer; const x0 = input.hidden; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const linear_attn = switch (self.attn) { @@ -482,12 +480,12 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); return .{ - .hidden = mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -501,18 +499,18 @@ pub const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ .up_proj = .init( - store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .gate_proj = .init( - store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .down_proj = .init( - store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .{ .d = .replicated, .dout = .model }), - store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .{ .d = .replicated }), + store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .model, .{ .d = .replicated, .dout = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), .dout, ), }; @@ -549,8 +547,8 @@ pub const SelfAttn = struct { fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { return .init( - store.createTensor("weight", .{ .dout, .d }, partitions), - store.maybeCreateTensor("bias", .{.dout}, bias_partitions), + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), .d, ); } @@ -594,26 +592,26 @@ pub const SelfAttn = struct { return .{ k, v }; } - fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - return switch (kv_head_sharding) { + fn partitionProjectedKv(kv: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { + return switch (kv_head_meshe) { .sharded => |heads| blk: { const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; - break :blk sharded_kv.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => kv.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), }; } - fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + fn partitionCachedKv(tensor: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { var cache_tensor = tensor.rename(.{ .s = .k }); - return switch (kv_head_sharding) { + return switch (kv_head_meshe) { .sharded => |heads| blk: { if (heads.factor != 1) { cache_tensor = cache_tensor.stutter1d(cache_tensor.axis(.h), heads.factor); } - break :blk cache_tensor.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + break :blk cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => cache_tensor.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), }; } @@ -623,18 +621,15 @@ pub const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = zml.Compiler.current().partitioning.shardableDim( - k.shape().withPartitioning(.{ .h = .model }), - .h, - q.dim(.h), - ) catch unreachable; + const kv_head_meshe = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -646,22 +641,22 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }); - k = partitionCachedKv(k, kv_head_sharding); - v = partitionCachedKv(v, kv_head_sharding); + k = partitionCachedKv(k, kv_head_meshe); + v = partitionCachedKv(v, kv_head_meshe); const attn_output = zml.attention.attention( q, k, v, token_index, - zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads), tp), zml.attention.Parameters.init(.fromBackend(.vanilla)), ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); @@ -669,7 +664,7 @@ pub const SelfAttn = struct { const projected_output = self.o_proj .forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(tp, .{ .d = .replicated }); return .{ projected_output, new_kv_cache }; } @@ -772,7 +767,7 @@ pub const GatedDeltaNet = struct { conv_kernel_size: i64, fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, partitions), null, .d); + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); } pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { @@ -784,9 +779,9 @@ pub const GatedDeltaNet = struct { .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), - .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), - .dt_bias = store.createTensor("dt_bias", .{.vh}, .{ .vh = .model }), - .aLog = store.createTensor("A_log", .{.vh}, .{ .vh = .model }), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), .num_k_heads = config.text_config.linear_num_key_heads, .num_v_heads = config.text_config.linear_num_value_heads, @@ -873,10 +868,10 @@ pub const GatedDeltaNet = struct { const conv_dim = 2 * key_dim + value_dim; const left_pad = self.conv_kernel_size - 1; - const x_in = x.withPartitioning(.{ .d = .replicated }); + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) .rename(.{ .dout = .mix }) - .withPartitioning(.{ .s = .replicated, .mix = .model }); + .withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; const conv_input = b: { if (use_cached_state) break :b zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s); @@ -906,7 +901,7 @@ pub const GatedDeltaNet = struct { if (use_cached_state) { mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); } - mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const z = self.in_proj_z.forward(x_in, x_in.dtype()) .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); @@ -959,7 +954,7 @@ pub const GatedDeltaNet = struct { const output = self.out_proj .forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(.model, .{ .d = .replicated }); const updated_cache = cache.update( if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), last_recurrent_state, @@ -973,7 +968,7 @@ pub const RmsNorm = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { @@ -994,7 +989,7 @@ pub const RmsNormGated = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { @@ -1026,7 +1021,7 @@ pub const KvCache = struct { pub const Buffers = zml.Bufferized(SelfAttnCache); - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_sharding: zml.Sharding) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -1035,10 +1030,10 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const kv_head_sharding = model_sharding.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_sharding) { - .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(.{ .h = .model }), - .replicated => kv_shape.withPartitioning(.{ .h = .replicated }), + const kv_head_meshe = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_meshe) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), }; return .{ .k = .fromShape(sharded_kv_shape), @@ -1047,10 +1042,10 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !SelfAttnCache.Buffers { + pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !SelfAttnCache.Buffers { return .{ - .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1127,7 +1122,7 @@ pub const KvCache = struct { pub const Buffers = zml.Bufferized(GatedDeltaNetCache); - pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + @@ -1145,19 +1140,19 @@ pub const KvCache = struct { .khd = config.text_config.linear_key_head_dim, .vhd = config.text_config.linear_value_head_dim, }, recurrent_dtype); - const sharded_conv_state_shape = conv_state_shape.withPartitioning(.{ .mix = .model }); - const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(.{ .vh = .model }); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); return .{ - .conv_state = .fromShape(sharded_conv_state_shape), - .recurrent_state = .fromShape(sharded_recurrent_state_shape), + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !GatedDeltaNetCache.Buffers { + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !GatedDeltaNetCache.Buffers { return .{ - .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), - .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1231,19 +1226,19 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, - model_sharding: zml.Sharding, + model_meshe: *const zml.Mesh, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_sharding), - .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype), + .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), + .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { return .{ - .self_attn = try self.self_attn.initBuffer(io, platform, sharding), - .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform, sharding), + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), }; } diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index a49940cc..84963773 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -31,11 +31,11 @@ pub const Session = struct { compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, compiled_model.params.shardings.model); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; @@ -111,9 +111,7 @@ pub const Session = struct { @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - const replicated_sharding: zml.Sharding = .replicated; - - var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice); defer prefill_tokens_buffer.deinit(); var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); @@ -141,9 +139,8 @@ pub const Session = struct { const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); - const replicated_sharding: zml.Sharding = .replicated; - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, replicated_sharding); + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 80c6be65..f29ecabd 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -12,17 +12,17 @@ pub const CompilationParameters = struct { rng: zml.Tensor.Rng, moe_backend: zml.moe.Backend, seqlen: u32, - shardings: common.Shardings, + meshes: common.Meshes, xla_dump_to: ?[]const u8, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, meshes: common.Meshes) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); return .{ - .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), .rng = .init(), .moe_backend = moe_backend, .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, .xla_dump_to = "/home/ubuntu/xla_dump", }; } @@ -274,7 +274,7 @@ fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.P fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardSelfAttn, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .token_index = zml.Tensor.init(.{}, .u32), .cache = .{ .k = parameters.kv_cache.self_attn.k, @@ -289,7 +289,7 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .active_length = zml.Tensor.init(.{}, .u32), .cache = .{ .conv_state = parameters.kv_cache.gated_delta_net.conv_state, @@ -304,7 +304,7 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { return compileExe(allocator, io, platform, model.Sampler.sampleTokens, .{.{ .sampler = mdl.text_model.sampler(), - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .rng = parameters.rng, .token_index = zml.Tensor.init(.{}, .u32), }}, parameters, progress, phase, "sampling"); @@ -329,16 +329,16 @@ fn compileExe( const now: std.Io.Timestamp = .now(io, .awake); defer log.info("Compiled {s} {s} [{f}]", .{ phase, component, now.untilNow(io, .awake) }); return zml.FnExe(function).compile(allocator, io, platform, .{ - .shardings = ¶meters.shardings.all(), + .meshes = ¶meters.meshes.all(), .xla_dump_to = parameters.xla_dump_to, }, args); } -fn hiddenTensor(mdl: model.Model, seqlen: usize) zml.Tensor { +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { return .fromShape(zml.Shape.init( .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, mdl.text_model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated })); + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); } fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 6b6f9bb0..59d52815 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -47,26 +47,26 @@ pub const RopeParameters = struct { rope_theta: f32, }; -fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - return switch (kv_head_sharding) { +fn partitionProjectedKv(kv: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { + return switch (kv_head_meshe) { .sharded => |heads| blk: { const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; - break :blk sharded_kv.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => kv.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), }; } -fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { +fn partitionCachedKv(tensor: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { var kv = tensor.rename(.{ .s = .k }); - return switch (kv_head_sharding) { + return switch (kv_head_meshe) { .sharded => |heads| blk: { if (heads.factor != 1) { kv = kv.stutter1d(kv.axis(.h), heads.factor); } - break :blk kv.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + break :blk kv.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => kv.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => kv.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), }; } @@ -107,7 +107,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -122,8 +121,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -145,7 +143,7 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { @@ -153,7 +151,7 @@ pub const LoadedModel = struct { const moe_gate_up = self.inner.text_model.layers[0].moe.gate_up_proj; log.info("Moe dtype : {}", .{moe_gate_up.weight.dtype()}); const moe_backend = try zml.moe.Backend.auto(platform, moe_gate_up.quantizationScheme(), moe_gate_up.weight.dtype()); - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, meshes); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } }; @@ -196,7 +194,7 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.Sharding, + meshes: []const *const zml.Mesh, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -212,7 +210,7 @@ pub const Model = struct { }); defer loader.deinit(); - loader.load(io, Model, self, &buffers, store, shardings); + loader.load(io, Model, self, &buffers, store, meshes); try loader.await(io); const took = now.untilNow(io, .awake); @@ -314,10 +312,10 @@ pub const TextModel = struct { } return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, .layers = layers, .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), - .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), null, .d), + .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), null, .d), .gen_options = gen_options, }; } @@ -447,7 +445,7 @@ pub const TransformerLayer = struct { const self = input.layer; const x0 = input.hidden; _ = input.config; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const self_attn = switch (self.attn) { @@ -456,13 +454,13 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); return .{ - .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -471,7 +469,7 @@ pub const TransformerLayer = struct { const self = input.layer; const x0 = input.hidden; _ = input.config; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const linear_attn = switch (self.attn) { @@ -480,13 +478,13 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); return .{ - .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -501,7 +499,7 @@ pub const TransformerLayer = struct { moe_backend: zml.moe.Backend, ) struct { zml.Tensor, KvCache } { _ = config; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; @@ -519,12 +517,12 @@ pub const TransformerLayer = struct { }, } - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, moe_backend); - return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }), updated_kv_cache }; + return .{ moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }), updated_kv_cache }; } }; @@ -550,8 +548,8 @@ pub const SelfAttn = struct { fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { return .init( - store.createTensor("weight", .{ .dout, .d }, partitions), - store.maybeCreateTensor("bias", .{.dout}, bias_partitions), + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), .d, ); } @@ -605,19 +603,16 @@ pub const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = zml.Compiler.current().partitioning.shardableDim( - k.shape().withPartitioning(.{ .h = .model }), - .h, - q.dim(.h), - ) catch unreachable; + const kv_head_meshe = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); - q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + gate = gate.withPartitioning(tp, .{ .s = .replicated, .d_out_proj = .model }); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -629,28 +624,28 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); - q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - k = partitionCachedKv(k, kv_head_sharding); - v = partitionCachedKv(v, kv_head_sharding); + q = q.rename(.{ .s = .q }).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }); + k = partitionCachedKv(k, kv_head_meshe); + v = partitionCachedKv(v, kv_head_meshe); const attn_output = zml.attention.attention( q, k, v, token_index, - zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads), tp), zml.attention.Parameters.init(.fromBackend(.vanilla)), - ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + ).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); - const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); return .{ projected_output, new_kv_cache }; } @@ -667,20 +662,20 @@ pub const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ .up_proj = .init( - store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .up_proj_scale = null, .gate_proj = .init( - store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .gate_proj_scale = null, .down_proj = .init( - store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .replicated, .d = .model }), - store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .{ .d = .replicated }), + store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .replicated, .d = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), .d, ), .down_proj_scale = null, @@ -708,8 +703,8 @@ const Router = struct { pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { return .{ .router = .init( - store.createTensor("weight", .{ .expert, .d }, .{ .expert = .replicated, .d = .replicated }), - store.maybeCreateTensor("bias", .{.expert}, .{ .expert = .replicated }), + store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), + store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), .d, ), .num_experts_per_tok = num_experts_per_tok, @@ -742,19 +737,21 @@ pub const Moe = struct { const gate_up_proj_tensor = experts_store.createTensor( "gate_up_proj", .{ .expert, .dout, .d }, + .experts, .{ .expert = .experts, .dout = .replicated, .d = .replicated }, ); const down_proj_tensor = experts_store.createTensor( "down_proj", .{ .expert, .d, .dout }, + .experts, .{ .expert = .experts, .d = .replicated, .dout = .replicated }, ); return .{ .shared_expert = Mlp.init(store.withPrefix("shared_expert")), .shared_expert_gate = .init( - store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .{ .dout = .replicated, .d = .replicated }), - store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .{ .dout = .replicated }), + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), .d, ), .gate_up_proj = .init(gate_up_proj_tensor, null, .d), @@ -786,7 +783,7 @@ pub const Moe = struct { ); const shared_gate = self.shared_expert_gate.forward(x, x.dtype()).sigmoid().broad(x.shape()); - const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.{ + const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.model, .{ .b = .replicated, .s = .replicated, .d = .replicated, @@ -896,7 +893,7 @@ pub const GatedDeltaNet = struct { conv_kernel_size: i64, fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, partitions), null, .d); + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); } pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { @@ -911,9 +908,9 @@ pub const GatedDeltaNet = struct { .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), .out_proj_scale = null, - .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), - .dt_bias = store.createTensor("dt_bias", .{.vh}, .{ .vh = .model }), - .aLog = store.createTensor("A_log", .{.vh}, .{ .vh = .model }), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), .num_k_heads = config.text_config.linear_num_key_heads, .num_v_heads = config.text_config.linear_num_value_heads, @@ -997,9 +994,9 @@ pub const GatedDeltaNet = struct { const conv_dim = 2 * key_dim + value_dim; const left_pad = self.conv_kernel_size - 1; - const x_in = x.withPartitioning(.{ .d = .replicated }); + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) - .rename(.{ .dout = .mix }).withPartitioning(.{ .s = .replicated, .mix = .model }); + .rename(.{ .dout = .mix }).withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; const conv_input = if (use_cached_state) zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) @@ -1029,26 +1026,26 @@ pub const GatedDeltaNet = struct { if (use_cached_state) { mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); } - mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const z = self.in_proj_z.forward(x_in, x_in.dtype()) .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); - const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); - const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); + const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); + const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); const query = mixed_qkv .slice(.mix, .{ .start = 0, .end = key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); const key = mixed_qkv .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); const value = mixed_qkv .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); const beta = b.sigmoid(); const aLog_type = self.aLog.dtype(); @@ -1073,10 +1070,10 @@ pub const GatedDeltaNet = struct { z.rename(.{ .vhd = .d }), ) .rename(.{ .d = .vhd }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) - .rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + .rename(.{ .dout = .d }).withPartitioning(.model, .{ .d = .replicated }); const updated_cache = cache.update( if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), last_recurrent_state, @@ -1090,7 +1087,7 @@ pub const RmsNorm = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { @@ -1111,7 +1108,7 @@ pub const RmsNormGated = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { @@ -1141,7 +1138,7 @@ pub const KvCache = struct { v: zml.Tensor, layer_index: zml.Tensor, - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_sharding: zml.Sharding) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -1150,22 +1147,22 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const kv_head_sharding = model_sharding.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_sharding) { - .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(.{ .h = .model }), - .replicated => kv_shape.withPartitioning(.{ .h = .replicated }), + const kv_head_meshe = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_meshe) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), }; return .{ - .k = .fromShape(sharded_kv_shape), - .v = .fromShape(sharded_kv_shape), + .k = zml.Tensor.fromShape(sharded_kv_shape), + .v = zml.Tensor.fromShape(sharded_kv_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(SelfAttnCache) { + pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(SelfAttnCache) { return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1238,7 +1235,7 @@ pub const KvCache = struct { recurrent_state: zml.Tensor, layer_index: zml.Tensor, - pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; const conv_state_shape = zml.Shape.init(.{ @@ -1254,19 +1251,19 @@ pub const KvCache = struct { .khd = config.text_config.linear_key_head_dim, .vhd = config.text_config.linear_value_head_dim, }, recurrent_dtype); - const sharded_conv_state_shape = conv_state_shape.withPartitioning(.{ .mix = .model }); - const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(.{ .vh = .model }); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); return .{ - .conv_state = .fromShape(sharded_conv_state_shape), - .recurrent_state = .fromShape(sharded_recurrent_state_shape), + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(GatedDeltaNetCache) { + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(GatedDeltaNetCache) { return .{ - .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), - .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1334,19 +1331,19 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, - model_sharding: zml.Sharding, + model_meshe: *const zml.Mesh, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_sharding), - .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype), + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), + .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { return .{ - .self_attn = try self.self_attn.initBuffer(io, platform, sharding), - .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform, sharding), + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), }; } diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index e1509df5..35998797 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -31,12 +31,11 @@ pub const Session = struct { compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - const shardings = compiled_model.params.shardings; - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; @@ -122,7 +121,7 @@ pub const Session = struct { @memset(tokens_slice.items(u32), 0); @memcpy(tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice, .replicated); + var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice); defer tokens_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); defer token_index_buffer.deinit(); @@ -149,7 +148,7 @@ pub const Session = struct { const out_tokens_buffer = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, .replicated); + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); defer token_index_buffer.deinit(); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 3b4349c4..d1291c1b 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -19,8 +19,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createTensor("weight", .{ .d_out, .d }, .replicated), - .bias = store.createTensor("bias", .{.d_out}, .replicated), + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), }; } @@ -49,7 +49,7 @@ const Mnist = struct { var loader: zml.io.Loader = try .init(allocator, platform, .default); errdefer loader.deinit(); - try loader.load(io, Mnist, self, &buffers, store, &.{}, .{}); + try loader.load(io, Mnist, self, &buffers, store, .{}); try loader.await(io); return buffers; @@ -66,7 +66,10 @@ const Mnist = struct { pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; - for (layers) |layer| { + const compiler = zml.Compiler.current(); + for (0.., layers) |i, layer| { + compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); + defer compiler.popLocation(); x = layer.forward(x); } return x.argMax(0).indices.convert(.u8); @@ -87,15 +90,15 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); defer registry.deinit(); - // Init model - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - const mnist_model: Mnist = .init(store.view()); - // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); + // Init model + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + defer store.deinit(); + const mnist_model: Mnist = .init(store.view()); + // // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); var exe = blk: { @@ -137,7 +140,7 @@ pub fn main(init: std.process.Init) !void { var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample), .replicated); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); defer input_buffer.deinit(); printDigit(sample); diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 61a16318..cc46a1bd 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -2,7 +2,7 @@ const std = @import("std"); const zml = @import("zml"); -const log = std.log.scoped(.sharding); +const log = std.log.scoped(.mesh); pub const std_options: std.Options = .{ .log_level = .info, @@ -34,10 +34,10 @@ const DemoModel = struct { var y = x.dot(self.w, .feature); y = y.add(self.b.broad(y.shape())); - y = y.withPartitioning(.{ .batch = .data, .hidden = .model }); + y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); y.print("dense_out"); - const gate = y.scale(0.01).sigmoid(); + const gate = y.scale(0.01).sigmoid().withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); return zml.ops.manualComputation( (struct { y: zml.Tensor, @@ -142,7 +142,6 @@ fn createSequenceBuffer( io: std.Io, platform: *const zml.Platform, shape: zml.Shape, - sharding: zml.Sharding, start: f32, ) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); @@ -152,7 +151,7 @@ fn createSequenceBuffer( e.* = start + @as(f32, @floatFromInt(i)); } - return zml.Buffer.fromSlice(io, platform, slice, sharding); + return zml.Buffer.fromSlice(io, platform, slice); } pub fn main(init: std.process.Init) !void { @@ -193,24 +192,24 @@ pub fn main(init: std.process.Init) !void { } log.info("{f}", .{platform.physical_mesh}); - const sharding: zml.Sharding = try platform.registerSharding( - "demo_mesh", + const mesh: *const zml.Mesh = try platform.registerMesh( + "dp_mp", .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), ); - log.info("{f}", .{sharding.data.logical}); - log.info("{f}", .{sharding}); + log.info("{f}", .{mesh.logical}); + log.info("{f}", .{mesh}); const input_shape = zml.Shape.init(.{ .batch = 16, .feature = 32 }, .f32) - .withPartitioning(.{ .batch = .data, .feature = .replicated }); + .withPartitioning(mesh, .{ .batch = .data, .feature = .replicated }); const w_shape = zml.Shape.init(.{ .feature = 32, .hidden = 64 }, .f32) - .withPartitioning(.{ .feature = .replicated, .hidden = .model }); + .withPartitioning(mesh, .{ .feature = .replicated, .hidden = .model }); const b_shape = zml.Shape.init(.{ .hidden = 64 }, .f32) - .withPartitioning(.{ .hidden = .model }); + .withPartitioning(mesh, .{ .hidden = .model }); - const input: zml.Tensor = .fromShape(input_shape); - const w: zml.Tensor = .fromShape(w_shape); - const b: zml.Tensor = .fromShape(b_shape); + const input: zml.Tensor = zml.Tensor.fromShape(input_shape); + const w: zml.Tensor = zml.Tensor.fromShape(w_shape); + const b: zml.Tensor = zml.Tensor.fromShape(b_shape); const model: DemoModel = .init(w, b); var exe = try platform.compile( @@ -221,16 +220,16 @@ pub fn main(init: std.process.Init) !void { .{input}, .{ .partitioner = args.partitioner, - .shardings = &.{sharding}, + .meshes = &.{mesh}, }, ); defer exe.deinit(); - var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, sharding, 0.0); + var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, 0.0); defer w_buf.deinit(); - var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, sharding, 100.0); + var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, 100.0); defer b_buf.deinit(); - var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, sharding, 1000.0); + var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, 1000.0); defer input_buf.deinit(); log.info("input placement: {f}", .{input_buf}); diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index b337cc9a..7f7aed55 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -20,6 +20,7 @@ zig_shared_library( cc_library( name = "libzml_cuda_repo", + target_compatible_with = ["@platforms//os:linux"], deps = select({ "@llvm//platforms/config:linux_x86_64": ["@libzml_cuda_linux_amd64//:libzml_cuda"], "@llvm//platforms/config:linux_aarch64": ["@libzml_cuda_linux_arm64//:libzml_cuda"], @@ -28,6 +29,7 @@ cc_library( cc_library( name = "cuda_nvtx_headers", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = select({ "@llvm//platforms/config:linux_x86_64": ["@cuda_nvtx_linux_x86_64//:headers"], diff --git a/stdx/debug.zig b/stdx/debug.zig index 4686b3b5..a217eca1 100644 --- a/stdx/debug.zig +++ b/stdx/debug.zig @@ -10,11 +10,12 @@ pub inline fn internalAssert(check: bool, comptime msg: []const u8, args: anytyp pub inline fn assert(check: bool, comptime msg: []const u8, args: anytype) void { if (!check) { + @branchHint(.cold); panic(msg, args); } } -pub inline fn panic(comptime format: []const u8, args: anytype) noreturn { +pub fn panic(comptime format: []const u8, args: anytype) noreturn { std.debug.panic(format, args); } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 47ec59aa..33351d93 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -5,6 +5,16 @@ pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { return .{ .slice = any_slice }; } +/// Properly format a slice of []const u8. +pub fn strings(strs: []const []const u8) FmtStrings { + return .{ .strings = strs }; +} + +/// Properly format a slice of [*:0]const u8. +pub fn stringsZ(strs: []const [*:0]const u8) FmtZeroStrings { + return .{ .strings = strs }; +} + fn FmtSlice(T: type) type { return struct { slice: []const T, @@ -31,6 +41,26 @@ fn FmtSlice(T: type) type { }; } +pub const FmtStrings = struct { + strings: []const []const u8, + + pub fn format(f: FmtStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStr, f.strings, .{}, 1, writer); + } + + pub fn formatNumber(f: FmtStrings, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + return try formatIntSlice(f.strings, n, 1, writer); + } +}; + +pub const FmtZeroStrings = struct { + strings: []const [*:0]const u8, + + pub fn format(f: FmtZeroStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStrZ, f.strings, .{}, 1, writer); + } +}; + pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), @@ -77,6 +107,14 @@ pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !vo try writer.alignBufferOptions(if (value) "1" else "0", .{ .alignment = spec.alignment, .fill = spec.fill }); } +pub fn formatStr(value: []const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(value); +} + +pub fn formatStrZ(value: [*:0]const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(std.mem.span(value)); +} + pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; const T = @TypeOf(value); diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index e8c8519d..844778f5 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -16,10 +16,11 @@ sh_binary( sh_binary( name = "rocprofv3", + srcs = ["rocprofv3.sh"], data = [ "@amdrocm-profiler-base//:rocprofv3_tool", "@libzml_rocm//:sandbox", ], - srcs = ["rocprofv3.sh"], + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], ) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 23259b1d..41f175ae 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -20,7 +20,6 @@ const Platform = @import("platform.zig").Platform; const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); -const Partitioning = Sharding.Partitioning; const Tensor = @import("tensor.zig").Tensor; const Compiler = @This(); @@ -35,7 +34,8 @@ mlir_ctx: *mlir.Context, mlir_pass_manager: *mlir.PassManager, module: *mlir.Module, platform: *const Platform, -partitioning: Sharding.Partitioning, +meshes: []const *const Sharding.Mesh, +partitioner: Sharding.Partitioner, mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), @@ -74,7 +74,7 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { pub const Options = struct { /// Incremented once after successful compilation. progress: std.Progress.Node = .none, - shardings: []const Sharding = &.{}, + meshes: []const *const Sharding.Mesh = &.{}, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, // Debugging options @@ -161,16 +161,13 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, } } - // Ensure replicated sharding is always included as a fallback option. - var shardings = std.ArrayList(Sharding).initCapacity(arena.allocator(), opts.shardings.len + 1) catch @panic("OOM"); - var needs_replicated: bool = true; - for (opts.shardings) |sharding| { - if (sharding.data == platform.replicated_sharding.data) needs_replicated = false; - shardings.appendAssumeCapacity(sharding.resolve(platform)); + var mesh_list: std.ArrayList(*const Sharding.Mesh) = .empty; + mesh_list.appendSlice(arena.allocator(), opts.meshes) catch @panic("OOM"); + if (std.mem.indexOfScalar(*const Sharding.Mesh, mesh_list.items, platform.replicated_mesh) == null) { + mesh_list.append(arena.allocator(), platform.replicated_mesh) catch @panic("OOM"); } - if (needs_replicated) shardings.appendAssumeCapacity(platform.replicated_sharding); - - const partitioning = Sharding.Partitioning.init(opts.partitioner orelse .fromTarget(platform.target), shardings.items) catch @panic("OOM"); + const meshes = mesh_list.items; + validateMeshes(meshes) catch |err| stdx.debug.panic("Incompatible meshes: {t}", .{err}); return .{ .allocator = allocator, @@ -182,7 +179,8 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, .mlir_known_types = mlir_known_types, .module = module, .platform = platform, - .partitioning = partitioning, + .partitioner = opts.partitioner orelse .fromTarget(platform.target), + .meshes = meshes, .location = unknown_location, .unknown_location = unknown_location, }; @@ -276,7 +274,8 @@ test pushLocation { .mlir_known_types = undefined, .module = undefined, .platform = undefined, - .partitioning = undefined, + .partitioner = undefined, + .meshes = undefined, .location = unknown_location, .unknown_location = unknown_location, }; @@ -328,6 +327,63 @@ pub fn allocPrint(self: *Compiler, comptime fmt: []const u8, args: anytype) []u8 return std.fmt.allocPrint(self.arena.allocator(), fmt, args) catch self.abortOOM(); } +/// Lookup a mesh by it's name +pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (compiler.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + if (name == .replicated) return compiler.platform.replicated_mesh; + std.debug.panic( + \\Found no meshes named {s}. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(compiler.meshes) }); +} + +/// Find a mesh that covers all the given axes +pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { + if (@TypeOf(logical_axes) != []const Shape.Tag) { + const comp_tags = comptime Shape.parseTags(logical_axes); + const parsed_tags: []const Shape.Tag = comptime comp_tags.constSlice(); + return compiler.resolveMesh(parsed_tags); + } + + var ok_meshe: ?*const Sharding.Mesh = null; + for (compiler.meshes) |shd| { + var covers_all: bool = true; + for (logical_axes) |ax| { + const input_axis = Shape.toTag(ax); + var covers_this: bool = false; + for (shd.logical.axes.slice()) |existing_axis| { + if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { + covers_this = true; + break; + } + } + covers_all = covers_all and covers_this; + } + if (covers_all) { + if (ok_meshe) |first_match| { + std.debug.panic( + \\Found two meshes covering axes: {any}, expected exacty one. + \\- First match: {f} + \\- Second match: {f} + , .{ logical_axes, first_match, shd }); + } + ok_meshe = shd; + } + } + + return ok_meshe orelse std.debug.panic( + \\Found no meshes covering axes: {any}, expected exacty one. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ logical_axes, stdx.fmt.slice(compiler.meshes) }); +} + pub fn Typed(comptime func: anytype) type { return struct { pub fn compile( @@ -402,9 +458,9 @@ pub fn compileInternal( _ = result.func.appendTo(compiler.module.body()); - const num_partitions = compiler.partitioning.numPartitions(); - const num_replicas = compiler.partitioning.numReplicas(); - const num_devices = compiler.partitioning.numDevices(); + const num_partitions = compiler.meshes[0].numPartitions(); + const num_replicas = compiler.meshes[0].numReplicas(); + const num_devices = num_partitions * num_replicas; compiler.module.operation().setAttributeByName( "mhlo.num_partitions", @@ -422,10 +478,7 @@ pub fn compileInternal( }, }; - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compiler.module, compiler.partitioning, opts) catch |err| { + const loaded_executable = compiler.compileModuleToPjrtExecutable(opts) catch |err| { compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Pjrt failed to compile the following MLIR"); return err; }; @@ -441,8 +494,6 @@ pub fn compileInternal( // This will get copied into exe result.input_info.items(.shape), result.output_info.items(.shape), - result.input_info.items(.sharding), - result.output_info.items(.sharding), result.input_info.items(.aliasing_output), ); errdefer exe.deinit(); @@ -455,16 +506,15 @@ fn addPartitionerOperations(ctx: *Compiler) !void { const allocator = ctx.arena.allocator(); const mlir_ctx = ctx.mlir_ctx; const module = ctx.module; - const partitioning = ctx.partitioning; - switch (partitioning.partitioner) { + switch (ctx.partitioner) { .gspmd => {}, .shardy => { - for (partitioning.shardings) |sharding| { - const attr_str = try sharding.data.sdyMeshAttr(allocator); + for (ctx.meshes) |shd| { + const attr_str = try shd.sdyMeshAttr(allocator); defer allocator.free(attr_str); - const name = sharding.data.name; + const name = shd.name; const mesh_attr = try mlir.Attribute.parse(mlir_ctx, attr_str); const mesh_op = mlir.Operation.make(mlir_ctx, "sdy.mesh", .{ @@ -491,7 +541,6 @@ const EmitMlirResult = struct { pub const TensorInfo = struct { id: Tensor.Id, shape: Shape, - sharding: Sharding, value: *const mlir.Value, // Only used for input tensors, stores which output tensor ends up with their buffer @@ -501,12 +550,12 @@ pub const TensorInfo = struct { var attrs: AttributeList = .empty; const mlir_ctx = compiler.mlir_ctx; - const sharding_attr = try compiler.partitioning.tensorShardingAttr(arena, mlir_ctx, info.shape, info.sharding); - const name = switch (compiler.partitioning.partitioner) { + const meshe_attr = try compiler.tensorShardingAttr(arena, mlir_ctx, info.shape); + const name = switch (compiler.partitioner) { .gspmd => "mhlo.sharding", .shardy => "sdy.sharding", }; - attrs.appendAssumeCapacity(.named(mlir_ctx, name, sharding_attr)); + attrs.appendAssumeCapacity(.named(mlir_ctx, name, meshe_attr)); const memory = scope.id_to_memory.get(info.id) orelse .device; if (memory != .device) { @@ -583,16 +632,9 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou defer ctx.current_argument_id += 1; - const input_sharding = ctx.compiler.partitioning.selectSharding(packed_shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "Failed to resolve sharding for input {f}({d}) because it's using unknown sharding. Pass more shardings to `platform.compile`. Known shardings: {f}", - .{ packed_shape, ctx.current_argument_id, stdx.fmt.slice(ctx.compiler.partitioning.shardings) }, - ), - }; try ctx.infos.append(ctx.compiler.allocator, .{ .id = tensor.id, .shape = og_shape, - .sharding = input_sharding, .value = value, }); } @@ -615,7 +657,6 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf fn cb(ctx: *@This(), tensor: *const Tensor) !void { const og_shape = tensor.shape(); - const packed_shape = og_shape.packedShape(); var value = ctx.scope.id_to_argument.get(tensor.id) orelse tensor._value orelse @panic("no value found for output tensor"); @@ -626,9 +667,9 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf try ctx.infos.append(ctx.compiler.allocator, .{ .id = tensor.id, + // const packed_shape = og_shape.packedShape(); + // TODO: clarify why this og_shape and not packedShape() .shape = og_shape, - // Note: the panic should have been triggered during createBlockArguments or emitMlir - .sharding = ctx.compiler.partitioning.selectSharding(packed_shape) catch @panic("failed to resolve output sharding"), .value = value, }); } @@ -666,14 +707,14 @@ fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.Multi } } - // Input sharding/memory/aliasing attributes + // Input mesh/memory/aliasing attributes const input_attributes = try arena.alloc(*const mlir.Attribute, input_info.len); for (0.., input_attributes) |i, *input_attrs| { const attrs_list = try input_info.get(i).attributes(arena, compiler, fn_scope); input_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); } - // Output sharding/memory attributes + // Output mesh/memory attributes const output_attributes = try arena.alloc(*const mlir.Attribute, output_info.len); for (0.., output_attributes) |i, *output_attrs| { const attrs_list = try output_info.get(i).attributes(arena, compiler, fn_scope); @@ -748,6 +789,38 @@ fn repack(compiler: *Compiler, scope: *Scope, og_shape: Shape, value: *const mli return bit_cast_op.result(0); } +fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { + stdx.debug.assert(meshes.len >= 1, "Waiting at leat 1 mesh strategy to be implemented", .{}); + + const first = meshes[0]; + const partitions = first.numPartitions(); + const replicas = first.numReplicas(); + + for (meshes[1..]) |s| { + if (s.numPartitions() != partitions or s.numReplicas() != replicas) { + // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers + return error.InconsistentMesheCardinality; + } + } +} + +pub fn tensorShardingAttr( + compiler: *const Compiler, + allocator: std.mem.Allocator, + mlir_ctx: *mlir.Context, + shape: Shape, +) error{OutOfMemory}!*const mlir.Attribute { + const mesh_ = shape._sharding.mesh orelse compiler.platform.replicated_mesh; + return switch (compiler.partitioner) { + .shardy => (try mesh_.sdyShardingAttrForShape(allocator, mlir_ctx, shape)).asAttr(), + .gspmd => mesh_.gspmdShardingAttrForShape(allocator, mlir_ctx, shape) catch |err| switch (err) { + error.WriteFailed => error.OutOfMemory, // We're writing to memory + error.OutOfMemory => error.OutOfMemory, + error.IncompatibleSharding => @panic("Shape dimensions must be divisible by their mesh partitions"), + }, + }; +} + fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { const result = c.upb_Map_Set( map, @@ -770,21 +843,25 @@ fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_are } } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, partitioning: Partitioning, opts: Options) !*pjrt.LoadedExecutable { - var upb_alloc: upb.Allocator = .init(arena); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - defer c.upb_Arena_Free(upb_arena); - - const use_shardy_partitioner = switch (partitioning.partitioner) { +fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.LoadedExecutable { + const use_shardy_partitioner = switch (compiler.partitioner) { .shardy => true, .gspmd => false, }; - const num_partitions = partitioning.numPartitions(); - const num_replicas = partitioning.numReplicas(); + const platform = compiler.platform; + const main_mesh = compiler.meshes[0]; + const num_partitions = main_mesh.numPartitions(); + const num_replicas = main_mesh.numReplicas(); - const device_assignment = try partitioning.deviceAssignment(arena); + const device_assignment = try main_mesh.deviceAssignment(compiler.allocator); + defer compiler.allocator.free(device_assignment); + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); const options = blk: { const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { @@ -918,9 +995,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: const loaded_executable = try pjrtx.Client.compile( platform.pjrt_client, platform.pjrt_api, - arena, - io, - module, + compiler.allocator, + compiler.io, + compiler.module, try upb.serialize(options, upb_arena), ); errdefer loaded_executable.deinit(); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index b510a9c4..761721fa 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -16,33 +16,152 @@ const Target = @import("platform.zig").Target; const Sharding = @This(); -data: *const Data, - -pub const Error = error{ MissingLogicalBinding, IncompatibleSharding }; +mesh: ?*const Mesh, +partition: Partitioning, pub const MAX_MESH_RANK = 4; +pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; -var _replicated: [11]u8 align(@alignOf(Data)) = "_replicated".*; - -// special value to make public apis more fluent. -pub const replicated: Sharding = .{ .data = @ptrCast(&_replicated) }; - -pub fn resolve(sharding: Sharding, platform: *const Platform) Sharding { - return if (sharding.data == replicated.data) platform.replicated_sharding else sharding; +pub fn get(sharding: Sharding, ax: usize) PartitionSpec { + return if (sharding.mesh == null) .replicated else sharding.partition.get(ax); } -pub fn numPartitionsForLogicalAxis(sharding: Sharding, logical_axis: anytype) i64 { - return sharding.data.numPartitionsForLogicalAxis(logical_axis); +pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.set(ax, spec); + return res; } -pub fn devicesInCanonicalOrder(sharding: Sharding) []const Device { - return sharding.data.physical.devices_in_canonical_order; +pub fn insert(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.insert(ax, spec); + return res; } -pub fn format(sharding: Sharding, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return sharding.data.format(writer); +pub fn orderedRemove(self: Sharding, ax: usize) Sharding { + var res = self; + res.partition = res.partition.orderedRemove(ax); + return res; } +pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { + var result: [Shape.MAX_RANK]PartitionSpec = undefined; + for (&result, 0..) |*spec, ax| spec.* = self.get(ax); + return result; +} + +pub fn isFullyReplicated(self: Sharding) bool { + if (self.mesh == null) return true; + for (self.partition.toArray()) |spec| { + if (spec != .replicated and spec != .out_of_bound) return false; + } + return true; +} + +/// An unspecified mesh is compatible with any sharding. On the same mesh, +/// open axes accept any in-bounds partition, while closed axes must match. +pub fn isCompatible(self: Sharding, other: Sharding) bool { + if (self.mesh == null or other.mesh == null) return true; + if (self.mesh != other.mesh) return false; + for (self.partition.toArray(), other.partition.toArray()) |lhs, rhs| { + if (lhs == rhs) continue; + if (lhs == .out_of_bound or rhs == .out_of_bound) return false; + if (lhs != .open and rhs != .open) return false; + } + return true; +} + +test isCompatible { + const tp: Mesh = .initTest(.tp, .{.model}); + const dp: Mesh = .initTest(.dp, .{ .data, .model }); + + const sharded: Sharding = .{ .mesh = &tp, .partition = .init(&.{ .mesh_axis_0, .replicated }) }; + const cases = [_]struct { lhs: Sharding, rhs: Sharding, compatible: bool }{ + .{ .lhs = .replicated, .rhs = .replicated, .compatible = true }, + .{ .lhs = .replicated, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = .{ .mesh = &tp, .partition = .open(2) }, .compatible = true }, + .{ .lhs = sharded.set(1, .open), .rhs = sharded.set(0, .open), .compatible = true }, + .{ .lhs = sharded, .rhs = sharded.set(0, .mesh_axis_1), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(0, .replicated), .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = sharded.partition }, .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = .open(2) }, .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(1, .out_of_bound), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(2, .open), .compatible = false }, + }; + for (cases) |case| { + errdefer std.log.err("{f}.isCompatible({f}) expected to be {}, got {}", .{ case.lhs, case.rhs, case.compatible, !case.compatible }); + + try std.testing.expectEqual(case.compatible, case.lhs.isCompatible(case.rhs)); + try std.testing.expectEqual(case.compatible, case.rhs.isCompatible(case.lhs)); + } +} + +pub fn format(self: Sharding, writer: *std.Io.Writer) !void { + const mesh = self.mesh orelse return writer.writeAll("{replicated}"); + const specs = self.partition.toArray(); + var rank = specs.len; + while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; + + try writer.writeByte('{'); + for (specs[0..rank], 0..) |spec, i| { + if (i > 0) try writer.writeByte(','); + switch (spec) { + .replicated => try writer.writeAll("replicated"), + .open => try writer.writeByte('?'), + .out_of_bound => try writer.writeByte('!'), + else => try writer.print("{s}", .{mesh.logical.axes.get(spec.meshAxis().?)}), + } + } + try writer.print("}}@{s}", .{mesh.name}); +} + +/// Describes how a given Shape axis behaves inside a mesh. +/// Is it replicated ? sharded along a specific logical axis ? or open to replication ? +pub const PartitionSpec = enum(u4) { + mesh_axis_0 = 0, + mesh_axis_1 = 1, + mesh_axis_2 = 2, + mesh_axis_3 = 3, + mesh_axis_4 = 4, + mesh_axis_5 = 5, + mesh_axis_6 = 6, + mesh_axis_7 = 7, + // an 8D mesh seems already a lot, the max we know about is 3D. + replicated = 8, + + // Partitioning don't encode the rank of corresponding tensor, so this is used to mark invalid positions. + out_of_bound = 0xa, + open = 15, + + pub fn sharded(mesh_axis: u3) PartitionSpec { + return @enumFromInt(mesh_axis); + } + + /// Extract the mesh axis along which we are sharded. Null if not sharded. + pub fn meshAxis(self: PartitionSpec) ?u3 { + const ax = @intFromEnum(self); + return if (ax < @intFromEnum(PartitionSpec.replicated)) @intCast(ax) else null; + } + + pub fn isSharded(self: PartitionSpec) bool { + return @intFromEnum(self) < @intFromEnum(PartitionSpec.replicated); + } + + pub fn isClosed(self: PartitionSpec) bool { + return @intFromEnum(self) <= @intFromEnum(PartitionSpec.replicated); + } + + test isClosed { + try std.testing.expect(PartitionSpec.mesh_axis_2.isClosed()); + try std.testing.expect(!PartitionSpec.open.isClosed()); + + try std.testing.expect(PartitionSpec.replicated.isClosed()); + + try std.testing.expect(!PartitionSpec.out_of_bound.isClosed()); + } +}; + pub const Partitioner = union(enum) { shardy, gspmd, @@ -54,121 +173,14 @@ pub const Partitioner = union(enum) { } }; -pub const Partitioning = struct { - partitioner: Partitioner, - shardings: []const Sharding, - - pub fn init(partitioner: Partitioner, shardings: []const Sharding) !Partitioning { - stdx.debug.assert(shardings.len >= 1, "Waiting at leat 1 sharding strategy to be implemented", .{}); - var plan: Partitioning = .{ .partitioner = partitioner, .shardings = shardings }; - - const first = plan.primarySharding(); - const partitions = first.data.numPartitions(); - const replicas = first.data.numReplicas(); - - for (shardings[1..]) |s| { - if (s.data.numPartitions() != partitions or s.data.numReplicas() != replicas) { - // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers - return error.InconsistentShardingCardinality; - } - } - - return plan; - } - - pub fn numPartitions(self: Partitioning) i32 { - const primary = self.primarySharding(); - return primary.data.numPartitions(); - } - - pub fn numReplicas(self: Partitioning) i32 { - const primary = self.primarySharding(); - return primary.data.numReplicas(); - } - - pub fn numDevices(self: Partitioning) i32 { - return self.numPartitions() * self.numReplicas(); - } - - pub fn deviceAssignment(self: Partitioning, allocator: std.mem.Allocator) ![]usize { - return switch (self.partitioner) { - .shardy, .gspmd => blk: { - const sharding = self.primarySharding(); - break :blk sharding.data.deviceAssignment(allocator); - }, - }; - } - - pub fn tensorShardingAttr( - self: Partitioning, - allocator: std.mem.Allocator, - ctx: *mlir.Context, - shape: Shape, - sharding: Sharding, - ) error{OutOfMemory}!*const mlir.Attribute { - return switch (self.partitioner) { - .shardy => (try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape)).asAttr(), - .gspmd => sharding.data.gspmdShardingAttrForShape(allocator, ctx, shape) catch |err| switch (err) { - error.WriteFailed => error.OutOfMemory, // We're writing to memory - error.OutOfMemory => error.OutOfMemory, - // TODO(hugomano): clarify what can trigger this and consider moving the check to the Sharding creation - error.MissingDeviceInTile => @panic("MissingDeviceInTile"), - }, - }; - } - - pub fn localShapeForShape(self: Partitioning, shape: Shape) !Shape { - const sharding = try self.selectSharding(shape); - const ordered_devices = sharding.devicesInCanonicalOrder(); - if (ordered_devices.len == 0) return error.InvalidPhysicalMesh; - return sharding.shardedShape(shape); - } - - pub fn numPartitionsForLogicalAxis(self: Partitioning, shape: Shape, logical_axis: anytype) !i64 { - const sharding = try self.selectSharding(shape); - return sharding.data.numPartitionsForLogicalAxis(logical_axis); - } - - pub fn shardableDim(self: Partitioning, shape: Shape, axis: anytype, must_divide: i64) !DimSharding { - const ax = shape.axis(axis); - const spec = shape.partition(ax); - if (spec != .axis) return .replicated; - - const sharding = try self.selectSharding(shape); - return sharding.shardableDim(shape.dim(ax), spec.axis, must_divide); - } - - pub fn selectSharding(self: Partitioning, shape: Shape) error{NoSuitableSharding}!Sharding { - return pickSharding(self.shardings, shape, .any_covering) orelse error.NoSuitableSharding; - } - - fn primarySharding(self: Partitioning) Sharding { - return self.shardings[0]; - } -}; - -pub const SelectShardingMode = enum { - any_covering, - explicit_axis_binding, -}; - -pub fn pickSharding(shardings: []const Sharding, shape: Shape, mode: SelectShardingMode) ?Sharding { - if (mode == .explicit_axis_binding and !shapeHasAxisPartition(shape)) return null; - - for (shardings) |sharding| { - if (sharding.data.covers(shape)) return sharding; - } - return null; -} - fn shapeHasAxisPartition(shape: Shape) bool { for (0..shape.rank()) |ax| { - if (shape.partition(ax) == .axis) return true; + if (shape.partition(ax).meshAxis()) |_| return true; } return false; } -pub const DimSharding = union(enum) { +pub const DimMeshe = union(enum) { sharded: struct { dim: i64, factor: u32, @@ -176,43 +188,25 @@ pub const DimSharding = union(enum) { replicated, }; -pub fn shardableDim(sharding: Sharding, dim: i64, logical_axis: anytype, must_divide: i64) DimSharding { - const partitions = sharding.numPartitionsForLogicalAxis(logical_axis); - - if (@mod(dim, partitions) == 0) { - return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; - } - - const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); - const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); - const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; - if (@mod(must_divide, materialized_dim) == 0) { - return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; - } else { - return .replicated; - } -} - pub fn sdyPerValueShardingAttr( allocator: std.mem.Allocator, ctx: *mlir.Context, shapes: []const Shape, - shardings: []const Sharding, + mesh: *const Mesh, ) error{OutOfMemory}!*const mlir.Attribute { - const sharding_attrs = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); - for (sharding_attrs, shapes, shardings) |*attr, shape, sharding| { - attr.* = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + const meshe_attrs = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); + for (meshe_attrs, shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); } - return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, sharding_attrs).asAttr(); + return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, meshe_attrs).asAttr(); } pub fn sdyManualAxesAttr( allocator: std.mem.Allocator, ctx: *mlir.Context, in_shapes: []const Shape, - in_shardings: []const Sharding, out_shapes: []const Shape, - out_shardings: []const Sharding, + mesh: *const Mesh, ) error{OutOfMemory}!*const mlir.Attribute { var axis_names = std.ArrayList([]const u8).empty; defer axis_names.deinit(allocator); @@ -226,8 +220,8 @@ pub fn sdyManualAxesAttr( } }; - for (in_shapes, in_shardings) |shape, sharding| { - const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (in_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); for (0..attr.numReplicatedAxes()) |i| { Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); } @@ -238,8 +232,8 @@ pub fn sdyManualAxesAttr( } } } - for (out_shapes, out_shardings) |shape, sharding| { - const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (out_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); for (0..attr.numReplicatedAxes()) |i| { Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); } @@ -259,6 +253,192 @@ pub fn sdyManualAxesAttr( return dialects.shardy.ManualAxesAttribute.init(ctx, axes).asAttr(); } +pub const Partitioning = packed struct { + _0: PartitionSpec, + _1: PartitionSpec, + _2: PartitionSpec, + _3: PartitionSpec, + _4: PartitionSpec, + _5: PartitionSpec, + _6: PartitionSpec, + _7: PartitionSpec, + + pub const out_of_bound: Partitioning = splat(.out_of_bound); + + const MAX_RANK = Shape.MAX_RANK; + const Vec = @Vector(MAX_RANK, u4); + + pub fn init(specs: []const PartitionSpec) Partitioning { + var res: Partitioning = out_of_bound; + for (0.., specs) |i, spec| { + res = res.set(i, spec); + } + return res; + } + + pub fn replicated(rank_: usize) Partitioning { + return repeat(.replicated, rank_); + } + + pub fn open(rank_: usize) Partitioning { + return repeat(.open, rank_); + } + + pub fn splat(spec: PartitionSpec) Partitioning { + const vec: Vec = @splat(@intFromEnum(spec)); + return @bitCast(vec); + } + + pub fn repeat(spec: PartitionSpec, rank_: usize) Partitioning { + std.debug.assert(rank_ <= MAX_RANK); + const splatted: Vec = @splat(@intFromEnum(spec)); + const mask = std.simd.iota(u4, 8) < @as(Vec, @splat(@truncate(rank_))); + return @bitCast(@select(u4, mask, splatted, @as(Vec, @bitCast(out_of_bound)))); + } + + pub fn get(p: Partitioning, ax: usize) PartitionSpec { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + return @enumFromInt(@as(u4, @truncate(pack >> shift))); + } + + pub fn set(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const mask = @as(u32, 0xf) << shift; + return @bitCast((pack & ~mask) | (@as(u32, @intFromEnum(spec)) << shift)); + } + + pub fn rank(p: Partitioning) u8 { + for (0..MAX_RANK) |i_usize| { + const i: u8 = @truncate(i_usize); + if (p.get(i) == .out_of_bound) return i; + } + return MAX_RANK; + } + + /// Inserts a spec, shifting subsequent slots right and discarding the last slot. + pub fn insert(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @intFromEnum(spec)) << shift)); + } + + /// Removes a spec, shifting subsequent slots left and filling the last with unknown. + pub fn orderedRemove(p: Partitioning, ax: usize) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @intFromEnum(PartitionSpec.out_of_bound)) << 28)); + } + + pub fn toArray(p: Partitioning) [MAX_RANK]PartitionSpec { + var res: [MAX_RANK]PartitionSpec = undefined; + for (&res, 0..) |*spec, ax| spec.* = p.get(ax); + return res; + } + + pub fn hasUniqueAxes(p: Partitioning) bool { + var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .initEmpty(); + for (0..MAX_RANK) |shape_ax| { + const spec = p.get(shape_ax); + if (spec.meshAxis()) |mesh_axis| { + if (used_mesh_axes.isSet(mesh_axis)) return false; + used_mesh_axes.set(mesh_axis); + } + } + return true; + } + + test hasUniqueAxes { + try std.testing.expect(hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_2 }))); + try std.testing.expect(!hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_0 }))); + } + + test "packed access and updates" { + try std.testing.expectEqual(32, @bitSizeOf(Partitioning)); + try std.testing.expectEqual(4, @sizeOf(Partitioning)); + inline for (std.meta.tags(PartitionSpec)) |spec| { + const filled = Partitioning.splat(spec); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(spec, filled.get(ax)); + const updated = Partitioning.out_of_bound.set(ax, spec); + for (0..MAX_RANK) |i| { + try std.testing.expectEqual(if (i == ax) spec else .out_of_bound, updated.get(i)); + } + } + } + for (0..MAX_RANK + 1) |rank_| { + const parts = Partitioning.replicated(rank_); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(if (ax < rank_) PartitionSpec.replicated else .out_of_bound, parts.get(ax)); + } + } + } + + test "insertion and removal at every slot" { + var parts: Partitioning = .out_of_bound; + for (0..MAX_RANK) |ax| parts = parts.set(ax, @enumFromInt(ax)); + const original = parts.toArray(); + for (0..MAX_RANK) |ax| { + var inserted = original; + std.mem.copyBackwards(PartitionSpec, inserted[ax + 1 ..], original[ax .. MAX_RANK - 1]); + inserted[ax] = .open; + try std.testing.expectEqualSlices(PartitionSpec, &inserted, &parts.insert(ax, .open).toArray()); + + var removed = original; + std.mem.copyForwards(PartitionSpec, removed[ax .. MAX_RANK - 1], original[ax + 1 ..]); + removed[MAX_RANK - 1] = .out_of_bound; + try std.testing.expectEqualSlices(PartitionSpec, &removed, &parts.orderedRemove(ax).toArray()); + try std.testing.expectEqual(parts, parts.orderedRemove(ax).insert(ax, parts.get(ax))); + } + try std.testing.expectEqual(Partitioning.out_of_bound, Partitioning.out_of_bound.orderedRemove(0)); + const comptime_parts = comptime Partitioning.out_of_bound.insert(7, .open).orderedRemove(0).set(0, .replicated); + try std.testing.expectEqual(PartitionSpec.replicated, comptime_parts.get(0)); + try std.testing.expectEqual(PartitionSpec.open, comptime_parts.get(6)); + try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); + } + + /// Parse a struct literal into a partitioning object. + /// The struct passed to partitioning must only have fields names present in `tags`. + /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. + /// The default value is important. + /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. + pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { + var partition_: Partitioning = .repeat(default_value, tags.len); + + const T = @TypeOf(partitioning); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "parsePartitioning expected a struct of enum literals eg {{ .b = .data, .d = .model }}, got: {any}", .{T}); + inline for (std.meta.fields(T)) |field| { + const shape_tag = Shape.toTag(field); + const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field.name }); + }; + + const value = @field(partitioning, field.name); + const spec: PartitionSpec = if (@TypeOf(value) == PartitionSpec) value else switch (value) { + .replicated => .replicated, + .out_of_bound => std.debug.panic("value out_of_bound not allowed", .{}), + .open => .open, + else => spec: { + const mesh_tag = Shape.toTag(value); + const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { + stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); + }; + break :spec .sharded(@intCast(mesh_ax)); + }, + }; + partition_ = partition_.set(shape_ax, spec); + } + return partition_; + } +}; + /// Device as part of a PhysicalMesh. pub const Device = struct { /// Unique device identifier in the mesh @@ -1127,7 +1307,7 @@ pub const PhysicalMesh = struct { }; pub const LogicalAxisIntent = enum { - /// Needs the fastest possible path (e.g. Weight sharding / All-Reduce). + /// Needs the fastest possible path (e.g. Weight mesh / All-Reduce). /// Requires high-bandwidth, low-latency interconnects (Cores/NVLink). high_bandwidth, @@ -1135,12 +1315,12 @@ pub const LogicalAxisIntent = enum { /// Suitable for chip-to-chip interconnects (NeuronLink/ICI). balanced, - /// Can tolerate lower bandwidth and higher latency (e.g. Batch sharding). + /// Can tolerate lower bandwidth and higher latency (e.g. Batch mesh). /// Suitable for system bus or network (PCIe/Ethernet). low_bandwidth, }; -// Defines the logical mesh used for sharding the model and data. +// Defines the logical mesh used for mesh the model and data. // E.g., batch, model, context, experts, etc. // Each axis represents a dimension along which the workload can be partitioned. The value is the intent of the axis. /// LogicalMesh defines *semantic* axes only (batch/model/context/...). @@ -1214,16 +1394,11 @@ pub const LogicalMesh = struct { } }; -pub const AxisList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); -pub const Binding = struct { - logical: Shape.Tag, - physical: AxisList, -}; -pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); +pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); pub const Fold = struct { target: PhysicalAxisTag, - sources: AxisList, + sources: PhysicalList, }; pub const Folds = stdx.BoundedArray(Fold, MAX_MESH_RANK); @@ -1260,38 +1435,73 @@ pub const PhysicalView = struct { } }; -pub const Data = struct { +pub const Mesh = struct { + pub const zml_no_meta_visit: void = {}; + name: []const u8, physical: *const PhysicalMesh, logical: LogicalMesh, /// Compact binding table: logical axis -> physical axes - bindings: Bindings, + bindings: stdx.BoundedArray(PhysicalList, MAX_MESH_RANK), /// Explicit folding rules: kept axis -> ordered source axes. folds: Folds, folds_consumed: std.EnumSet(PhysicalAxisTag), - pub fn binding(self: *const Data, tag: Shape.Tag) ?[]const PhysicalAxisTag { - for (self.bindings.slice()) |*b| { - if (b.logical == tag) return b.physical.slice(); + pub fn shardableDim(mesh: *const Mesh, dim: i64, logical_axis: anytype, must_divide: i64) DimMeshe { + const partitions = mesh.numPartitionsForLogicalAxis(logical_axis); + + if (@mod(dim, partitions) == 0) { + return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; + } + + const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); + const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); + const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; + if (@mod(must_divide, materialized_dim) == 0) { + return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; + } else { + return .replicated; + } + } + + pub fn binding(self: *const Mesh, mesh_axis: usize) []const PhysicalAxisTag { + return self.bindings.constSlice()[mesh_axis].constSlice(); + } + + pub fn resolveLogicalAxis(self: *const Mesh, tag: Shape.Tag) ?u8 { + for (0.., self.logical.axes.slice()) |i, logical_axis| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(tag))) return @truncate(i); } return null; } pub fn init( - name: []const u8, + name_: []const u8, physical: *const PhysicalMesh, logical: LogicalMesh, strategy: Strategy, - ) !Data { + ) !Mesh { const axis_order = physical.axisOrder().slice(); if (axis_order.len == 0) return error.InvalidPhysicalMesh; if (strategy.bindings.len == 0) return error.InvalidStrategy; - var bindings: Bindings = .empty; - for (strategy.bindings.slice()) |bind| { - bindings.appendAssumeCapacity(.{ .logical = bind.logical, .physical = bind.physical }); + // Create one binding per logical axis + var bindings: @FieldType(Mesh, "bindings") = .empty; + for (logical.axes.slice()) |_| { + bindings.appendAssumeCapacity(.empty); + } + + for (strategy.bindings.slice()) |strat_binding| { + for (logical.axes.slice(), bindings.slice()) |logical_axis, *binding_| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(strat_binding.logical))) { + binding_.* = strat_binding.physical; + break; + } + } else { + std.debug.panic("{f} expects strategy to only list known logical axes, got unknown {s}", .{ logical, strat_binding.logical }); + } } var folds: Folds = .empty; @@ -1305,7 +1515,7 @@ pub const Data = struct { } return .{ - .name = name, + .name = name_, .logical = logical, .physical = physical, .bindings = bindings, @@ -1314,14 +1524,34 @@ pub const Data = struct { }; } - fn foldSources(self: *const Data, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { + pub fn initTest(name_: @EnumLiteral(), tagz: anytype) Mesh { + if (!builtin.is_test) @compileError("Cannot use zml.Sharding.Mesh.initTest outside of a test block"); + const axes_tags = Shape.parseTags(tagz); + return .{ + .name = @tagName(name_), + .logical = .{ + .axes = .init(axes_tags.slice()), + .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, + }, + .physical = undefined, + .bindings = .empty, + .folds = .empty, + .folds_consumed = .empty, + }; + } + + fn foldSources(self: *const Mesh, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { for (self.folds.slice()) |*f| { if (f.target == tag) return f.sources.slice(); } return null; } - pub fn physicalView(self: *const Data) PhysicalView { + pub fn devicesInCanonicalOrder(mesh: *const Mesh) []const Device { + return mesh.physical.devices_in_canonical_order; + } + + pub fn physicalView(self: *const Mesh) PhysicalView { const physical = self.physical; var view: PhysicalView = .{ .axes = .empty, @@ -1359,17 +1589,18 @@ pub const Data = struct { return view; } - pub fn logicalIndexFromCoords(self: *const Data, coords: []const u8) usize { + pub fn logicalIndexFromCoords(self: *const Mesh, coords: []const u8) usize { return self.physical.linearIndexFromCoords(coords); } - pub fn numPartitions(self: *const Data) i32 { + pub fn numPartitions(self: *const Mesh) i32 { return @intCast(self.physicalView().total_devices); } - pub fn numPartitionsForLogicalAxis(self: *const Data, logical_axis: anytype) i64 { + pub fn numPartitionsForLogicalAxis(self: *const Mesh, logical_axis: anytype) i64 { const logical_tag = Shape.toTag(logical_axis); - const bound_axes = self.binding(logical_tag) orelse return 1; + const logical_ax = self.resolveLogicalAxis(logical_tag) orelse std.debug.panic("mesh {f} has no axis {s}", .{ self, logical_tag }); + const bound_axes = self.binding(logical_ax); var physical_axes: std.EnumSet(PhysicalAxisTag) = .empty; for (bound_axes) |bound_axis| { @@ -1389,25 +1620,15 @@ pub const Data = struct { return partitions; } - pub fn numReplicas(_: *const Data) i32 { + pub fn numReplicas(_: *const Mesh) i32 { return 1; } - pub fn numDevices(self: *const Data) i32 { + pub fn numDevices(self: *const Mesh) i32 { return self.numPartitions() * self.numReplicas(); } - pub fn covers(sharding: *const Data, shape: Shape) bool { - for (shape._partitioning.slice()) |partitioning| { - switch (partitioning) { - .axis => |tag| if (sharding.binding(tag) == null) return false, - else => {}, - } - } - return true; - } - - pub fn sdyMeshAttr(self: *const Data, allocator: std.mem.Allocator) ![]const u8 { + pub fn sdyMeshAttr(self: *const Mesh, allocator: std.mem.Allocator) ![]const u8 { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); @@ -1422,7 +1643,7 @@ pub const Data = struct { } const DimMapping = struct { - /// For each tensor dimension, the indices of physical axes (in PhysicalView) sharding it. + /// For each tensor dimension, the indices of physical axes (in PhysicalView) mesh it. axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK), /// Indices of physical axes used for replication. replicated_axes: stdx.BoundedArray(usize, Shape.MAX_RANK), @@ -1431,7 +1652,7 @@ pub const Data = struct { }; /// Common logic to map tensor dimensions to physical mesh indices. - fn getDimMapping(self: *const Data, shape: Shape) DimMapping { + fn getDimMapping(self: *const Mesh, shape: Shape) DimMapping { const view = self.physicalView(); var axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK) = .empty; var used_mask: [Shape.MAX_RANK]bool = @splat(false); @@ -1441,16 +1662,15 @@ pub const Data = struct { var dim_axes: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; const spec = shape.partition(ax); - if (spec == .axis) { - if (self.binding(spec.axis)) |binding_| { - for (binding_) |p_tag| { - for (view.axes.slice(), 0..) |v_ax, i| { - // Only use the axis if it's bound and hasn't been consumed by a previous dimension - if (v_ax.contains(p_tag) and !globally_used.contains(v_ax.tag)) { - dim_axes.appendAssumeCapacity(i); - globally_used.insert(v_ax.tag); - used_mask[i] = true; - } + if (spec.meshAxis()) |mesh_ax| { + const binding_ = self.bindings.get(mesh_ax); + for (binding_.slice()) |p_tag| { + for (view.axes.slice(), 0..) |v_ax, i| { + // Only use the axis if it's bound and hasn't been consumed by a previous dimension + if (v_ax.contains(p_tag) and !globally_used.contains(v_ax.tag)) { + dim_axes.appendAssumeCapacity(i); + globally_used.insert(v_ax.tag); + used_mask[i] = true; } } } @@ -1471,7 +1691,7 @@ pub const Data = struct { } pub fn sdyShardingAttrForShape( - data: *const Data, + data: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape, @@ -1479,39 +1699,28 @@ pub const Data = struct { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); - var any_explicit = false; - for (0..shape.rank()) |ax| { - if (shape.partition(ax) != .unknown) { - any_explicit = true; - break; - } - } - const all_replicated = !any_explicit; - const mapping = data.getDimMapping(shape); const dimensions = try allocator.alloc(*const dialects.shardy.DimensionShardingAttribute, shape.rank()); + // Note: here we map from the logical axis to physical. Is this a good idea ? + // why not stay in logical realm ? for (0.., dimensions) |ax, *d| { - const spec = shape.partition(ax); - d.* = switch (spec) { - .axis => |logical_tag| d: { - if (data.binding(logical_tag)) |_| { - const dim_phys_indices = mapping.axes_per_dim.get(ax); - if (dim_phys_indices.len == 0) { - break :d .replicated(ctx); - } else { - const axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, dim_phys_indices.len); - for (dim_phys_indices.slice(), 0..) |p_idx, i| { - axes[i] = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); - } - break :d .closed(ctx, axes); - } + d.* = switch (shape.partition(ax)) { + .out_of_bound => @panic("undefined"), + .replicated => .replicated(ctx), + .open => .open(ctx, &.{}), + else => d: { + const dim_phys_indices = mapping.axes_per_dim.get(ax); + if (dim_phys_indices.len == 0) { + break :d .replicated(ctx); } else { - break :d .open(ctx, &.{}); + const axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, dim_phys_indices.len); + for (dim_phys_indices.slice(), 0..) |p_idx, i| { + axes[i] = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); + } + break :d .closed(ctx, axes); } }, - .replicated => .replicated(ctx), - .open, .unknown => if (all_replicated) .replicated(ctx) else .open(ctx, &.{}), }; } @@ -1523,14 +1732,14 @@ pub const Data = struct { return .init(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes }); } - pub fn gspmdShardingAttrForShape(self: *const Data, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { + pub fn gspmdShardingAttrForShape(self: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); var has_sharding = false; for (0..shape.rank()) |ax| { - if (shape.partition(ax) == .axis) { + if (shape.partition(ax).isSharded()) { has_sharding = true; break; } @@ -1562,9 +1771,7 @@ pub const Data = struct { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); - // Emit explicit device assignment list to avoid XLA requiring - // a sharding attribute on the custom-call instruction. - const ids = try self.deviceAssignment(allocator); + const ids = try self.gspmdTileAssignment(allocator, shape); defer allocator.free(ids); try out.writer.writeAll("{devices=["); @@ -1591,12 +1798,36 @@ pub const Data = struct { return .string(ctx, try out.toOwnedSlice()); } - pub fn deviceAssignment(self: *const Data, allocator: std.mem.Allocator) ![]usize { - const view = self.physicalView(); - const count: usize = @intCast(view.total_devices); + fn gspmdTileAssignment(self: *const Mesh, allocator: std.mem.Allocator, shape: Shape) ![]u32 { + const placement = try Placement.init(shape); + const devices = self.devicesInCanonicalOrder(); + var tile_count: usize = 1; + for (placement.axis_plans.constSlice()) |plan| tile_count *= @intCast(plan.num_devices); + const replicas_per_tile = @divExact(devices.len, tile_count); + const replica_indices = try allocator.alloc(usize, tile_count); + defer allocator.free(replica_indices); + @memset(replica_indices, 0); + const ids = try allocator.alloc(u32, devices.len); - var ids = try allocator.alloc(usize, count); - @memset(ids, std.math.maxInt(usize)); + for (devices, 0..) |device, partition_index| { + var tile_index: usize = 0; + for (placement.axis_plans.constSlice()) |plan| { + tile_index = tile_index * @as(usize, @intCast(plan.num_devices)) + @as(usize, @intCast(plan.linearIndex(device.coords))); + } + // HLO shardings refer to logical partitions; executable device assignment + // separately maps those partitions to physical PJRT device IDs. + ids[tile_index * replicas_per_tile + replica_indices[tile_index]] = @intCast(partition_index); + replica_indices[tile_index] += 1; + } + return ids; + } + + pub fn deviceAssignment(self: *const Mesh, allocator: std.mem.Allocator) ![]u32 { + const view = self.physicalView(); + const count: u32 = @intCast(view.total_devices); + + var ids = try allocator.alloc(u32, count); + @memset(ids, std.math.maxInt(u32)); for (self.physical.devices_in_canonical_order) |d| { const coords = d.coords; @@ -1605,31 +1836,35 @@ pub const Data = struct { } for (ids) |id| { - if (id == std.math.maxInt(usize)) return error.MissingDeviceInTile; + if (id == std.math.maxInt(u32)) return error.MissingDeviceInTile; } return ids; } - pub fn format(self: Data, writer: *std.Io.Writer) !void { - try writer.print("Sharding(name={s})\n", .{self.name}); + /// Extract coordinate data from the physical mesh for a specific axis. + fn addPhysicalToSplit(mesh: *const Mesh, plan: *AxisSplit, tag: PhysicalAxisTag) void { + const physical = mesh.physical; + const info = physical.axisInfo(tag) orelse return; + const depth = physical.axis_traversal.depth(tag) orelse return; + plan.add(@intCast(info.size), depth); + } + + pub fn format(self: Mesh, writer: *std.Io.Writer) !void { + try writer.print("Mesh(name={s})\n", .{self.name}); try writer.writeAll("Bindings:\n"); - for (self.logical.axes.slice(), self.logical.intents.slice()) |l_tag, l_intent| { + for (self.logical.axes.slice(), self.logical.intents.slice(), self.bindings.slice()) |l_tag, l_intent, bind| { try writer.print(" - {s} ({s}) -> ", .{ l_tag, @tagName(l_intent) }); - if (self.binding(l_tag)) |axes| { - if (axes.len == 0) { - try writer.writeAll("replicated\n"); - } else { - for (axes, 0..) |p, i| { - if (i > 0) try writer.writeAll(", "); - try writer.writeAll(@tagName(p)); - } - try writer.writeAll("\n"); - } - } else { + if (bind.len == 0) { try writer.writeAll("unbound\n"); + } else { + for (bind.slice(), 0..) |physical, i| { + if (i > 0) try writer.writeAll(", "); + try writer.writeAll(@tagName(physical)); + } + try writer.writeAll("\n"); } } @@ -1665,19 +1900,19 @@ pub const Data = struct { }; pub const Strategy = struct { - pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); - pub const Binding = struct { - logical: Shape.Tag, - physical: PhysicalList, - }; - pub const Bindings = stdx.BoundedArray(Strategy.Binding, MAX_MESH_RANK); pub const Fold = struct { target: PhysicalAxisTag, sources: PhysicalList, }; + pub const Binding = struct { + logical: Shape.Tag, + physical: PhysicalList, + }; + + pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); pub const Folding = stdx.BoundedArray(Strategy.Fold, MAX_MESH_RANK); - bindings: Strategy.Bindings, + bindings: Bindings, folding: Folding, // TODO: remove .init, promote .parseBindings to .init @@ -1784,16 +2019,19 @@ pub const Strategy = struct { }; /// For a given shape, compute the shape of the slice each shard will receive. -pub fn shardedShape(sharding: Sharding, shape: Shape) Error!Shape { - const pl = try sharding.placement(shape); - return pl.shape; +pub fn shardedShape(shape: Shape) error{IncompatibleSharding}!Shape { + if (shape._sharding.mesh != null) { + const pl: Placement = try .init(shape); + var local_shape = pl.shape; + local_shape._sharding = .replicated; + return local_shape; + } else { + // fully replicated + return shape; + } } -pub fn placement(sharding: Sharding, shape: Shape) Error!Placement { - return .init(sharding, shape); -} - -/// Precompute sharding information common to all devices for a given shape. +/// Precompute mesh information common to all devices for a given shape. pub const Placement = struct { pub const Slices = stdx.BoundedArray(Slice1d, Shape.MAX_RANK); pub const Slice1d = struct { @@ -1801,26 +2039,26 @@ pub const Placement = struct { size: i64, }; - sharding: Sharding, // Note this shape is mainly use to indicate the dims of the sharded buffer. // We may want to only store that. shape: Shape, global_shape: if (builtin.mode == .Debug) Shape else void, - axis_plans: stdx.BoundedArray(AxisSplit, Shape.MAX_RANK), - pub fn init(sharding: Sharding, shape: Shape) Error!Placement { + pub fn init(shape: Shape) error{IncompatibleSharding}!Placement { var pl: Placement = .{ - .sharding = sharding, .shape = shape, // modified below .global_shape = if (builtin.mode == .Debug) shape else {}, .axis_plans = .empty, // set below }; + // mesh == null => replicated => no axis plan + if (shape._sharding.mesh == null) return pl; + const mesh_ = shape._sharding.mesh.?; var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; for (0..shape.rank()) |ax| { const axis_index: u8 = @intCast(ax); - pl.axis_plans.appendAssumeCapacity(try axisSplit(sharding, shape, &used_axes, axis_index)); + pl.axis_plans.appendAssumeCapacity(try axisSplit(mesh_, shape, &used_axes, axis_index)); } for (pl.shape._dims.slice(), shape.dims(), pl.axis_plans.slice()) |*shard_dim, dim, plan| { @@ -1829,8 +2067,16 @@ pub const Placement = struct { return pl; } + pub fn mesh(pl: *const Placement) ?*const Mesh { + return pl.shape._sharding.mesh; + } + pub fn slices(pl: *const Placement, device: Device.Coords) Slices { var res: Slices = .{ .buffer = undefined, .len = pl.shape.rank() }; + if (pl.mesh() == null) { + for (res.slice(), pl.shape.dims()) |*slice, dim| slice.* = .{ .start = 0, .size = dim }; + return res; + } for (res.slice(), pl.shape.dims(), pl.axis_plans.slice()) |*r, size, plan| { const start = plan.linearIndex(device) * size; r.* = .{ @@ -1848,6 +2094,8 @@ pub const Placement = struct { } var ptr: [*]const u8 = slice.constData().ptr; + // mesh = null => implies replicated => each device read from the start. + if (pl.shape._sharding.mesh == null) return ptr; for (pl.shape.dims(), pl.axis_plans.slice(), slice.byte_strides.slice()) |size, plan, stride| { const start = plan.linearIndex(device) * size; ptr = ptr[@intCast(start * stride)..]; @@ -1870,7 +2118,9 @@ pub const Placement = struct { } pub fn format(pl: Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const ordered_devices = pl.sharding.devicesInCanonicalOrder(); + const ordered_devices = if (pl.mesh()) |m| m.devicesInCanonicalOrder() else { + return try writer.print("Placement(shape={f}, replicated)\n", .{pl.shape}); + }; try pl.formatPlacementSummary(pl.shape, ordered_devices.len, writer); try writer.writeAll("Per-shard placement:\n"); @@ -1945,7 +2195,7 @@ const AxisSplit = struct { }; fn axisSplit( - sharding: Sharding, + mesh: *const Mesh, shape: Shape, used_axes: *std.EnumSet(PhysicalAxisTag), axis_index: u8, @@ -1953,37 +2203,34 @@ fn axisSplit( const dim = shape.dim(axis_index); const spec = shape.partition(axis_index); - switch (spec) { - .axis => |logical_tag| { - const binding = sharding.data.binding(logical_tag) orelse return error.MissingLogicalBinding; + return if (spec.meshAxis()) |mesh_axis| { + const binding = mesh.binding(mesh_axis); - // Calculate the split based on the physical coordinates of the current device. - return try calculateSplit(sharding, dim, binding, used_axes); - }, - else => return .empty, - } + // Calculate the split based on the physical coordinates of the current device. + return try calculateSplit(mesh, dim, binding, used_axes); + } else .empty; } fn calculateSplit( - sharding: Sharding, + mesh: *const Mesh, dim: i64, binding: []const PhysicalAxisTag, used_axes: *std.EnumSet(PhysicalAxisTag), -) !AxisSplit { +) error{IncompatibleSharding}!AxisSplit { var plan: AxisSplit = .empty; for (binding) |p_tag| { // Expand the physical tag: check if it's a target for folded source axes. - if (sharding.data.foldSources(p_tag)) |sources| { + if (mesh.foldSources(p_tag)) |sources| { for (sources) |src| { if (used_axes.contains(src)) continue; - addPhysicalToSplit(sharding, &plan, src); + mesh.addPhysicalToSplit(&plan, src); used_axes.insert(src); } } else { // Standard case: directly bound, not part of an explicit fold. if (used_axes.contains(p_tag)) continue; - addPhysicalToSplit(sharding, &plan, p_tag); + mesh.addPhysicalToSplit(&plan, p_tag); used_axes.insert(p_tag); } } @@ -1994,14 +2241,6 @@ fn calculateSplit( return plan; } -/// Extract coordinate data from the physical mesh for a specific axis. -fn addPhysicalToSplit(sharding: Sharding, plan: *AxisSplit, tag: PhysicalAxisTag) void { - const physical = sharding.data.physical; - const info = physical.axisInfo(tag) orelse return; - const depth = physical.axis_traversal.depth(tag) orelse return; - plan.add(@intCast(info.size), depth); -} - const ShardingTest = struct { pub const ExpectedShard = struct { // TODO: remove device_id from here, we always have ids from 0 to N @@ -2010,7 +2249,7 @@ const ShardingTest = struct { }; pub const Scenario = struct { - sharding: Sharding.Data, + mesh: *const Mesh, shape: Shape, expect_error: ?anyerror = null, @@ -2064,8 +2303,8 @@ const ShardingTest = struct { } pub fn run(self: ShardingTest, s: Scenario) !void { - const sharding = s.sharding; - errdefer std.log.warn("Failed sharding {f}", .{s}); + const mesh = s.mesh; + errdefer std.log.warn("Failed mesh {f}", .{s}); // Verify MLIR String if (s.expected_sdy) |expected_attr| { @@ -2079,7 +2318,7 @@ const ShardingTest = struct { defer ctx.deinit(); ctx.loadAllAvailableDialects(); - const actual_attr = try sharding.sdyShardingAttrForShape(self.allocator, ctx, s.shape); + const actual_attr = try mesh.sdyShardingAttrForShape(self.allocator, ctx, s.shape); var writer: std.Io.Writer.Allocating = .init(self.allocator); defer writer.deinit(); try actual_attr.asAttr().format(&writer.writer); @@ -2087,18 +2326,17 @@ const ShardingTest = struct { } // Verify Placement logic / Error - const sharding_ref: Sharding = .{ .data = &sharding }; - const ordered_devices = sharding_ref.devicesInCanonicalOrder(); + const ordered_devices = mesh.devicesInCanonicalOrder(); if (s.expect_error) |err| { errdefer std.log.warn("Expected error failed", .{}); try std.testing.expect(ordered_devices.len > 0); - try std.testing.expectError(err, sharding_ref.placement(s.shape)); + try std.testing.expectError(err, Sharding.Placement.init(s.shape)); return; } // Verify Shard Slices (Math) if (s.expected_shards.len > 0) { - const pl = try sharding_ref.placement(s.shape); + const pl: Placement = try .init(s.shape); errdefer std.log.warn("Expected shards failed. Got {f}", .{pl}); try std.testing.expectEqual(s.expected_shards.len, ordered_devices.len); for (s.expected_shards, ordered_devices) |expected, device| { @@ -2132,7 +2370,7 @@ const ShardingTest = struct { } }; -test "sharding: unknown partitioning implies replication" { +test "mesh: unknown partitioning implies replication" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2141,9 +2379,10 @@ test "sharding: unknown partitioning implies replication" { const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth }); const strategy: Strategy = .parseBindings(.{ .batch = .link_x }); + const mesh: Sharding.Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), + .mesh = &mesh, // No partitioning on shape .shape = Shape.init(.{ .batch = 8 }, .f32), .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", @@ -2156,7 +2395,7 @@ test "sharding: unknown partitioning implies replication" { }); } -test "sharding: suggest strategy realization" { +test "mesh: suggest strategy realization" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2169,11 +2408,12 @@ test "sharding: suggest strategy realization" { }); const strategy: Strategy = .suggest(logical, &physical); + const mesh: Mesh = try .init("suggested_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("suggested_mesh", &physical, logical, strategy), + .mesh = &mesh, .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) - .withPartitioning(.{ .batch = .batch, .model = .model }), + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model }), .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y0, z0 @@ -2188,7 +2428,7 @@ test "sharding: suggest strategy realization" { }); } -test "sharding: suggest folds logical axes with same intent" { +test "mesh: suggest folds logical axes with same intent" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2202,10 +2442,11 @@ test "sharding: suggest folds logical axes with same intent" { const strategy: Strategy = .suggest(logical, &physical); + const mesh: Mesh = try .init("fold_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("fold_mesh", &physical, logical, strategy), + .mesh = &mesh, .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) - .withPartitioning(.{ .model = .model, .experts = .experts }), + .withPartitioning(&mesh, .{ .model = .model, .experts = .experts }), .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 1 }, .{ 0, 4 } } }, @@ -2216,7 +2457,7 @@ test "sharding: suggest folds logical axes with same intent" { }); } -test "sharding: multiple physical axes on one logical dimension (folding)" { +test "mesh: multiple physical axes on one logical dimension (folding)" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2226,9 +2467,10 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { { const strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_y } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, @@ -2243,9 +2485,10 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { // Swap link_x and link_y order: // -> the roles of devices 1 (0,1) and 2 (1,0) are swapped const strategy: Strategy = .parseBindings(.{ .model = .{ .link_y, .link_x } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_y\", \"link_x\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, @@ -2257,7 +2500,7 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { } } -test "sharding: explicit strategy folding" { +test "mesh: explicit strategy folding" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2268,9 +2511,10 @@ test "sharding: explicit strategy folding" { var strategy: Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_z }); + const mesh: Mesh = try .init("strategy_fold", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("strategy_fold", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, @@ -2285,7 +2529,7 @@ test "sharding: explicit strategy folding" { }); } -test "sharding: open and replicated dimension mix" { +test "mesh: open and replicated dimension mix" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2295,9 +2539,10 @@ test "sharding: open and replicated dimension mix" { const strategy: Strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }); + const mesh: Mesh = try .init("mix_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("mix_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(.{ .batch = .open, .model = .replicated }), + .mesh = &mesh, + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, @@ -2308,7 +2553,7 @@ test "sharding: open and replicated dimension mix" { }); } -test "sharding: full 3D cluster sharding" { +test "mesh: full 3D cluster mesh" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2322,11 +2567,12 @@ test "sharding: full 3D cluster sharding" { .context = .link_z, }); + const mesh: Mesh = try .init("3d_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("3d_mesh", &physical, logical, strategy), + .mesh = &mesh, // Note: .context and .model are swapped compared to .link_x and .link_z .shape = Shape.init(.{ .batch = 4, .context = 4, .model = 4 }, .f32) - .withPartitioning(.{ .batch = .batch, .model = .model, .context = .context }), + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model, .context = .context }), .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_z\"}, {\"link_y\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, @@ -2341,7 +2587,7 @@ test "sharding: full 3D cluster sharding" { }); } -test "sharding: num partitions for logical axis" { +test "mesh: num partitions for logical axis" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2355,8 +2601,35 @@ test "sharding: num partitions for logical axis" { var strategy: Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_z }); - const sharding: Sharding.Data = try .init("axis_parts_mesh", &physical, logical, strategy); + const mesh: Mesh = try .init("axis_parts_mesh", &physical, logical, strategy); - try std.testing.expectEqual(4, sharding.numPartitionsForLogicalAxis(.model)); - try std.testing.expectEqual(1, sharding.numPartitionsForLogicalAxis(.batch)); + try std.testing.expectEqual(4, mesh.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(1, mesh.numPartitionsForLogicalAxis(.batch)); +} + +test "localized shapes do not retain global sharding" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{2}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("local", &physical, .mesh(.{ .batch = .balanced }), .parseBindings(.{ .batch = .link_x })); + try std.testing.expectEqual(mesh_.bindings.constSlice()[0].constSlice().ptr, mesh_.binding(0).ptr); + const global = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }); + const local = try shardedShape(global); + try std.testing.expectEqual(4, local.dim(.batch)); + try std.testing.expectEqual(null, local._sharding.mesh); + try std.testing.expectEqual(8, global.dim(.batch)); + try std.testing.expectEqual(&mesh_, global._sharding.mesh); +} + +test "GSPMD tiles use tensor axis order and logical partition indices" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); + const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + const shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }); + const ids = try mesh_.gspmdTileAssignment(arena.allocator(), shape); + try std.testing.expectEqualSlices(u32, &.{ 0, 2, 4, 6, 1, 3, 5, 7 }, ids); } diff --git a/zml/attention.zig b/zml/attention.zig index b3701b48..f5399c5e 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -116,22 +116,22 @@ pub const Metadata = union(Backend) { } }; - pub fn init(opts: InitOptions) Metadata { + pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { return switch (opts) { .vanilla => .{ .vanilla = {} }, .attnd => @panic("Must be initialized manually"), .nki => .{ .nki = {} }, - .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, - .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, + .cuda_fa2 => |o| .{ .cuda_fa2 = .init(o, mesh) }, + .cuda_fa3 => |o| .{ .cuda_fa3 = .init(o, mesh) }, .metal_fa => .{ .metal_fa = .init() }, }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return switch (self) { .vanilla => .{ .vanilla = {} }, .nki => .{ .nki = {} }, - inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform, sharding)), + inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform)), }; } @@ -179,6 +179,31 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T }; } +test "FlashAttention metadata initializes with explicit sharding outside compilation" { + try std.testing.expect(zml.Compiler.currentOrNull() == null); + const mesh: zml.Sharding.Mesh = .{ + .name = "metadata_mesh", + .physical = undefined, + .logical = .mesh(.{ .model = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .folds = .empty, + .folds_consumed = .empty, + }; + inline for (.{ Backend.cuda_fa2, Backend.cuda_fa3 }) |backend| { + const metadata: Metadata = .init(.fromBackend(backend, 16, 8), &mesh); + const cuda_metadata = @field(metadata, @tagName(backend)); + inline for (std.meta.fields(@TypeOf(cuda_metadata))) |field| { + const shape = @field(cuda_metadata, field.name).shape(); + try std.testing.expectEqual(&mesh, shape._sharding.mesh); + if (shape.hasTag(.h)) |axis| { + try std.testing.expectEqual(zml.Sharding.PartitionSpec.sharded(0), shape.partition(axis)); + } else { + try std.testing.expectEqual(zml.Sharding.PartitionSpec.replicated, shape.partition(.meta)); + } + } + } +} + test "attention: q=1,qh=64,kh=8" { try testAttention( .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), @@ -270,13 +295,13 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co defer k.deinit(); var v = try rng_k.eval(allocator, io, {}); defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, .replicated, @ptrCast(token_index_h)); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); defer token_index.deinit(); - const shardings = platform.shardings.values(); + const meshes = platform.meshes.values(); const vanilla_exe = try platform.compileFn(allocator, io, attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .vanilla, .vanilla }, .{ .program_name = "attention_vanilla", - .shardings = shardings, + .meshes = meshes, }); defer vanilla_exe.deinit(); @@ -292,7 +317,7 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co else => if (!backend.isAvailable(platform)) continue, } - const metadata: Metadata = .init(.fromBackend(backend, tensors.k.dim(.k), tensors.q.dim(.h))); + const metadata: Metadata = .init(.fromBackend(backend, tensors.k.dim(.k), tensors.q.dim(.h)), platform.meshes.get("model").?); const parameters: Parameters = .init(.fromBackend(backend)); const exe = try platform.compileFn( allocator, @@ -301,12 +326,12 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, .{ .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .shardings = shardings, + .meshes = meshes, }, ); defer exe.deinit(); - var metadata_d = try metadata.initBuffer(io, platform, platform.shardings.get("model").?); + var metadata_d = try metadata.initBuffer(io, platform); defer Metadata.deinitBuffer(&metadata_d); var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); diff --git a/zml/attention/attnd.zig b/zml/attention/attnd.zig index b1c39cd1..a986ab7b 100644 --- a/zml/attention/attnd.zig +++ b/zml/attention/attnd.zig @@ -74,12 +74,11 @@ pub const Metadata = struct { self: Metadata, io: std.Io, platform: *const zml.Platform, - sharding: zml.Sharding, ) !zml.Bufferized(Metadata) { return .{ .conversation_id = try zml.Buffer.scalar(io, platform, 749, .u64), - .layer_id = try zml.Buffer.uninitialized(io, platform, self.layer_id.shape(), sharding, .{}), - .num_tokens = try zml.Buffer.uninitialized(io, platform, self.num_tokens.shape(), sharding, .{}), + .layer_id = try zml.Buffer.uninitialized(io, platform, self.layer_id.shape(), .{}), + .num_tokens = try zml.Buffer.uninitialized(io, platform, self.num_tokens.shape(), .{}), }; } @@ -140,8 +139,8 @@ const Context = struct { }; pub fn causalAttention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_offset: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - const ctx = zml.Compiler.current(); - const num_partitions = ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable; + const tp = zml.Compiler.current().mesh(.model); + const num_partitions = tp.numPartitionsForLogicalAxis(.model); const actual_k, const actual_v = if (parameters.is_prefill) .{ k, v } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index f533b008..dd010217 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -236,27 +236,24 @@ pub const fa2 = struct { num_heads: i64, }; - pub fn init(opts: InitOptions) Metadata { + pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { return .{ - .softmax_lse = .fromShape(zml.Shape.init(.{ opts.seqlen, opts.num_heads, 1 }, .f32) - .withTags(.{ .s, .h, .dummy }) - .withPartitioning(.{ .h = .model })), + .softmax_lse = zml.Tensor.init(.{ .s = opts.seqlen, .h = opts.num_heads, .dummy = 1 }, .f32) + .withPartitioning(mesh, .{ .h = .model }), - .softmax_lse_accum = .fromShape(zml.Shape.init(.{ 1, opts.num_heads, 128 }, .f32) - .withTags(.{ .dummy, .h, .hd }) - .withPartitioning(.{ .h = .model })), + .softmax_lse_accum = zml.Tensor.init(.{ .dummy = 1, .h = opts.num_heads, .hd = 128 }, .f32) + .withPartitioning(mesh, .{ .h = .model }), - .out_accum = .fromShape(zml.Shape.init(.{ opts.seqlen, opts.num_heads, 128 }, .f32) - .withTags(.{ .s, .h, .hd }) - .withPartitioning(.{ .h = .model })), + .out_accum = zml.Tensor.init(.{ .s = opts.seqlen, .h = opts.num_heads, .hd = 128 }, .f32) + .withPartitioning(mesh, .{ .h = .model }), }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), sharding, .{}), + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), }; } @@ -313,20 +310,21 @@ pub const fa2 = struct { .merge(.{ .tot = .{ .tot, .ngroups } }); } - const q_sharded = q.withPartitioning(.{ .h = .model }); - const model_partitions: i32 = @intCast(ctx.partitioning.numPartitionsForLogicalAxis(q_sharded.shape(), .model) catch std.debug.panic("cu_fa2 attention backend requires a .model sharding", .{})); + const attn_sdy = ctx.resolveMesh(.{.model}); + const q_sharded = q.withPartitioning(attn_sdy, .{ .h = .model }); + const model_partitions: i32 = @intCast(attn_sdy.numPartitionsForLogicalAxis(.model)); const output = fa2_mha_varlen_fwd.call( .{ .q = q_sharded, - .k = k.withPartitioning(.{ .h = .model }), - .v = v.withPartitioning(.{ .h = .model }), + .k = k.withPartitioning(attn_sdy, .{ .h = .model }), + .v = v.withPartitioning(attn_sdy, .{ .h = .model }), .cu_seqlens_q = cu_seqlens_q, .cu_seqlens_k = cu_seqlens_k, .seqused_k = seqused_k, - .softmax_lse = metadata.softmax_lse.withPartitioning(.{ .h = .model }), - .softmax_lse_accum = metadata.softmax_lse_accum.withPartitioning(.{ .h = .model }), - .out_accum = metadata.out_accum.withPartitioning(.{ .h = .model }), + .softmax_lse = metadata.softmax_lse.withPartitioning(attn_sdy, .{ .h = .model }), + .softmax_lse_accum = metadata.softmax_lse_accum.withPartitioning(attn_sdy, .{ .h = .model }), + .out_accum = metadata.out_accum.withPartitioning(attn_sdy, .{ .h = .model }), }, .{ .o = q_sharded.shape(), @@ -451,25 +449,25 @@ pub const fa3 = struct { num_heads: i64, }; - pub fn init(opts: InitOptions) Metadata { + pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { return .{ - .softmax_lse = .fromShape(zml.Shape.init(.{opts.num_heads * opts.seqlen * 4}, .i8) - .withTags(.{.h}).withPartitioning(.{ .h = .model })), - .softmax_lse_accum = .fromShape(zml.Shape.init(.{opts.num_heads * 128 * 4}, .i8) - .withTags(.{.h}).withPartitioning(.{ .h = .model })), - .out_accum = .fromShape(zml.Shape.init(.{opts.num_heads * opts.seqlen * 128 * 4}, .i8) - .withTags(.{.h}).withPartitioning(.{ .h = .model })), - .scheduler_metadata = .fromShape(zml.Shape.init(.{2}, .i32) - .withTags(.{.meta}).withPartitioning(.{ .meta = .replicated })), + .softmax_lse = zml.Tensor.init(.{ .h = opts.num_heads * opts.seqlen * 4 }, .i8) + .withPartitioning(mesh, .{ .h = .model }), + .softmax_lse_accum = zml.Tensor.init(.{ .h = opts.num_heads * 128 * 4 }, .i8) + .withPartitioning(mesh, .{ .h = .model }), + .out_accum = zml.Tensor.init(.{ .h = opts.num_heads * opts.seqlen * 128 * 4 }, .i8) + .withPartitioning(mesh, .{ .h = .model }), + .scheduler_metadata = zml.Tensor.init(.{ .meta = 2 }, .i32) + .withPartitioning(mesh, .{ .meta = .replicated }), }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), sharding, .{}), - .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), sharding, .{}), + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), }; } @@ -496,7 +494,7 @@ pub const fa3 = struct { const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); // TODO(Corendos): replace with cumsum const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })) - .withPartitioning(.{ ._0 = .replicated }); + .withPartitioning(.model, .{ ._0 = .replicated }); var o = zml.ops.customCall( custom_call_name, @@ -909,15 +907,18 @@ pub const paged_fa2 = struct { const head_dim = q.dim(.hd); const num_heads = num_head_groups * num_kv_heads; // FIXME: remove unreachable and propagate error correctly. - const num_heads_per_shard = @divExact(num_heads, ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable); + + const attn_sdy = ctx.resolveMesh(.{.model}); + const model_partitions: i32 = @intCast(attn_sdy.numPartitionsForLogicalAxis(.model)); + const num_heads_per_shard = @divExact(num_heads, model_partitions); const o = switch (parameters) { .decode => |decode_parameters| b: { const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; - const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); - const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); + const block_table = decode_parameters.block_table.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k = decode_parameters.seqused_k.withPartitioning(.model, .{ .b = .replicated }); const out_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -925,27 +926,27 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const dummy_cu_seqlens_k: zml.Tensor = .zeroes(cu_seqlens_q.shape()); const batch_dim = q.dim(.b); var q2 = q; if (seqlenq_ngroups_swapped) { - q2 = q2.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); + q2 = q2.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.model, .{ .hkv = .model }); } else { - q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); } const output_shape = q2.shape(); @@ -1003,12 +1004,12 @@ pub const paged_fa2 = struct { .mixed => |mixed_parameters| b: { const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; - const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.{ .b = .replicated }); - const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.{ .b = .replicated }); - const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); - const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); + const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.model, .{ .b = .replicated }); + const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.model, .{ .b = .replicated }); const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -1016,22 +1017,22 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const dummy_cu_seqlens_k_prefill: zml.Tensor = .zeroes(cu_seqlens_q_prefill.shape()); var q2 = q; - q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const output_shape = q2.shape(); var o = zml.ops.manualComputation( @@ -1088,25 +1089,25 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = batch_dim_decode, .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = batch_dim_decode, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum_decode = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = batch_dim_decode, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const dummy_cu_seqlens_k_decode = zml.Tensor.zeroes(cu_seqlens_q_decode.shape()); var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_dim_decode)); if (seqlenq_ngroups_swapped) { - q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); + q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.model, .{ .hkv = .model }); } else { - q_decode = q_decode.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q_decode = q_decode.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); } const output_shape_decode = q_decode.shape(); @@ -1563,9 +1564,9 @@ pub const paged_fa3 = struct { .decode => |decode_parameters| b: { const batch_size = decode_parameters.block_table.dim(0); - const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); - const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); + const block_table = decode_parameters.block_table.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k = decode_parameters.seqused_k.withPartitioning(.model, .{ .b = .replicated }); const out_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -1573,21 +1574,21 @@ pub const paged_fa3 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .q = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .q = q.dim(.b), .hkv = num_kv_heads, .hg = num_head_groups, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const scheduler_metadata = zml.Tensor.zeroes(.init(.{ .b = batch_size + 1 }, .i32)); - var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const output_shape = q2.shape(); var o = zml.ops.manualComputation( @@ -1636,12 +1637,12 @@ pub const paged_fa3 = struct { .mixed => |mixed_parameters| b: { const batch_size_prefill = mixed_parameters.block_table_prefill.dim(0); - const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.{ .b = .replicated }); - const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.{ .b = .replicated }); - const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); - const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); + const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.model, .{ .b = .replicated }); + const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.model, .{ .b = .replicated }); const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -1649,21 +1650,21 @@ pub const paged_fa3 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .q = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .q = q.dim(.b), .hkv = num_kv_heads, .hg = num_head_groups, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const scheduler_metadata_prefill = zml.Tensor.zeroes(.init(.{ .b = batch_size_prefill + 1 }, .i32)); - var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const output_shape = q2.shape(); var o = zml.ops.manualComputation( @@ -1714,7 +1715,7 @@ pub const paged_fa3 = struct { .hg = num_head_groups, .b = batch_size_decode, .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, @@ -1727,9 +1728,9 @@ pub const paged_fa3 = struct { .hg = num_head_groups, }, .f32)); const scheduler_metadata_decode = zml.Tensor.zeroes(.init(.{ .b = batch_size_decode + 1 }, .i32)); - var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_size_decode)).withPartitioning(.{ .hkv = .model }); + var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_size_decode)).withPartitioning(.model, .{ .hkv = .model }); - q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const decode_output_shape = q_decode.shape(); var o_decode = zml.ops.manualComputation( diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig index 11942afe..42821a1f 100644 --- a/zml/attention/metal_attention.zig +++ b/zml/attention/metal_attention.zig @@ -14,9 +14,8 @@ pub const Metadata = struct { return .{ .num_tokens = .fromShape(.scalar(.u32)) }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { _ = self; - _ = sharding; return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; } diff --git a/zml/attention/nki/attention.zig b/zml/attention/nki/attention.zig index 6553b463..cdbad308 100644 --- a/zml/attention/nki/attention.zig +++ b/zml/attention/nki/attention.zig @@ -43,13 +43,14 @@ pub const Parameters = struct { /// Attention for Neuron. Both prefill and decode lower to NKI custom calls. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - const q_sharded = q.withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - const k_sharded = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - const v_sharded = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + const tp_mesh = zml.Compiler.current().mesh(.model); + const q_sharded = q.withPartitioning(tp_mesh, .{ .q = .replicated, .h = .model, .hd = .replicated }); + const k_sharded = k.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); + const v_sharded = v.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); if (q.dim(.q) == 1) { const token_index_2d = token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) - .withPartitioning(.{ .row = .replicated, .col = .replicated }); + .withPartitioning(tp_mesh, .{ .row = .replicated, .col = .replicated }); return zml.ops.manualComputation( (struct { diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 97960240..403b5a97 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -339,8 +339,8 @@ test "Backend.auto selects mosaic_tpu on TPU" { .devices = &.{}, .memories = &.{}, .physical_mesh = undefined, - .replicated_sharding = undefined, - .shardings = .empty, + .replicated_mesh = undefined, + .meshes = .empty, .io_impl = .threaded, }; @@ -357,8 +357,8 @@ test "Backend.auto selects triton on oneAPI" { .devices = &.{}, .memories = &.{}, .physical_mesh = undefined, - .replicated_sharding = undefined, - .shardings = .empty, + .replicated_mesh = undefined, + .meshes = .empty, .io_impl = .threaded, }; @@ -383,25 +383,26 @@ test pagedAttention { const batch_size = active_batch_size + 1; const query_token_count = prefill_token_count + num_decode; const dt: zml.DataType = .bf16; + const model_meshe = platform.meshes.get("model").?; const partition = .{ .hkv = .model }; const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt), partition), - .k = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt), partition), - .v = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt), partition), + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), .kv_cache = .{ .split = .{ - .k = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt), partition), - .v = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt), partition), + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), }, }, }; - const shardings: []const zml.Sharding = &.{ platform.replicated_sharding, platform.shardings.get("model").? }; - const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .shardings = shardings }); + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, platform.meshes.get("model").? }; + const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); defer rng_q.deinit(); - const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .shardings = shardings }); + const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); defer rng_k.deinit(); - const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .shardings = shardings }); + const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .meshes = meshes }); defer rng_kv_cache.deinit(); const triton_options_args: Options.Args = .{ @@ -458,9 +459,9 @@ test pagedAttention { const query_start_len: [batch_size + 1]i32 = .{ 0, 32, 33, 34, 35, 36, 37, 38, 38 }; var triton_parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ - .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), .replicated, @ptrCast(&block_table)), - .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), .replicated, @ptrCast(&seq_lens)), - .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), .replicated, @ptrCast(&query_start_len)), + .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), @ptrCast(&block_table)), + .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), @ptrCast(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), @ptrCast(&query_start_len)), } }; defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); @@ -514,7 +515,7 @@ test pagedAttention { io, pagedAttention, .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, - .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .shardings = shardings }, + .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, ); defer exe.deinit(); @@ -540,13 +541,13 @@ test pagedAttention { @memcpy(&seqused_k_decode, seq_lens[num_prefill .. num_prefill + num_decode]); break :cuda_fa2 .{ .cuda_fa2 = .{ .mixed = .{ - .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), .replicated, @ptrCast(&block_table_prefill)), - .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), .replicated, @ptrCast(&cu_seqlens_q_prefill)), - .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), .replicated, @ptrCast(&seqused_k_prefill)), + .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), @ptrCast(&block_table_prefill)), + .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), @ptrCast(&cu_seqlens_q_prefill)), + .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), @ptrCast(&seqused_k_prefill)), - .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), .replicated, @ptrCast(&block_table_decode)), - .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), .replicated, @ptrCast(&cu_seqlens_q_decode)), - .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), .replicated, @ptrCast(&seqused_k_decode)), + .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), @ptrCast(&block_table_decode)), + .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), @ptrCast(&cu_seqlens_q_decode)), + .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), @ptrCast(&seqused_k_decode)), .metadata = .{ .decode_offset = try .scalar(io, platform, prefill_token_count, .i32) }, } } }; @@ -1060,20 +1061,20 @@ test "Triton sparse MLA value ranks and padded queries" { const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); var parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ - .block_table = try .fromBytes(std.testing.io, platform, parameters.triton.block_table.shape(), .replicated, std.mem.sliceAsBytes(&block_table)), - .seq_lens = try .fromBytes(std.testing.io, platform, parameters.triton.seq_lens.shape(), .replicated, std.mem.sliceAsBytes(&seq_lens)), - .query_start_len = try .fromBytes(std.testing.io, platform, parameters.triton.query_start_len.shape(), .replicated, std.mem.sliceAsBytes(&query_start_len)), + .block_table = try .fromBytes(std.testing.io, platform, parameters.triton.block_table.shape(), std.mem.sliceAsBytes(&block_table)), + .seq_lens = try .fromBytes(std.testing.io, platform, parameters.triton.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(std.testing.io, platform, parameters.triton.query_start_len.shape(), std.mem.sliceAsBytes(&query_start_len)), } }; defer zml.Buffer.deinitAll(Parameters, ¶meters_d); - var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); + var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, std.mem.sliceAsBytes(&q_data)); defer q_d.deinit(); - var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)); + var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, std.mem.sliceAsBytes(&kv_data)); defer kv_d.deinit(); - var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); + var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, std.mem.sliceAsBytes(&sink_data)); defer sink_d.deinit(); - var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); + var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, std.mem.sliceAsBytes(&topk_data)); defer topk_d.deinit(); - var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, .replicated, std.mem.sliceAsBytes(&tokens_pos_data)); + var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, std.mem.sliceAsBytes(&tokens_pos_data)); defer tokens_pos_d.deinit(); const TestCase = struct { @@ -1174,20 +1175,20 @@ test "execute stablehlo mla kernel" { const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); var parameters_d: zml.Bufferized(Parameters) = .{ .stablehlo = .{ - .block_table = try .fromBytes(std.testing.io, platform, parameters.stablehlo.block_table.shape(), .replicated, std.mem.sliceAsBytes(&block_table)), - .seq_lens = try .fromBytes(std.testing.io, platform, parameters.stablehlo.seq_lens.shape(), .replicated, std.mem.sliceAsBytes(&seq_lens)), - .query_start_len = try .fromBytes(std.testing.io, platform, parameters.stablehlo.query_start_len.shape(), .replicated, std.mem.sliceAsBytes(&query_start_len)), + .block_table = try .fromBytes(std.testing.io, platform, parameters.stablehlo.block_table.shape(), std.mem.sliceAsBytes(&block_table)), + .seq_lens = try .fromBytes(std.testing.io, platform, parameters.stablehlo.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(std.testing.io, platform, parameters.stablehlo.query_start_len.shape(), std.mem.sliceAsBytes(&query_start_len)), } }; defer zml.Buffer.deinitAll(Parameters, ¶meters_d); - var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); + var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, std.mem.sliceAsBytes(&q_data)); defer q_d.deinit(); - var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)); + var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, std.mem.sliceAsBytes(&kv_data)); defer kv_d.deinit(); - var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); + var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, std.mem.sliceAsBytes(&sink_data)); defer sink_d.deinit(); - var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); + var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, std.mem.sliceAsBytes(&topk_data)); defer topk_d.deinit(); - var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, .replicated, std.mem.sliceAsBytes(&tokens_pos_data)); + var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, std.mem.sliceAsBytes(&tokens_pos_data)); defer tokens_pos_d.deinit(); const exe = try platform.compileFn( diff --git a/zml/attention/sparse_mla.zig b/zml/attention/sparse_mla.zig index fbb9573b..3edbbe67 100644 --- a/zml/attention/sparse_mla.zig +++ b/zml/attention/sparse_mla.zig @@ -103,7 +103,7 @@ pub fn launchConfig(paged_opts: Options, topk_count: usize, cu_count_: usize) Co } // Full 256-query prefill chunks already expose enough query parallelism. // Retain BLOCK_M=16 and the generic split model, but process wider sparse - // tiles. The predicate uses the post-sharding flattened query shape. + // tiles. The predicate uses the post-mesh flattened query shape. if (is_sm103 and !paged_opts.all_decode and paged_opts.total_q_blocks >= 256 and diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index d1806730..82b39006 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -155,12 +155,12 @@ pub const mosaic_tpu = struct { .{ q.dim(.hd), target_head_dim }, ); return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }) - .withPartitioning(.{ .h = .model }); + .withPartitioning(.model, .{ .h = .model }); } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const kv_pages_partitioned = kv_pages.withPartitioning(.{ .hkv = .model }); - const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + const kv_pages_partitioned = kv_pages.withPartitioning(.model, .{ .hkv = .model }); + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); const cache_head_dim: usize = @intCast(kv_pages_partitioned.dim(.hd)); stdx.debug.assert( @@ -169,15 +169,15 @@ pub const mosaic_tpu = struct { .{ cache_head_dim, logical_head_dim }, ); const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); - const query_start_len = parameters.query_start_len.withPartitioning(.{ .b = .replicated }); + const query_start_len = parameters.query_start_len.withPartitioning(.model, .{ .b = .replicated }); return .{ .q = q_ragged, .kv_pages = kv_pages_partitioned, - .seq_lens = parameters.seq_lens.withPartitioning(.{ ._0 = .replicated }), - .block_table = parameters.block_table.withPartitioning(.{ ._0 = .replicated, ._1 = .replicated }), + .seq_lens = parameters.seq_lens.withPartitioning(.model, .{ ._0 = .replicated }), + .block_table = parameters.block_table.withPartitioning(.model, .{ ._0 = .replicated, ._1 = .replicated }), .query_start_len = query_start_len, - .num_seqs = activeSequenceCount(query_start_len).withPartitioning(.{ ._0 = .replicated }), + .num_seqs = activeSequenceCount(query_start_len).withPartitioning(.model, .{ ._0 = .replicated }), }; } diff --git a/zml/buffer.zig b/zml/buffer.zig index 5b86cae3..6f7acd8c 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -31,7 +31,6 @@ test { pub const Buffer = struct { _platform: *const Platform, _shape: Shape, - _sharding: Sharding, _shards: Shards, pub const MAX_NUM_SHARDS: u16 = Platform.MAX_NUM_DEVICES; @@ -115,10 +114,10 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - const placement = self._sharding.placement(self._shape) catch { - return try writer.print("sharding error {} vs {}", .{ self._sharding, self._shape }); + const pl = Sharding.Placement.init(self._shape) catch |err| { + return try writer.print("Buffer({f}) with invalid sharding {} ({})", .{ self._shape, self._shape._sharding, err }); }; - try writer.print("{f}", .{placement}); + try writer.print("{f}", .{pl}); } /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -126,7 +125,6 @@ pub const Buffer = struct { io: std.Io, platform: *const Platform, shape_: Shape, - sharding: Sharding, data_: []const u8, opts: FromOptions, ) !Buffer { @@ -135,7 +133,6 @@ pub const Buffer = struct { var res: Buffer = .{ ._platform = platform, ._shape = shape_, - ._sharding = sharding.resolve(platform), ._shards = .empty, }; errdefer for (res._shards.slice()) |shard| { @@ -146,7 +143,7 @@ pub const Buffer = struct { const slice = Slice.init(sh, data_); const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - const placement = placementOrPanic(res._sharding, sh); + const placement = placementOrPanic(platform, sh); const shard_dims: []const i64 = placement.shape.dims(); const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); @@ -179,28 +176,28 @@ pub const Buffer = struct { /// Copies the given Zig bytes to the accelerator memory and /// return a Buffer with the given dimensions. - pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, data: []const u8) !Buffer { - return fromBytesOpts(io, platform, sh, sharding, data, .{}); + pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8) !Buffer { + return fromBytesOpts(io, platform, sh, data, .{}); } - pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, data: []const u8, opts: FromOptions) !Buffer { - return from(io, platform, sh, sharding, data, opts); + pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { + return from(io, platform, sh, data, opts); } /// Copies the given zml.Slice to the accelerator memory and /// return a Buffer. - pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice, sharding: Sharding) !Buffer { - return fromSliceOpts(io, platform, slice, sharding, .{}); + pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice) !Buffer { + return fromSliceOpts(io, platform, slice, .{}); } - pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, sharding: Sharding, opts: FromOptions) !Buffer { - return from(io, platform, slice.shape, sharding, std.mem.sliceAsBytes(slice.constData()), opts); + pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, opts: FromOptions) !Buffer { + return from(io, platform, slice.shape, std.mem.sliceAsBytes(slice.constData()), opts); } /// Creates a Buffer with a single element. pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); - return fromBytes(io, platform, .scalar(dtype_), .replicated, x.asBytes()); + return fromBytes(io, platform, .scalar(dtype_), x.asBytes()); } pub fn await(self: Buffer, io: std.Io) !void { @@ -217,7 +214,6 @@ pub const Buffer = struct { _: std.Io, platform: *const Platform, shape_: Shape, - sharding: Sharding, opts: UnitializedOptions, ) !Buffer { std.log.debug("uninitialized {f}", .{shape_}); @@ -225,7 +221,6 @@ pub const Buffer = struct { var res: Buffer = .{ ._platform = platform, ._shape = shape_, - ._sharding = sharding.resolve(platform), ._shards = .empty, }; errdefer for (res._shards.slice()) |shard| { @@ -234,7 +229,7 @@ pub const Buffer = struct { stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); const element_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - const placement = placementOrPanic(res._sharding, sh); + const placement = placementOrPanic(platform, sh); const shard_dims: []const i64 = placement.shape.dims(); const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); @@ -257,14 +252,13 @@ pub const Buffer = struct { } /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, sharding: Sharding, pjrt_buffers: []const *pjrt.Buffer) Buffer { + pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); return .{ ._platform = platform, ._shape = sh, - ._sharding = sharding, ._shards = Shards.fromSlice(pjrt_buffers) catch unreachable, }; } @@ -283,8 +277,8 @@ pub const Buffer = struct { pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); - const placement = placementOrPanic(self._sharding, self._shape); - for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { + const placement = placementOrPanic(self._platform, self._shape); + for (self._platform.physical_mesh.devices_in_canonical_order, 0..) |device, shard_index| { // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed const sub_slice = placement.shardSlice(device.coords, slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; @@ -306,12 +300,12 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - const placement = placementOrPanic(self._sharding, self._shape); + const placement = placementOrPanic(self._platform, self._shape); var shard_slice = try Slice.alloc(allocator, placement.shape); defer shard_slice.free(allocator); - for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { + for (self._platform.physical_mesh.devices_in_canonical_order, 0..) |device, shard_index| { const sub_slice = placement.shardSlice(device.coords, slice); const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { @@ -331,8 +325,8 @@ pub const Buffer = struct { /// ie: `num_devices * shard_byte_size` /// `shard_byte_size` can be up to `self.shape().byteSize()` when the buffer is fully replicated. pub fn byteSize(self: Buffer) usize { - const placement = placementOrPanic(self._sharding, self._shape); - return placement.shape.byteSize() * self._sharding.devicesInCanonicalOrder().len; + const placement = placementOrPanic(self._platform, self._shape); + return placement.shape.byteSize() * self._shards.len; } pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { @@ -360,7 +354,7 @@ pub const Buffer = struct { view_shard_list.appendAssumeCapacity(view_shard); } - return fromPjrtBuffers(platform, self._shape, self._sharding, view_shard_list.items); + return fromPjrtBuffers(platform, self._shape, view_shard_list.items); } }; @@ -381,13 +375,14 @@ test "device round-trip" { .{ 56, 57, 58, 59, 60, 61, 62, 63 }, }; + const mp: *const zml.Mesh = platform.meshes.get("model").?; const x_h: zml.Slice = .init(.withPartitioning( .init(.{ .b = 8, .d = 8 }, .u32), + mp, .{ .b = .model }, ), std.mem.asBytes(&x)); // no free: x_h is stack allocated - const model_sharding: zml.Sharding = platform.shardings.get("model").?; - const x_d: zml.Buffer = try .fromSlice(io, platform, x_h, model_sharding); + const x_d: zml.Buffer = try .fromSlice(io, platform, x_h); try std.testing.expectEqual(platform.devices.len, x_d.numShards()); { @@ -409,29 +404,11 @@ test "device round-trip" { } } -fn placementOrPanic(sharding: Sharding, shape: Shape) Sharding.Placement { - return sharding.placement(shape) catch |err| { +fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { + var sh = shape; + if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; + return Sharding.Placement.init(sh) catch |err| { @branchHint(.cold); - switch (err) { - error.MissingLogicalBinding => { - log.err( - \\Failed to shard Buffer of shape {f}, with sharding: - \\{f} - \\ - \\The Buffer is probably inheriting a partitionned shape from a Tensor, - \\So Buffer creation must pass a Sharding, that maps the logical sharding of the Tensor to the physical mesh. - , .{ shape, sharding }); - @panic("Buffer shape and sharding should be consistent"); - }, - error.IncompatibleSharding => { - log.err( - \\Failed to shard Buffer of shape {f}, with sharding: - \\{f} - \\ - \\The Buffer dimension isn't properly divisible by the number of devices along the sharded axis. - , .{ shape, sharding }); - @panic("Buffer shape should be divisible by the number of devices along the sharded axis."); - }, - } + std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); }; } diff --git a/zml/exe.zig b/zml/exe.zig index f97b8f7d..1db04837 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -23,9 +23,6 @@ pub const Exe = struct { input_shapes: []const Shape, output_shapes: []const Shape, - input_shardings: []const Sharding, - output_shardings: []const Sharding, - /// Inputs whose buffers are donated to an output (`reuseBuffer`). /// Runners destroy their PJRT handles after the raw call returns. donated_input_indices: []const usize, @@ -43,8 +40,6 @@ pub const Exe = struct { num_partitions: i32, input_shapes: []const Shape, output_shapes: []const Shape, - input_shardings: []const Sharding, - output_shardings: []const Sharding, input_aliasing: []const ?u32, ) !Exe { var arena = std.heap.ArenaAllocator.init(allocator); @@ -53,10 +48,6 @@ pub const Exe = struct { const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); - // Re-home sharding pointers into arena-owned values so exe doesn't depend on caller lifetimes. - const input_shardings_copy = try arena.allocator().dupe(Sharding, input_shardings); - const output_shardings_copy = try arena.allocator().dupe(Sharding, output_shardings); - var donated_count: usize = 0; for (input_aliasing) |aliasing| donated_count += @intFromBool(aliasing != null); @@ -74,8 +65,6 @@ pub const Exe = struct { .exe = exe, .input_shapes = input_shapes_copy, .output_shapes = output_shapes_copy, - .input_shardings = input_shardings_copy, - .output_shardings = output_shardings_copy, .donated_input_indices = donated_input_indices, .num_devices = num_devices, .num_partitions = num_partitions, @@ -90,11 +79,11 @@ pub const Exe = struct { } pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { - return Arguments.init(allocator, self.input_shapes, self.input_shardings, self.num_devices); + return Arguments.init(allocator, self.input_shapes, self.num_devices); } pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { - return Results.init(allocator, self.output_shapes, self.output_shardings, self.platform, self.num_devices); + return Results.init(allocator, self.output_shapes, self.platform, self.num_devices); } pub const FlatBuffers = struct { @@ -131,9 +120,8 @@ pub const Exe = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, baked_count: usize = 0, - shardings: []const Sharding, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, num_devices: usize) error{OutOfMemory}!Arguments { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) error{OutOfMemory}!Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -143,7 +131,6 @@ pub const Exe = struct { return .{ .flat_buffers = flat_buffers, .expected_shapes = expected_shapes, - .shardings = shardings, }; } @@ -219,9 +206,8 @@ pub const Exe = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, - shardings: []const Sharding, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, platform: *const Platform, num_devices: usize) !Results { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: *const Platform, num_devices: usize) !Results { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -232,7 +218,6 @@ pub const Exe = struct { .platform = platform, .flat_buffers = flat_buffers, .expected_shapes = expected_shapes, - .shardings = shardings, }; } @@ -254,7 +239,7 @@ pub const Exe = struct { for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], context_.self.shardings[context_.current_index], shards.constSlice()); + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); context_.current_index += 1; } }.cb, &context, &result); @@ -274,7 +259,7 @@ pub const Exe = struct { for (0..ctx.results.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(ctx.results.flat_buffers.buffers[device_index][ctx.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], ctx.results.shardings[ctx.current_index], shards.constSlice()); + buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], shards.constSlice()); ctx.current_index += 1; } }.cb, &context, &v); @@ -639,7 +624,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { defer exe.deinit(); try std.testing.expectEqualSlices(usize, &.{0}, exe.donated_input_indices); - var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), .replicated, std.mem.asBytes(&[2]i32{ 10, 20 })); + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); defer weight_buffer.deinit(); var args = try exe.args(allocator); defer args.deinit(allocator); @@ -647,7 +632,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { defer results.deinit(allocator); for ([_]bool{ false, true }) |wait| { - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); args.set(.{ value_buffer, weight_buffer }); exe.call(io, args, &results, .{ .wait = wait }); @@ -679,9 +664,9 @@ test "FnExe runner replaces donated buffers with and without waiting" { const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .weight = weight, .value = value }}); defer exe.deinit(); - var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), .replicated, std.mem.asBytes(&[2]i32{ 10, 20 })); + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); defer weight_buffer.deinit(); - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); var runner = try Model.Runner(.{.weight}).init(&exe, allocator, .{ .weight = weight_buffer }); defer runner.deinit(allocator); @@ -714,7 +699,7 @@ test "FnExe runner replaces unchanged donated buffers" { defer exe.deinit(); try std.testing.expectEqualSlices(usize, &.{0}, exe.raw.donated_input_indices); - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); var runner = try exe.runner(allocator); defer runner.deinit(allocator); diff --git a/zml/io.zig b/zml/io.zig index 589abd47..5fce0793 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -31,14 +31,16 @@ pub const TensorStore = struct { }; registry: *safetensors.TensorRegistry, + meshes: []const *const Sharding.Mesh, id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, - pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry) TensorStore { - const arena: std.heap.ArenaAllocator = .init(allocator); + pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { + var arena: std.heap.ArenaAllocator = .init(allocator); return .{ .registry = registry, + .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), .id_to_sources = .empty, .allocator = allocator, .arena = arena, @@ -104,6 +106,20 @@ pub const TensorStore = struct { return .{ .store = self }; } + pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (store.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + std.debug.panic( + \\Found no meshes named {s} in TensorStore. + \\Try passing more meshes to `zml.TensorStore.fromRegistry`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(store.meshes) }); + } + pub const View = struct { store: *TensorStore, @@ -162,19 +178,50 @@ pub const TensorStore = struct { } else false; } - pub const CreateTensorOpts = struct { - memory: Memory.Kind = .default, - }; - - pub fn maybeCreateTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{}); + /// Creates a zml.Tensor from a specific entry in the store. + pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{ .memory = .host_pinned }); + pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); + const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + if (mesh_ == .replicated) { + return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .replicated); + } + const resolved_mesh = self.store.mesh(mesh_); + // Unspecified tensor axes are treated as replicated + const partition: Sharding.Partitioning = if (has_tags) + .parse(parsed_tags.constSlice(), resolved_mesh, .replicated, partitioning) + else + partitioning; + return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .{ .mesh = resolved_mesh, .partition = partition }); } - fn maybeCreateTensorInternal(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype, opts: CreateTensorOpts) ?Tensor { + pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { + const tensor = self.maybeCreateTensor(subkey, tags, mesh_, partitioning); + if (tensor) |t| { + const storage = self.store.id_to_sources.getPtr(t.id); + storage.?.memory = .host_pinned; + } + return tensor; + } + + pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { + const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); + const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + + return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.slice() else null, .replicated) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { var buffer: [256]u8 = undefined; const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); const source = self.store.dupeSource(key) orelse return null; @@ -184,57 +231,19 @@ pub const TensorStore = struct { sources[0] = source; var shape = source.shape; - shape = applyTags(shape, tagz); - shape = applyPartitioning(shape, partitioning); + if (tags) |user_tags| { + stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); + @memcpy(shape._tags.slice(), user_tags); + } + shape._sharding = sharding; + if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = opts.memory }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); return tensor; } - pub fn createTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{}) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - pub fn createHostPinnedTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{ .memory = .host_pinned }) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - fn applyTags(shape_: Shape, tagz: anytype) Shape { - var shape = shape_; - if (@TypeOf(tagz) != @TypeOf(null)) { - switch (@typeInfo(@TypeOf(tagz))) { - .optional => if (tagz) |t| { - shape = shape.withTags(t); - }, - else => shape = shape.withTags(tagz), - } - } - return shape; - } - - fn applyPartitioning(shape_: Shape, partitioning: anytype) Shape { - var shape = shape_; - - if (@TypeOf(partitioning) == @TypeOf(null)) { - @compileError("TensorStore.View.createTensor partitioning cannot be null; pass .replicated or an explicit partitioning"); - } - - switch (@typeInfo(@TypeOf(partitioning))) { - .optional => @compileError("TensorStore.View.createTensor partitioning cannot be optional; pass .replicated or an explicit partitioning"), - .enum_literal => switch (partitioning) { - .replicated => shape = shape.withReplicatedPartitioning(), - else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), - }, - else => shape = shape.withPartitioning(partitioning), - } - - return shape; - } - pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { const arena = self.store.arena.allocator(); @@ -365,7 +374,7 @@ pub const Loader = struct { pub const LoadError = error{TransformedTensorNotDelivered}; - pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) LoadError!void { + pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { const tensor_count = meta.count(Tensor, model); var arena: std.heap.ArenaAllocator = .init(self.allocator); @@ -406,7 +415,7 @@ pub const Loader = struct { if (sources.transformed) continue; } - self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, shardings, opts }); + self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); } } @@ -424,25 +433,25 @@ pub const Loader = struct { log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); } - pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: Tensor, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { - loader.group.async(io, defaultCallback, .{ loader, io, &tensor, buffer, store, shardings, opts }); + pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { + loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); } - fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { + fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { const sources = store.getSourcesById(tensor.id) orelse { std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); return; }; stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); - self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, shardings, opts) catch |e| { + self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); unreachable; }; } - fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, shardings: []const Sharding, opts: LoadOpts) void { - self.loadSingleInner(io, source, shape, buffer, .device, shardings, opts) catch |e| { + fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { + self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { log.err("Failed to load tensor {s}: {}", .{ source.name, e }); loaded.* = false; return; @@ -457,17 +466,11 @@ pub const Loader = struct { shape: Shape, buffer: *Buffer, memory: Memory.Kind, - shardings: []const Sharding, opts: LoadOpts, ) !void { var reader = try source.reader(io, &.{}, .{}); defer reader.deinit(); - const sharding = Sharding.pickSharding(shardings, shape, .explicit_axis_binding) orelse blk: { - log.debug("No sharding strategy found for tensor {s} with shape {f}, using replicated sharding", .{ reader.tensor.name, shape }); - break :blk self.platform.replicated_sharding; - }; - var writer = try MemoryWriter.init( self.allocator, io, @@ -476,7 +479,6 @@ pub const Loader = struct { self.dma_allocators, self.dma_chunk_size, shape, - sharding, buffer, memory, ); @@ -507,7 +509,6 @@ pub const Loader = struct { tensor: Tensor, buffer: *Buffer, store: *const TensorStore, - shardings: []const Sharding, exe: *const Exe, opts: LoadOpts, ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { @@ -530,7 +531,7 @@ pub const Loader = struct { defer if (node) |*n| n.end(); for (sources, 0..) |source, i| { - self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], shardings, .{} }); + self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); } self.group.await(io) catch |err| switch (err) { error.Canceled => return error.Canceled, @@ -566,7 +567,7 @@ test "Loader.loadExecute cleans up donated and non-donated inputs" { defer registry.deinit(); try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: TensorStore = .fromRegistry(allocator, ®istry); + var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); defer store.deinit(); var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); defer loader.deinit(); @@ -586,7 +587,7 @@ test "Loader.loadExecute cleans up donated and non-donated inputs" { try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); var output: Buffer = undefined; - try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &.{}, &exe, .{}); + try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); defer output.deinit(); try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); try std.testing.expect(loader.delivered.contains(tensor.id)); @@ -681,18 +682,17 @@ pub const MemoryWriter = union(enum) { dma_allocators: []const mem.DmaAllocator, dma_chunk_size: usize, shape: Shape, - sharding: Sharding, buffer: *Buffer, memory: Memory.Kind, ) !MemoryWriter { return switch (platform.target) { .cuda, .rocm, .oneapi => .{ - .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer, memory), + .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), }, .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) else - .{ .buffered = try .init(allocator, io, platform, shape, sharding, buffer) }, + .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, }; } @@ -722,16 +722,14 @@ pub const BufferedMemoryWriter = struct { io: std.Io, platform: *const Platform, shape: Shape, - sharding: Sharding, buffer: *Buffer, interface: std.Io.Writer, - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, sharding: Sharding, buffer: *Buffer) !BufferedMemoryWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { return .{ .io = io, .platform = platform, .shape = shape, - .sharding = sharding, .buffer = buffer, .interface = .{ .buffer = try allocator.alloc(u8, shape.byteSize()), @@ -757,7 +755,6 @@ pub const BufferedMemoryWriter = struct { self.io, self.platform, self.shape, - self.sharding, @ptrCast(self.interface.buffer), .{ .wait = true }, ) catch return std.Io.Writer.Error.WriteFailed; @@ -982,9 +979,9 @@ const DispatchSpans = struct { spans: []DispatchSpan, mirror_writers: []usize, - fn init(allocator: std.mem.Allocator, shape: Shape, sharding: Sharding) !DispatchSpans { - const placement = try sharding.placement(shape); - const ordered_devices = sharding.devicesInCanonicalOrder(); + fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { + const placement = try Sharding.Placement.init(shape); + const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); var placement_span_count: usize = 0; for (ordered_devices) |device| { @@ -1197,11 +1194,14 @@ pub const DirectMemoryWriter = struct { dma_allocators: []const mem.DmaAllocator, dma_chunk_size: usize, shape: Shape, - sharding: Sharding, buffer: *Buffer, memory: Memory.Kind, ) !DirectMemoryWriter { - const ordered_devices = sharding.devicesInCanonicalOrder(); + var placed_shape = shape; + if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; + const placement = try Sharding.Placement.init(placed_shape); + const mesh = placement.mesh().?; + const ordered_devices = mesh.devicesInCanonicalOrder(); var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); errdefer allocator.free(shard_writers); @@ -1211,7 +1211,6 @@ pub const DirectMemoryWriter = struct { }; var pjrt_buffers: Buffer.Shards = .empty; - const placement = try sharding.placement(shape); for (ordered_devices, 0..) |device, i| { const pool = &pools[device.id]; const shard_dma_allocator = dma_allocators[device.id].allocator(); @@ -1223,9 +1222,9 @@ pub const DirectMemoryWriter = struct { pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); } - buffer.* = .fromPjrtBuffers(platform, shape, sharding, pjrt_buffers.constSlice()); + buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); - const dispatch_spans: DispatchSpans = try .init(allocator, shape, sharding); + const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); errdefer dispatch_spans.deinit(allocator); const first_span = dispatch_spans.spans[0]; @@ -1535,15 +1534,14 @@ const DirectMemoryWriterDeviceTest = struct { allocator: std.mem.Allocator, io: std.Io, - fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario) !void { + fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; defer platform.deinit(self.allocator, self.io); - const sharding: Sharding.Data = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); try self.runDirectMemoryWriter( platform, - scenario.shape, - .{ .data = &sharding }, + scenario.shape.withPartitioning(&mesh, partitioning), scenario.write_mode, scenario.writable_slice_min_len, scenario.pool_chunks, @@ -1556,7 +1554,6 @@ const DirectMemoryWriterDeviceTest = struct { self: DirectMemoryWriterDeviceTest, platform: *const Platform, shape: Shape, - sharding: Sharding, write_mode: WriteMode, writable_slice_min_len: usize, pool_chunks: usize, @@ -1595,7 +1592,6 @@ const DirectMemoryWriterDeviceTest = struct { dma_allocators, pool_chunk_size, shape, - sharding, &written_buffer, memory, ); @@ -1643,11 +1639,10 @@ test "DirectMemoryWriter: replicated with auto topology" { .physical_mesh = .auto, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32) - .withPartitioning(.{ .rows = .replicated, .cols = .replicated }), + .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), .logical_mesh = .mesh(.{ .x = .high_bandwidth }), .strategy = .parseBindings(.{ .x = .link_x }), - }); + }, .{ .rows = .replicated, .cols = .replicated }); } test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { @@ -1662,11 +1657,10 @@ test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) - .withPartitioning(.{ .rows = .replicated, .cols = .model }), + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = .parseBindings(.{ .model = .link_x }), - }); + }, .{ .rows = .replicated, .cols = .model }); } test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { @@ -1681,17 +1675,16 @@ test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32) - .withPartitioning(.{ .batch = .batch, .model = .model }), + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, }), .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), - }); + }, .{ .batch = .batch, .model = .model }); } -test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { +test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { const case: DirectMemoryWriterDeviceTest = .{ .allocator = std.testing.allocator, .io = std.testing.io, @@ -1703,14 +1696,14 @@ test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .model = 4096 }, .f32).withPartitioning(.{ .model = .model }), + .shape = Shape.init(.{ .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = blk: { var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_y }); break :blk strategy; }, - }); + }, .{ .model = .model }); } test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { @@ -1725,14 +1718,13 @@ test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) - .withPartitioning(.{ .rows = .replicated, .cols = .model }), + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = .parseBindings(.{ .model = .link_x }), .write_mode = .writable_slice_greedy, .writable_slice_min_len = 64, .pool_chunk_size = 1024, - }); + }, .{ .rows = .replicated, .cols = .model }); } test "DirectMemoryWriter: 3D topology folded model + replicated batch" { @@ -1747,8 +1739,7 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { .physical_mesh = .{ .custom = buildMesh2x2x2 }, .cpu = .{ .device_count = 8 }, }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) - .withPartitioning(.{ .batch = .replicated, .model = .model }), + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, @@ -1758,7 +1749,7 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { strategy.addFold(.link_x, &.{ .link_x, .link_z }); break :blk strategy; }, - }); + }, .{ .batch = .replicated, .model = .model }); } test "MemoryWriter can produce a host pinned buffer" { @@ -1772,10 +1763,9 @@ test "MemoryWriter can produce a host pinned buffer" { .create_options = .{ .physical_mesh = .{ .custom = buildMesh2 }, }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) - .withPartitioning(.{ .batch = .replicated, .model = .model }), + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = .parseBindings(.{ .model = .link_x }), .memory = .host_pinned, - }); + }, .{ .batch = .replicated, .model = .model }); } diff --git a/zml/kernel.zig b/zml/kernel.zig index c2114001..2d8945e4 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -955,9 +955,9 @@ test "fly kernels run on rocm" { fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), .replicated, std.mem.sliceAsBytes(ha)); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), .replicated, std.mem.sliceAsBytes(hb)); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); defer bb.deinit(); var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); defer out.deinit(); diff --git a/zml/mem.zig b/zml/mem.zig index 5ba82c69..f471a83a 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -333,7 +333,6 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu // I'm not sure I like that. I'd rather set all fields but _shards than leaving most of the m undefined. ._shape = model._shape, ._shards = .empty, - ._sharding = undefined, ._platform = undefined, }; return; diff --git a/zml/meta.zig b/zml/meta.zig index 9ff4d39a..08738e11 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -456,6 +456,7 @@ const Visitor = struct { pub const Action = enum { callback, recurse, + exit, }; pub fn determineAction(comptime callback: anytype, comptime PtrTypeOfV: type) Action { @@ -473,6 +474,13 @@ const Visitor = struct { return .callback; } + switch (@typeInfo(ptr_info.child)) { + .@"struct", .@"union" => { + if (@hasDecl(ptr_info.child, "zml_no_meta_visit")) return .exit; + }, + else => {}, + } + return .recurse; } }; @@ -487,6 +495,7 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) const can_error = stdx.meta.FnReturnErrorSet(callback) != null; return switch (action) { + .exit => {}, .callback => callback(ctx, v), .recurse => { const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { @@ -628,8 +637,8 @@ pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: const err_args = .{ChildTypeV}; switch (action) { + .exit => {}, .callback => return callback(ctx, v), - .recurse => { if (comptime !Contains(ChildTypeV, TargetType)) return; @@ -929,6 +938,7 @@ pub fn Contains(Haystack: type, T: type) bool { return switch (@typeInfo(Haystack)) { .@"struct" => |info| { + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; inline for (info.fields) |field| { if (!field.is_comptime and Contains(field.type, T)) { return true; @@ -937,6 +947,7 @@ pub fn Contains(Haystack: type, T: type) bool { return false; }, .@"union" => |info| { + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; inline for (info.fields) |field| { if (Contains(field.type, T)) return true; diff --git a/zml/moe/cute_mxfp4.zig b/zml/moe/cute_mxfp4.zig index 7cb235d0..9928aeb4 100644 --- a/zml/moe/cute_mxfp4.zig +++ b/zml/moe/cute_mxfp4.zig @@ -47,7 +47,7 @@ pub fn fusedExperts( stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); + const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); const hidden = down.weight.dim(1); const intermediate = down.weight.dim(2) * 2; diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index 18527910..f90a7882 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -252,7 +252,7 @@ fn ffiCallNvfp4( const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ .name = "flashinfer_cutlass_nvfp4_routed_moe", - // Expert sharding is owned by forwardMoe outer manual computation. + // Expert mesh is owned by forwardMoe outer manual computation. .sharding_aware = false, .has_side_effect = false, }); @@ -694,7 +694,7 @@ pub fn fusedExperts( // TODO(Corentin): Do error checking on nvfp4 // Also, maybe pass `zml.nn.Linear` directly - if (expert_partition.eql(.init(.experts))) { + if (expert_partition.isSharded()) { return zml.ops.manualComputation( (struct { input: zml.Tensor, @@ -799,7 +799,7 @@ pub fn fusedExperts( ); } - if (expert_partition.eql(.init(.experts))) { + if (expert_partition.isSharded()) { return zml.ops.manualComputation( (struct { input: zml.Tensor, diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index 2c58baa8..b59e892b 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -46,7 +46,7 @@ pub fn fusedExperts( }; const expert_partition = gate_up.weight.shape().partition(.expert); - if (!expert_partition.eql(.init(.experts))) { + if (!expert_partition.isSharded()) { return fusedExpertsImpl(args, backend); } @@ -238,7 +238,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); values[if (layout == .concatenated) i + 3 else 2 * i + 1] = .fromF32(up); } - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; @@ -270,7 +270,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { // }; // const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); // const values = [_]f32{ -3, 0, 1, 2, 4 }; -// var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); +// var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); // defer input.deinit(); // for ([_]?f32{ null, 2 }) |threshold| { // var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); @@ -412,7 +412,7 @@ test "FP8 routed GEMM with bias matches dequantized weights" { @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), } } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); initialized += 1; } var output: zml.Bufferized(Local.Outputs) = undefined; @@ -491,7 +491,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const host = try zml.Slice.alloc(allocator, x.shape()); defer host.free(allocator); @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); - var input = try zml.Buffer.fromSlice(io, platform, host, .replicated); + var input = try zml.Buffer.fromSlice(io, platform, host); defer input.deinit(); for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { diff --git a/zml/moe/mosaic_tpu.zig b/zml/moe/mosaic_tpu.zig index 6f6a7bc5..f43d1b84 100644 --- a/zml/moe/mosaic_tpu.zig +++ b/zml/moe/mosaic_tpu.zig @@ -238,7 +238,7 @@ pub fn fusedExperts( const expert_partition = gate_up.weight.shape().partition(.expert); - if (expert_partition.eql(.init(.experts))) { + if (expert_partition.isSharded()) { const global_num_experts = down.weight.dim(.expert); const partial_output = zml.ops.manualComputation( (struct { diff --git a/zml/moe/triton_mxfp4.zig b/zml/moe/triton_mxfp4.zig index bf6bbdf5..d20ffe8f 100644 --- a/zml/moe/triton_mxfp4.zig +++ b/zml/moe/triton_mxfp4.zig @@ -39,7 +39,7 @@ pub fn fusedExperts( stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); + const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); const context: Context = .{ .input = input, diff --git a/zml/nn.zig b/zml/nn.zig index a49865fa..1403788f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -141,7 +141,7 @@ pub fn scaledDot( const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - var res_shape: Shape = .{ ._dtype = out_dtype }; + var res_shape: Shape = .scalar(out_dtype); var lhs_batching_axes: Axes = .empty; var rhs_batching_axes: Axes = .empty; for (dot_axes.batching.constSlice()) |b_axes| { @@ -290,7 +290,7 @@ test "block128 scaled dot layouts" { slice.items(zml.floats.BFloat16)[j] = .fromF32(value); } } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); initialized += 1; } var output: zml.Bufferized(Local.Outputs) = undefined; @@ -443,7 +443,7 @@ test normalizeL2 { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, .{}); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); defer input_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -1041,7 +1041,7 @@ test rope { defer exe_sequential.deinit(); const x_values: [5][4]f32 = @splat(.{ 1.0, 0.1, -1.0, -0.5 }); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_values)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_values)); defer x_buffer.deinit(); var res1 = try exe_interleaved.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -1075,7 +1075,7 @@ test "rope: Proportional" { defer exe.deinit(); const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); - var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), .replicated, @ptrCast(&x_h)); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); defer x_buffer.deinit(); const expected_h: [5][16]f32 = .{ @@ -1127,7 +1127,7 @@ test "rope: Yarn with partial_rotary_factor" { defer exe.deinit(); const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); - var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), .replicated, @ptrCast(&x_h)); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); defer x_buffer.deinit(); const expected_h: [5][16]f32 = .{ @@ -1223,7 +1223,6 @@ test nearest { std.testing.io, platform, input_3d_basic.shape(), - .replicated, std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), ); defer input_3d_basic_buffer.deinit(); @@ -1248,7 +1247,7 @@ test nearest { ); defer exe.deinit(); - var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), .replicated, std.mem.sliceAsBytes(&[2][3][4]i32{ + var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, })); @@ -1285,7 +1284,7 @@ test nearest { ); defer exe.deinit(); - var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_basic_buffer}); @@ -1314,7 +1313,7 @@ test nearest { ); defer exe.deinit(); - var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), .replicated, std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ + var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ .{ .{ 1, 2 }, .{ 3, 4 } }, .{ .{ 5, 6 }, .{ 7, 8 } }, }, .{ @@ -1371,7 +1370,7 @@ test nearest { ); defer exe.deinit(); - var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_5d_buffer}); @@ -1821,7 +1820,6 @@ test "gated delta net" { std.testing.io, platform, queries.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, .{ .{ 1.0, 1.0 }, .{ 1.0, -1.0 } }, @@ -1832,7 +1830,6 @@ test "gated delta net" { std.testing.io, platform, keys.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 2.0 }, .{ 0.0, 1.0 } }, .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, @@ -1843,7 +1840,6 @@ test "gated delta net" { std.testing.io, platform, values.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 3.0, 1.0 }, .{ 2.0, 4.0 } }, .{ .{ 1.0, 5.0 }, .{ 3.0, 0.0 } }, @@ -1854,7 +1850,6 @@ test "gated delta net" { std.testing.io, platform, alphas.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2]f32{ .{ 0.5, 0.25 }, .{ 0.8, 0.6 }, @@ -1865,7 +1860,6 @@ test "gated delta net" { std.testing.io, platform, betas.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2]f32{ .{ 1.0, 0.5 }, .{ 0.75, 1.0 }, @@ -1876,7 +1870,6 @@ test "gated delta net" { std.testing.io, platform, initial_s.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, @@ -2004,7 +1997,7 @@ test sampleTokens { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, .{}); defer exe.deinit(); - var rng_buffer = try zml.Tensor.Rng.initBuffer(std.testing.io, platform, .replicated, 0xdeadbeef); + var rng_buffer = try rng.initBuffer(std.testing.io, platform, 0xdeadbeef); defer rng_buffer._state.deinit(); const inf = std.math.inf(f32); @@ -2016,7 +2009,7 @@ test sampleTokens { }) |logits_expected| { const logits, const expected: i32 = logits_expected; - var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), .replicated, std.mem.sliceAsBytes(&logits)); + var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); var sampled: zml.Buffer = undefined; @@ -2149,7 +2142,7 @@ test sampleTokensDynamic { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, .{}); defer exe.deinit(); - var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), .replicated, std.mem.sliceAsBytes(&logits_data)); + var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); defer logits_buffer.deinit(); const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; @@ -2193,7 +2186,7 @@ test sampleTokensDynamic { const boost = bf16.inf; const nerf = bf16.minus_inf; - var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), .replicated, std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); + var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); defer logits_bf16_buffer.deinit(); var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); diff --git a/zml/ops.zig b/zml/ops.zig index fd97ceb1..186805ea 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -21,6 +21,7 @@ const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; +const log = std.log.scoped(.@"zml/Compiler"); pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); @@ -66,7 +67,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), }; - const num_devices = ctx.partitioning.numPartitions(); + const num_devices = ctx.meshes[0].numPartitions(); if (num_devices <= 1) return inputs; const reducer_block = b: { @@ -635,9 +636,9 @@ test @"while" { ); defer exe.deinit(); - var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{1})); + var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), std.mem.sliceAsBytes(&[1]i64{1})); defer i_buffer.deinit(); - var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{0})); + var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), std.mem.sliceAsBytes(&[1]i64{0})); defer sum_buffer.deinit(); var results: zml.Bufferized(While.State) = undefined; @@ -865,10 +866,10 @@ fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(b fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); - // copy tags and sharding info over + // copy tags and mesh info over // some ops can change dims eg reduceWindow, so we trust mlir here. new._shape._tags = tensor._shape._tags; - new._shape._partitioning = tensor._shape._partitioning; + new._shape._sharding = tensor._shape._sharding; tensor.* = new; inner_ctx.index += 1; } @@ -1055,7 +1056,7 @@ test "fly custom call, both entry ABIs" { fn f(comptime forward: anytype, p: *const zml.Platform, t: Tensor, h: []const f32) !zml.Slice { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{t}, p, .{}); defer exe.deinit(); - var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), .replicated, std.mem.sliceAsBytes(h)); + var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), std.mem.sliceAsBytes(h)); defer buf.deinit(); var out = try exe.eval(std.testing.allocator, std.testing.io, .{buf}); defer out.deinit(); @@ -1274,9 +1275,9 @@ test "triton" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{1})); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); defer a_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{3})); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); var results: [2]zml.Buffer = undefined; @@ -1497,7 +1498,7 @@ test "cute" { var input: [128]f32 = undefined; for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -1556,7 +1557,7 @@ test "cuda_tile" { var input: [128]f32 = undefined; for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -1623,7 +1624,7 @@ test "cuda_tile grid" { defer exe.deinit(); const input = [_]f32{0} ** n; - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -1683,7 +1684,7 @@ test "cuda_tile zeroed_outputs" { var input: [128]f32 = undefined; for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -2109,7 +2110,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, if (indices_shape.count() == 1 and idx_axes.len == 1) { return self .slice(idx_axes[0], .dyn(indices_per_axis.get(0).asScalar(), 1)) - // Keep downstream resharding after the slice. SPMD engines like Neuron otherwise + // Keep downstream remeshe after the slice. SPMD engines like Neuron otherwise // may hoist an all-gather before the slice and materialize the full tensor. .optimizationBarrier() .reshape(res_shape); @@ -2522,6 +2523,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; } +/// Runs a per-shard body using one mesh for all input and output partition specs. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), @@ -2576,6 +2578,47 @@ fn manualComputationLocalizeInputs(allocator: std.mem.Allocator, inputs: anytype return local_inputs; } +fn manualComputationMesh(inputs: []const Shape, outputs: []const Shape, fallback: *const Sharding.Mesh) error{IncompatibleMeshes}!*const Sharding.Mesh { + var mesh: ?*const Sharding.Mesh = null; + for ([_][]const Shape{ inputs, outputs }) |shapes| { + for (shapes) |shape| { + if (shape._sharding.mesh) |shape_mesh| { + if (mesh) |m| { + if (m != shape_mesh) return error.IncompatibleMeshes; + } else mesh = shape_mesh; + } + } + } + return mesh orelse fallback; +} + +test "manualComputation infers one mesh from inputs and outputs" { + const fallback: Sharding.Mesh = .{ + .name = "fallback", + .physical = undefined, + .logical = .mesh(.{ .batch = .balanced }), + .bindings = .empty, + .folds = .empty, + .folds_consumed = .empty, + }; + var first = fallback; + first.name = "first"; + var second = fallback; + second.name = "second"; + const replicated: Shape = .init(.{8}, .f32); + var sharded = replicated; + sharded._sharding = .{ .mesh = &first, .partition = .init(&.{.mesh_axis_0}) }; + var other = sharded; + other._sharding.mesh = &second; + + try std.testing.expectEqual(&fallback, try manualComputationMesh(&.{replicated}, &.{replicated}, &fallback)); + try std.testing.expectEqual(&first, try manualComputationMesh(&.{ replicated, sharded, sharded }, &.{sharded}, &fallback)); + try std.testing.expectEqual(&first, try manualComputationMesh(&.{replicated}, &.{sharded}, &fallback)); + try std.testing.expectEqual(&first, try manualComputationMesh(&.{}, &.{sharded}, &fallback)); + try std.testing.expectError(error.IncompatibleMeshes, manualComputationMesh(&.{ sharded, other }, &.{}, &fallback)); + try std.testing.expectError(error.IncompatibleMeshes, manualComputationMesh(&.{sharded}, &.{other}, &fallback)); +} + fn manualComputationInternal( inputs: anytype, outputs: []const Shape, @@ -2596,35 +2639,21 @@ fn manualComputationInternal( const local_input_shapes = try arena.alloc(Shape, input_shapes.len); const local_output_shapes = try arena.alloc(Shape, outputs.len); - const input_shardings = try arena.alloc(Sharding, input_shapes.len); - const output_shardings = try arena.alloc(Sharding, outputs.len); + const manual_mesh = manualComputationMesh(input_shapes, outputs, ctx.platform.replicated_mesh) catch + @panic("manualComputation expects all inputs and outputs to use the same mesh"); for (input_shapes, 0..) |shape, i| { - const sharding = ctx.partitioning.selectSharding(shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "failed to shard manualComputation input {f}({d}) because it's using unknown sharding. Pass more shardings to `.compile`. Known shardings: {f}", - .{ shape, i, stdx.fmt.slice(ctx.partitioning.shardings) }, - ), - }; - input_shardings[i] = sharding; - local_input_shapes[i] = sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, sharding }); + local_input_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); } for (outputs, 0..) |shape, i| { - const sharding = ctx.partitioning.selectSharding(shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "failed to shard manualComputation output {f}({d}) because it's using unknown sharding. Pass more shardings to `.compile`. Known shardings: {f}", - .{ shape, i, stdx.fmt.slice(ctx.partitioning.shardings) }, - ), - }; - output_shardings[i] = sharding; - local_output_shapes[i] = sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, sharding }); + local_output_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); } - return switch (ctx.partitioning.partitioner) { + return switch (ctx.partitioner) { .shardy => { - const in_shardings_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, input_shapes, input_shardings); - const out_shardings_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, outputs, output_shardings); - const manual_axes_attr = try Sharding.sdyManualAxesAttr(arena, ctx.mlir_ctx, input_shapes, input_shardings, outputs, output_shardings); + const in_meshes_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, input_shapes, manual_mesh); + const out_meshes_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, outputs, manual_mesh); + const manual_axes_attr = try Sharding.sdyManualAxesAttr(arena, ctx.mlir_ctx, input_shapes, outputs, manual_mesh); const block_types = try arena.alloc(*const mlir.Type, input_shapes.len); for (local_input_shapes, 0..) |input_shape, i| { @@ -2675,8 +2704,8 @@ fn manualComputationInternal( .results = .{ .flat = global_result_types }, .blocks = &.{manual_block}, .attributes = &.{ - .named(ctx.mlir_ctx, "in_shardings", in_shardings_attr), - .named(ctx.mlir_ctx, "out_shardings", out_shardings_attr), + .named(ctx.mlir_ctx, "in_shardings", in_meshes_attr), + .named(ctx.mlir_ctx, "out_shardings", out_meshes_attr), .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr), }, .verify = true, @@ -2686,7 +2715,7 @@ fn manualComputationInternal( // Use the compiler allocator to return memory to the parent const sharded_outputs = ctx.alloc(Tensor, outputs.len); for (outputs, 0..) |output, i| { - sharded_outputs[i] = Tensor._result(output, op.result(i)); + sharded_outputs[i] = Tensor._resultPropagateSharding(output, op.result(i)); } return sharded_outputs; }, @@ -2732,8 +2761,8 @@ fn manualComputationInternal( const global_values = try arena.alloc(*const mlir.Value, outputs.len); const global_types = try arena.alloc(*const mlir.Type, outputs.len); - for (outputs, output_shardings, 0..) |output_shape, output_sharding, i| { - const gspmd_attr = try ctx.partitioning.tensorShardingAttr(arena, ctx.mlir_ctx, output_shape, output_sharding); + for (outputs, 0..) |output_shape, i| { + const gspmd_attr = try ctx.tensorShardingAttr(arena, ctx.mlir_ctx, output_shape); global_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, output_shape); const shard_to_full = dialects.stablehlo.custom_call( @@ -2762,7 +2791,7 @@ fn manualComputationInternal( const sharded_outputs = ctx.alloc(Tensor, outputs.len); for (outputs, 0..) |output_shape, i| { - sharded_outputs[i] = Tensor._result(output_shape, barrier.result(i)); + sharded_outputs[i] = Tensor._resultPropagateSharding(output_shape, barrier.result(i)); } return sharded_outputs; }, @@ -3153,7 +3182,7 @@ pub fn typedCustomCall( } break :b &input_tensors; }, - // Extra case to support []const Tensor from shardingAwareTypedCustomCall + // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -3169,7 +3198,7 @@ pub fn typedCustomCall( } break :b &output_shapes; }, - // Extra case to support []const Shape from shardingAwareTypedCustomCall + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -3234,7 +3263,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioning.partitioner == .gspmd) { + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -3246,7 +3275,7 @@ pub fn typedCustomCall( } return out; }, - // Extra case to support []const Shape from shardingAwareTypedCustomCall + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -3307,7 +3336,7 @@ test customCall { const scope = comp.pushBlock(block); defer scope.pop(); - const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(.{ ._0 = .x }); + const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(platform.meshes.get("model").?, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ .has_side_effect = false, diff --git a/zml/platform.zig b/zml/platform.zig index 826eeaee..7a9ee710 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -296,8 +296,8 @@ pub const Platform = struct { devices: []const Device, memories: []const Memory, physical_mesh: zml.Sharding.PhysicalMesh, - replicated_sharding: zml.Sharding, - shardings: std.StringArrayHashMapUnmanaged(zml.Sharding), + meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), + replicated_mesh: *const zml.Mesh, io_impl: IoImpl, pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; @@ -337,12 +337,12 @@ pub const Platform = struct { .pjrt_api = api, .pjrt_client = pjrt_client, .state = State.init(target), - .shardings = .empty, + .meshes = .empty, + .replicated_mesh = undefined, // set below .devices = undefined, .memories = undefined, .physical_mesh = undefined, - .replicated_sharding = undefined, .io_impl = options.io_impl, }; break :platform platform; @@ -350,7 +350,7 @@ pub const Platform = struct { const arena = platform.arena.allocator(); errdefer platform.arena.deinit(); - try platform.shardings.ensureTotalCapacity(arena, 8); + try platform.meshes.ensureTotalCapacity(arena, 8); { const devices = try arena.alloc(Device, pjrt_devices.len); @@ -375,7 +375,7 @@ pub const Platform = struct { .auto => zml.Sharding.PhysicalMesh.auto(arena, target, devices), .custom => |builder| builder(arena, target, devices), }; - platform.replicated_sharding = try platform.registerSharding("replicated", .mesh(.{ .x = .high_bandwidth })); + platform.replicated_mesh = try platform.registerMesh("replicated", .mesh(.{ .all = .high_bandwidth })); } switch (target) { @@ -639,38 +639,37 @@ pub const Platform = struct { return try profiler_.profiler(self.pjrt_api, allocator, io, options); } - /// Create a Sharding based on the given logical mesh and the default strategy. + /// Create a Mesh based on the given logical mesh and the default strategy. /// Memory is owned by the platform, making it safe to copy around. - pub fn registerSharding(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!Sharding { - return platform.registerShardingWithStrategy( + pub fn registerMesh(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!*const Sharding.Mesh { + return platform.registerMeshWithStrategy( name, logical, .suggest(logical, &platform.physical_mesh), ) catch |err| switch (err) { error.InvalidPhysicalMesh, error.InvalidStrategy, error.InvalidPhysicalAxis => { - std.debug.panic("ZML failed to create a valid sharding for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); + std.debug.panic("ZML failed to create a valid mesh for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); }, error.OutOfMemory => |e| return e, }; } - /// Create a Sharding based on the given logical mesh and a strategy. + /// Create a Mesh based on the given logical mesh and a strategy. /// Memory is owned by the platform, making it safe to copy around. - pub fn registerShardingWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !Sharding { + pub fn registerMeshWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !*const Sharding.Mesh { const arena = platform.arena.allocator(); - const entry = try platform.shardings.getOrPut(arena, name); + const entry = try platform.meshes.getOrPut(arena, name); if (entry.found_existing) { - std.debug.panic("Another sharding already exists with this name: {s}", .{name}); + std.debug.panic("Another mesh already exists with this name: {s}", .{name}); } const owned_name = try arena.dupe(u8, name); - const owned_data = try arena.create(Sharding.Data); - owned_data.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); - const sharding: Sharding = .{ .data = owned_data }; + const owned_mesh = try arena.create(Sharding.Mesh); + owned_mesh.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); entry.key_ptr.* = owned_name; - entry.value_ptr.* = sharding; + entry.value_ptr.* = owned_mesh; - return sharding; + return owned_mesh; } fn memoryFromPjrt(self: *const Platform, pjrt_memory: *const pjrt.Memory) *const Memory { diff --git a/zml/quantization.zig b/zml/quantization.zig index 489162fb..b125513a 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -436,7 +436,7 @@ test "block FP8 quantization preserves axes and reconstructs constant blocks in else => -magnitude, }); } - var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); defer buffer.deinit(); var output: zml.Bufferized(Local.Outputs) = undefined; var runner = try exe.runner(allocator); @@ -495,7 +495,7 @@ test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" const exponent = (bits >> 23) + @intFromBool(bits & 0x7fffff != 0); scale.* = @bitCast(exponent << 23); } - var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); defer buffer.deinit(); var output: zml.Bufferized(Local.Outputs) = undefined; var runner = try exe.runner(allocator); diff --git a/zml/shape.zig b/zml/shape.zig index 8afae990..11a3efac 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -6,141 +6,26 @@ const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const Sharding = @import("Sharding.zig"); const log = std.log.scoped(.shape); +test { + std.testing.refAllDecls(Shape); +} + /// Represent the shape of a tensor. +/// Contains also the sharding information, +/// so it describes exactly how the corresponding tensor/buffer will be layed out +/// across all devices of the current Platform. +/// +/// By default all created Shape are replicated, meaning the corresponding tensor / buffer will be copied on each device. +/// `.withPartitioning` allows to specify how it should be split. pub const Shape = struct { - pub const MAX_RANK: u8 = 8; - - pub const PartitionSpec = union(enum) { - axis: Tag, - open, - replicated, - unknown, - - pub fn init(comptime value: anytype) PartitionSpec { - const T = @TypeOf(value); - - if (T == PartitionSpec) { - return value; - } - - if (comptime T == @EnumLiteral()) { - const name = @tagName(value); - inline for (std.meta.fields(PartitionSpec)) |field| { - if (field.type == void and std.mem.eql(u8, field.name, name)) { - return @field(PartitionSpec, field.name); - } - } - } - - if (comptime isTagConvertible(T)) { - const tag_ = Shape.toTag(value); - - return .{ .axis = tag_ }; - } - - stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); - } - - test "PartitionSpec.init" { - const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; - try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); - - const spec_axis_open: PartitionSpec = .open; - try testing.expect(spec_axis_open.eql(.init(.open))); - } - - pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { - if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { - return false; - } - - switch (self) { - .axis => |t1| { - const t2 = other.axis; - return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); - }, - else => return true, - } - } - - test "PartitionSpec.eql" { - const spec_axis: PartitionSpec = .init(.a); - const spec_axis_b: PartitionSpec = .init(.b); - const spec_open: PartitionSpec = .open; - const spec_replicated: PartitionSpec = .replicated; - const spec_unknown: PartitionSpec = .unknown; - - try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); - try testing.expect(!spec_axis.eql(spec_axis_b)); - try testing.expect(!spec_axis.eql(.open)); - try testing.expect(!spec_axis.eql(.replicated)); - try testing.expect(!spec_axis.eql(.unknown)); - - try testing.expect(spec_open.eql(.open)); - try testing.expect(!spec_open.eql(spec_axis)); - try testing.expect(!spec_open.eql(.replicated)); - try testing.expect(!spec_open.eql(.unknown)); - - try testing.expect(spec_replicated.eql(.replicated)); - try testing.expect(!spec_replicated.eql(spec_axis)); - try testing.expect(!spec_replicated.eql(.open)); - try testing.expect(!spec_replicated.eql(.unknown)); - - try testing.expect(spec_unknown.eql(.unknown)); - try testing.expect(!spec_unknown.eql(spec_axis)); - try testing.expect(!spec_unknown.eql(.open)); - try testing.expect(!spec_unknown.eql(.replicated)); - } - - pub fn isClosed(self: PartitionSpec) bool { - return switch (self) { - .axis => true, - .open => false, - .replicated => true, - .unknown => false, - }; - } - - test isClosed { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.isClosed()); - - const spec_open: PartitionSpec = .open; - try testing.expect(!spec_open.isClosed()); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.isClosed()); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(!spec_unknown.isClosed()); - } - - pub fn toTag(self: PartitionSpec) Tag { - return switch (self) { - .axis => |t| return t, - .open => TagOpen, - .replicated => TagReplicated, - .unknown => TagUnknown, - }; - } - - test "PartitionSpec.toTag" { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); - - const spec_open: PartitionSpec = .open; - try testing.expect(spec_open.toTag() == TagOpen); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.toTag() == TagReplicated); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(spec_unknown.toTag() == TagUnknown); - } - }; + _dtype: DataType, + _dims: DimsArray, + _tags: TagsArray, + _sharding: Sharding, pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; @@ -148,17 +33,10 @@ pub const Shape = struct { const TagReplicated = "replicated".ptr; const TagOpen = "open".ptr; + pub const MAX_RANK: u8 = constants.MAX_RANK; pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); - pub const PartitionArray = stdx.BoundedArray(PartitionSpec, constants.MAX_RANK); - - const UnknownTags: TagsArray = .{ .len = 0, .buffer = @splat(TagUnknown) }; - - _dtype: DataType, - _dims: DimsArray = .empty, - _tags: TagsArray = UnknownTags, - _partitioning: PartitionArray = .empty, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -252,28 +130,39 @@ pub const Shape = struct { /// Create a shape from a struct literal, eg: /// Shape.init(.{ .h = 1024, .w = 512, .c = 3 }); /// Shape.init(.{ 1024, 512, 3 }); + /// + /// The created shape is replicated. pub fn init(dimz: anytype, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; - res._dims, res._tags = parseDimensions(dimz); - res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); - - return res; + const _dims, const _tags = parseDimensions(dimz); + return .{ + ._dtype = dt, + ._dims = _dims, + ._tags = _tags, + ._sharding = .replicated, + }; } pub fn scalar(dt: DataType) Shape { - return .{ ._dtype = dt }; + return .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .len = 0, .buffer = @splat(TagUnknown) }, + ._sharding = .replicated, + }; } /// Creates a Shape with dims set to `.{0, 1, 2, ..., rank-1}`. pub fn range(rank_: usize, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; + stdx.debug.assert(rank_ <= MAX_RANK, "zml.Shape.range expects a maximum rank of {d}, got {}", .{ MAX_RANK, rank_ }); + var res: Shape = .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .buffer = @splat(TagUnknown), .len = rank_ }, + ._sharding = .replicated, + }; for (0..rank_) |i| { - res._dims.append(@intCast(i)) catch { - stdx.debug.panic("Too many dimensions! Max: {d}, passed: {d}", .{ res._dims.capacity(), rank_ }); - }; - res._tags.append(TagUnknown) catch unreachable; + res._dims.appendAssumeCapacity(@intCast(i)); } - res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -319,7 +208,7 @@ pub const Shape = struct { fn ensureAttributesAreSync(self: Shape) void { if (builtin.mode == .Debug) { - stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); + stdx.debug.assert(self._dims.len == self._tags.len, "Tags, dims and partitioning have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); } } @@ -349,7 +238,7 @@ pub const Shape = struct { } pub fn hasTag(self: Shape, tag_: anytype) ?u3 { - return self.axisFromTagMaybe(toTag(tag_)); + return axisFromTagMaybe(self.tags(), toTag(tag_)); } pub fn hasTags(self: Shape, tagz: anytype) bool { @@ -443,7 +332,7 @@ pub const Shape = struct { @intCast(a); } - fn axisFromTagMaybe(self: Shape, t: Tag) ?u3 { + pub fn axisFromTagMaybe(tagz: []const Tag, t: Tag) ?u3 { if (t == TagUnknown) return null; if (axisFromLiteralInt(t)) |ax| return ax; @@ -451,7 +340,7 @@ pub const Shape = struct { if (@inComptime()) { // At comptime two duplicated strings may have two different representations const t_bytes: []const u8 = std.mem.span(t); - for (self.tags(), 0..) |self_tag, ax| { + for (tagz, 0..) |self_tag, ax| { if (std.mem.eql(u8, t_bytes, std.mem.span(self_tag))) { return @truncate(ax); } @@ -460,7 +349,7 @@ pub const Shape = struct { } // But at runtime the comptime strings have been deduplicated and ptr match is enough. - if (std.mem.indexOfScalar(Tag, self.tags(), t)) |ax| { + if (std.mem.indexOfScalar(Tag, tagz, t)) |ax| { return @truncate(ax); } return null; @@ -477,7 +366,7 @@ pub const Shape = struct { fn axisFromTag(self: Shape, d: Tag) u3 { stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); - return self.axisFromTagMaybe(d) orelse { + return axisFromTagMaybe(self.tags(), d) orelse { stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); }; } @@ -519,8 +408,7 @@ pub const Shape = struct { } /// Format the shape. - /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" - /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" + /// "Shape({.a=10, .b=20}, dtype=.f32)" pub fn format(self: Shape, writer: *std.Io.Writer) !void { _ = try writer.writeByte('{'); @@ -534,9 +422,16 @@ pub const Shape = struct { try writer.print("{d}", .{d}); } - const part = self._partitioning.get(i); - if (part.toTag() != TagUnknown) { - try writer.print("/{s}", .{part.toTag()}); + const partitioning = self._sharding.get(i); + switch (partitioning) { + .replicated => {}, + .open => try writer.writeAll("/?"), + // this state is invalid, but don't panic cause we may be called from a panic + .out_of_bound => try writer.writeAll("/!"), + else => { + const mesh_ax = partitioning.meshAxis().?; + try writer.print("/{s}", .{self._sharding.mesh.?.logical.axes.get(mesh_ax)}); + }, } need_comma = true; @@ -544,6 +439,10 @@ pub const Shape = struct { if (need_comma) try writer.writeByte(','); _ = try writer.write(@tagName(self.dtype())); _ = try writer.writeByte('}'); + if (self._sharding.mesh) |m| { + try writer.writeByte('@'); + try writer.writeAll(m.name); + } } /// Broadcasts a Tensor to the given shape, extending dimensions if needed. @@ -569,10 +468,17 @@ pub const Shape = struct { return true; } + /// Reshape a given shape but keep the same total number of elements. + /// Partitioning information is generally NOT preserved, + /// unless input was fully replicated. pub fn reshape(self: Shape, new_shape_: anytype) Shape { - var new_shape: Shape = .{ ._dtype = self.dtype() }; - new_shape._dims, new_shape._tags = parseDimensions(new_shape_); - new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); + const _dims, const _tags = parseDimensions(new_shape_); + var new_shape: Shape = .{ + ._dtype = self._dtype, + ._dims = _dims, + ._tags = _tags, + ._sharding = .{ .mesh = self._sharding.mesh, .partition = if (self.isFullyReplicated()) .replicated(_dims.len) else .open(_dims.len) }, + }; new_shape.inferMissingAxis(self.count()) catch |err| { std.debug.panic("Can't reshape {any} to {any}: {t}", .{ self.dims(), new_shape.dims(), err }); }; @@ -655,7 +561,7 @@ pub const Shape = struct { res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; for (0..dims_.len) |_| { - res._partitioning.insert(ax, .unknown) catch unreachable; + res._sharding = res._sharding.insert(ax, .replicated); } return res; @@ -678,7 +584,7 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; - res._partitioning.insert(ax, .unknown) catch unreachable; + res._sharding = res._sharding.insert(ax, .replicated); return res; } @@ -688,7 +594,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); - res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); + for (self.rank()..res.rank()) |ax| res._sharding = res._sharding.set(ax, .replicated); return res; } @@ -710,7 +616,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); - res._partitioning.appendAssumeCapacity(.unknown); + res._sharding = res._sharding.set(self.rank(), .replicated); return res; } @@ -719,7 +625,7 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); - _ = res._partitioning.orderedRemove(a); + res._sharding = res._sharding.orderedRemove(a); return res; } @@ -736,6 +642,7 @@ pub const Shape = struct { std.mem.sort(u3, to_remove.slice(), {}, std.sort.asc(u3)); var sh: Shape = self; + sh._sharding.partition = .out_of_bound; const rk = self.rank(); var res_ax: u32 = 0; for (0..rk) |ax| { @@ -745,12 +652,11 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; - sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; + sh._sharding = sh._sharding.set(res_ax, self._sharding.get(ax)); res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; - sh._partitioning.len = rk - to_remove.len; return sh; } @@ -773,12 +679,12 @@ pub const Shape = struct { var res = self; const original_dims = self._dims; const original_tags = self._tags; - const original_parts = self._partitioning; + const original_parts = self._sharding; for (permutations_.constSlice(), 0..) |permutation, i| { res._dims.set(i, original_dims.get(permutation)); res._tags.set(i, original_tags.get(permutation)); - res._partitioning.set(i, original_parts.get(permutation)); + res._sharding = res._sharding.set(i, original_parts.get(permutation)); } return res; @@ -797,9 +703,9 @@ pub const Shape = struct { ), ); - const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); + try testing.expectEqualDeep(Sharding.Partitioning.init(&.{ .mesh_axis_0, .open, .mesh_axis_2 }), shape._sharding.partition); } /// Tag each ax of this shape with tags from a tuple. @@ -908,154 +814,131 @@ pub const Shape = struct { return res; } - pub fn withPartitioning(self: Shape, specs: anytype) Shape { - const T = @TypeOf(specs); + // These tests only resolve logical names; placement does not use this fixture. + const test_mesh: Sharding.Mesh = .{ + .name = "test_mesh", + .physical = undefined, + .logical = .mesh(.{ .batch = .balanced, .feature = .balanced, .colors = .balanced }), + .bindings = .init(&.{ + .init(&.{.link_x}), + .init(&.{.link_y}), + .init(&.{.link_z}), + }), + .folds = .empty, + .folds_consumed = .empty, + }; - var res = self.withDefaultPartitioning(); // todo add test for this new change - - if (stdx.meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - const partition_axis = @field(specs, field.name); - const axis_ = res.axisFromTagMaybe(toTag(field)); - - if (axis_) |ax| { - res._partitioning.set(ax, PartitionSpec.init(partition_axis)); - } else { - stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); - } - } - } else { - stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { + if (@TypeOf(partitioning) != Sharding.Partitioning) { + return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); } - - // Check that no mesh axis is used to partition multiple tensor dimensions. - var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .empty; - - for (res._partitioning.constSlice()) |spec| { - const axis_tag = spec.toTag(); - - if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { - stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); - } - - switch (spec) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), - } + var res = self; + res._sharding = .{ .mesh = mesh, .partition = partitioning }; + for (0..self.rank()) |ax| { + if (res._sharding.get(ax) == .out_of_bound) res._sharding = res._sharding.set(ax, .replicated); } + for (self.rank()..MAX_RANK) |ax| res._sharding.partition = res._sharding.partition.set(ax, .out_of_bound); + + stdx.debug.assert(res._sharding.partition.hasUniqueAxes(), "{f}.withPartitioning({s}, ...) expects mesh axes to be used a most once, got {any}", .{ self, mesh.name, partitioning.toArray()[0..self.rank()] }); return res; } test withPartitioning { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .replicated, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withDefaultPartitioning(); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .open, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); } - pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { + pub fn mapPartitioningAxes(self: Shape, mesh: *const Sharding.Mesh, mapping: anytype) Shape { const T = @TypeOf(mapping); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); - var res = self; - for (res._partitioning.slice()) |*spec| { - switch (spec.*) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => |*old_axis_tag| { - inline for (std.meta.fields(T)) |field| { - const logical_axis_in_map = toTag(field); - - if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { - const physical_axis_from_map = @field(mapping, field.name); - spec.* = PartitionSpec.init(physical_axis_from_map); - break; - } - } - }, + for (0..self.rank()) |ax| { + const mesh_axis = self.partition(ax).meshAxis() orelse continue; + inline for (std.meta.fields(T)) |field| { + const old_axis = mesh.resolveLogicalAxis(toTag(field)) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field.name }); + if (mesh_axis == old_axis) { + const new_tag = toTag(@field(mapping, field.name)); + const new_axis = mesh.resolveLogicalAxis(new_tag) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, new_tag }); + res._sharding = res._sharding.set(ax, .sharded(@intCast(new_axis))); + break; + } } } return res; } test mapPartitioningAxes { - const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ - .batch = .data, - .seq = .replicated, - .features = .model, - }); - - const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); - - try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); - try testing.expect(physical_shape.partition(.features).eql(.init(.y))); - try testing.expect(physical_shape.partition(.seq).eql(.replicated)); - - const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); - try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); - try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); + const shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32) + .withPartitioning(&test_mesh, .{ .batch = .batch, .seq = .replicated, .features = .feature }); + const mapped = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .feature, .feature = .batch }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), mapped.partition(.batch)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), mapped.partition(.features)); + try testing.expectEqual(Sharding.PartitionSpec.replicated, mapped.partition(.seq)); + const partial = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .colors }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(2), partial.partition(.batch)); + try testing.expectEqual(shape.partition(.features), partial.partition(.features)); } pub fn withDefaultPartitioning(self: Shape) Shape { var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + res._sharding.partition = .replicated(self.rank()); return res; } test withDefaultPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withDefaultPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); } pub fn withReplicatedPartitioning(self: Shape) Shape { var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + res._sharding.partition = .replicated(self._dims.len); return res; } test withReplicatedPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withReplicatedPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); } pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec == .axis) return true; + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (spec.isSharded()) return true; } return false; } test hasAtLeastOnePartitionedAxis { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(shape.hasAtLeastOnePartitionedAxis()); - shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); shape = shape.withDefaultPartitioning(); @@ -1063,20 +946,18 @@ pub const Shape = struct { } pub fn isFullyPartitioned(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - switch (spec) { - .replicated, .open, .unknown => return false, - .axis => {}, - } + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (!spec.isSharded()) return false; } return true; } test isFullyPartitioned { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .feature }); try testing.expect(shape.isFullyPartitioned()); - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(!shape.isFullyPartitioned()); shape = shape.withDefaultPartitioning(); @@ -1084,54 +965,47 @@ pub const Shape = struct { } pub fn isFullyReplicated(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec != .replicated and spec != .unknown) { - return false; - } - } - return true; + return self._sharding.isFullyReplicated(); } test isFullyReplicated { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); try testing.expect(shape.isFullyReplicated()); - shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); - try testing.expect(shape.isFullyReplicated()); - - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(!shape.isFullyReplicated()); } - pub fn partition(self: Shape, ax: anytype) PartitionSpec { - return self._partitioning.get(self.axis(ax)); + pub fn partition(self: Shape, ax: anytype) Sharding.PartitionSpec { + return self._sharding.get(self.axis(ax)); } test partition { - const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), shape.partition(.a)); try testing.expectEqual(.open, shape.partition(.b)); - try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), shape.partition(.c)); } - pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { - for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { - if (dim_part.eql(part)) return true; + pub fn containsPartitionSpec(self: Shape, part: Sharding.PartitionSpec) bool { + for (0..self.rank()) |ax| { + const dim_part = self._sharding.get(ax); + if (dim_part == part) return true; } return false; } test containsPartitionSpec { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expect(shape.containsPartitionSpec(.init(.batch))); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.sharded(0))); try testing.expect(shape.containsPartitionSpec(.open)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); - try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); shape = shape.withDefaultPartitioning(); - try testing.expect(!shape.containsPartitionSpec(.init(.batch))); - try testing.expect(shape.containsPartitionSpec(.unknown)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.sharded(0))); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); + try testing.expect(shape.containsPartitionSpec(.replicated)); try testing.expect(!shape.containsPartitionSpec(.open)); } @@ -1329,10 +1203,10 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; - _ = new_shape._partitioning.orderedRemove(ax); + new_shape._sharding = new_shape._sharding.orderedRemove(ax); for (0..dims_.len) |_| { - new_shape._partitioning.insert(ax, .unknown) catch unreachable; + new_shape._sharding = new_shape._sharding.insert(ax, .replicated); } new_shape.inferMissingAxis(self.count()) catch |err| { @@ -1383,39 +1257,34 @@ pub const Shape = struct { const axes_to_merge = self.axes(axes_); stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); - var new_dim: i64 = 1; // Merging partitioned axes is complex. If ANY axis being merged is partitioned, - // the resulting sharding is ambiguous. A propagation pass must resolve this. + // the resulting mesh is ambiguous. A propagation pass must resolve this. // Marking it 'open' is the correct signal. If all are replicated/unknown, the // result is also replicated/unknown. - // rule: .replicated wins over .unknown. .open wins over everything. - var resulting_partition_spec: PartitionSpec = .unknown; - - for (0..axes_to_merge.len) |i| { - const current_axis = axes_to_merge.get(i); - new_dim *= self.dim(current_axis); - - if (i > 0) { - stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); - } - - switch (self.partition(current_axis)) { - .axis, .open => resulting_partition_spec = .open, - .replicated => if (resulting_partition_spec != .open) { - resulting_partition_spec = .replicated; - }, - .unknown => {}, - } - } + // rule: .replicated wins over .out_of_bound. .open wins over everything. const first_axis_to_merge = axes_to_merge.get(0); const num_axes_to_merge = axes_to_merge.len; - var new_shape = self; + var merged_dim: i64 = 1; + var merged_spec: Sharding.PartitionSpec = .out_of_bound; + for (axes_to_merge.constSlice()) |ax| { + merged_dim *= self.dim(ax); + const spec = self.partition(ax); + if (spec.isSharded() or spec == .open) { + merged_spec = .open; + } else if (spec == .replicated and merged_spec != .open) { + merged_spec = .replicated; + } + } - new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); + var new_shape = self; + for (1..num_axes_to_merge) |_| { + new_shape._sharding = new_shape._sharding.orderedRemove(first_axis_to_merge); + } + new_shape._sharding = new_shape._sharding.set(first_axis_to_merge, merged_spec); + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{merged_dim}) catch @panic("mergeAxis failed on dims"); new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); - new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); return new_shape; } @@ -1451,23 +1320,23 @@ pub const Shape = struct { test "mergeAxis partitioning" { // Merging a partitioned axis with a replicated one -> open const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .batch, .c = .replicated }); const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged1.partition(.ab).eql(.open)); + try testing.expect(merged1.partition(.ab) == .open); try testing.expect(merged1.dim(.ab) == 10); try testing.expect(merged1.rank() == 2); // Merging two replicated axes -> replicated const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated, .c = .feature }); const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged2.partition(.ab).eql(.replicated)); - try testing.expect(merged2.partition(.c).eql(.init(.y))); + try testing.expect(merged2.partition(.ab) == .replicated); + try testing.expect(merged2.partition(.c) == Sharding.PartitionSpec.sharded(1)); // Merging two unknown axes -> unknown - const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32).withDefaultPartitioning(); const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged3.partition(.ab).eql(.unknown)); + try testing.expect(merged3.partition(.ab) == .replicated); } pub fn mergeAxes(self: Shape, axes_: anytype) Shape { @@ -1641,6 +1510,37 @@ pub const Shape = struct { const x: Shape = .init(.{ 4, 8 }, .u2); try expectEqualShapes(.init(.{ 4, 2 }, .u8), x.packedShape()); } + + test "shape transformations preserve partition specs and clear unused slots" { + var shape = Shape.init(.{ 2, 3, 4 }, .f32); + shape._sharding = .{ .mesh = &test_mesh, .partition = Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .open).set(2, .mesh_axis_1) }; + for ([_]Shape{ + shape.insert(1, .{5}), + shape.append(.{5}), + shape.remove(1), + shape.removeMany(.{ 0, 1 }), + shape.transpose(.{ 2, 0, 1 }), + shape.reshape(.{24}), + shape.splitAxis(1, .{ 1, 3 }), + shape.mergeAxis(.merged, .{ 0, 1 }), + shape.withDefaultPartitioning(), + }) |transformed| { + try testing.expectEqual(&test_mesh, transformed._sharding.mesh); + } + const parts = shape._sharding.partition; + try testing.expectEqual(parts.insert(1, .replicated), shape.insert(1, .{5})._sharding.partition); + try testing.expectEqual(parts.insert(1, .replicated), shape.insertTag(1, 5, .new)._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.append(.{5})._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.appendDim(5, null)._sharding.partition); + try testing.expectEqual(parts.orderedRemove(1), shape.remove(1)._sharding.partition); + const removed = shape.removeMany(.{ 0, 1 }); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1), removed._sharding.partition); + try testing.expectEqual(Sharding.PartitionSpec.replicated, removed.append(.{5}).partition(1)); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1).set(1, .mesh_axis_0).set(2, .open), shape.transpose(.{ 2, 0, 1 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .replicated).set(2, .replicated).set(3, .mesh_axis_1), shape.splitAxis(1, .{ 1, 3 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withDefaultPartitioning()._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withReplicatedPartitioning()._sharding.partition); + } }; /// Iterates over all coordinates of a shape, yielding the multi-dimensional diff --git a/zml/tensor.zig b/zml/tensor.zig index 83d4beba..e630af1f 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -77,8 +77,11 @@ pub const Tensor = struct { /// Internal use /// /// Creates a tensor from a Shape and an mlir.Value. + /// Note: sharding information is discarded, the output tensor is considered open for replication. + /// Use _resultPropagateSharding when the shape sharding should be propagated. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { - const res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; + var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; + res._shape._sharding.partition = .open(sh.rank()); if (builtin.mode == .Debug) { // Check that the MLIR value actually have the same shape. @@ -91,9 +94,16 @@ pub const Tensor = struct { return res; } + pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { + var res = _result(sh, val); + res._shape._sharding = sh._sharding; + return res; + } + /// Creates a Tensor from a mlir.Value /// /// The shape is derived from the type of the mlir.Value. + /// mesh is `unknown` pub fn fromMlirValue(val: *const mlir.Value) Tensor { const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; @@ -101,13 +111,15 @@ pub const Tensor = struct { stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); - var sh: Shape = .{ ._dtype = ctx.dtype(ranked_tensor.elementType()) }; + var sh: Shape = .{ + ._dtype = ctx.dtype(ranked_tensor.elementType()), + ._dims = .empty, + ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, + ._sharding = .replicated, + }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } - sh._tags.appendNTimes(Shape.TagUnknown, n) catch unreachable; - sh._partitioning.appendNTimes(.unknown, n) catch unreachable; - return .{ ._shape = sh, ._value = val, .id = nextTensorId() }; } @@ -153,8 +165,24 @@ pub const Tensor = struct { return res; } - pub fn withPartitioning(self: Tensor, axes_: anytype) Tensor { - const partitioned_shape = self._shape.withPartitioning(axes_); + /// Specify the mesh of the input tensor. + /// * mesh: *const zml.Mesh, but during compilation the `.meshe_name` syntax can used to get + /// a known mesh from the compilation options. + /// * partition spec: a struct where the field names match the axis of the given mesh + /// + /// eg `x.withPartitioning(tp, .{ .h = .model }))` or `x.withPartitioning(.tp, .{ .h = .model }))` + pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { + if (@TypeOf(mesh_) == @EnumLiteral()) { + const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); + return self.withPartitioning(compiler.mesh(mesh_), partitioning); + } + + if (@TypeOf(partitioning) != Sharding.Partitioning) { + const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, .open, partitioning); + return self.withPartitioning(mesh_, parsed); + } + + const partitioned_shape = self._shape.withPartitioning(mesh_, partitioning); const ctx = Compiler.currentOrNull() orelse { var res = self; @@ -162,15 +190,9 @@ pub const Tensor = struct { return res; }; - const sharding = ctx.partitioning.selectSharding(partitioned_shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "{f}.withPartitioning({f}) failed to resolve because it's using unknown sharding. Pass more shardings to `zml.compile`. Known shardings: {f}", - .{ self, partitioned_shape, stdx.fmt.slice(ctx.partitioning.shardings) }, - ), - }; - const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape, sharding) catch @panic("OOM"); + const attr = ctx.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape) catch @panic("OOM"); - const op_result = switch (ctx.partitioning.partitioner) { + const op_result = switch (ctx.partitioner) { .shardy => blk: { const op = mlir.Operation.make(ctx.mlir_ctx, "sdy.sharding_constraint", .{ .operands = .{ .flat = &.{self.value()} }, @@ -201,7 +223,52 @@ pub const Tensor = struct { }, }; - return _result(partitioned_shape, op_result); + std.log.warn("{f}.withPartitioning({f}) -> {f}", .{ self, partitioned_shape, attr }); + return _resultPropagateSharding(partitioned_shape, op_result); + } + + /// Force the input tensor to be fully replicated. + /// This can be detrimental for performance. + pub fn replicate(self: Tensor) Tensor { + const compiler = Compiler.current(); + const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); + return self.withPartitioning(compiler.platform.replicated_mesh, partitioning); + } + + test withPartitioning { + // Create two similar meshes, + // Then check that tensor.withPartitioning don't confuse the meshes + const dp_mp: Sharding.Mesh = .{ + .name = "dp_mp", + .physical = undefined, + .logical = .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .folds = .empty, + .folds_consumed = .empty, + }; + + const mp_dp: Sharding.Mesh = .{ + .name = "mp_dp", + .physical = undefined, + .logical = .mesh(.{ .model = .low_bandwidth, .data = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .folds = .empty, + .folds_consumed = .empty, + }; + + const x = Tensor.init(.{ .h = 8 }, .f32).withPartitioning(&dp_mp, .{ .h = .model }); + try std.testing.expectEqual(&dp_mp, x.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(1), x.shape().partition(.h)); + try std.testing.expectEqual(&dp_mp, x.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2 = x.withPartitioning(&mp_dp, .{ .h = .model }); + try std.testing.expectEqual(&mp_dp, x2.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(0), x2.shape().partition(.h)); + try std.testing.expectEqual(&mp_dp, x2.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2_replicated = x.withPartitioning(&mp_dp, .{ .h = .replicated }); + try std.testing.expectEqual(&mp_dp, x2_replicated.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); } /// Copy the given tensor to the specified memory. @@ -257,7 +324,7 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcpyH2D, .{x_t}, platform, .{}); defer exe.deinit(); - var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), .replicated, @ptrCast(&inputs), .{ .memory = .host_pinned }); + var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), @ptrCast(&inputs), .{ .memory = .host_pinned }); defer x_h.deinit(); const x_h_ptr: [*]f32 = @ptrCast(@alignCast(x_h.opaqueDevicePtr(0))); @@ -359,6 +426,8 @@ pub const Tensor = struct { /// cannot use the donated input's contents after execution. Runners destroy its /// handle automatically; raw `Exe.call` leaves it for the caller to deinit. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. + /// + /// The sharding of the origin tensor is propagated to this one. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { const compilation_context = Compiler.current(); const scope = compilation_context.currentScope(); @@ -369,8 +438,30 @@ pub const Tensor = struct { .explicit => og_donation, }; scope.id_to_donation.put(scope.arena.allocator(), self.id, donation) catch @panic("OOM"); + + // The origin is mapped to a physical buffer either because it's an input or through `reuseBuffer` + // We can now look at its sharding. + if (origin._shape.isFullyReplicated()) { + stdx.debug.assert(self.byteSize() == origin.byteSize(), ".reuseBuffer expects two tensors with same byteSize, got {f} ({B}) and {f} ({B})", .{ self, self.byteSize(), origin, origin.byteSize() }); + } else { + // Origin is sharded, we can't just use the global byte size, we actually need the same byte size per shard. + // We are stricter here and require the same shape. + stdx.debug.assert(self._shape.eql(origin._shape), ".reuseBuffer expects two sharded tensors with same shape, got {f} and {f}", .{ self, origin }); + + // Normally self sharding is unspecified and we can just propagate origin sharding, + // but if the user called `.withPartitioning` explicitly it would be weird to override it silently, hence the check. + stdx.debug.assert(self._shape._sharding.isCompatible(origin._shape._sharding), ".reuseBuffer expects compatible shardings, got {f} and {f}", .{ self, origin }); + } + + var res = self; + res._shape._sharding = origin._shape._sharding; + return res; + } else { + // Origin tensor doesn't has any physical buffer associated so we can move on. + // This is allowed to allow a function like KVCache.update to call `reuseBuffer`, + // even though the caller may have constructed the inputs internally. + return self; } - return self; } test reuseBuffer { @@ -393,13 +484,13 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcopy, .{ x_t, y_t, z_t }, platform, .{}); defer exe.deinit(); - var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), .replicated, @ptrCast(&inputs)); + var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), @ptrCast(&inputs)); defer x_d.deinit(); - var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .replicated, .{}); + var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .{}); defer y_d.deinit(); - var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .replicated, .{}); + var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .{}); defer z_d.deinit(); var y_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; @@ -532,7 +623,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&i)); + var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -729,9 +820,9 @@ pub const Tensor = struct { return .{ ._state = .init(.{2}, .u64) }; } - pub fn initBuffer(io: std.Io, platform: *const Platform, sharding: Sharding, seed: u128) !Buffer { + pub fn initBuffer(rng: Rng, io: std.Io, platform: *const Platform, seed: u128) !Buffer { return .{ - ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), sharding, std.mem.asBytes(&seed)), + ._state = try .fromBytes(io, platform, rng._state._shape, std.mem.asBytes(&seed)), }; } @@ -753,12 +844,12 @@ pub const Tensor = struct { mlirx.Type.rankedTensor(mlirCtx(), sh), currentLoc(), ).appendTo(currentBlock()); - return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; + return .{ self.update(op.result(0)), _resultPropagateSharding(sh, op.result(1)) }; } fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ - ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), + ._state = _resultPropagateSharding(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, }; } @@ -799,7 +890,7 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - return .{ rng, floats.convert(shape_.dtype()) }; + return .{ rng, _resultPropagateSharding(shape_, floats.convert(shape_.dtype()).value()) }; } test uniform { @@ -834,7 +925,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); defer rng_buffer._state.deinit(); var stats: zml.Bufferized(Stats) = undefined; @@ -878,7 +969,7 @@ pub const Tensor = struct { const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, currentLoc()).appendTo(currentBlock()); - return _result(sh, op.result(0)); + return _resultPropagateSharding(sh, op.result(0)); } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. @@ -896,7 +987,8 @@ pub const Tensor = struct { // We don't want 0 to be sampled otherwise `log` will return -inf. .{ .min = std.math.floatEps(f32), .max = 1 }, ); - return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; + const result = u.log().scale(-1).log().scale(-1).convert(shape_.dtype()); + return .{ rand, _resultPropagateSharding(shape_, result.value()) }; } test gumbel { @@ -944,9 +1036,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); defer rng_buffer._state.deinit(); - var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), .replicated, @ptrCast(&tgt_dist_data)); + var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), @ptrCast(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); var stats: zml.Bufferized(Stats) = undefined; @@ -1294,7 +1386,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, @ptrCast(&x_f32)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), @ptrCast(&x_f32)); defer x_d_buffer.deinit(); var x_f4_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); @@ -1326,7 +1418,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, std.mem.sliceAsBytes(&x)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); var x_f8e3_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); @@ -1356,7 +1448,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), .replicated, @ptrCast(&x_u2)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), @ptrCast(&x_u2)); defer x_d_buffer.deinit(); var x_u8_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); @@ -1384,7 +1476,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), .replicated, @ptrCast(&x_f4_packed)); + var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), @ptrCast(&x_f4_packed)); defer x_f4_d.deinit(); var x_f32_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_f4_d}); @@ -1526,7 +1618,7 @@ pub const Tensor = struct { const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + var res_shape: Shape = .scalar(lhs.dtype()); // Validate batching axes var lhs_batching_axes: Axes = .empty; var rhs_batching_axes: Axes = .empty; @@ -1626,7 +1718,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -1670,27 +1762,27 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); defer input_buffer.deinit(); - var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), .replicated, std.mem.sliceAsBytes(&[3][4]f32{ + var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), std.mem.sliceAsBytes(&[3][4]f32{ .{ 1, 0, 2, 0 }, .{ 2, 1, 3, 3 }, .{ 1, 1, 0, 0 }, })); defer w_buffer.deinit(); - var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), .replicated, std.mem.sliceAsBytes(&[3][4]f32{ + var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), std.mem.sliceAsBytes(&[3][4]f32{ .{ 0, 0, 2, 1 }, .{ 1, 2, 1, 3 }, .{ 0, 0, 2, 0 }, })); defer v_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), .replicated, std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); defer b_buffer.deinit(); - var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), .replicated, std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); + var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); defer c_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ input_buffer, w_buffer, v_buffer, b_buffer, c_buffer }); @@ -1948,7 +2040,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -2236,7 +2328,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 2, 5 }, .f32); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); // Wrap slice to hide the anytype in the signature. @@ -2291,9 +2383,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.dynSliceResolved, .{ x, ax, .dyn(z, 2) }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), .replicated, std.mem.asBytes(&z_value)); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); defer z_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .dyn_start = z_buffer } }); @@ -2417,7 +2509,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -2481,7 +2573,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -3173,7 +3265,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), .replicated, std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); + var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); defer indices_buffer.deinit(); var operand_buffer: zml.Buffer = b: { const temp_func = struct { @@ -3391,11 +3483,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), .replicated, @ptrCast(&[2]i32{ 0, 2 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), @ptrCast(&[2]i32{ 0, 2 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); @@ -3414,11 +3506,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), .replicated, std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); defer updates_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); @@ -3436,9 +3528,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform, .{}); defer exe.deinit(); - var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), .replicated, std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); + var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); defer operand_buffer.deinit(); - var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), .replicated, std.mem.sliceAsBytes(&[2][2][3][2]i32{ + var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ .{ .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, @@ -3449,7 +3541,7 @@ pub const Tensor = struct { }, })); defer start_indices_buffer.deinit(); - var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), .replicated, std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); + var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); defer values_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, start_indices_buffer, values_buffer }); defer result.deinit(); @@ -3554,7 +3646,7 @@ pub const Tensor = struct { defer exe.deinit(); { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; @@ -3571,7 +3663,7 @@ pub const Tensor = struct { } { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; @@ -3628,7 +3720,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); defer x_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -3647,7 +3739,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f16{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, @@ -3678,7 +3770,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 89, 31, 22, 42, 64, 39, 0, 30, 64, 71, 46, 31, @@ -4015,7 +4107,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._split, .{ x, ax, &args }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); const res = try std.testing.allocator.alloc(zml.Buffer, expectation.len); @@ -4143,11 +4235,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{4})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); @@ -4169,11 +4261,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); @@ -4198,11 +4290,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); @@ -4228,13 +4320,13 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{-1})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); @@ -4260,13 +4352,13 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); @@ -4306,12 +4398,12 @@ pub const Tensor = struct { stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); - const p = self.shape()._partitioning.get(a); + const p = self.shape()._sharding.partition.get(a); var res_shape = self._shape; res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; - // TODO(Corentin): Not sure about that - res_shape._partitioning.replaceRange(a, 1, &.{ p, p }) catch unreachable; + + res_shape._sharding.partition = res_shape._sharding.partition.insert(a, p); const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); @@ -4340,7 +4432,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -4388,7 +4480,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 3, 3 }, .u8); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[3][3]u8{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ .{ 1, 1, 1 }, .{ 1, 1, 1 }, .{ 1, 1, 1 }, @@ -4570,9 +4662,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var xs: zml.Buffer = undefined; @@ -4647,9 +4739,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var z = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, y_buffer }); @@ -4799,7 +4891,7 @@ test "Tensor.maxPool1d" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; @@ -4841,7 +4933,7 @@ test "Tensor.maxPool2d" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; diff --git a/zml/testing.zig b/zml/testing.zig index e2e84502..366351d7 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -26,8 +26,8 @@ pub fn env() *const Platform { }, ) catch @panic("Pjrt not available"); - // Register shardings required by flash attention. - _ = platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; + // Register meshes required by flash attention. + _ = platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; _platform = platform; } @@ -394,7 +394,7 @@ pub fn testLayer( activation_store: zml.io.TensorStore.View, name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), - shardings: []const *const zml.Sharding, + meshes: []const *const zml.Mesh, opts: CompareOpts, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); @@ -424,12 +424,12 @@ pub fn testLayer( var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; - tensor.* = ctx_.activation_store.createTensor(subkey, null, .replicated); + tensor.* = ctx_.activation_store.createReplicatedTensor(subkey, null); ctx_.index += 1; } }.cb, &ctx, &args); - const exe = try platform.compile(allocator, io, layer, func, args, .{ .shardings = if (shardings.len != 0) shardings else &.{} }); + const exe = try platform.compile(allocator, io, layer, func, args, .{ .meshes = if (meshes.len != 0) meshes else &.{} }); defer exe.deinit(); const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); diff --git a/zml/zml.zig b/zml/zml.zig index 995318eb..2eb84a17 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -45,6 +45,7 @@ pub const tracer = @import("profiling/tracer.zig"); pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; +pub const Mesh = Sharding.Mesh; pub const Sharding = @import("Sharding.zig"); pub const slice = @import("slice.zig"); pub const Slice = slice.Slice;