diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 67513227..8af28b00 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -pub export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { +export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = envp; var threaded: std.Io.Threaded = .init_single_threaded; diff --git a/dsl/fly/README.md b/dsl/fly/README.md index 687c99dc..b886988b 100644 --- a/dsl/fly/README.md +++ b/dsl/fly/README.md @@ -81,7 +81,7 @@ const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ ``` Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` -and `zml/kernel/fly/test.zig`. +and `zml/kernel.zig`. Model kernels live beside their consumers: diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md index b01b7189..ec1ae61f 100644 --- a/dsl/tcl/README.md +++ b/dsl/tcl/README.md @@ -104,7 +104,8 @@ def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16 across the partitions; other kernels stay per chip. Its arguments and results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis `C`, sized to the chip count, across chips (each partition holds its share), - the default `.broadcast` keeps a copy per chip. + the default `.broadcast` keeps a copy per chip. Call it inside + `zml.ops.manualComputation` so it sees each partition's shards. * **Convolutions** pad and slide the input in its read: a slide replaces its axis by `window_axis, frame_axis`, then `contract` with the filter. On its own (without a slide) the padding is dropped. diff --git a/examples/io/main.zig b/examples/io/main.zig index c9dc05ea..1cec23b0 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -132,9 +132,13 @@ pub fn main(init: std.process.Init) !void { const root = try TensorNode.init(init.arena.allocator(), ""); - for (registry.tensors.values()) |tensor| { + var registry_it = registry.iterator(); + while (registry_it.next()) |kv| { + const name = kv.key_ptr.*; + const tensor = kv.value_ptr.*; + var current = root; - var parts = std.mem.tokenizeScalar(u8, tensor.name, '.'); + var parts = std.mem.tokenizeScalar(u8, name, '.'); while (parts.next()) |part| { const gop = try current.children.getOrPut(current.allocator, part); if (!gop.found_existing) { @@ -179,16 +183,16 @@ pub fn main(init: std.process.Init) !void { const tensors = try allocator.alloc(zml.Tensor, tensor_count); defer allocator.free(tensors); + var registry_it = registry.iterator(); var load_count: usize = 0; - for (registry.tensors.keys(), registry.tensors.values()) |name, tensor| { + while (registry_it.next()) |entry| : (load_count += 1) { tensors[load_count] = switch (mesh_type) { - .replicated => store.view().createReplicatedTensor(name, null), - .sharded => if (tensor.shape.rank() > 0) - store.view().createTensor(name, null, .tp, tp_partitioning) + .replicated => store.view().createReplicatedTensor(entry.key_ptr.*, null), + .sharded => if (entry.value_ptr.shape.rank() > 0) + store.view().createTensor(entry.key_ptr.*, null, .tp, tp_partitioning) else - store.view().createReplicatedTensor(name, null), + store.view().createReplicatedTensor(entry.key_ptr.*, null), }; - load_count += 1; } const model: AllTensorsModel = .{ .tensors = tensors }; diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 34a78bfe..7e519bf0 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -42,7 +42,7 @@ const DemoModel = struct { const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); - return zml.ops.manualComputation( + return zml.ops.manualComputation2( (struct { y: zml.Tensor, gate: zml.Tensor, @@ -70,7 +70,7 @@ fn usage() void { \\Simple sharding demo: \\ - explicit partitioning on data/model axes \\ - tensor print - \\ - manualComputation with multiple inputs and one output + \\ - manualComputation2 with multiple inputs and one output \\ \\Mesh modes: \\ - auto: physical_mesh=auto, inferred cpu device_count=2 diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md index 9673870f..b66b7abe 100644 --- a/mlir/dialects/fly/REGENERATING.md +++ b/mlir/dialects/fly/REGENERATING.md @@ -71,7 +71,7 @@ dialect today; when one appears, extend the generator and this file. `test.zig`. - Callers that no longer compile are the point of generating: an operation whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, + the same change (`dsl/fly/builder.zig`, `zml/kernel.zig`, `zml/kernels/*/fly/`, and downstream llmd's `llmd/kernels/*/fly/`). - Emit the IR of the fly kernels before and after (for instance llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 5370afdc..1747e0c9 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "pjrt", @@ -22,3 +22,8 @@ zig_library( "@xla//xla/pjrt/c:pjrt_c_api_stream_extension_hdrs", ], ) + +zig_test( + name = "profiler_collect_data_test", + deps = [":pjrt"], +) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 3715361f..75cb29ba 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1832,11 +1832,8 @@ pub const Profiler = struct { } if (args.buffer != null) { - var data = args.buffer[0..args.buffer_size_in_bytes]; - if (data.len > 0 and data[data.len - 1] == 0) { - data = data[0 .. data.len - 1]; - } - return allocator.dupe(u8, data); + // CollectData returns binary serialized bytes, not a C string. + return allocator.dupe(u8, args.buffer[0..args.buffer_size_in_bytes]); } if (args.buffer_size_in_bytes == 0) return &[_]u8{}; @@ -1850,15 +1847,52 @@ pub const Profiler = struct { return pjrt_err.getCode(self.pjrt_api).toApiError(); } - // Trim trailing null byte logic - var data = buffer[0..args.buffer_size_in_bytes]; - if (data.len > 0 and data[data.len - 1] == 0) { - data = data[0 .. data.len - 1]; - } - - const pb = try allocator.dupe(u8, data); + const pb = try allocator.dupe(u8, buffer[0..args.buffer_size_in_bytes]); allocator.free(buffer); return pb; } }; + +test "Profiler collectData preserves binary protobuf terminal zero in both API paths" { + const Fixture = struct { + // A valid XSpace wire message: planes[0] has an explicit int64 id = 0. + const protobuf = [_]u8{ 0x0a, 0x02, 0x08, 0x00 }; + + fn direct(args: [*c]c.PLUGIN_Profiler_CollectData_Args) callconv(.c) ?*c.PLUGIN_Profiler_Error { + args.*.buffer = @constCast(&protobuf); + args.*.buffer_size_in_bytes = protobuf.len; + return null; + } + + fn twoPass(args: [*c]c.PLUGIN_Profiler_CollectData_Args) callconv(.c) ?*c.PLUGIN_Profiler_Error { + if (args.*.buffer) |buffer| @memcpy(buffer[0..protobuf.len], &protobuf); + args.*.buffer_size_in_bytes = protobuf.len; + return null; + } + }; + inline for (.{ Fixture.direct, Fixture.twoPass }) |collect| { + var api = std.mem.zeroes(c.PLUGIN_Profiler_Api); + api.collect_data = &collect; + var instance: Profiler = .{ .pjrt_api = undefined, .api = &api, .inner = @ptrFromInt(1) }; + const data = try instance.collectData(std.testing.allocator); + defer std.testing.allocator.free(data); + try std.testing.expectEqualSlices(u8, &Fixture.protobuf, data); + } +} + +test "Profiler collectData accepts empty serialized output" { + const Fixture = struct { + fn collect(args: [*c]c.PLUGIN_Profiler_CollectData_Args) callconv(.c) ?*c.PLUGIN_Profiler_Error { + args.*.buffer = null; + args.*.buffer_size_in_bytes = 0; + return null; + } + }; + var api = std.mem.zeroes(c.PLUGIN_Profiler_Api); + api.collect_data = &Fixture.collect; + var instance: Profiler = .{ .pjrt_api = undefined, .api = &api, .inner = @ptrFromInt(1) }; + const data = try instance.collectData(std.testing.allocator); + defer std.testing.allocator.free(data); + try std.testing.expectEqual(@as(usize, 0), data.len); +} diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index afb26864..81518d53 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", - sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-furiosa-linux-amd64.tar.zst", + sha256 = "6da755570140c9f8959ba477fef3df4c48b0619274c97429d204e177b8649e12", ) return mctx.extension_metadata( diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index ee33fd58..ebbd9e3c 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -50,6 +50,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); + try setEnv("FURIOSA_VISIBLE_DEVICES", "0", 0); const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); log.info("Loading Furiosa plugin: {s}", .{library}); diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index a3c968d3..6016e389 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" ONEAPI_VERSION = "2026.1" @@ -248,7 +248,6 @@ filegroup( name = "mkl_core_runtime", srcs = [ "{ONEAPI_MKL_LIB}/libmkl_core.so.3", - "{ONEAPI_MKL_LIB}/libmkl_def.so.3", "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index 140ce102..a3e69ad4 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -216,7 +216,7 @@ "name": "libigc2", "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -227,7 +227,7 @@ "name": "libigdfcl2", "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -249,7 +249,7 @@ "name": "libze1", "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" ], "version": "1.32.0-1~24.04~ppa1" }, diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 5a829ac5..272bc02c 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -15,11 +15,9 @@ sources: - channel: noble main url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z - # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics - # packages are pinned below to keep their versions from it. Use snapshots, - # not the live PPA: the PPA deletes superseded packages from its pool. + # IGC 2.41.15 is newer than the graphics snapshot above. - channel: noble main - url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z + url: https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu archs: - "amd64" @@ -45,9 +43,9 @@ packages: - "libnl-genl-3-200" - "libnl-route-3-200" - "libnuma1" - - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" + - "libigdgmm12" - "libigc2 (= 2.41.15-1~24.04)" - "libigdfcl2 (= 2.41.15-1~24.04)" - - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" + - "libze-intel-gpu1" - "libze1" - "zlib1g" diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index 6a0fdfae..4c1c50ed 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -482,7 +482,7 @@ fn flashAttention( } /// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in -/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the +/// `zml/kernel.zig` opens the builder, calls this `run`, and finalizes the /// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ diff --git a/stdx/flags.zig b/stdx/flags.zig index d0c4352d..52a9e6a4 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -591,19 +591,25 @@ fn fields_to_comma_list(comptime E: type) []const u8 { } } -/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. pub fn flag_name(comptime field_name: []const u8) []const u8 { - comptime assert(!std.mem.eql(u8, field_name, "positional")); - comptime var name = field_name[0..field_name.len].*; - comptime std.mem.replaceScalar(u8, &name, '_', '-'); - const dashed = name; - return "--" ++ dashed; + // TODO(Zig): Cleanup when this is fixed after Zig 0.11. + // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. + return comptime blk: { + assert(!std.mem.eql(u8, field_name, "positional")); + + var result: []const u8 = "--"; + var index = 0; + while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { + result = result ++ field_name[index..][0..i] ++ "-"; + index = index + i + 1; + } + result = result ++ field_name[index..]; + break :blk result; + }; } test flag_name { - try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); - try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); - try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); + try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); } fn flag_name_positional(comptime field_name: []const u8) []const u8 { diff --git a/tools/safetensors/BUILD.bazel b/tools/safetensors/BUILD.bazel deleted file mode 100644 index aff4954d..00000000 --- a/tools/safetensors/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") - -zig_binary( - name = "safetensors", - main = "main.zig", - visibility = ["//visibility:public"], - zigopts = ["-lc"], - deps = [ - "//stdx", - "//vfs", - "//zml", - ], -) diff --git a/tools/safetensors/main.zig b/tools/safetensors/main.zig deleted file mode 100644 index cf902eef..00000000 --- a/tools/safetensors/main.zig +++ /dev/null @@ -1,91 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); -const VFS = @import("vfs"); -const zml = @import("zml"); - -const log = std.log.scoped(.safetensors_convert); - -const Args = struct { - output: ?[]const u8 = null, - positional: struct { model: []const u8 }, - - pub const help = - \\Usage: [--output=] - \\ - \\Read safetensors headers and write model.safetensors.full_index.json in the working directory - \\Example: bazel run //tools/safetensors -- --output="model.safetensors.full_index.json" hf://deepseek-ai/DeepSeek-V4.1-Flash - ; -}; - -pub fn main(init: std.process.Init) !void { - const args = stdx.flags.parse(init.minimal.args, Args); - const allocator = init.gpa; - - const start = std.Io.Clock.now(.awake, init.io); - var client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - defer client.deinit(); - try client.initDefaultProxies(allocator, init.environ_map); - - const input = args.positional.model; - - var file_vfs: VFS.File = .init(allocator, init.io, .{ .direct_io = false }); - defer file_vfs.deinit(); - var http_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .http); - defer http_vfs.deinit(); - var https_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .https); - defer https_vfs.deinit(); - var hf_vfs: ?VFS.HF = if (std.mem.startsWith(u8, input, "hf://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (hf_vfs) |*backend| backend.deinit(); - var s3_vfs: ?VFS.S3 = if (std.mem.startsWith(u8, input, "s3://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (s3_vfs) |*backend| backend.deinit(); - var gcs_vfs: ?VFS.GCS = if (std.mem.startsWith(u8, input, "gs://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (gcs_vfs) |*backend| backend.deinit(); - - var vfs: VFS = try .init(allocator, init.io); - const io = vfs.io(); - defer vfs.deinit(); - try vfs.register("file", file_vfs.io()); - try vfs.register("http", http_vfs.io()); - try vfs.register("https", https_vfs.io()); - if (hf_vfs) |*backend| try vfs.register("hf", backend.io()); - if (s3_vfs) |*backend| try vfs.register("s3", backend.io()); - if (gcs_vfs) |*backend| try vfs.register("gs", backend.io()); - - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - const repo = try zml.safetensors.resolveModelRepo(io, input); - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - const loaded = std.Io.Clock.now(.awake, init.io); - - const index_json_file = if (args.output) |output| - try std.Io.Dir.cwd().createFile(io, output, .{}) - else - try repo.createFile(io, full_index_path, .{}); - const out_buffer = try allocator.alloc(u8, 4096); - defer allocator.free(out_buffer); - var out_writer = index_json_file.writer(io, out_buffer); - try registry.saveAsFullIndexJson(io, allocator, repo, &out_writer.interface); - const done = std.Io.Clock.now(.awake, init.io); - - log.info("Wrote {s}: {d} tensors; load {f}s, write {f}s, total {f}s", .{ - args.output orelse full_index_path, - registry.tensors.count(), - start.durationTo(loaded), - loaded.durationTo(done), - start.durationTo(done), - }); -} - -const full_index_path = "model.safetensors.full_index.json"; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f52ecdd5..757b83e7 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,5 +1,5 @@ load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") -load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") upb_c_proto_library( @@ -65,6 +65,16 @@ cc_library( }), ) +cc_test( + name = "profiling_traceme_test", + srcs = ["profiling/traceme_test.cc"], + deps = [ + ":profiling_traceme_lib", + "@tsl//tsl/profiler/protobuf:xplane_proto_cc", + "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", + ], +) + zig_library( name = "logo", srcs = ["logo.zig"], @@ -87,10 +97,11 @@ zig_library( "exe.zig", "floats.zig", "io.zig", + "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", - "kernels/moe/api.zig", "kernels/moe/moe.zig", + "kernels/moe/options.zig", "kernels/moe/tests.zig", "kernels/moe/weights.zig", "kernels/paged_attention/options.zig", @@ -102,6 +113,7 @@ zig_library( "nn.zig", "ops.zig", "ops/manual_computation.zig", + "ops/manual_computation2.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", @@ -113,7 +125,6 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", - "//zml/kernel", "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index edd9b4ca..0ac2a95c 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -97,6 +97,7 @@ pub const Options = @import("Options.zig"); pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; + const InputSpecs = @FieldType(zml.ops.ManualPartitioning(Attention.forward), "in_specs"); in_specs: struct { q: Logical, @@ -104,7 +105,7 @@ pub const Partitioning = struct { v: Logical, token_index: Logical = .replicated, /// Omit to replicate backend metadata tensors. - metadata: ?zml.ops.PartitionSpecs(Metadata) = null, + metadata: ?@FieldType(InputSpecs, "metadata") = null, }, out_specs: Logical, @@ -134,13 +135,20 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T /// `attention`, causal or not depending on `opts`. pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; - return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ + return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, .{ .in_specs = .{ .q = partitioning.in_specs.q, .k = partitioning.in_specs.k, .v = partitioning.in_specs.v, .token_index = partitioning.in_specs.token_index, - .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), + .metadata = partitioning.in_specs.metadata orelse switch (metadata) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |metadata_| .{ .flash_attention = switch (metadata_) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = .{ .num_tokens = .replicated } }, + } }, + }, }, .out_specs = partitioning.out_specs, }); diff --git a/zml/kernel.zig b/zml/kernel.zig new file mode 100644 index 00000000..d85b6a56 --- /dev/null +++ b/zml/kernel.zig @@ -0,0 +1,1627 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +const cute_builder = @import("dsl/cute/builder"); +const fly_builder = @import("dsl/fly/builder"); +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +const triton_builder = @import("dsl/triton/builder"); + +const Compiler = @import("Compiler.zig"); +const DataType = @import("dtype.zig").DataType; +const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); +const platform_ = @import("platform.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub const triton = struct { + pub const Builder = triton_builder.Builder; + pub const Value = triton_builder.Value; + pub const DType = triton_builder.DType; + pub const FinishError = triton_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; + } +}; + +pub const mosaic_tpu = struct { + const Builder = mosaic_tpu_builder.Builder; + const DType = mosaic_tpu_builder.DType; + const FinishError = mosaic_tpu_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; + } + + const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + }; + + const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, + }; + + const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, + }; + + const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + }; + + fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, + ) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); + } + + const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, + }; + + fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); + } +}; + +pub const cuda_tile = struct { + pub const Builder = cuda_tile_builder.Builder; + pub const Value = cuda_tile_builder.Value; + pub const DType = cuda_tile_builder.DType; + pub const FinishError = cuda_tile_builder.FinishError; + pub const EntryHint = cuda_tile_builder.EntryHint; + pub const Arch = cuda_tile_builder.Arch; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const cute = struct { + pub const AlgebraExpr = cute_builder.AlgebraExpr; + pub const Builder = cute_builder.Builder; + pub const ArgSpec = cute_builder.ArgSpec; + pub const Value = cute_builder.Value; + pub const View = cute_builder.View; + pub const Atom = cute_builder.Atom; + pub const DType = cute_builder.DType; + pub const AlgebraToken = cute_builder.AlgebraToken; + pub const LayoutSpec = cute_builder.LayoutSpec; + pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; + pub const FinishError = cute_builder.FinishError; + pub const TmaConfig = cute_builder.TmaConfig; + pub const TmaFormat = cute_builder.TmaFormat; + pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; + pub const CtaGroup = cute_builder.CtaGroup; + pub const ctaCount = cute_builder.ctaCount; + pub const MBarrierScope = cute_builder.MBarrierScope; + pub const OperandMajorMode = cute_builder.OperandMajorMode; + pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; + pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; + pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; + pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; + pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; + pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } + + fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + /// A complete CuTe program: one or more `cuda.kernel` operations inside + /// a `gpu.module`, followed by a public `func.func` which constructs the + /// launch-time CuTe objects and calls `cuda.launch_ex`. + /// + /// This is the source-level equivalent of invoking a Python `@cute.jit` + /// object whose `__call__` builds TMA descriptors and launches an + /// `@cute.kernel`. XLA executes the generated host function on its CUDA + /// stream, so the custom call deliberately carries no external grid or + /// block dimensions. + pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const tcl = struct { + pub const Builder = tcl_builder.Builder; + pub const TensorOperation = tcl_builder.TensorOperation; + pub const Value = tcl_builder.Value; + pub const Axis = tcl_builder.Axis; + pub const Expr = tcl_builder.Expr; + pub const Dram = tcl_builder.Dram; + pub const Context = tcl_builder.Context; + pub const DType = tcl_builder.DType; + pub const VeOp = tcl_builder.VeOp; + pub const ReduceMode = tcl_builder.ReduceMode; + pub const Predicate = tcl_builder.Predicate; + pub const Tactic = tcl_builder.Tactic; + pub const FinishError = tcl_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + /// A TCL function compiled by the Furiosa plugin as one native unit: + /// `declareArgs` declares the inputs in order, `ret` the outputs. + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; + } +}; + +pub const metal = struct { + pub const Builder = metal_builder.Builder; + pub const Value = metal_builder.Value; + pub const Type = metal_builder.Type; + pub const Struct = metal_builder.Struct; + pub const ArgSpec = metal_builder.ArgSpec; + pub const LoopOpts = metal_builder.LoopOpts; + pub const FinishError = metal_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); + } + + pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; + } + + /// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host + /// `extern struct` laid out like a `constant S &` argument. + pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } + }; + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; + } + + /// A Metal kernel: `run` declares its arguments with `declareArgs`, by + /// name, and builds its body. + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const fly = struct { + /// Threads per wavefront: 64 on CDNA, 32 on RDNA. + pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; + } + + /// The plugin sizes a block in wavefronts, so a thread count has to divide + /// evenly: a kernel's layouts are built for an exact number of threads and + /// spare ones would partition out of range. + fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); + } + + pub const Builder = fly_builder.Builder; + pub const Value = fly_builder.Value; + pub const DType = fly_builder.DType; + pub const FinishError = fly_builder.FinishError; + pub const TiledCopy = fly_builder.TiledCopy; + pub const TiledMma = fly_builder.TiledMma; + pub const Arch = fly_builder.Arch; + pub const MmaFlavor = fly_builder.MmaFlavor; + pub const fly_rocdl = fly_builder.fly_rocdl; + + /// The matrix atom this device provides, or null when it has none: WMMA + /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. + pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; + } + pub const layout = fly_builder.layout; + pub const Layout = fly_builder.Layout; + pub const Tile = fly_builder.Tile; + pub const IntTuple = fly_builder.IntTuple; + pub const L = fly_builder.L; + pub const rowMajor = fly_builder.rowMajor; + pub const colMajor = fly_builder.colMajor; + pub const ordered = fly_builder.ordered; + pub const tile = fly_builder.tile; + pub const it = fly_builder.it; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernels run on rocm" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: Tensor, b: Tensor) Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: Tensor, b: Tensor) Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: Tensor) Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: Tensor }, shapes: [2]Shape) [2]Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const tp = zml.testing.tpMesh(); + const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: Tensor) [2]Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); + return .{ r[0], r[1] }; + } + }; + const x: Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{tp} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: Tensor, y: Tensor) Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel deleted file mode 100644 index 61ab44f8..00000000 --- a/zml/kernel/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "kernel", - srcs = [ - "kernel.zig", - "//zml/kernel/cuda_tile", - "//zml/kernel/cute", - "//zml/kernel/fly", - "//zml/kernel/metal", - "//zml/kernel/mosaic_tpu", - "//zml/kernel/tcl", - "//zml/kernel/triton", - ], -) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel deleted file mode 100644 index 89b7977e..00000000 --- a/zml/kernel/cuda_tile/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "cuda_tile", - srcs = [ - "cuda_tile.zig", - "test.zig", - ], -) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig deleted file mode 100644 index 3f6d8734..00000000 --- a/zml/kernel/cuda_tile/cuda_tile.zig +++ /dev/null @@ -1,163 +0,0 @@ -const std = @import("std"); - -const cuda_tile_builder = @import("dsl/cuda_tile/builder"); -pub const Builder = cuda_tile_builder.Builder; -pub const Value = cuda_tile_builder.Value; -pub const DType = cuda_tile_builder.DType; -pub const FinishError = cuda_tile_builder.FinishError; -pub const EntryHint = cuda_tile_builder.EntryHint; -pub const Arch = cuda_tile_builder.Arch; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cuda_tile_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i4, .u4 => .i4, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - .f4e2m1 => .f4e2m1fn, - else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// No `num_warps`/`num_stages`: warp and CTA tuning is - /// `optimization_hints` inside the IR (`Opts.hints`). - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Outputs XLA zeroes before the launch, by position in - /// `spec.outputs`, ascending. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(&.{}); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig deleted file mode 100644 index c929b0b9..00000000 --- a/zml/kernel/cuda_tile/test.zig +++ /dev/null @@ -1,28 +0,0 @@ -const std = @import("std"); - -const cuda_tile = @import("cuda_tile.zig"); - -test "cuda_tile kernel emits a module XLA can parse" { - const Cfg = struct { n: i64 }; - const AddOne = cuda_tile.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); -} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel deleted file mode 100644 index 7a613072..00000000 --- a/zml/kernel/cute/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "cute", - srcs = [ - "cute.zig", - "test.zig", - ], -) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig deleted file mode 100644 index 2e8c919f..00000000 --- a/zml/kernel/cute/cute.zig +++ /dev/null @@ -1,264 +0,0 @@ -const std = @import("std"); - -const cute_builder = @import("dsl/cute/builder"); -pub const AlgebraExpr = cute_builder.AlgebraExpr; -pub const Builder = cute_builder.Builder; -pub const ArgSpec = cute_builder.ArgSpec; -pub const Value = cute_builder.Value; -pub const View = cute_builder.View; -pub const Atom = cute_builder.Atom; -pub const DType = cute_builder.DType; -pub const AlgebraToken = cute_builder.AlgebraToken; -pub const LayoutSpec = cute_builder.LayoutSpec; -pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; -pub const FinishError = cute_builder.FinishError; -pub const TmaConfig = cute_builder.TmaConfig; -pub const TmaFormat = cute_builder.TmaFormat; -pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; -pub const CtaGroup = cute_builder.CtaGroup; -pub const ctaCount = cute_builder.ctaCount; -pub const MBarrierScope = cute_builder.MBarrierScope; -pub const OperandMajorMode = cute_builder.OperandMajorMode; -pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; -pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; -pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; -pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; -pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; -pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cute_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4e2m1fn, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared - /// memory is what the kernel declares. - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - block: [3]i32, - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - // XLA passes one pointer per input then per output; it does - // not check the count against the kernel. - const expected = spec.inputs.len + spec.outputs.len; - if (b.args.len != expected) { - std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); - } - - return b.finish(block); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| - std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .block = opts.block, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} - -fn ProgramSpec(comptime Config: type) type { - return struct { - /// Public host entry selected by XLA. The callback may give the - /// nested CUDA kernel a different (private) symbol. - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -/// A complete CuTe program: one or more `cuda.kernel` operations inside -/// a `gpu.module`, followed by a public `func.func` which constructs the -/// launch-time CuTe objects and calls `cuda.launch_ex`. -/// -/// This is the source-level equivalent of invoking a Python `@cute.jit` -/// object whose `__call__` builds TMA descriptors and launches an -/// `@cute.kernel`. XLA executes the generated host function on its CUDA -/// stream, so the custom call deliberately carries no external grid or -/// block dimensions. -pub fn Program( - comptime ConfigT: type, - comptime spec: ProgramSpec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.openProgram(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - return b.finishProgram(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .zeroed_outputs = opts.zeroed_outputs, - .scalars = opts.scalars, - .symmetric_workspace = opts.symmetric_workspace, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig deleted file mode 100644 index f4d457a3..00000000 --- a/zml/kernel/cute/test.zig +++ /dev/null @@ -1,34 +0,0 @@ -const std = @import("std"); - -const cute = @import("cute.zig"); - -test "cute kernel emits the module the CuTe compiler takes" { - const Cfg = struct { n: i64 }; - const AddOne = cute.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - }); - const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); - var guard = b.openIf(i.lt(cfg.n)); - { - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.close(); - } - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); -} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel deleted file mode 100644 index 57203521..00000000 --- a/zml/kernel/fly/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "fly", - srcs = [ - "fly.zig", - "test.zig", - ], -) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig deleted file mode 100644 index b7d6c943..00000000 --- a/zml/kernel/fly/fly.zig +++ /dev/null @@ -1,248 +0,0 @@ -const std = @import("std"); - -const fly_builder = @import("dsl/fly/builder"); -pub const Builder = fly_builder.Builder; -pub const Value = fly_builder.Value; -pub const DType = fly_builder.DType; -pub const FinishError = fly_builder.FinishError; -pub const TiledCopy = fly_builder.TiledCopy; -pub const TiledMma = fly_builder.TiledMma; -pub const Arch = fly_builder.Arch; -pub const MmaFlavor = fly_builder.MmaFlavor; -pub const fly_rocdl = fly_builder.fly_rocdl; -pub const layout = fly_builder.layout; -pub const Layout = fly_builder.Layout; -pub const Tile = fly_builder.Tile; -pub const IntTuple = fly_builder.IntTuple; -pub const L = fly_builder.L; -pub const rowMajor = fly_builder.rowMajor; -pub const colMajor = fly_builder.colMajor; -pub const ordered = fly_builder.ordered; -pub const tile = fly_builder.tile; -pub const it = fly_builder.it; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -/// Threads per wavefront: 64 on CDNA, 32 on RDNA. -pub fn waveSize(p: *const platform_.Platform) i32 { - const cc = platform_.rocm.computeCapability(p) orelse return 64; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => 64, - .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, - }; -} - -/// The plugin sizes a block in wavefronts, so a thread count has to divide -/// evenly: a kernel's layouts are built for an exact number of threads and -/// spare ones would partition out of range. -fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { - const wave = waveSize(p); - if (@rem(threads, wave) != 0) { - std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); - } - return @divExact(threads, wave); -} - -/// The matrix atom this device provides, or null when it has none: WMMA -/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. -pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { - const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, - .rdna3, .rdna3_5 => .gfx11_wmma, - .rdna4 => .gfx120x_wmma, - .rdna2 => null, - }; -} -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(fly_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - .f8e5m2fnuz => .f8e5m2fnuz, - else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const arg_names = spec.inputs ++ spec.outputs; - /// By name, as `!fly.memref` views. - pub const Args = StructOf(arg_names, Value); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Threads per block. Converted to the wavefronts the plugin - /// wants using the device's wave size, which is 64 on CDNA and - /// 32 on RDNA, so a kernel states the thread count its layouts - /// were built for and runs on both. - threads: i32, - /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. - waves_per_eu: i32 = 0, - /// Required when the body uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Inputs first, then outputs; see `zeroedOutput`. A no-op - /// inside command buffers. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - /// The arguments `run` was declared with. - pub fn args(b: *Builder) Args { - var out: Args = undefined; - inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); - return out; - } - - /// `zeroed_args` index of an output. - pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); - } - - /// `shapes` is inputs, then outputs. - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { - std.debug.assert(shapes.len == arg_names.len); - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try fly_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - var specs: [arg_names.len]Builder.ArgSpec = undefined; - inline for (arg_names, 0..) |fname, i| { - const shape = shapes[i]; - specs[i] = .{ - .name = fname, - .dtype = from(shape.dtype()), - .dims = if (shape.rank() > 0) shape.dims() else null, - }; - } - try b.declareArgsLow(&specs); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var shapes: [arg_names.len]Shape = undefined; - inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); - inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; - - const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| - std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .num_warps = warpsFor(cur.platform, opts.threads), - .waves_per_eu = opts.waves_per_eu, - .shared_mem_bytes = opts.shared_mem_bytes, - .zeroed_args = opts.zeroed_args, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig deleted file mode 100644 index ae788d96..00000000 --- a/zml/kernel/fly/test.zig +++ /dev/null @@ -1,149 +0,0 @@ -const std = @import("std"); - -const fly_builder = @import("dsl/fly/builder"); - -const fly = @import("fly.zig"); -const zml = @import("../../zml.zig"); - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -const FlyVectorAdd = struct { - const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), - }; - - const K = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); - } -}; - -/// C = A @ B^T with one MFMA-tiled block. -const FlyTiledMma = struct { - const Cfg = struct { flavor: fly.MmaFlavor }; - const K = fly.Kernel(Cfg, .{ - .name = "tiled_mma", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); - } -}; - -test "fly kernel emits a module XLA can parse" { - const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; - const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); -} - -test "fly kernels run on rocm" { - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward`, upload both inputs, run, and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); - defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); - defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); - defer bb.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // vector add: elementwise over a ragged (100, 1000), predicated at the edge. - { - const M, const N = .{ 100, 1000 }; - const Mod = struct { - pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, - .threads = 8 * 16, - }).c; - } - }; - - const ha = try allocator.alloc(f32, M * N); - defer allocator.free(ha); - const hb = try allocator.alloc(f32, M * N); - defer allocator.free(hb); - for (ha, hb, 0..) |*va, *vb, i| { - va.* = @floatFromInt(i % 977); - vb.* = @floatFromInt(1000 + (i % 313)); - } - - var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); - } - - // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix - // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, - // whose verifier rejects f32 operands, so the operand type and K come from - // the flavour rather than being written into the test. - if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { - inline else => |f| { - const Elem = switch (comptime f.operandDType()) { - .f32 => f32, - .f16 => f16, - else => @compileError("unhandled MMA operand type"), - }; - const M, const N = .{ 64, 64 }; - const Kd = comptime f.blockK(); - const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { - .f32 => .f32, - .f16 => .f16, - else => unreachable, - }; - - const Mod = struct { - pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ - .cfg = .{ .flavor = f }, - .grid = .{ 1, 1, 1 }, - .threads = comptime f.blockThreads(), - }).c; - } - }; - - var ha: [M * Kd]Elem = undefined; - var hb: [N * Kd]Elem = undefined; - for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; - for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; - - var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); - defer host.free(allocator); - - // f16 operands accumulate in f32, so the error is set by the input - // rounding rather than the sum. - const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; - const out = host.items(f32); - for (0..M) |m| for (0..N) |n| { - var acc: f32 = 0; - for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); - try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); - }; - }, - }; -} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig deleted file mode 100644 index 204c6204..00000000 --- a/zml/kernel/kernel.zig +++ /dev/null @@ -1,12 +0,0 @@ -pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); -pub const cute = @import("cute/cute.zig"); -pub const fly = @import("fly/fly.zig"); -pub const metal = @import("metal/metal.zig"); -pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const tcl = @import("tcl/tcl.zig"); -pub const triton = @import("triton/triton.zig"); - -test { - const std = @import("std"); - std.testing.refAllDecls(@This()); -} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel deleted file mode 100644 index 4007beb4..00000000 --- a/zml/kernel/metal/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "metal", - srcs = [ - "metal.zig", - "test.zig", - ], -) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig deleted file mode 100644 index 9626a1de..00000000 --- a/zml/kernel/metal/metal.zig +++ /dev/null @@ -1,237 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -pub const Builder = metal_builder.Builder; -pub const Value = metal_builder.Value; -pub const Type = metal_builder.Type; -pub const Struct = metal_builder.Struct; -pub const ArgSpec = metal_builder.ArgSpec; -pub const LoopOpts = metal_builder.LoopOpts; -pub const FinishError = metal_builder.FinishError; -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(metal_builder.registerDialects); -} - -pub fn from(dt: DataType) Type { - return switch (dt) { - .bool => .bool, - .i8 => .char, - .i16 => .short, - .i32 => .int, - .i64 => .long, - .u8 => .uchar, - .u16 => .ushort, - .u32 => .uint, - .u64 => .ulong, - .f16 => .half, - .bf16 => .bfloat, - .f32 => .float, - else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), - }; -} - -/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host -/// `extern struct` laid out like a `constant S &` argument. -pub const Constant = struct { - buffer: [256]u8 = undefined, - len: usize = 0, - - pub fn of(value: anytype) Constant { - const raw = std.mem.asBytes(&value); - var c: Constant = .{ .len = raw.len }; - @memcpy(c.buffer[0..raw.len], raw); - return c; - } - - pub fn bytes(self: *const Constant) []const u8 { - return self.buffer[0..self.len]; - } -}; - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - /// The arguments bound to input tensors. - inputs: []const [:0]const u8, - /// The arguments bound to output tensors; one named like an - /// input updates it in place. - outputs: []const [:0]const u8, - /// The `constant` arguments given by value at the call. - constants: []const [:0]const u8 = &.{}, - run: *const fn (*Builder, Config) FinishError!void, - fast_math: bool = true, - }; -} - -/// A Metal kernel: `run` declares its arguments with `declareArgs`, by -/// name, and builds its body. -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const Constants = StructOf(spec.constants, Constant); - - pub const CallOpts = struct { - cfg: ConfigT, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - constants: Constants = std.mem.zeroes(Constants), - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - return build(allocator, cfg, &arguments); - } - - fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); - defer b.deinit(); - try spec.run(&b, cfg); - const ir = try b.finish(); - errdefer allocator.free(ir); - try arguments.appendSlice(allocator, b.arguments.items); - return ir; - } - - fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { - for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; - return null; - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const allocator = cur.allocator; - - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - const ir = build(allocator, opts.cfg, &arguments) catch |err| - std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); - var constants: [spec.constants.len][]const u8 = undefined; - inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); - - var bound: [spec.outputs.len]bool = @splat(false); - var bound_inputs: [spec.inputs.len]bool = @splat(false); - var bound_constants: [spec.constants.len]bool = @splat(false); - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - var bindings: std.ArrayList(i32) = .empty; - defer bindings.deinit(allocator); - for (arguments.items) |arg| { - if (arg.kind == .threadgroup) { - // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. - if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); - continue; - } - std.debug.assert(arg.index == bindings.items.len); - const input = indexOf(spec.inputs, arg.name); - const output = indexOf(spec.outputs, arg.name); - if (input) |i| bound_inputs[i] = true; - if (output) |o| bound[o] = true; - const binding: i32 = if (input) |i| blk: { - if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); - break :blk @intCast(i); - } else if (output) |o| - @intCast(spec.inputs.len + o) - else if (indexOf(spec.constants, arg.name)) |k| blk: { - if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); - bound_constants[k] = true; - break :blk -1 - @as(i32, @intCast(k)); - } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); - bindings.append(allocator, binding) catch @panic("OOM"); - } - for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); - for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); - for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); - - const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .threadgroup = opts.threadgroup, - .threadgroup_memory = opts.threadgroup_memory, - .arguments = bindings.items, - .constants = &constants, - .output_operand_aliases = aliases.constSlice(), - }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig deleted file mode 100644 index 0a342af0..00000000 --- a/zml/kernel/metal/test.zig +++ /dev/null @@ -1,71 +0,0 @@ -const std = @import("std"); - -const metal = @import("metal.zig"); -const zml = @import("../../zml.zig"); - -/// `out[i] = a[i] * scale + b[i]` over `n` floats. -const MetalAxpy = struct { - const Cfg = struct {}; - const K = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -test "metal kernels run on metal" { - const allocator = std.testing.allocator; - - const ir = try MetalAxpy.K.emit(allocator, .{}); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); - - const platform = zml.testing.env(); - if (platform.target != .metal) return error.SkipZigTest; - - const n = 1000; - const Mod = struct { - pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { - return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ (n + 255) / 256, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, - }).out; - } - }; - const x: zml.Tensor = .init(.{n}, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); - defer exe.deinit(); - - var hx: [n]f32 = undefined; - var hy: [n]f32 = undefined; - for (&hx, &hy, 0..) |*a, *b, i| { - a.* = @floatFromInt(i); - b.* = 1; - } - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); - defer by.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); -} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel deleted file mode 100644 index ba3b6b66..00000000 --- a/zml/kernel/mosaic_tpu/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "mosaic_tpu", - srcs = ["mosaic_tpu.zig"], -) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig deleted file mode 100644 index a09e8368..00000000 --- a/zml/kernel/mosaic_tpu/mosaic_tpu.zig +++ /dev/null @@ -1,284 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); -const Builder = mosaic_tpu_builder.Builder; -const DType = mosaic_tpu_builder.DType; -const FinishError = mosaic_tpu_builder.FinishError; -const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(mosaic_tpu_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - dynamic_grid_bounds: []const Tensor = &.{}, - }; - - pub const CallOpts = struct { - cfg: ConfigT, - extras: CallExtras = .{}, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finishOpts(&.{}, .{ .canonicalize = true }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const backend_config = buildBackendConfig(cur.allocator, ir, .{ - .vmem_limit_bytes = opts.extras.vmem_limit_bytes, - .disable_bounds_checks = opts.extras.disable_bounds_checks, - .disable_semaphore_checks = opts.extras.disable_semaphore_checks, - .has_communication = opts.extras.has_communication, - }) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); - defer cur.allocator.free(backend_config); - - var values: [spec.inputs.len]*const mlir.Value = undefined; - inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); - - var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (&res_types, spec.outputs) |*r, fname| { - r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); - } - - const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); - - const op = callTpuCustomCall(.{ - .inputs = &values, - .result_types = &res_types, - .backend_config = backend_config, - .aliases = aliases.constSlice(), - .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, - .additional_attributes = opts.extras.additional_attributes, - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - @field(results, fname) = Tensor._result(out, op.result(i)); - } - return results; - } - }; -} - -const CustomCallConfig = struct { - body: []const u8, - has_communication: ?bool = null, - serialization_format: i64 = 1, - needs_layout_passes: bool = true, - shape_invariant_numerics: bool = false, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, -}; - -const ScopedMemoryConfig = struct { - memory_space: i64 = 1, - offset: i64 = 0, - size: i64, -}; - -const BackendConfig = struct { - custom_call_config: CustomCallConfig, - scoped_memory_configs: ?[]const ScopedMemoryConfig = null, -}; - -const BackendConfigOpts = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, -}; - -fn buildBackendConfig( - allocator: std.mem.Allocator, - ir: [:0]const u8, - opts: BackendConfigOpts, -) ![]u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - const module = try mlir.Module.parse(ctx, ir); - defer module.deinit(); - - mosaic_tpu_dialect.registerMosaicSerdePass(); - const prev_allow_unregistered = ctx.allowUnregisteredDialects(); - ctx.setAllowUnregisteredDialects(true); - defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); - - const pm = mlir.PassManager.init(ctx); - defer pm.deinit(); - try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); - try pm.runOnOp(module.operation()); - - const Encoder = std.base64.standard.Encoder; - const b64 = b64: { - var bytecode: std.Io.Writer.Allocating = .init(allocator); - defer bytecode.deinit(); - try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); - const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); - break :b64 Encoder.encode(buf, bytecode.written()); - }; - defer allocator.free(b64); - - const config = BackendConfig{ - .custom_call_config = .{ - .body = b64, - .has_communication = opts.has_communication, - .disable_bounds_checks = opts.disable_bounds_checks, - .disable_semaphore_checks = opts.disable_semaphore_checks, - }, - .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| - &.{.{ .size = vmem }} - else - null, - }; - - var json: std.Io.Writer.Allocating = .init(allocator); - errdefer json.deinit(); - - try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - return try json.toOwnedSlice(); -} - -const TpuCustomCallArgs = struct { - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - backend_config: []const u8, - aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, - dynamic_grid_bounds: []const Tensor, - additional_attributes: []const mlir.NamedAttribute, -}; - -fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = Compiler.current(); - - var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; - for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); - all_inputs.appendSliceAssumeCapacity(args.inputs); - - return stablehlo.custom_call( - cur.mlir_ctx, - all_inputs.constSlice(), - args.result_types, - .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = args.backend_config }, - .has_side_effect = false, - .api_version = .original, - .additional_attributes = args.additional_attributes, - .output_operand_aliases = args.aliases, - }, - cur.location, - ).appendTo(cur.currentScope().block); -} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel deleted file mode 100644 index bc7a0aa2..00000000 --- a/zml/kernel/tcl/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "tcl", - srcs = [ - "tcl.zig", - "test.zig", - ], -) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig deleted file mode 100644 index 023e862f..00000000 --- a/zml/kernel/tcl/tcl.zig +++ /dev/null @@ -1,206 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const tcl_builder = @import("dsl/tcl/builder"); -pub const Builder = tcl_builder.Builder; -pub const TensorOperation = tcl_builder.TensorOperation; -pub const Value = tcl_builder.Value; -pub const Axis = tcl_builder.Axis; -pub const Expr = tcl_builder.Expr; -pub const Dram = tcl_builder.Dram; -pub const Context = tcl_builder.Context; -pub const DType = tcl_builder.DType; -pub const VeOp = tcl_builder.VeOp; -pub const ReduceMode = tcl_builder.ReduceMode; -pub const Predicate = tcl_builder.Predicate; -pub const Tactic = tcl_builder.Tactic; -pub const FinishError = tcl_builder.FinishError; - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(tcl_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .bool, - .i4 => .i4, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4_e2, - .f8e4m3fn => .f8_e4, - .f8e5m2 => .f8_e5, - else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -/// A TCL function compiled by the Furiosa plugin as one native unit: -/// `declareArgs` declares the inputs in order, `ret` the outputs. -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - return build(allocator, cfg, null, null, null); - } - - /// `updated`, when given, receives the input each output updates - /// in place (`Builder.update`), or null. - fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try Builder.open(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - - const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); - if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { - std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); - } - - if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); - if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); - if (updated) |u| for (u, results) |*dst, r| { - dst.* = b.updatedArgument(r); - }; - - return b.finish(); - } - - fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { - var same = from(s.dtype()) == t.dtype; - if (t.shards > 1) { - // A partition holds one chip's share of a chip-split DRAM tensor. - var n: i64 = 1; - for (t.axes) |a| n *= a.size; - same = same and n == @as(i64, @intCast(s.count())) * t.shards; - } else { - same = same and s.rank() == t.axes.len; - if (same) for (t.axes, 0..) |a, i| { - same = same and a.size == s.dim(i); - }; - } - - if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - var input_shapes: [spec.inputs.len]Shape = undefined; - inline for (spec.inputs, 0..) |fname, i| { - inputs_arr[i] = @field(inputs, fname); - input_shapes[i] = inputs_arr[i].shape(); - } - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var updated: [spec.outputs.len]?usize = undefined; - const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| - std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ - .output_index = @intCast(output), - .operand_index = @intCast(i), - }); - const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - @field(results, fname) = tensor_results[i]; - } - - return results; - } - }; -} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig deleted file mode 100644 index b9ae1875..00000000 --- a/zml/kernel/tcl/test.zig +++ /dev/null @@ -1,123 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const tcl = @import("tcl.zig"); - -/// `furiosa.tcl.examples.softmax` over f32. -const TclSoftmax = struct { - const Cfg = struct { b: i64, s: i64, d: i64 }; - const K = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"x"}, - .outputs = &.{"y"}, - .run = run, - }); - - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.x, .{}); - const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); - b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); - } -}; - -test "tcl kernels run on furiosa" { - const allocator = std.testing.allocator; - const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; - - const ir = try TclSoftmax.K.emit(allocator, cfg); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); - - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - - const Mod = struct { - pub fn forward(x: zml.Tensor) zml.Tensor { - return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; - } - }; - const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); - defer exe.deinit(); - - var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); - defer x_buffer.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - - const out = host.items(f32); - var row: usize = 0; - while (row < cfg.b * cfg.s) : (row += 1) { - const in = input[row * cfg.d ..][0..cfg.d]; - var max: f32 = in[0]; - for (in) |v| max = @max(max, v); - var sum: f32 = 0; - for (in) |v| sum += @exp(v - max); - for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); - } -} - -/// exp over a tensor split across two chips in one launch, returning the -/// split result and its all_gather. -const TclSplitExp = struct { - const Cfg = struct {}; - const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); - fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - } - fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { - const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); - return .{ r.y, r.all }; - } -}; - -// Needs FURIOSA_VISIBLE_DEVICES=0,1. -test "tcl multi-chip kernels run on two furiosa chips" { - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const tp = zml.testing.tpMesh(); - const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); - const Mod = struct { - pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ - .in_specs = .{ .x = .parse(.{ .c = .model }) }, - .out_specs = .{ .parse(.{ .c = .model }), .replicated }, - }); - return .{ r[0], r[1] }; - } - }; - const x: zml.Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); - defer exe.deinit(); - var hx: [2 * 1024]f32 = undefined; - for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var res: [2]zml.Buffer = undefined; - runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); - defer for (&res) |*r| r.deinit(); - for (res) |r| { - var host = try r.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); - } -} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel deleted file mode 100644 index 1e992352..00000000 --- a/zml/kernel/triton/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "triton", - srcs = ["triton.zig"], -) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig deleted file mode 100644 index 6a91a03f..00000000 --- a/zml/kernel/triton/triton.zig +++ /dev/null @@ -1,165 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const triton_builder = @import("dsl/triton/builder"); -pub const Builder = triton_builder.Builder; -pub const Value = triton_builder.Value; -pub const DType = triton_builder.DType; -pub const FinishError = triton_builder.FinishError; - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const ops = @import("../../ops.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(triton_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - debug: bool = false, - /// Bytes for Triton's implicit global scratch argument, across all CTAs. - global_scratch_memory_size: i32 = 0, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try triton_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ttir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ttir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const call_opts: ops.TritonOps = .{ - .debug = opts.debug, - .name = name, - .ir = ttir, - .grid = opts.grid, - .num_stages = opts.num_stages, - .num_warps = opts.num_warps, - .output_operand_aliases = aliases.constSlice(), - .is_tma_allowed = opts.global_scratch_memory_size > 0, - .global_scratch_memory_size = opts.global_scratch_memory_size, - }; - - const triton_results = if (opts.global_scratch_memory_size > 0) blk: { - // XLA maps the final custom-call result to Triton's implicit - // scratch pointer; it must not appear in the TTIR signature. - const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; - const res = ops.triton(inputs_arr, extended, call_opts); - break :blk res[0..spec.outputs.len].*; - } else ops.triton(inputs_arr, outputs_arr, call_opts); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; - return results; - } - }; -} diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig index de4b68da..4faa1e82 100644 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -4,8 +4,9 @@ const std = @import("std"); -const metal = @import("../../../kernel/kernel.zig").metal; +const kernel = @import("../../../kernel.zig"); +const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index 8bc03c9c..bd482093 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,9 +3,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const Options = @import("../api.zig").Options; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; +const Options = @import("../options.zig").Options; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -42,7 +41,7 @@ pub fn validateOptions(opts: Options) void { /// interleave by 8 and its scale atoms are MN-major packed i32 words. pub fn fusedExperts( input: zml.Tensor, - routing: LocalRouting, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index ff6edec2..f651813f 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -4,17 +4,16 @@ //! rows in route order, and the top-k reduction. MXFP4 applies routing weights //! before the down projection, with the reference model's BF16 rounding. const std = @import("std"); - const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; -const triton_kernels = @import("../../triton/moe.zig"); pub const mxfp4 = @import("mxfp4.zig"); pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); +const RoutingWeightPlacement = @import("../../options.zig").RoutingWeightPlacement; +const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { /// `[tokens, hidden]` BF16. diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig index 92317c7c..d1a0e1b5 100644 --- a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -5,12 +5,10 @@ const platforms = @import("platforms"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const LocalRouting = @import("../api.zig").LocalRouting; +const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../options.zig").Activation; const unpackedWeight = @import("../weights.zig").unpackedWeight; const contract = @import("contract.zig"); -const Attributes = contract.Attributes; const runtime = @import("runtime.zig"); const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); @@ -52,6 +50,8 @@ const Inputs = contract.Inputs(zml.Tensor); const Outputs = contract.Outputs(zml.Shape); +const Attributes = contract.Attributes; + fn currentRunners() !*runtime.Runners { const platform = zml.Compiler.current().platform; const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; @@ -311,7 +311,7 @@ pub fn fusedExpertsNvfp4( pub fn fusedExperts( input: zml.Tensor, - routing: LocalRouting, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index ba3d3c00..d041a5ab 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,12 +6,11 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; +const ExpertsLayout = @import("../options.zig").ExpertsLayout; +const ProjectionLayout = @import("../options.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Activation = @import("../options.zig").Activation; +const Options = @import("../options.zig").Options; const fly = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -22,7 +21,7 @@ test { pub fn fusedExperts( hidden_states: Tensor, - routing_: LocalRouting, + routing_: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options, @@ -136,7 +135,7 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { +pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index f835c889..988a4f51 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -4,9 +4,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; +const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../options.zig").Options; const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); @@ -69,7 +68,7 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso pub fn fusedExperts( hidden_states: zml.Tensor, - routing: LocalRouting, + routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index eb697f36..33b6da55 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,18 +4,17 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; -pub const Activation = @import("api.zig").Activation; -pub const ActivationKind = @import("api.zig").ActivationKind; pub const cute = @import("cute/cute.zig"); -pub const ExpertsLayout = @import("api.zig").ExpertsLayout; +pub const ExpertsLayout = @import("options.zig").ExpertsLayout; pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); pub const fly = @import("fly/fly.zig"); -pub const LocalRouting = @import("api.zig").LocalRouting; pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const Options = @import("api.zig").Options; -pub const ProjectionLayout = @import("api.zig").ProjectionLayout; -pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; +pub const Options = @import("options.zig").Options; +pub const Activation = @import("options.zig").Activation; +pub const ActivationKind = @import("options.zig").ActivationKind; +pub const ProjectionLayout = @import("options.zig").ProjectionLayout; +pub const RoutingWeightPlacement = @import("options.zig").RoutingWeightPlacement; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); @@ -38,6 +37,8 @@ pub const Backend = enum { tcl, pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { + // Keep the dtype as an argument because non scheme-specific backends may depend on it later + _ = dtype; return switch (platform.target) { .cuda => b: { const s = scheme orelse break :b .triton; @@ -71,13 +72,7 @@ pub const Backend = enum { }; }, .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, - .furiosa => b: { - const s = scheme orelse break :b .tcl; - break :b switch (s) { - .fp8_block128 => if (dtype == .f8e4m3fn) .tcl else error.UnsupportedQuantization, - .mxfp4, .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block32 => error.UnsupportedQuantization, - }; - }, + .furiosa => if (scheme == null) .tcl else error.UnsupportedQuantization, .metal => b: { const s = scheme orelse break :b .metal; break :b switch (s) { @@ -139,6 +134,38 @@ pub const Backend = enum { } }; +/// Routing for a local token batch and a contiguous range of experts. +/// IDs retain the model's global expert numbering, while the projection weights +/// contain only the local experts. Kernels compute those experts contributions and +/// `forwardMoe` sums them along the expert axis. +pub const LocalRouting = struct { + /// Global expert IDs for local tokens, with shape {.t, .topk} + ids: zml.Tensor, + weights: zml.Tensor, + global_num_experts: u32, + local_num_experts: u32, + /// Scalar i32 containing the first local expert's global ID. + /// Null when all experts are local and global IDs already index the weights. + expert_start: ?zml.Tensor = null, + + /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. + /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. + pub fn localIds(self: LocalRouting) zml.Tensor { + const start = self.expert_start orelse return self.ids; + const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(self.ids.sub(start), .scalar(-1, .i32)); + } + + /// Returns a lookup table from every global expert ID to its local weight + /// index or -1 if nonlocal. Returns null when every expert is local. + pub fn expertMap(self: LocalRouting) ?zml.Tensor { + const start = self.expert_start orelse return null; + const ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(ids.sub(start), .scalar(-1, .i32)); + } +}; + /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); @@ -218,7 +245,7 @@ pub fn forwardMoe( } const expert_axis = experts.meshAxis(); const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; - if (backend == .metal) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); + if (backend == .metal or backend == .tcl) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); const Call = struct { input: zml.Tensor, @@ -266,7 +293,7 @@ pub fn forwardMoe( .global_experts = @intCast(gate_up.weight.dim(.expert)), }; const Logical = zml.Sharding.Partitioning.Logical; - var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; + var projection_specs: [2]zml.meta.MapRestrict(zml.Tensor, Logical).map(zml.nn.Linear) = undefined; for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { zml.meta.mapAlloc(struct { fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { @@ -285,7 +312,7 @@ pub fn forwardMoe( .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, }, linear, specs) catch @panic("OOM"); } - return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ + return zml.ops.manualComputation2(Call.forward, args, output_shape, mesh, .{ .in_specs = .{ .input = partitioning.in_specs.input, .ids = partitioning.in_specs.ids, diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index bb60594b..4b33b585 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,9 +5,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; +const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../options.zig").Options; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -215,7 +214,7 @@ pub fn callGmmEp( pub fn fusedExperts( hidden_states: zml.Tensor, - routing: LocalRouting, + routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/api.zig b/zml/kernels/moe/options.zig similarity index 75% rename from zml/kernels/moe/api.zig rename to zml/kernels/moe/options.zig index b66593c1..7f8d26de 100644 --- a/zml/kernels/moe/api.zig +++ b/zml/kernels/moe/options.zig @@ -130,35 +130,3 @@ pub const ExpertsLayout = struct { mxfp4_interleaved8, }; }; - -/// Routing for a local token batch and a contiguous range of experts. -/// IDs retain the model's global expert numbering, while the projection weights -/// contain only the local experts. Kernels compute those experts contributions and -/// `forwardMoe` sums them along the expert axis. -pub const LocalRouting = struct { - /// Global expert IDs for local tokens, with shape {.t, .topk} - ids: Tensor, - weights: Tensor, - global_num_experts: u32, - local_num_experts: u32, - /// Scalar i32 containing the first local expert's global ID. - /// Null when all experts are local and global IDs already index the weights. - expert_start: ?Tensor = null, - - /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. - /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. - pub fn localIds(self: LocalRouting) Tensor { - const start = self.expert_start orelse return self.ids; - const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(self.ids.sub(start), .scalar(-1, .i32)); - } - - /// Returns a lookup table from every global expert ID to its local weight - /// index or -1 if nonlocal. Returns null when every expert is local. - pub fn expertMap(self: LocalRouting) ?Tensor { - const start = self.expert_start orelse return null; - const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(ids.sub(start), .scalar(-1, .i32)); - } -}; diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index ba723234..7f84835b 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -7,10 +7,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; +const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../options.zig").Activation; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; pub fn activate(x: Tensor, activation: Activation) Tensor { return switch (activation) { @@ -192,7 +191,7 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel index 44b56e08..4057eea3 100644 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -2,8 +2,5 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "tcl", - srcs = [ - "moe.zig", - "tcl.zig", - ], + srcs = ["tcl.zig"], ) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig deleted file mode 100644 index 23fb2274..00000000 --- a/zml/kernels/moe/tcl/moe.zig +++ /dev/null @@ -1,457 +0,0 @@ -//! TCL kernel schedules for Furiosa RNGD fused experts. -const std = @import("std"); - -const builder = @import("dsl/tcl/builder"); - -const zml = @import("../../../zml.zig"); -const tcl = zml.kernel.tcl; -const ActivationKind = @import("../api.zig").ActivationKind; - -pub const Format = enum { bf16, fp8_block128 }; - -/// Dispatch in `tcl.zig` uses the first matching schedule. -/// -/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected -/// expert on the whole batch to reuse its weights, with zero routing weight -/// for unrouted pairs. -/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather -/// weights for each route to avoid the overhead of grouping routes. -/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to -/// reuse weights within each block, then gather results back by route. -pub const Schedule = enum { per_route, blockwise, small_batch }; - -pub const Cfg = struct { - num_tokens: i64, - top_k: i64, - local_num_experts: i64, - hidden_size: i64, - expert_intermediate_size: i64, - weight_scale_dtype: tcl.DType = .bf16, - activation: ActivationKind = .swiglu, -}; - -const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; -const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; - -pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { - if (schedule == .small_batch and format != .fp8_block128) { - @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); - } - return tcl.Kernel(Cfg, .{ - .name = "moe_" ++ @tagName(schedule) ++ switch (format) { - .bf16 => "", - .fp8_block128 => "_fp8", - }, - .inputs = switch (format) { - .bf16 => base_inputs, - .fp8_block128 => base_inputs ++ scale_inputs, - }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, schedule); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, format); - switch (schedule) { - .per_route => try perRoute(b, a, in, cfg.activation, format), - .blockwise => try blockwise(b, a, in, cfg, format), - .small_batch => try fp8SmallBatch(b, a, in, cfg), - } - } - }.run, - }); -} - -const Axes = struct { - T: tcl.Axis, - K: tcl.Axis, - E: tcl.Axis, - D: tcl.Axis, - I: tcl.Axis, - I2: tcl.Axis, - S2: tcl.Axis, - S1: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.num_tokens), - .K = b.axis("K", cfg.top_k), - .E = b.axis("E", cfg.local_num_experts), - .D = b.axis("D", cfg.hidden_size), - .I = b.axis("I", cfg.expert_intermediate_size), - .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), - .S2 = b.axis("S2", 2), - .S1 = b.axis("S1", 1), - }; - } -}; - -fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { - const weight_dtype: tcl.DType = switch (format) { - .bf16 => .bf16, - .fp8_block128 => .f8_e4, - }; - const spec = .{ - .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, - .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, - .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, - }; - if (format == .bf16) { - const t = try b.declareArgs(spec); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; - } - const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); - const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); - const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); - const t = try b.declareArgs(.{ - .x = spec.x, - .ids = spec.ids, - .w = spec.w, - .gate_up = spec.gate_up, - .down = spec.down, - .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, - .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, - }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; -} - -const Args = struct { - x: builder.Tensor, - ids: builder.Tensor, - w: builder.Tensor, - gate_up: builder.Tensor, - down: builder.Tensor, - gate_up_scale: ?builder.Tensor = null, - down_scale: ?builder.Tensor = null, -}; - -const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; - -fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { - if (format == .bf16) return .{}; - const IG = b.axis("IG", @divExact(a.I.size, 128)); - const DG = b.axis("DG", @divExact(a.D.size, 128)); - const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); - var gu_axes: [8]tcl.Axis = undefined; - var dn_axes: [8]tcl.Axis = undefined; - @memcpy(gu_axes[0..batch.len], batch); - @memcpy(dn_axes[0..batch.len], batch); - @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); - @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); - return .{ - .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), - .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), - }; -} - -fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { - var n: usize = 0; - for (axes) |a| { - if (std.mem.eql(u8, a.name, axis.name)) { - storage[n] = group; - n += 1; - storage[n] = lane; - } else storage[n] = a; - n += 1; - } - return storage[0..n]; -} - -/// Block scales multiply each K-block partial before K-block reduction. -/// Only the requested experts are gathered; their bank remains FP8 in HBM. -fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { - if (format == .bf16) { - const op = b.tensorOperation(.{}); - return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); - } - const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); - const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); - const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); - const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); - var x_axes: [16]tcl.Axis = undefined; - var w_axes: [16]tcl.Axis = undefined; - var w2_axes: [16]tcl.Axis = undefined; - var out_axes: [16]tcl.Axis = undefined; - var partial_axes: [16]tcl.Axis = undefined; - const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); - const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); - const out = blockedAxes(to, output, OG, OL, &out_axes); - @memcpy(partial_axes[0..out.len], out); - partial_axes[out.len] = CG; - - // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. - var op = b.tensorOperation(.{}); - const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); - op = b.tensorOperation(.{}); - // Native F32 reciprocal accuracy is qualified separately (within one ULP). - const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); - const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); - op = b.tensorOperation(.{}); - const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); - return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); -} - -/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` -/// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { - var sliced: [4]tcl.Axis = undefined; - var flat: [3]tcl.Axis = undefined; - @memcpy(sliced[0..batch.len], batch); - @memcpy(flat[0..batch.len], batch); - sliced[batch.len] = a.S1; - sliced[batch.len + 1] = a.I; - flat[batch.len] = a.I; - const s = sliced[0 .. batch.len + 2]; - const f = flat[0 .. batch.len + 1]; - const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); - const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // The builder can infer the TCL `Interleaving` tactic for two inputs with - // matching axes, such as `gate` and `up`. That tactic allows exactly one - // binary instruction, so force `.elementwise` for this activation chain. - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const g = op.fetch(gate, .{}); - const activated = if (format == .fp8_block128) blk: { - if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); - - // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal - // BF16 SiLU write, even through as_dram/as_logical. Make its RNE - // rounding observable as integer bits before multiplying by up. - // Remove once the native reproducer preserves the BF16 boundary. - const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const v = op.fetch(bits, .{}); - const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); - const silu = try op.commit(rounded, .{ .dtype = .bf16 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - break :blk op.fetch(silu, .{}); - } else switch (activation) { - .swiglu => g.sigmoid().mulf(g), - .geglu_tanh => geluTanh(g), - else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), - }; - var v = activated.mulf(op.fetch(up, .{})); - if (weights) |w| v = v.mulf(op.fetch(w, .{})); - return op.commit(v, .{ .dtype = .bf16 }); -} - -fn geluTanh(g: builder.Value) builder.Value { - const cube = g.mulf(g).mulf(g); - const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); - return inner.tanh().addf(1.0).mulf(g).mulf(0.5); -} - -/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): -/// Use IO-bound tactics for per-route and FP8 small-batch schedules and -/// compute-bound tactics for blockwise. -/// Keep the compiler's default lowering mode and omit the adaptive-einsum and -/// half-VRF settings removed in TCC 2026.5. -fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { - const config = comptime blk: { - const common = .{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .propagate_sparse_axis_from_op = "None", - }; - const tactics = switch (schedule) { - .blockwise => .{ - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, - .dma_preference = 1.2, - }, - .per_route, .small_batch => .{ - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - }, - }; - // The transaction-simulation setting is only emitted for IO-bound schedules. - const common_info = @typeInfo(@TypeOf(common)).@"struct"; - const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; - const Config = @Struct( - .auto, - null, - common_info.field_names ++ tactics_info.field_names, - common_info.field_types ++ tactics_info.field_types, - common_info.field_attrs ++ tactics_info.field_attrs, - ); - var merged: Config = undefined; - for (common_info.field_names) |field| @field(merged, field) = @field(common, field); - for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); - break :blk merged; - }; - b.compilerConfig(config); -} - -/// Gather the selected experts for every token. -fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - - const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); - const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); - if (format == .fp8_block128) { - const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); - } else { - // The BF16 recipe weights activations and contracts experts together. - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } -} - -/// Rows per expert block: the next power of two of the routes per expert. -fn blockRows(cfg: Cfg) i64 { - const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); -} - -const blocks_per_chunk = 32; - -/// Group routes into expert-major blocks, then run both projections in a -/// device loop over chunks of blocks. -fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. - const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); - op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); - op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); - - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); - { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); - const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); - const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); - const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); -} - -/// Number of experts processed per device-loop iteration, based on the -/// batch's route count (`cfg.num_tokens * cfg.top_k`). -fn expertsPerChunk(cfg: Cfg) i64 { - const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); - return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); -} - -/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. -/// Load each selected expert once and run it for every token. Routing weights -/// scale the F32 down-projection results before summing the experts and steps. -fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { - const chunk = expertsPerChunk(cfg); - const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; - const C = b.axis("C", chunk); - const Go = b.axis("Go", steps); - const J = b.axis("J", steps * chunk); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Each token's routing weight per expert, and the used experts. - const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); - op = b.tensorOperation(.{}); - const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); - // A used expert's rank among them, -1 for the others. - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - - // Slot j holds the used expert of rank j and its tokens' weights; - // slots past the last hold expert 0 with no weight. - const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); - op = b.tensorOperation(.{}); - const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); - // A reduction's condition is read plainly, so broadcast it first. - op = b.tensorOperation(.{}); - const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); - op = b.tensorOperation(.{}); - const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); - const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); - const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); - { - const ids = try b.indexRead(step_experts, loop.iv); - const weights = try b.indexRead(step_gate, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); - const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); - const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); - const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); - op = b.tensorOperation(.{}); - const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); -} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index f05727ce..698c4d30 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -1,165 +1,272 @@ //! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. +//! +//! Few tokens (at most `experts / top_k`): activation stationary, each token +//! gathers the weights of its experts and the down projection contracts over +//! the experts and the intermediate axis at once. +//! +//! More tokens: blockwise. The kernel routes on the device: routes are placed +//! in expert-major blocks of `BB` rows (one-hot, cumulative sums), then a +//! device loop over the used chunks of `BGi` blocks gathers each block's +//! expert weights and runs both projections; outputs are gathered back per +//! route and weighted. const std = @import("std"); +const builder = @import("dsl/tcl/builder"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const kernels = @import("moe.zig"); -const api = @import("../api.zig"); +const Options = @import("../options.zig").Options; -/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` -/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is -/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with -/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. -pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { - const topk_ids = routing.localIds(); - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); +/// `input` is `{.t, .d}`, routing IDs and weights `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` +/// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). +/// SwiGLU, routing weights applied after the down projection. +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { + const topk_ids = routing.ids; + const topk_weights = routing.weights; + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); + stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); + stdx.debug.assert(gate_up_linear.quantization == null and down_linear.quantization == null, "TCL MoE takes bf16 experts", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); - stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); - const format = expertFormat(gate_up_linear, down_linear); - const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); - stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); - const gate_up = gate_up_linear.weight; const down = down_linear.weight; - const cfg: kernels.Cfg = .{ - .num_tokens = input.dim(.t), - .top_k = topk_ids.dim(.topk), - .local_num_experts = gate_up.dim(.expert), - .hidden_size = input.dim(.d), - .expert_intermediate_size = down.dim(.dout), - .weight_scale_dtype = switch (format) { - .bf16 => .bf16, - .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), - }, - .activation = std.meta.activeTag(opts.activation), + stdx.debug.assert(input.dtype() == .bf16 and gate_up.dtype() == .bf16 and down.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes bf16 activations and experts, i32 ids", .{}); + const t = input.dim(.t); + const k = topk_ids.dim(.topk); + const cfg: Cfg = .{ + .t = t, + .k = k, + .e = gate_up.dim(.expert), + .d = input.dim(.d), + .i = down.dim(.dout), }; - stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); - const weights = routing.weights.convert(.f32); - const local_weights = switch (format) { - .bf16 => weights, - .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), - }; - return switch (format) { - .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), - .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ - .x = input, - .ids = topk_ids, - .w = local_weights, - .gate_up = gate_up, - .down = down, - .gate_up_scale = s.gate_up, - .down_scale = s.down, - }), - }; + const x = input.reshape(.{ t, cfg.d }); + const ids = topk_ids.reshape(.{ t, k }); + const weights = topk_weights.convert(.f32).reshape(.{ t, k }); + const out_shape = x.shape(); + const y = if (cfg.t * cfg.k <= cfg.e) + ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y + else + Blockwise.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; + return y.reshape(input.shape()); } -const Scales = struct { gate_up: Tensor, down: Tensor }; +pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64 }; -const Format = union(kernels.Format) { - bf16: void, - fp8_block128: Scales, +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.t), + .K = b.axis("K", cfg.k), + .E = b.axis("E", cfg.e), + .D = b.axis("D", cfg.d), + .I = b.axis("I", cfg.i), + .I2 = b.axis("I2", 2 * cfg.i), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } }; -fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { - const scheme = gate_up.quantizationScheme(); - stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); - return switch (scheme orelse { - stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); - return .bf16; - }) { - .fp8_block128 => blk: { - const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; - stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); - break :blk .{ .fp8_block128 = scales }; - }, - else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), - }; +fn args(b: *tcl.Builder, a: Axes) tcl.FinishError!Args { + const t = try b.declareArgs(.{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = .bf16, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = .bf16, .axes = &.{ a.E, a.D, a.I } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; } -fn fp8Scales(linear: zml.nn.Linear) Tensor { - const q = linear.quantization.?; - stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); - stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); - const w = linear.weight; - const scale = q.scales; - stdx.debug.assert(@mod(w.dim(.d), 128) == 0 and @mod(w.dim(.dout), 128) == 0 and scale.dim(.expert) == w.dim(.expert) and scale.dim(.d) * 128 == w.dim(.d) and scale.dim(.dout) * 128 == w.dim(.dout), "TCL MoE FP8 scales must cover 128x128 weight blocks", .{}); - return scale; +const Args = struct { x: builder.Tensor, ids: builder.Tensor, w: builder.Tensor, gate_up: builder.Tensor, down: builder.Tensor }; + +/// `silu(gate) * up` of `h = [batch..., S2, I]` (gate first), times +/// `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // Interleaving requires one binary instruction; SwiGLU chains several. + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + var v = g.sigmoid().mulf(g).mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); } -// Maximum total input tokens per MoE call before padding for the FP8 -// small-batch schedule. It loads each used expert's weights once and reuses -// them across the batch. -// Larger batches fall back to per-route or blockwise dispatch. -const fp8_small_batch_max_tokens = 64; - -fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { - if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { - return run(format, .small_batch, cfg, inputs); - } - if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { - return run(format, .per_route, cfg, inputs); - } - return run(format, .blockwise, cfg, inputs); +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// IO-bound tactics for activation stationary, compute-bound for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, blockwise: bool) void { + if (blockwise) b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + .propagate_sparse_axis_from_op = "None", + }) else b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + .propagate_sparse_axis_from_op = "None", + }); } -fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { - const call_cfg, const prepared = if (schedule == .small_batch) - prepareFp8SmallBatch(cfg, inputs) - else - .{ cfg, prepareRoutedInputs(cfg, inputs) }; +pub const ActivationStationary = tcl.Kernel(Cfg, .{ + .name = "moe_activation_stationary", + .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, false); + const a: Axes = .init(b, cfg); + const in = try args(b, a); + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - const shape = prepared.x.shape(); - var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; + var op = b.tensorOperation(.{}); + const h = try op.commit(op.contract(in.x, gu, &.{ a.T, a.K, a.S2, a.I }), .{}); + // Routing weights scale the activations, so the down projection + // also sums over the experts. + const act = try gated(b, a, h, &.{ a.T, a.K }, in.w); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } + }.run, +}); - if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); - return y.reshape(inputs.x.shape()); +/// Rows per expert block: the next power of two of the routes per expert. +pub fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.t * cfg.k, cfg.e) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); } -fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { - var prepared = inputs; - prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); - const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); - const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); - // These kernels gather every route, and blockwise also scatters each route - // into an expert slot. Give nonlocal routes a valid slot and zero weight. - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); - prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); - prepared.w = valid.select(w, Tensor.scalar(0, .f32)); - return prepared; -} +const blocks_per_chunk = 32; -fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { - // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two - // panics its adder tree ("Fail to cover valid range", 5 tokens padded - // to 8): pad with unrouted tokens. - var padded = cfg; - padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); - const pad = padded.num_tokens - cfg.num_tokens; - var prepared = inputs; - prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); - prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); - prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); - return .{ padded, prepared }; -} +pub const Blockwise = tcl.Kernel(Cfg, .{ + .name = "moe_blockwise", + .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, true); + const a: Axes = .init(b, cfg); + const in = try args(b, a); + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(.bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + op = b.tensorOperation(.{}); + const h = try op.commit(op.contract(chunk, gu, &.{ BGi, BB, a.S2, I }), .{}); + const act = try gated(b, a, h, &.{ BGi, BB }, null); + op = b.tensorOperation(.{}); + const y = try op.commit(op.contract(act, dn, &.{ BGi, BB, D }), .{ .dtype = .bf16 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); + } + }.run, +}); test "tcl fused experts run on furiosa" { - try testFusedExperts(.{ .swiglu = .{} }); - try testFusedExperts(.geglu_tanh); -} - -fn testFusedExperts(activation: zml.moe.Activation) !void { const platform = zml.testing.env(); if (platform.target != .furiosa) return error.SkipZigTest; const allocator = std.testing.allocator; const io = std.testing.io; const bf16 = zml.floats.BFloat16; - // With two routes per token and eight experts, four tokens use per-route - // dispatch and sixteen use blockwise dispatch. + // 2 * 4 <= 8 experts is activation stationary, 16 tokens is blockwise. for ([_]i64{ 4, 16 }) |t| { const k = 2; const e = 8; @@ -171,15 +278,15 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ - .activation = act, + .activation = .{ .swiglu = .{} }, .quantize_input = false, .routing_weight_placement = .after_down, }); } }; - var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); + var exe = try zml.module.compile(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, platform, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); @@ -231,12 +338,7 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); } - const gate = switch (activation) { - .swiglu => g / (1 + @exp(-g)), - .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), - else => unreachable, - }; - act[n] = gate * u; + act[n] = g / (1 + @exp(-g)) * u; } for (0..d) |m| { var y: f32 = 0; diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index c74406c3..f78cd1ef 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,10 +5,10 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const Activation = @import("api.zig").Activation; -const ExpertsLayout = @import("api.zig").ExpertsLayout; -const GlobalOptions = @import("api.zig").Options; -const Placement = @import("api.zig").RoutingWeightPlacement; +const ExpertsLayout = @import("options.zig").ExpertsLayout; +const GlobalOptions = @import("options.zig").Options; +const Activation = @import("options.zig").Activation; +const Placement = @import("options.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); const Case = struct { diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 6e4006fa..0776a590 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,7 +12,7 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 8ba799a5..ed6c9f5f 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -7,12 +7,11 @@ const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; const toDType = zml.kernel.triton.from; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; +const ExpertsLayout = @import("../options.zig").ExpertsLayout; +const ProjectionLayout = @import("../options.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Activation = @import("../options.zig").Activation; +const Options = @import("../options.zig").Options; const kernels = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/triton"); @@ -39,7 +38,7 @@ test { pub fn fusedExperts( input: zml.Tensor, - routing: LocalRouting, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, @@ -59,7 +58,7 @@ pub fn fusedExperts( pub fn fusedExpertsBase( hidden_states: Tensor, - routing_: LocalRouting, + routing_: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, layout: ProjectionLayout, @@ -901,7 +900,7 @@ fn validateInputs( /// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. pub fn fusedExpertsMxfp4( input: zml.Tensor, - routing: LocalRouting, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 587af69f..3e05fed8 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -125,7 +125,7 @@ fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation2(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 04f4c1ac..337432e7 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,8 +3,9 @@ const std = @import("std"); -const metal = @import("../../../kernel/kernel.zig").metal; +const kernel = @import("../../../kernel.zig"); +const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 4a97a38e..aaa42b20 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -207,7 +207,7 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z _ = k; _ = v; const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; - return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); + return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, partitioning); } const Attention = struct { @@ -435,9 +435,12 @@ test pagedAttention { const parameters: Parameters = .init(.fromBackend(backend_options_args)); const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + const ParameterSpecs = zml.meta.MapRestrict(zml.Tensor, zml.Sharding.Partitioning.Logical).map(Parameters); const partitioning: Partitioning = .{ .in_specs = .{ - .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), + .parameters = switch (parameters) { + inline else => |_, tag| @unionInit(ParameterSpecs, @tagName(tag), .{ .block_table = .replicated, .seq_lens = .replicated, .query_start_len = .replicated }), + }, .q = head_spec, .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, .opts = .{ .mask = .{ .standard = {} }, .sink = null }, diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index d04b684e..881593aa 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -67,9 +67,22 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); + if (compiler.meshes[0].numDevices() > 1) return zml.ops.manualComputation(Shard.body, .{ .parameters = parameters, .q = q, .k_cache = k_cache, .v_cache = v_cache, .opts = opts }, q.shape()); return attend(parameters, q, k_cache, v_cache, opts); } + const Shard = struct { + parameters: Parameters, + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + opts: AttentionOptions, + + fn body(self: Shard, _: zml.Shape) Tensor { + return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); + } + }; + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); diff --git a/zml/nn.zig b/zml/nn.zig index eceb316b..a2b582ee 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,13 +2,11 @@ const std = @import("std"); const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; @@ -322,23 +320,13 @@ fn onesGrid(shape: Shape, dt: DataType) Shape { } fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { - // Keep the unmatched-composite marker in the graph for backend lowering. - const ctx = zml.Compiler.current(); - const op = stablehlo.custom_call( - ctx.mlir_ctx, - &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, - &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, - .{ - .call_target_name = "zml$scaled_dot_unmatched", - .has_side_effect = false, - .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, - }, - ctx.location, - ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { - op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); - } - return Tensor._result(out_shape, op.result(0)); + return ops.customCall( + "zml$scaled_dot_unmatched", + .{ in[0], in[1], in[2], in[3] }, + out_shape, + {}, + .{ .has_side_effect = false }, + ); } fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { diff --git a/zml/ops.zig b/zml/ops.zig index f8fdba7e..a9ae0f8d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -18,10 +18,9 @@ const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const manual_computation = @import("ops/manual_computation.zig"); pub const manualComputation = manual_computation.manualComputation; -pub const ManualPartitioning = manual_computation.ManualPartitioning; -pub const PartitionSpecs = manual_computation.PartitionSpecs; -pub const partitionSpecs = manual_computation.partitionSpecs; -pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; +const manual_computation2 = @import("ops/manual_computation2.zig"); +pub const manualComputation2 = manual_computation2.manualComputation2; +pub const ManualPartitioning = manual_computation2.ManualPartitioning; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; @@ -31,6 +30,7 @@ const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; test { _ = manual_computation; + _ = manual_computation2; } const log = std.log.scoped(.@"zml/Compiler"); @@ -89,7 +89,7 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ /// ``` /// /// Each expert shard now has the complete output for its data shard. To expose -/// this result through `manualComputation`, an output spec such as +/// this result through `manualComputation2`, an output spec such as /// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the /// result replicated along `.experts`. pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { @@ -2760,6 +2760,11 @@ pub fn CustomCall( comptime params: struct { /// Name of the custom call, must be unique. name: [:0]const u8, + /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies + /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, + /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output + /// sharding, this setting will avoid the shuffling of the buffer's data. + sharding_aware: bool, /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. has_side_effect: bool, /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. @@ -2793,7 +2798,23 @@ pub fn CustomCall( .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), .compute_on_host = params.compute_on_host, }; - return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); + if (params.sharding_aware) { + return shardingAwareTypedCustomCall( + params.name, + opts, + input_tensors, + output_shapes, + attributes, + ); + } else { + return typedCustomCall( + params.name, + opts, + input_tensors, + output_shapes, + attributes, + ); + } } fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { @@ -2821,6 +2842,44 @@ pub fn CustomCallOutputOperandAliases(I: type, O: type) type { ); } +pub fn shardingAwareTypedCustomCall( + comptime target_name: [:0]const u8, + comptime opts: CustomCallOptions, + input: anytype, + output: anytype, + attributes: anytype, +) ShapeToTensor(@TypeOf(output)) { + const Input = @TypeOf(input); + const Output = @TypeOf(output); + const Attributes = @TypeOf(attributes); + + var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + output_shapes[i] = @field(output, field_name); + } + + const Handler = struct { + input: Input, + attributes: Attributes, + + fn body(self: @This(), sharded_output_shapes: []const Shape) []const Tensor { + return typedCustomCall(target_name, opts, self.input, sharded_output_shapes, self.attributes); + } + }; + const output_tensors = manualComputation( + Handler.body, + .{ .input = input, .attributes = attributes }, + @as([]const Shape, &output_shapes), + ); + + // Convert the slice back to a struct + var out: ShapeToTensor(Output) = undefined; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + @field(out, field_name) = output_tensors[i]; + } + return out; +} + pub fn typedCustomCall( target_name: [:0]const u8, opts: CustomCallOptions, @@ -2835,7 +2894,7 @@ pub fn typedCustomCall( const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); - stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); + stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { @@ -2845,6 +2904,7 @@ pub fn typedCustomCall( } break :b &input_tensors; }, + // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -2860,6 +2920,7 @@ pub fn typedCustomCall( } break :b &output_shapes; }, + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -2924,7 +2985,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.partitioner == .gspmd) { + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -2936,6 +2997,7 @@ pub fn typedCustomCall( } return out; }, + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -2988,7 +3050,7 @@ test customCall { const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2999,17 +3061,9 @@ test customCall { const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); - const Local = struct { - fn body(x: Tensor, output_shape: Shape) Tensor { - return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, - }); - } - }; - const output = manualComputation(Local.body, input, shape, tp, .{ - .in_specs = .parse(.{ ._0 = .model }), - .out_specs = .parse(.{ ._0 = .model }), + const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index a0ec0037..45400206 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); @@ -6,126 +7,44 @@ const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("../Compiler.zig"); -const meta = @import("../meta.zig"); +const DataType = @import("../dtype.zig").DataType; const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; +const zml = struct { + // Nested to disambiguate from stdx.meta + const meta = @import("../meta.zig"); + const testing = @import("../testing.zig"); +}; + const log = std.log.scoped(.@"zml/Compiler"); -/// Partitioning declarations for the tensors in `T`. -/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. -/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes -/// `struct { x: Logical, bias: ?Logical }`. +/// ZML operator describes computation across several devices, +/// but `manualComputation` allows to partition the devices in small group, +/// split the `Tensor` on those partitions, +/// and describes sub-computation that work inside a partition. /// -/// Initialize individual fields to declare different shardings, or use -/// `partitionSpecs` to apply one spec to every tensor in a value. -pub fn PartitionSpecs(comptime T: type) type { - return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); -} - -/// Declares the same sharding for a group of tensors at a `manualComputation` -/// boundary. +/// See: +/// https://github.com/openxla/shardy/blob/main/docs/sdy_dialect.md#sdymanual_computation-sdymanualcomputationop /// -/// ```zig -/// .in_specs = .{ -/// .query = .parse(.{ .h = .model }), -/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), -/// .metadata = partitionSpecs(metadata, .replicated), -/// } -/// ``` +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R` where: +/// - `I` is user defined and contains inputs tensors and any metadata required by the computation +/// - `O` contains the partionned output shapes, types must be `void`, `Shape` or `[]const Shape` for 0, 1 or many outputs +/// - `R` must be `void`, `Tensor, or `[]const Tensor` for 0, 1 or many outputs +/// * `inputs` must be using the same mesh (using .withPartitioning), or be fully replicated +/// * `outputs` contains the global output shapes, +/// and `manualComputation` will pass to `body_fn` the shapes of each local output shard. /// -/// Requires an active compiler and uses its arena for allocations. Use -/// `partitionSpecsAlloc` outside graph construction. Use named struct fields -/// when mixing tensors with non-tensor metadata. -pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { - return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); -} - -/// Builds the same declarations as `partitionSpecs` using an explicit allocator. -pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { - var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; - try meta.mapAlloc(struct { - fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { - return spec_; - } - }.assign, allocator, spec, tensors, &specs); - return specs; -} - -/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. -/// Returns the type of the `partitioning` argument to `manualComputation`. -/// -/// * `in_specs` specifies the sharding of each tensor in the body's first argument. -/// It follows the argument's structure, omitting non-tensor metadata. -/// * `out_specs` specifies the sharding of each shape in the body's second argument. -/// It follows the output shapes' structure and can be omitted for a `void` argument. -/// -/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as -/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across -/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` -/// replicates the entire tensor. The specs must not contain open dimensions. -/// Slice lengths, optional presence and active union tags must match the arguments. -/// -/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single -/// output `Shape`, the partitioning can be declared as: -/// -/// ```zig -/// const partitioning: ManualPartitioning(body) = .{ -/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, -/// .out_specs = .parse(.{ .hidden = .model }), -/// }; -/// ``` -pub fn ManualPartitioning(comptime body_fn: anytype) type { - const Logical = Sharding.Partitioning.Logical; - - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - - const InputSpecs = PartitionSpecs(Inputs); - const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); - - if (OutputSpecs == void) return struct { - in_specs: InputSpecs, - out_specs: void = {}, - }; - - return struct { - in_specs: InputSpecs, - out_specs: OutputSpecs, - }; -} - -/// Describes a computation on local shards of tensors distributed across `mesh`. -/// ZML operations normally describe work on global tensors. The partitioner -/// distributes the inputs according to `in_specs`, and `body_fn` describes the work -/// performed on each shard. The results are exposed to the surrounding graph as -/// global tensors with the sharding declared by `out_specs`. -/// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: -/// - `I` contains the local input tensors and any metadata needed by the body. -/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. -/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes -/// in traversal order. -/// * `inputs` contains the global tensors and metadata. The partitioner reshards -/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. -/// * `output_shapes` contains the global output shapes. The body receives their -/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. -/// * `mesh` is the logical mesh whose axis names are referenced by the specs. -/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. -/// Each partitioned dimension must be divisible by the size of its mesh axis. -/// -/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives -/// the body a dimension of size 8. A replicated dimension keeps its global size. -/// The caller supplies the global output shapes because the body only describes -/// local results: their shapes alone do not determine the global tensor shapes. +/// This depart from the rest of ZML where output shapes are generally inferred, +/// but in this case since `body_fn` explains how to compute parts of the result, +/// it's not clear how to put them back together, +/// hence why the user is asked to explicitly provided the expected output shape. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), output_shapes: stdx.meta.FnParam(body_fn, 1), - mesh: *const Sharding.Mesh, - partitioning: ManualPartitioning(body_fn), ) stdx.meta.FnReturn(body_fn) { const Inputs = stdx.meta.FnParam(body_fn, 0); const OutputShapes = stdx.meta.FnParam(body_fn, 1); @@ -136,42 +55,13 @@ pub fn manualComputation( defer arena_state.deinit(); const arena = arena_state.allocator(); - // The body may return containers borrowed from its inputs or output shapes. - const body_arena = compiler.arena.allocator(); + var owned_inputs: Inputs = inputs; + const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + defer arena.free(flat_input_tensors); - var owned_inputs: Inputs = undefined; - meta.mapAlloc((struct { - fn copy(_: void, tensor: Tensor) Tensor { - return tensor; - } - }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); - - const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); - stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); - - for (flat_input_tensors, in_specs) |tensor, spec| { - tensor._shape = tensor.shape().withPartitioning(mesh, spec); - stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); - } - - var local_output_shapes: OutputShapes = undefined; - meta.mapAlloc((struct { - fn copy(_: void, shape: Shape) Shape { - return shape; - } - }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); - - const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); - const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); - stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); - - const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); - for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { - global.* = local.withPartitioning(mesh, spec); - stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); - local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } + var owned_output_shapes: OutputShapes = output_shapes; + const flat_output_shapes: []*Shape = zml.meta.collectMutPtrs(Shape, arena, &owned_output_shapes) catch @panic("OOM"); + defer arena.free(flat_output_shapes); const WrappedInputs = struct { inputs: *const Inputs, @@ -183,17 +73,15 @@ pub fn manualComputation( outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); } }; var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; - + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &owned_output_shapes }; _ = manualComputationInternal( compiler, arena, - mesh, @typeName(@TypeOf(body_fn)), &wrapped_inputs, flat_input_tensors, @@ -203,7 +91,6 @@ pub fn manualComputation( ) catch |err| switch (err) { error.OutOfMemory => @panic("OOM"), }; - return outputs_struct; } @@ -214,21 +101,27 @@ const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyo noinline fn manualComputationInternal( compiler: *Compiler, arena: std.mem.Allocator, - mesh: *const Sharding.Mesh, functor_name: []const u8, input_struct: *anyopaque, input_tensors: []*Tensor, output_struct: *anyopaque, - global_output_shapes: []const Shape, + output_shapes: []*Shape, local_fn: LocalComputationFn, ) error{OutOfMemory}![]*Tensor { const global_scope = compiler.currentScope(); const mlir_ctx = compiler.mlir_ctx; - log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); + const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { + .mesh => |m| m, + .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {d}: {f} and input {d}: {f}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), + }; + const manual_axes = collectManualAxes(mesh, input_tensors); + log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); const global_input_shapes = try arena.alloc(Shape, input_tensors.len); for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + const global_output_shapes = try arena.alloc(Shape, output_shapes.len); + for (global_output_shapes, output_shapes) |*shape, shape_ptr| shape.* = shape_ptr.*; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -242,17 +135,11 @@ noinline fn manualComputationInternal( const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); for (input_values, input_tensors) |*input_val, tensor_ptr| { - if (compiler.partitioner == .gspmd) { - tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); - } input_val.* = global_scope.value(tensor_ptr.*); } - const manual_block = switch (compiler.partitioner) { - .shardy => mlir.Block.init(input_types, input_locs), - .gspmd => global_scope.block, - }; - errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); + const manual_block = mlir.Block.init(input_types, input_locs); + errdefer manual_block.deinit(); const local_scope = compiler.pushBlock(manual_block); defer local_scope.pop(); @@ -264,14 +151,17 @@ noinline fn manualComputationInternal( }; } + for (output_shapes, global_output_shapes) |shape_ptr, global| { + shape_ptr.* = Sharding.shardedShape(global) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + compiler.manual_computation_depth += 1; defer compiler.manual_computation_depth -= 1; const local_outputs = try local_fn(arena, input_struct, output_struct); stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { - const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); - stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + for (0.., local_outputs, output_shapes) |i, local_tensor, local_shape| { + stdx.debug.assert(local_shape.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, local_shape.*, local_tensor._shape }); } const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); @@ -328,11 +218,95 @@ noinline fn manualComputationInternal( // Returned tensors describe the operation's global output shapes and sharding. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape._sharding = output_shape._sharding; + output_tensor_ptr._shape = output_shape; } return local_outputs; } +const ChosenMesh = union(enum) { + mesh: *const Sharding.Mesh, + incompatible: [2]u32, +}; + +fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ChosenMesh { + var mesh: ?*const Sharding.Mesh = null; + var mesh_source_id: u32 = @intCast(inputs.len); + for (0.., inputs) |i, input| { + if (input.shape()._sharding.mesh) |shape_mesh| { + if (mesh) |m| { + if (m != shape_mesh) + return .{ .incompatible = .{ mesh_source_id, @intCast(i) } }; + } else { + mesh = shape_mesh; + mesh_source_id = @intCast(i); + } + } + } + return .{ .mesh = mesh orelse fallback }; +} + +test chooseCommonMesh { + const fallback: Sharding.Mesh = .initTest(.fallback, .{.all}); + const first: Sharding.Mesh = .initTest(.first, .{ .data, .model }); + const second: Sharding.Mesh = .initTest(.second, .{ .experts, .model }); + const replicated = Tensor.init(.{ .a = 8 }, .f32); + const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); + const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); + + try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); +} + +fn collectManualAxes(mesh: *const Sharding.Mesh, inputs: []const *const Tensor) stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) { + var used: [Sharding.MAX_MESH_RANK]bool = @splat(false); + for (inputs) |input| { + const sharding = input.shape()._sharding; + if (sharding.mesh == null) continue; + std.debug.assert(sharding.mesh.? == mesh); + for (0..input.rank()) |dim| { + if (sharding.partition.get(dim).meshAxis()) |axis| used[axis] = true; + } + } + + var axes: stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) = .empty; + for (mesh.logical.axes.constSlice(), 0..) |axis, i| { + if (used[i]) axes.appendAssumeCapacity(axis); + } + return axes; +} + +test collectManualAxes { + const mesh: Sharding.Mesh = .initTest(.manual, .{ .data, .model, .unused }); + var data = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .data }); + var model = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .model }); + const replicated = Tensor.init(.{ .a = 8 }, .f32); + const open = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .open }); + + const axes = collectManualAxes(&mesh, &.{ &model, &replicated, &data, &model, &open }); + data = replicated; + model = replicated; + try std.testing.expectEqual(@as(usize, 2), axes.len); + try std.testing.expectEqualStrings("data", std.mem.span(axes.get(0))); + try std.testing.expectEqualStrings("model", std.mem.span(axes.get(1))); + try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{ &replicated, &open }).len); + try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{}).len); + + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const expected = shardy.ManualAxesAttr.get(ctx, &.{ + .init(ctx, "data"), + .init(ctx, "model"), + }).attribute(); + try std.testing.expect(shardy_.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); +} + const gspmd = struct { fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); @@ -375,172 +349,54 @@ const gspmd = struct { } }; -test manualComputation { - const zml = @import("../zml.zig"); +const shardy_ = struct { + pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { + var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; + for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); + return shardy.ManualAxesAttr.get(mlir_ctx, attrs.constSlice()).attribute(); + } +}; + +test "manualComputation passes local output shapes to the callback" { const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const mesh = zml.testing.tpMesh(); + const partitions = mesh.numPartitionsForLogicalAxis(.model); + if (partitions == 1) return error.SkipZigTest; - const LocalSum = struct { - x: Tensor, - bias: Tensor, - scale: i32, + var compiler: Compiler = .init(allocator, io, platform, .{ .meshes = &.{mesh} }); + defer compiler.deinit(); + compiler.activate(); + defer compiler.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + const scope = compiler.pushBlock(block); + defer scope.pop(); + + const global_shape = Shape.init(.{ .b = 4, .h = 3 * partitions, .hd = 128 }, .bf16) + .withPartitioning(mesh, .{ .h = .model }); + const local_shape = Shape.init(.{ .b = 4, .h = 3, .hd = 128 }, .bf16); + const input = Tensor.zeroes(global_shape).withPartitioning(mesh, .{ .h = .model }); + var received_shape: ?Shape = null; + const Context = struct { + input: Tensor, + received_shape: *?Shape, fn body(self: @This(), output_shape: Shape) Tensor { - return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); - } - - fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { - return manualComputation( - body, - .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, - x.shape(), - mesh_, - .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, - ); + self.received_shape.* = output_shape; + return self.input; } }; + const output = manualComputation( + Context.body, + .{ .input = input, .received_shape = &received_shape }, + global_shape, + ); - // Each shard sums two elements and broadcasts the result to its local output shape. - const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); - const values = try allocator.alloc(i32, shape.count()); - defer allocator.free(values); - - for (values, 0..) |*value, i| value.* = @intCast(i + 1); - - var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - const actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - - for (actual.constItems(i32), 0..) |value, i| { - const pair = values[i / 2 * 2 ..][0..2]; - try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); - } - - try std.testing.expect(output.shape().eqlWithTags(shape)); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } -} - -test "manualComputation composes across logical and physical meshes" { - const zml = @import("../zml.zig"); - if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - - const Topologies = enum { flat, torus }; - - const Model = struct { - fn increment(x: Tensor, _: Shape) Tensor { - return x.addConstant(1); - } - - fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { - const partitioning: ManualPartitioning(increment) = .{ - .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), - .out_specs = .parse(.{ .x = .model }), - }; - - const first = manualComputation(increment, x, x.shape(), mesh, partitioning); - const converted = first.convert(.f32); - return manualComputation(increment, converted, converted.shape(), mesh, partitioning); - } - }; - - const shape: Shape = .init(.{ .x = 16 }, .i32); - - var values: [16]i32 = undefined; - var expected: [16]f32 = undefined; - for (&values, &expected, 0..) |*value, *result, i| { - value.* = @intCast(i); - result.* = @floatFromInt(i + 2); - } - - for ([_]Topologies{ .flat, .torus }) |topology| { - const platform = try zml.Platform.init(allocator, io, .cpu, .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, - }); - defer platform.deinit(allocator, io); - - // Fold three physical axes into one logical axis, or split one into three. - const logical: Sharding.LogicalMesh = if (topology == .torus) - try .init(.{.model}, &.{.balanced}, &.{8}) - else - try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); - - const mesh = try platform.registerMesh("manual", logical); - - var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); - try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } - } -} - -test "partitionSpecs preserves optional tensors and active cache variants" { - const zml = @import("../zml.zig"); - const Logical = Sharding.Partitioning.Logical; - const Inputs = struct { - layers: []const Tensor, - bias: ?Tensor, - cache: union(enum) { disabled, values: [2]Tensor }, - scale: f32, - }; - - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - - const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); - var inputs: Inputs = .{ - .layers = &.{ tensor, tensor }, - .bias = null, - .cache = .{ .values = .{ tensor, tensor } }, - .scale = 2, - }; - const heads: Logical = .parse(.{ .h = .model }); - const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); - const expected: zml.ops.PartitionSpecs(Inputs) = .{ - .layers = &.{ heads, heads }, - .bias = null, - .cache = .{ .values = .{ heads, heads } }, - }; - try std.testing.expectEqualDeep(expected, specs); - - inputs.layers = &.{}; - inputs.bias = tensor; - inputs.cache = .disabled; - const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); - const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ - .layers = &.{}, - .bias = .replicated, - .cache = .disabled, - }; - try std.testing.expectEqualDeep(expected_replicated, replicated); + try std.testing.expect(received_shape != null); + try std.testing.expect(received_shape.?.eql(local_shape)); + try std.testing.expect(output.shape().eql(global_shape)); + try std.testing.expectEqualDeep(global_shape._sharding, output.shape()._sharding); } diff --git a/zml/ops/manual_computation2.zig b/zml/ops/manual_computation2.zig new file mode 100644 index 00000000..2d3f0aec --- /dev/null +++ b/zml/ops/manual_computation2.zig @@ -0,0 +1,465 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../Compiler.zig"); +const meta = @import("../meta.zig"); +const mlirx = @import("../mlirx.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/Compiler"); + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = meta.MapRestrict(Tensor, Logical).map(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. +pub fn manualComputation2( + comptime body_fn: anytype, + inputs: stdx.meta.FnParam(body_fn, 0), + output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), +) stdx.meta.FnReturn(body_fn) { + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + const Outputs = stdx.meta.FnReturn(body_fn); + + const compiler = Compiler.current(); + var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); + + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + + const WrappedInputs = struct { + inputs: *const Inputs, + output_shapes: *const OutputShapes, + + fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { + const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); + const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); + + outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); + + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + } + }; + + var outputs_struct: Outputs = undefined; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + + _ = manualComputationInternal( + compiler, + arena, + mesh, + @typeName(@TypeOf(body_fn)), + &wrapped_inputs, + flat_input_tensors, + @ptrCast(&outputs_struct), + flat_output_shapes, + WrappedInputs.wrap, + ) catch |err| switch (err) { + error.OutOfMemory => @panic("OOM"), + }; + + return outputs_struct; +} + +const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; + +/// Reified version of manualComputation2. +/// This one is compiled exactly once, user code just need to compile the `collectPtrs` +noinline fn manualComputationInternal( + compiler: *Compiler, + arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, + functor_name: []const u8, + input_struct: *anyopaque, + input_tensors: []*Tensor, + output_struct: *anyopaque, + global_output_shapes: []const Shape, + local_fn: LocalComputationFn, +) error{OutOfMemory}![]*Tensor { + const global_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + + log.debug("manualComputation2({s}) will use mesh {s}", .{ functor_name, mesh.name }); + + const global_input_shapes = try arena.alloc(Shape, input_tensors.len); + for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + + const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); + for (input_types, input_tensors) |*input_ty, tensor_ptr| { + const global = tensor_ptr._shape; + const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); + log.debug("\t- {f} -> {f}", .{ global, local }); + input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); + } + const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); + @memset(input_locs, compiler.unknown_location); + + const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); + for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } + input_val.* = global_scope.value(tensor_ptr.*); + } + + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); + + const local_scope = compiler.pushBlock(manual_block); + defer local_scope.pop(); + for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { + const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above + tensor_ptr.* = switch (compiler.partitioner) { + .shardy => Tensor._result(local, manual_block.argument(i)), + .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), + }; + } + + compiler.manual_computation_depth += 1; + defer compiler.manual_computation_depth -= 1; + + const local_outputs = try local_fn(arena, input_struct, output_struct); + stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation2 body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation2 body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + } + + const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |output_shape, i| { + global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); + } + + const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); + for (local_output_values, local_outputs) |*output_val, tensor_ptr| { + output_val.* = local_scope.value(tensor_ptr.*); + } + + const op: *const mlir.Operation = switch (compiler.partitioner) { + .shardy => op: { + // Shardy creates a sdy.manual_computation block. + _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); + + const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); + for (out_mesh_attrs, global_output_shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); + } + + break :op shardy.manual_computation( + mlir_ctx, + input_values, + global_output_types, + manual_block, + try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), + .get(mlir_ctx, out_mesh_attrs), + try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), + compiler.location, + ).appendTo(global_scope.block); + }, + .gspmd => op: { + // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier + // Shortcut to avoid empty optimization barrier + if (global_output_shapes.len == 0) return &.{}; + + const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); + const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |global_shape, i| { + global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); + } + + break :op stablehlo.optimizationBarrier( + mlir_ctx, + global_values, + global_types, + compiler.location, + ).appendTo(global_scope.block); + }, + }; + + // Returned tensors describe the operation's global output shapes and sharding. + for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { + output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); + output_tensor_ptr._shape._sharding = output_shape._sharding; + } + return local_outputs; +} + +const gspmd = struct { + fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { + const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); + const full_to_shard = stablehlo.custom_call( + mlir_ctx, + &.{scope.value(local_tensor)}, + &.{local_type}, + .{ + .call_target_name = "SPMDFullToShardShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), + }, + }, + location, + ).appendTo(scope.block); + return Tensor._result(local_shape, full_to_shard.result(0)); + } + + fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { + const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); + const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); + + const shard_to_full = stablehlo.custom_call( + compiler.mlir_ctx, + &.{local_value}, + &.{global_type}, + .{ + .call_target_name = "SPMDShardToFullShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), + }, + }, + compiler.location, + ).appendTo(scope.block); + return .{ shard_to_full.result(0), global_type }; + } +}; + +test manualComputation2 { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const mesh = zml.testing.tpMesh(); + + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, + + fn body(self: @This(), output_shape: Shape) Tensor { + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation2( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); + } + }; + + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation2 composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = .parse(.{ .x = .model }), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation2(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation2(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} diff --git a/zml/platform.zig b/zml/platform.zig index 54fb90e4..6c6081aa 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1015,15 +1015,12 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; - const buffers = call_frame.args.buffers(); - if (buffers.len <= 0) return null; - const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = buffers[0]; + const buffer = call_frame.args.buffers()[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -1047,13 +1044,9 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); - defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - - const event = pjrt_buffer.readyEvent(pjrt_api); - defer event.deinit(pjrt_api); - try event.awaitRaw(pjrt_api); + try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); diff --git a/zml/profiling/traceme_lib.cc b/zml/profiling/traceme_lib.cc index ca37dee7..9840807d 100644 --- a/zml/profiling/traceme_lib.cc +++ b/zml/profiling/traceme_lib.cc @@ -1,5 +1,6 @@ #include "zml/profiling/traceme.h" +#include #include #include #include @@ -58,6 +59,35 @@ zig_slice CopyToOwnedSlice(const std::string& data) { return {ptr, data.size()}; } +// PJRT's generic contract is binary data with an explicit length. Some +// providers append one NUL sentinel; accept that only if the exact bytes fail +// and removing that sentinel produces a valid XSpace. +bool ParseXSpace(zig_slice bytes, tensorflow::profiler::XSpace* space) { + if (bytes.len > INT_MAX || (bytes.len != 0 && bytes.ptr == nullptr)) { + return false; + } + if (bytes.len == 0) { + return true; + } + const int size = static_cast(bytes.len); + if (space->ParseFromArray(bytes.ptr, size)) { + return true; + } + const auto* data = static_cast(bytes.ptr); + if (data[bytes.len - 1] != 0) { + return false; + } + space->Clear(); + return space->ParseFromArray(bytes.ptr, size - 1); +} + +void DiscardLocalHostTraceSession() { + if (g_local_host_trace_session.active) { + (void)tsl::profiler::TraceMeRecorder::Stop(); + g_local_host_trace_session.active = false; + } +} + #if defined(ZML_RUNTIME_CUDA) struct CudaNvtxRangeBackend { explicit CudaNvtxRangeBackend(const std::string& name) @@ -215,10 +245,9 @@ extern "C" void zml_traceme_session_start(int level, uint64_t filter_mask, extern "C" zig_slice zml_traceme_session_merge(zig_slice xspace) { tensorflow::profiler::XSpace space; - if (xspace.len != 0 && - !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { - return CopyToOwnedSlice( - std::string(static_cast(xspace.ptr), xspace.len)); + if (!ParseXSpace(xspace, &space)) { + DiscardLocalHostTraceSession(); + return {}; } if (g_local_host_trace_session.active) { @@ -238,8 +267,8 @@ extern "C" zig_slice zml_traceme_session_postprocess(zig_slice xspace, uint64_t start_time_ns, uint64_t stop_time_ns) { tensorflow::profiler::XSpace space; - if (xspace.len != 0 && - !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { + if (!ParseXSpace(xspace, &space)) { + DiscardLocalHostTraceSession(); return {}; } diff --git a/zml/profiling/traceme_test.cc b/zml/profiling/traceme_test.cc new file mode 100644 index 00000000..2d855382 --- /dev/null +++ b/zml/profiling/traceme_test.cc @@ -0,0 +1,87 @@ +#include "zml/profiling/traceme.h" + +#include +#include +#include + +#include "tsl/profiler/protobuf/xplane.pb.h" +#include "xla/tsl/profiler/backends/cpu/traceme_recorder.h" + +namespace { + +int failures = 0; + +void Check(bool condition, const char* name) { + if (!condition) { + std::cerr << "FAIL: " << name << '\n'; + ++failures; + } +} + +zig_slice Slice(const char* bytes, size_t size) { + return {const_cast(bytes), size}; +} + +void CheckAccepted(zig_slice input, const char* name) { + const bool modes[] = {false, true}; + for (bool postprocess : modes) { + zig_slice result = postprocess + ? zml_traceme_session_postprocess(input, 0, 1) + : zml_traceme_session_merge(input); + tensorflow::profiler::XSpace space; + Check(result.len > 0 && result.len <= INT_MAX && + space.ParseFromArray(result.ptr, static_cast(result.len)) && + space.planes_size() > 0 && space.planes(0).id() == 0, + name); + zml_traceme_str_free(result); + } +} + +void CheckRejected(zig_slice input, const char* name) { + zig_slice merged = zml_traceme_session_merge(input); + Check(merged.len == 0, name); + zml_traceme_str_free(merged); + zig_slice normalized = zml_traceme_session_postprocess(input, 0, 1); + Check(normalized.len == 0, name); + zml_traceme_str_free(normalized); +} + +} // namespace + +int main() { + // A real XSpace wire message: planes[0] has an explicit int64 id = 0. + const char terminal_zero[] = {0x0a, 0x02, 0x08, 0x00}; + const char sentinel_zero[] = {0x0a, 0x02, 0x08, 0x00, 0x00}; + const char two_sentinels[] = {0x0a, 0x02, 0x08, 0x00, 0x00, 0x00}; + const char malformed[] = {0x0a, 0x02, 0x08}; + const char malformed_zero[] = {static_cast(0xff), 0x00}; + CheckAccepted(Slice(terminal_zero, sizeof(terminal_zero)), + "valid binary terminal zero survives"); + CheckAccepted(Slice(sentinel_zero, sizeof(sentinel_zero)), + "one provider sentinel is accepted after validated retry"); + CheckRejected(Slice(two_sentinels, sizeof(two_sentinels)), + "two sentinels are rejected"); + CheckRejected(Slice(malformed, sizeof(malformed)), "truncated protobuf rejected"); + CheckRejected(Slice(malformed_zero, sizeof(malformed_zero)), + "invalid bytes remain invalid after sentinel retry"); + CheckRejected(Slice(terminal_zero, static_cast(INT_MAX) + 1), + "oversized input rejected before reading or int conversion"); + CheckRejected({nullptr, 1}, "nonempty null input rejected"); + + const bool modes[] = {false, true}; + for (bool postprocess : modes) { + zml_traceme_session_start(1, 0, false); + zig_slice input = Slice(malformed, sizeof(malformed)); + zig_slice result = postprocess + ? zml_traceme_session_postprocess(input, 0, 1) + : zml_traceme_session_merge(input); + Check(result.len == 0, "malformed active session rejected"); + zml_traceme_str_free(result); + const bool restarted = tsl::profiler::TraceMeRecorder::Start(1); + Check(restarted, "parse failure releases local host TraceMe session"); + if (restarted) (void)tsl::profiler::TraceMeRecorder::Stop(); + } + + if (failures == 0) std::cout << "All XSpace compatibility checks passed.\n"; + return failures == 0 ? 0 : 1; +} diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 3827fe27..f854f02f 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,337 +2,68 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; +const Dims = Shape.DimsArray; + +const StringBuilder = std.ArrayListUnmanaged(u8); + +pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); +pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); const log = std.log.scoped(.@"zml/safetensors"); -// TODO: consider promoting this to the top level struct -// TODO: Feels like we are missing a mapping file -> tensors that would be helfpful for loading weights -/// Maps tensor names to their shape and positions in different sub files. -pub const TensorRegistry = struct { - arena: std.heap.ArenaAllocator, +const BYTES_HEADER = 8; - tensors: std.StringArrayHashMapUnmanaged(Tensor), - - /// Metadata found in `__metadata__` field of the safetensors header. - /// This is rarely use and the information there is not standardized, so we let user code handle it. - /// Here we just store the plain bytes, reach out if you need this to be already parsed. - metadata: []const u8, - - pub fn init(allocator: std.mem.Allocator) TensorRegistry { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = "", - }; - } - - pub fn deinit(self: *TensorRegistry) void { - self.arena.deinit(); - } - - pub fn fromRepo(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !TensorRegistry { - const file_type, const file = try resolveModelEntrypoint(io, repo); - defer file.close(io); - - var registry: TensorRegistry = .init(allocator); - switch (file_type) { - .safetensors => try registry.parseSafetensors(io, allocator, file), - .index => try registry.parseIndexJson(io, allocator, repo, file), - .full_index => try registry.parseFullIndexJson(io, allocator, repo, file), - .unknown => return error.InvalidPath, - } - return registry; - } - - pub fn fromPath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !TensorRegistry { - const file_type: FileType = .resolve(path); - switch (file_type) { - .safetensors => { - // open directly - const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); - var registry: TensorRegistry = .init(allocator); - try registry.parseSafetensors(io, allocator, file); - return registry; - }, - .full_index, - .index, - => { - // resolve repo as parent folder then open - const repo_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const repo = std.Io.Dir.openDir(.cwd(), io, repo_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); - - var registry: TensorRegistry = .init(allocator); - try switch (file_type) { - .index => registry.parseIndexJson(io, allocator, repo, file), - .full_index => registry.parseFullIndexJson(io, allocator, repo, file), - .safetensors, .unknown => unreachable, - }; - return registry; - }, - .unknown => { - // the path is possibly a folder, let's try that - const repo = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - return try fromRepo(allocator, io, repo); - }, - } - } - - /// Register a tensor, duplicate tensor name and uri. - /// If you create a lot of tensors sharing the same uri, - /// considere calling `registry.tensors` directly and avoid duplicated uris. - pub fn registerTensor(self: *TensorRegistry, tensor: Tensor) !void { - const allocator = self.arena.allocator(); - - const owned_name = try allocator.dupe(u8, tensor.name); - errdefer allocator.free(owned_name); - const owned_uri = try allocator.dupe(u8, tensor.file_uri); - errdefer allocator.free(owned_uri); - - const owned_tensor: Tensor = .{ - .name = owned_name, - .file_uri = owned_uri, - .shape = tensor.shape, - .offset = tensor.offset, - }; - try self.tensors.put(allocator, owned_name, owned_tensor); - } - - pub fn reader( - self: *TensorRegistry, - io: std.Io, - tensor_name: []const u8, - buffer: []u8, - ) !TensorReader { - const tensor = self.tensors.get(tensor_name) orelse { - log.err("Tensor {s} not found in registry", .{tensor_name}); - return error.TensorNotFound; - }; - - return try .init(io, tensor, buffer, .{}); - } - - pub fn totalBytes(self: *TensorRegistry) u64 { - var total: u64 = 0; - - var it = self.tensors.iterator(); - while (it.next()) |entry| { - total += entry.value_ptr.byteSize(); - } - - return total; - } - - /// Parse a given .safetensors file and create entries for each tensor - pub fn parseSafetensors( - self: *TensorRegistry, - io: std.Io, - scratch_allocator: std.mem.Allocator, - file: std.Io.File, - ) !void { - var tmp_state = std.heap.ArenaAllocator.init(scratch_allocator); - defer tmp_state.deinit(); - const tmp_arena = tmp_state.allocator(); - - const self_arena = self.arena.allocator(); - var owned_file_name = try self_arena.alloc(u8, std.fs.max_path_bytes); - const file_name_len = try file.realPath(io, owned_file_name); - // Try to release extra memory, but we don't care if it fails - _ = self_arena.resize(owned_file_name, file_name_len); - owned_file_name = owned_file_name[0..file_name_len]; - - const header_buf = try tmp_arena.alloc(u8, 64 * 1024); - var file_reader: std.Io.File.Reader = file.reader(io, header_buf); - - const json_header_length: u64 = try file_reader.interface.takeInt(u64, .little); - const BYTES_HEADER = 8; - const data_start_offset = BYTES_HEADER + json_header_length; - const json_data = try tmp_arena.alloc(u8, @intCast(json_header_length)); - - try file_reader.interface.readSliceAll(json_data); - const headers: std.json.Value = try std.json.parseFromSliceLeaky( - std.json.Value, - tmp_arena, - json_data, - .{}, - ); - - try self.tensors.ensureUnusedCapacity(self_arena, headers.object.count()); - for (headers.object.keys(), headers.object.values()) |key, value| { - if (std.mem.eql(u8, key, "__metadata__")) { - // value belong to tmp arena, duplicate it into self arena. - var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); - try std.json.fmt(value, .{ .whitespace = .minified }).format(&owned_metadata.writer); - self.metadata = owned_metadata.written(); - continue; - } - - const owned_name = try self_arena.dupe(u8, key); - const tensor: Tensor = try .parse(value.object, owned_file_name, owned_name, data_start_offset); - self.tensors.putAssumeCapacity(owned_name, tensor); - } - } - - pub fn parseIndexJson( - self: *TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - index_json_file: std.Io.File, - ) !void { - var tmp_state: std.heap.ArenaAllocator = .init(gpa); - defer tmp_state.deinit(); - const tmp = tmp_state.allocator(); - - const file_reader_buf = try tmp.alloc(u8, 64 * 1024); - var file_reader: std.Io.File.Reader = index_json_file.reader(io, file_reader_buf); - var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - tmp, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - var safetensors_files: std.StringArrayHashMapUnmanaged(void) = .empty; - try safetensors_files.ensureUnusedCapacity(tmp, 64); - if (index.object.get("weight_map")) |wm| { - for (wm.object.values()) |file_name| { - try safetensors_files.put(tmp, file_name.string, {}); - } - } else { - log.warn("No weight_map attribute found in index", .{}); - } - - for (safetensors_files.keys()) |file_name| { - const file = try repo.openFile(io, file_name, .{}); - defer file.close(io); - try self.parseSafetensors(io, tmp, file); - } - } - - pub fn parseFullIndexJson( - self: *TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - index_json_file: std.Io.File, - ) !void { - var tmp_state: std.heap.ArenaAllocator = .init(gpa); - defer tmp_state.deinit(); - const tmp = tmp_state.allocator(); - - const self_arena = self.arena.allocator(); - - var repo_uri_buf = try tmp.alloc(u8, 1024); - const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; - - // Note: this is relatively inefficient, we read the full file, copy most of it's content, - // then reduce to TensorRegistry format. - // We could update TensorRegistry as we stream the json. - // Zig std json API are a bit awkward around streaming though. - const file_reader_buf = try tmp.alloc(u8, 64 * 1024); - var file_reader = index_json_file.reader(io, file_reader_buf); - var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - tmp, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - var realpaths: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - try realpaths.ensureUnusedCapacity(tmp, 64); - const wm = index.object.get("weight_map") orelse return error.InvalidJson; - - try self.tensors.ensureUnusedCapacity(self_arena, wm.object.count()); - for (wm.object.keys(), wm.object.values()) |key, value| { - const tensor_name = try self_arena.dupe(u8, key); - const object = value.object; - const filename = value.object.get("filename").?.string; - const filename_entry = try realpaths.getOrPut(tmp, filename); - const file_uri = if (filename_entry.found_existing) - filename_entry.value_ptr.* - else uri: { - const uri = try std.fs.path.join(self_arena, &.{ repo_name, filename }); - filename_entry.value_ptr.* = uri; - break :uri uri; - }; - - const tensor_info: Tensor = try .parse(object, file_uri, tensor_name, 0); - self.tensors.putAssumeCapacity(tensor_name, tensor_info); - } - - if (index.object.get("metadata")) |metadata| { - // value belong to tmp arena, duplicate it into self arena. - var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); - try std.json.fmt(metadata, .{ .whitespace = .minified }).format(&owned_metadata.writer); - self.metadata = owned_metadata.written(); - } - } - - pub fn saveAsFullIndexJson( - self: *const TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - out: *std.Io.Writer, - ) !void { - var repo_uri_buf = try gpa.alloc(u8, std.fs.max_path_bytes); - defer gpa.free(repo_uri_buf); - const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; - var total_size: u64 = 0; - for (self.tensors.values()) |value| total_size += value.byteSize(); - - try out.print( - \\{{ - \\"metadata": {{"num_tensors":{d},"total_size":{d}}}, - \\"weight_map": {{ - \\ - , .{ self.tensors.count(), total_size }); - - const n = self.tensors.count(); - for (0.., self.tensors.keys(), self.tensors.values()) |i, key, value| { - std.debug.assert(std.mem.startsWith(u8, value.file_uri, repo_name)); - const local_name = value.file_uri[repo_name.len + 1 ..]; - const end_offset = value.offset + value.byteSize(); - const safetensors_dtype: []const u8 = dt: { - for (safetensors_dtype_to_zml_dtype.keys(), safetensors_dtype_to_zml_dtype.values()) |st_dt, zml_dt| { - if (zml_dt == value.shape._dtype) break :dt st_dt; - } - // Note: we could bitcast to u8, - // but reaching this panic implies the user didn't loaded this TensorRegistry - // from a safetensors file, and manually created the dict. - // In which case, they know what they are doing, - // and can easily insert the bitcast themselves. - std.debug.panic("Safetensors doesn't support data type {}", .{value.shape._dtype}); - }; - const comma = if (i + 1 == n) "" else ","; - try out.print( - \\ "{s}": {{"filename":"{s}","shape":{f},"data_offsets":[{d},{d}],"dtype":"{s}"}}{s} - \\ - , .{ key, local_name, std.json.fmt(value.shape.dims(), .{ .whitespace = .minified }), value.offset, end_offset, safetensors_dtype, comma }); - } - - try out.writeAll( - \\} - \\} - \\ - ); - try out.flush(); - } +pub const FileType = enum { + index, + safetensors, + unknown, }; +pub fn fetchRegistry( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + model: std.Io.File, + model_path: []const u8, +) !TensorRegistry { + const file_type = resolveFiletype(model_path); + + return switch (file_type) { + .index => blk: { + const file_reader_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(file_reader_buf); + + var index_reader = model.reader(io, file_reader_buf); + + var safetensors_index = try parseSafetensorsIndex( + allocator, + &index_reader.interface, + ); + defer safetensors_index.deinit(); + + var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); + + try parseSafetensorsIndexFiles( + allocator, + io, + ®istry, + safetensors_index, + repo, + ); + + break :blk registry; + }, + .safetensors => blk: { + var registry: TensorRegistry = .init(allocator); + + try parseSafetensors(allocator, io, ®istry, model); + + break :blk registry; + }, + else => return error.InvalidPath, + }; +} + pub const TensorReader = struct { tensor: Tensor, file: std.Io.File, @@ -507,30 +238,6 @@ pub const Tensor = struct { return TensorReader.init(io, self, buffer, opts); } - pub fn parse(object: std.json.ObjectMap, file_uri: []const u8, name: []const u8, base_offset: u64) !Tensor { - const shape_field = object.get("shape").?.array; - if (shape_field.items.len > Shape.MAX_RANK) { - log.warn("Can't load tensor {s}, too many dims: {}", .{ name, shape_field.items.len }); - return error.TooManyDimensions; - } - - const offset_field = object.get("data_offsets").?; - const start: u64 = @intCast(offset_field.array.items[0].integer); - const dtype = try stringToDtype(object.get("dtype").?.string); - - var dims: Shape.DimsArray = .empty; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); - } - - return .{ - .file_uri = file_uri, - .name = name, - .shape = .init(dims.slice(), dtype), - .offset = base_offset + start, - }; - } - pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ self.name, @@ -542,100 +249,620 @@ pub const Tensor = struct { } }; -pub const FileType = enum { - full_index, - index, - safetensors, - unknown, +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, - pub fn resolve(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors.full_index.json")) { - return .full_index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { - return .safetensors; - } else { - return .unknown; + tensors: Tensors, + metadata: Metadatas, + + mutex: std.Io.Mutex = .init, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .{}, + }; + } + + pub fn initWithMetadata( + allocator: std.mem.Allocator, + metadata: Metadatas, + ) !TensorRegistry { + var self: TensorRegistry = .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .empty, + }; + + try self.mergeMetadata(metadata); + + return self; + } + + pub fn fromRepo( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + ) !TensorRegistry { + const entrypoint_file, const entrypoint_path = try resolveModelEntrypoint(io, repo); + defer entrypoint_file.close(io); + return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); + } + + pub fn fromPath( + allocator: std.mem.Allocator, + io: std.Io, + path: []const u8, + ) !TensorRegistry { + var repo = try resolveModelRepo(io, path); + defer repo.close(io); + + const entrypoint_file, const entrypoint_path = if (std.mem.endsWith(u8, path, ".safetensors.index.json") or std.mem.endsWith(u8, path, ".safetensors")) + .{ try repo.openFile(io, path, .{ .mode = .read_only }), path } + else + try resolveModelEntrypoint(io, repo); + defer entrypoint_file.close(io); + + return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); + } + + pub fn deinit(self: *TensorRegistry) void { + const allocator = self.arena.allocator(); + self.tensors.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } + + pub fn mergeMetadata( + self: *TensorRegistry, + other: Metadatas, + ) !void { + // self.mutex.lock(); + // defer self.mutex.unlock(); + + const allocator = self.arena.allocator(); + + var it = other.iterator(); + while (it.next()) |entry| { + const key = try allocator.dupe(u8, entry.key_ptr.*); + const value = try entry.value_ptr.*.clone(allocator); + + const gop = try self.metadata.getOrPut(allocator, key); + if (gop.found_existing) { + gop.value_ptr.*.deinit(allocator); + gop.value_ptr.* = value; + log.debug("Overwrote existing metadata key={s} with value={f}", .{ key, value }); + } else { + gop.value_ptr.* = value; + log.debug("Added new metadata key={s} with value={f}", .{ key, value }); + } } } + + pub fn registerTensor( + self: *TensorRegistry, + tensor: Tensor, + ) !void { + const allocator = self.arena.allocator(); + + // self.mutex.lock(); + // defer self.mutex.unlock(); + + var tensor_copy = tensor; + + tensor_copy.name = try allocator.dupe(u8, tensor.name); + tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); + + try self.tensors.put(allocator, tensor_copy.name, tensor_copy); + } + + pub fn reader( + self: *TensorRegistry, + io: std.Io, + tensor_name: []const u8, + buffer: []u8, + ) !TensorReader { + const tensor = self.tensors.get(tensor_name) orelse { + log.err("Tensor {s} not found in registry", .{tensor_name}); + return error.TensorNotFound; + }; + + return try .init(io, tensor, buffer, .{}); + } + + pub fn iterator(self: *TensorRegistry) Tensors.Iterator { + return self.tensors.iterator(); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } }; +pub fn parseSafetensors( + allocator: std.mem.Allocator, + io: std.Io, + registry: *TensorRegistry, + file: std.Io.File, +) !void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const header_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(header_buf); + + var reader = file.reader(io, header_buf); + + var file_uri_buf: [1024]u8 = undefined; + const file_uri_len = try file.realPath(io, &file_uri_buf); + const file_uri = file_uri_buf[0..file_uri_len]; + + const json_header_length: u64 = try reader.interface.takeInt(u64, .little); + const data_start_offset = BYTES_HEADER + json_header_length; + const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); + + try reader.interface.readSliceAll(json_data); + + const headers: std.json.Value = try std.json.parseFromSliceLeaky( + std.json.Value, + arena.allocator(), + json_data, + .{}, + ); + + var it = headers.object.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + + if (std.mem.eql(u8, key, "__metadata__")) { + // Parse metadata into a local arena to avoid concurrent allocations + // against registry.arena (which is not thread-safe). + var local_arena = std.heap.ArenaAllocator.init(allocator); + defer local_arena.deinit(); + + const metas = try parseMetadata(local_arena.allocator(), value); + try registry.mergeMetadata(metas); + + continue; + } + + const shape_field = value.object.get("shape").?.array; + + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); + return error.TooManyDimensions; + } + + const offset_field = value.object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const dtype = try stringToDtype(value.object.get("dtype").?.string); + + var dims: Dims = .empty; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + const tensor: Tensor = .{ + .file_uri = file_uri, + .name = key, + .shape = .init(dims.slice(), dtype), + .offset = data_start_offset + start, + }; + + try registry.registerTensor(tensor); + } +} + +const SafetensorsIndex = struct { + pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); + + arena: std.heap.ArenaAllocator, + + map: Map, + metadata: Metadatas, + + pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .map = .empty, + .metadata = .empty, + }; + } + + pub fn iterator(self: *SafetensorsIndex) Map.Iterator { + return self.map.iterator(); + } + + pub fn deinit(self: *SafetensorsIndex) void { + const allocator = self.arena.allocator(); + self.map.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } +}; + +fn parseSafetensorsIndex( + allocator: std.mem.Allocator, + reader: *std.Io.Reader, +) !SafetensorsIndex { + var safetensors_index: SafetensorsIndex = .init(allocator); + errdefer safetensors_index.deinit(); + + const arena_allocator = safetensors_index.arena.allocator(); + var json_reader: std.json.Reader = .init(arena_allocator, reader); + + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + arena_allocator, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + const weight_map = index.object.get("weight_map"); + + if (weight_map) |wm| { + var it = wm.object.iterator(); + + while (it.next()) |entry| { + const weight_name = entry.key_ptr.*; + const filename = entry.value_ptr.string; + + const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); + + if (!map_entry.found_existing) { + map_entry.value_ptr.* = .empty; + } + + try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + if (index.object.get("metadata")) |metadata_val| { + safetensors_index.metadata = try parseMetadata(arena_allocator, metadata_val); + } + + return safetensors_index; +} + +fn parseSafetensorsIndexFiles( + allocator: std.mem.Allocator, + io: std.Io, + registry: *TensorRegistry, + safetensors_index: SafetensorsIndex, + repo: std.Io.Dir, +) !void { + const filenames = safetensors_index.map.keys(); + for (filenames) |filename| { + const file = try repo.openFile(io, filename, .{}); + defer file.close(io); + try parseSafetensors(allocator, io, registry, file); + } +} + +pub fn resolveFiletype(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path, ".safetensors")) { + return .safetensors; + } else { + return .unknown; + } +} + pub const ModelPathResolutionError = error{ FileNotFound, InvalidPath, } || std.mem.Allocator.Error; -pub fn resolveModelRepo(io: std.Io, path: []const u8) ModelPathResolutionError!std.Io.Dir { - switch (FileType.resolve(path)) { - .full_index, .index, .safetensors => { - // The path is a file, let's open it's folder - const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; +pub fn resolveModelRepo( + io: std.Io, + path: []const u8, +) ModelPathResolutionError!std.Io.Dir { + if (std.mem.endsWith(u8, path, ".safetensors.index.json") or + std.mem.endsWith(u8, path, ".safetensors")) + { + const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, }; + }; - return dir; - }, - .unknown => { - // The path is probably a directory, let's try to open it as such - const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - return dir; - }, + return dir; } -} -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { FileType, std.Io.File } { - const default_entrypoints: []const struct { FileType, []const u8 } = &.{ - .{ .full_index, "model.safetensors.full_index.json" }, - .{ .full_index, "consolidated.safetensors.full_index.json" }, - .{ .index, "model.safetensors.index.json" }, - .{ .index, "consolidated.safetensors.index.json" }, - .{ .safetensors, "model.safetensors" }, - .{ .safetensors, "consolidated.safetensors" }, + const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; }; - for (default_entrypoints) |entrypoint| { - const file_type, const file_name = entrypoint; - const file = repo.openFile(io, file_name, .{ .mode = .read_only }) catch continue; - - return .{ file_type, file }; - } - - return error.FileNotFound; + return dir; } -const safetensors_dtype_to_zml_dtype = std.StaticStringMap(DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E8M0", .f8e8m0 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "F4_E2M1", .f4e2m1 }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, -}); +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, []const u8 } { + const default_entrypoints = .{ + "model.safetensors.index.json", + "consolidated.safetensors.index.json", + "model.safetensors", + "consolidated.safetensors", + }; -fn stringToDtype(safetensor_dtype: []const u8) !DataType { - return safetensors_dtype_to_zml_dtype.get(safetensor_dtype) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_dtype}); + inline for (default_entrypoints) |entrypoint| { + if (repo.openFile(io, entrypoint, .{ .mode = .read_only })) |index_file| { + return .{ index_file, entrypoint }; + } else |_| {} + } + + return ModelPathResolutionError.FileNotFound; +} + +const Metadata = union(enum) { + null: void, + int: i64, + float: f64, + bool: bool, + string: []const u8, + + array_bool: []const bool, + array_int: []const i64, + array_float: []const f64, + array_string: []const []const u8, + + pub const ItemType = enum { + int, + float, + bool, + string, + + pub fn toZigType(comptime kind: ItemType) type { + return switch (kind) { + .int => i64, + .float => f64, + .bool => bool, + .string => []const u8, + }; + } + }; + + pub fn wrap(x: anytype) Metadata { + return switch (@TypeOf(x)) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, + inline f16, f32, f64 => .{ .float = @floatCast(x) }, + bool => .{ .bool = x }, + []const u8 => .{ .string = x }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), + }; + } + + pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { + return switch (@TypeOf(any_slice[0])) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => { + const res = try allocator.alloc(i64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @intCast(val); + return .{ .array_int = res }; + }, + inline f16, f32, f64 => { + const res = try allocator.alloc(f64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @floatCast(val); + return .{ .array_float = res }; + }, + bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, + []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), + }; + } + + pub fn format( + self: Metadata, + writer: *std.Io.Writer, + ) !void { + switch (self) { + .null => _ = try writer.write("null"), + .string => |s| try writer.print("{s}", .{s}), + .bool => |b| try writer.print("{}", .{b}), + .int => |i| try writer.print("{d}", .{i}), + .float => |f| try writer.print("{d}", .{f}), + .array_bool => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{}", .{v}); + } + try writer.writeByte(']'); + }, + .array_int => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_float => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_string => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("\"{s}\"", .{v}); + } + try writer.writeByte(']'); + }, + } + } + + pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { + switch (self) { + .string => |s| allocator.free(s), + .array_bool => |s| allocator.free(s), + .array_int => |s| allocator.free(s), + .array_float => |s| allocator.free(s), + .array_string => |s| { + for (s) |str| allocator.free(str); + allocator.free(s); + }, + else => {}, + } + } + + pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { + return switch (self) { + .null => .null, + .int => |v| .{ .int = v }, + .float => |v| .{ .float = v }, + .bool => |v| .{ .bool = v }, + .string => |s| .{ .string = try allocator.dupe(u8, s) }, + .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, + .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, + .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, + .array_string => |s| blk: { + const new_slice = try allocator.alloc([]const u8, s.len); + errdefer allocator.free(new_slice); + for (s, 0..) |str, i| { + new_slice[i] = try allocator.dupe(u8, str); + } + break :blk .{ .array_string = new_slice }; + }, + }; + } +}; + +fn stringToDtype(safetensor_type: []const u8) !DataType { + const map = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E8M0", .f8e8m0 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "F4_E2M1", .f4e2m1 }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, + }); + + return map.get(safetensor_type) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); return error.UnsupportedDataType; }; } + +fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { + var metadatas: Metadatas = .{}; + var prefix_buf: [1024]u8 = undefined; + var prefix = StringBuilder.initBuffer(&prefix_buf); + + try populateMetadata(allocator, &prefix, val, &metadatas); + + return metadatas; +} + +fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { + const key = prefix.items; + return switch (val) { + .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), + .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), + .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), + .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), + .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), + .array => |v| { + if (v.items.len == 0) return; + if (validSlice(v)) |item_type| { + const data: Metadata = switch (item_type) { + .bool => blk: { + const values = try allocator.alloc(bool, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.bool; + break :blk .{ .array_bool = values }; + }, + .integer => blk: { + const values = try allocator.alloc(i64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.integer; + break :blk .{ .array_int = values }; + }, + .float => blk: { + const values = try allocator.alloc(f64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.float; + break :blk .{ .array_float = values }; + }, + inline .string, .number_string => |tag| blk: { + const values = try allocator.alloc([]const u8, v.items.len); + for (v.items, 0..) |item, i| { + values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); + } + break :blk .{ .array_string = values }; + }, + .null, .array, .object => unreachable, + }; + try metadatas.put(allocator, try allocator.dupe(u8, key), data); + } else { + for (v.items, 0..) |item, i| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + try populateMetadata(allocator, prefix, item, metadatas); + prefix.items.len = old_len; + } + } + }, + .object => |v| { + var obj_iter = v.iterator(); + while (obj_iter.next()) |entry| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.appendSliceAssumeCapacity(entry.key_ptr.*); + try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); + prefix.items.len = old_len; + } + }, + }; +} + +/// We can only create a Zig slice out of json array, if all values +/// in the array have the same type. +fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { + if (v.items.len == 0) return null; + + const item_type: std.meta.Tag(std.json.Value) = v.items[0]; + switch (item_type) { + .null, .array, .object => return null, + else => {}, + } + + for (v.items[1..]) |item| { + if (item != item_type) + return null; + } + + return item_type; +} diff --git a/zml/tensor.zig b/zml/tensor.zig index ee084835..a7782844 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -282,13 +282,7 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { - .cpu, .neuron, .metal => { - // A single memory needs no copy, but the result must still be a distinct - // tensor: `reuseBuffer` donations are keyed by tensor id. - var res = _result(self._shape, self.value()); - res._shape._sharding = self._shape._sharding; - return res; - }, + .cpu, .neuron, .metal => return self, .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, } @@ -4819,7 +4813,7 @@ pub const Tensor = struct { defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation((struct { + ops.manualComputation2((struct { input: Tensor, name: []const u8, diff --git a/zml/zml.zig b/zml/zml.zig index dafc3a0f..23d12e75 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -26,7 +26,7 @@ pub const Exe = exe.Exe; pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); -pub const kernel = @import("kernel/kernel.zig"); +pub const kernel = @import("kernel.zig"); pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; @@ -81,9 +81,3 @@ pub const KiB = 1024; pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; - -comptime { - // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. - // Otherwise exported functions declared in `bazel` aren't compiled. - std.testing.refAllDecls(bazel); -}