From 7378ca8acf610133cebf9cf0f1563dc5f1241bb3 Mon Sep 17 00:00:00 2001 From: Erwan Vivien Date: Fri, 13 Nov 2026 11:23:01 +0100 Subject: [PATCH 01/14] stdx/flags: support `--flag value` along `--flag=value` --- stdx/flags.zig | 48 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/stdx/flags.zig b/stdx/flags.zig index 52a9e6a4..cfda232a 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -12,8 +12,9 @@ //! - In particular, check that no options are repeated. //! - Use only long options (`--addresses`). //! - Exception: `-h/--help` is allowed. -//! - Use `--key=value` syntax for an option with an argument. -//! Don't use `--key value`, as that can be ambiguous (e.g., `--key --verbose`). +//! - Use `--key=value` or `--key value` syntax for an option with an argument. +//! With `--key value`, a value starting with `--` is rejected to avoid ambiguity +//! (e.g., `--key --verbose`); use `--key=--value` for such values. //! - Use subcommand syntax when appropriate. //! - Use positional arguments when appropriate. //! @@ -94,7 +95,7 @@ pub fn parseProcessArgs(init_: std.process.Init.Minimal, comptime CliArgs: type) fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Commands { comptime assert(@typeInfo(Commands) == .@"union"); - const field_names = std.meta.fieldNames(Commands); + const field_names = comptime std.meta.fieldNames(Commands); comptime assert(field_names.len >= 2); const first_arg = args.next() orelse fatal( @@ -110,7 +111,7 @@ fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Com } } - inline for (field_names, std.meta.fieldTypes(Commands)) |field_name, FieldType| { + inline for (field_names, comptime std.meta.fieldTypes(Commands)) |field_name, FieldType| { comptime assert(std.mem.indexOf(u8, field_name, "_") == null); if (std.mem.eql(u8, first_arg, field_name)) { return @unionInit(Commands, field_name, parse_flags(args, FieldType)); @@ -246,8 +247,15 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { } @field(counts, field_name) += 1; - const flag_value = parse_flag(FieldType, flag, arg); - @field(result, field_name) = flag_value; + // `--flag=value` carries its value inline; a bare non-bool `--flag` takes + // the next argument as its value. + const value: ?[:0]const u8 = if (arg.len > flag.len) + parse_flag_split_value(flag, arg) + else if (FieldType == bool) + null + else + parse_flag_next_value(flag, args); + @field(result, field_name) = parse_flag(FieldType, flag, value); continue :next_arg; } } @@ -331,20 +339,34 @@ fn assert_valid_value_type(comptime T: type) void { } } -/// Parse, e.g., `--cluster=123` into `123` integer -fn parse_flag(comptime T: type, flag: []const u8, arg: [:0]const u8) T { +/// Parse the value of a matched flag into `T`, e.g. `123` for `--cluster=123` or `--cluster 123`. +/// +/// - `flag`: the flag name with its leading dashes, e.g. `--cluster`. +/// - `value`: the flag's value with the separator stripped, e.g. `123`, or `null` if the flag was +/// given without a value (only valid for `bool` flags, where presence means `true`). +fn parse_flag(comptime T: type, flag: []const u8, value: ?[:0]const u8) T { assert(flag[0] == '-' and flag[1] == '-'); if (T == bool) { - if (!std.mem.eql(u8, arg, flag)) { - fatal("{s}: argument does not require a value in '{s}'", .{ flag, arg }); + if (value) |v| { + fatal("{s}: argument does not require a value, but found '{s}'", .{ flag, v }); } return true; } - const value = parse_flag_split_value(flag, arg); - assert(value.len > 0); - return parse_value(T, flag, value); + // The caller always resolves a value for non-bool flags (or exits). + assert(value.?.len > 0); + return parse_value(T, flag, value.?); +} + +/// Takes the value from the next argument for `--flag value` syntax. +fn parse_flag_next_value(flag: []const u8, args: *std.process.Args.Iterator) [:0]const u8 { + const value = args.next() orelse fatal("{s}: argument requires a value", .{flag}); + if (value.len == 0) fatal("{s}: argument requires a value", .{flag}); + if (std.mem.startsWith(u8, value, "--")) { + fatal("{s}: argument requires a value, but found '{s}' (use {s}=)", .{ flag, value, flag }); + } + return value; } /// Splits the value part from a `--arg=value` syntax. From bb2ded0e2125e7edecbeec6fe093e1037f2183aa Mon Sep 17 00:00:00 2001 From: Erwan Vivien Date: Fri, 13 Nov 2026 11:23:35 +0100 Subject: [PATCH 02/14] stdx/flags: add cheap tests --- stdx/flags.zig | 106 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/stdx/flags.zig b/stdx/flags.zig index cfda232a..e39c6768 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -643,3 +643,109 @@ pub fn defaultValue(comptime FieldType: type, default_value_ptr: ?*const anyopaq const dp: *const FieldType = @ptrCast(@alignCast(default_value_ptr orelse return null)); return dp.*; } + +/// Builds a real args iterator over `args` (POSIX only, where `Args` is a plain argv vector). +fn testArgsIterator(args: []const [*:0]const u8) std.process.Args.Iterator { + const process_args: std.process.Args = .{ .vector = args }; + return process_args.iterate(); +} + +fn testParseFlags(comptime Flags: type, args: []const [*:0]const u8) Flags { + var it = testArgsIterator(args); + const result = parse_flags(&it, Flags); + std.debug.assert(it.next() == null); + return result; +} + +const TestFlags = struct { + model: []const u8, + boolean_flag: bool = false, + count: u32 = 1, + ratio: f32 = 0.5, + mode: enum { fast, slow } = .fast, + seed: ?i64 = null, +}; + +test "parse_flags: --flag=value syntax" { + const flags = testParseFlags(TestFlags, &.{ "--model=AA", "--count=3", "--ratio=0.25", "--mode=slow", "--seed=-7" }); + try std.testing.expectEqualStrings("AA", flags.model); + try std.testing.expectEqual(false, flags.boolean_flag); + try std.testing.expectEqual(3, flags.count); + try std.testing.expectEqual(0.25, flags.ratio); + try std.testing.expectEqual(.slow, flags.mode); + try std.testing.expectEqual(-7, flags.seed); +} + +test "parse_flags: --flag value syntax" { + const flags = testParseFlags(TestFlags, &.{ "--model", "AA", "--count", "3", "--ratio", "0.25", "--mode", "slow", "--seed", "-7" }); + try std.testing.expectEqualStrings("AA", flags.model); + try std.testing.expectEqual(3, flags.count); + try std.testing.expectEqual(0.25, flags.ratio); + try std.testing.expectEqual(.slow, flags.mode); + try std.testing.expectEqual(-7, flags.seed); +} + +test "parse_flags: bool flag after spaced value is not consumed as a value" { + const flags = testParseFlags(TestFlags, &.{ "--model", "AA", "--boolean-flag" }); + try std.testing.expectEqualStrings("AA", flags.model); + try std.testing.expectEqual(true, flags.boolean_flag); +} + +test "parse_flags: bool flag before spaced value" { + const flags = testParseFlags(TestFlags, &.{ "--boolean-flag", "--model", "AA" }); + try std.testing.expectEqualStrings("AA", flags.model); + try std.testing.expectEqual(true, flags.boolean_flag); +} + +test "parse_flags: mixed syntaxes and defaults" { + const flags = testParseFlags(TestFlags, &.{ "--count=2", "--model", "hf://org/repo", "--boolean-flag" }); + try std.testing.expectEqualStrings("hf://org/repo", flags.model); + try std.testing.expectEqual(true, flags.boolean_flag); + try std.testing.expectEqual(2, flags.count); + try std.testing.expectEqual(0.5, flags.ratio); + try std.testing.expectEqual(.fast, flags.mode); + try std.testing.expectEqual(null, flags.seed); +} + +test "parse_flags: --flag=value allows values starting with --" { + const flags = testParseFlags(TestFlags, &.{"--model=--AA"}); + try std.testing.expectEqualStrings("--AA", flags.model); +} + +test "parse_flags: spaced value containing '='" { + const flags = testParseFlags(TestFlags, &.{ "--model", "a=b" }); + try std.testing.expectEqualStrings("a=b", flags.model); +} + +test "parse_flags: longer flag names are not confused with prefixes" { + const Flags = struct { + model: []const u8 = "", + model_path: []const u8 = "", + }; + const flags = testParseFlags(Flags, &.{ "--model-path", "P", "--model", "M" }); + try std.testing.expectEqualStrings("M", flags.model); + try std.testing.expectEqualStrings("P", flags.model_path); +} + +test "parse_flags: spaced value followed by positional" { + const Flags = struct { + model: []const u8, + positional: struct { + path: []const u8, + }, + }; + const flags = testParseFlags(Flags, &.{ "--model", "AA", "some/path" }); + try std.testing.expectEqualStrings("AA", flags.model); + try std.testing.expectEqualStrings("some/path", flags.positional.path); +} + +test "parse_commands: subcommand with spaced value" { + const Commands = union(enum) { + run: struct { model: []const u8, verbose: bool = false }, + list: void, + }; + var it = testArgsIterator(&.{ "run", "--model", "AA", "--verbose" }); + const cmd = parse_commands(&it, Commands); + try std.testing.expectEqualStrings("AA", cmd.run.model); + try std.testing.expectEqual(true, cmd.run.verbose); +} From 5253ea1362d04d976734e7fdc6e417422e6e693d Mon Sep 17 00:00:00 2001 From: Erwan Vivien Date: Fri, 13 Nov 2026 11:36:07 +0100 Subject: [PATCH 03/14] stdx/flags: cleanup flag_name using mem.replaceScalar --- stdx/flags.zig | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/stdx/flags.zig b/stdx/flags.zig index e39c6768..13f509a4 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -613,25 +613,19 @@ fn fields_to_comma_list(comptime E: type) []const u8 { } } +/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. pub fn flag_name(comptime field_name: []const u8) []const u8 { - // TODO(Zig): Cleanup when this is fixed after Zig 0.11. - // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. - return comptime blk: { - assert(!std.mem.eql(u8, field_name, "positional")); - - var result: []const u8 = "--"; - var index = 0; - while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { - result = result ++ field_name[index..][0..i] ++ "-"; - index = index + i + 1; - } - result = result ++ field_name[index..]; - break :blk result; - }; + comptime assert(!std.mem.eql(u8, field_name, "positional")); + comptime var name = field_name[0..field_name.len].*; + comptime std.mem.replaceScalar(u8, &name, '_', '-'); + const dashed = name; + return "--" ++ dashed; } test flag_name { - try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); + try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); + try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); + try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); } fn flag_name_positional(comptime field_name: []const u8) []const u8 { From 2d45ef0804132eb4309b71f9b261b52de4ddd02c Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 13 Nov 2026 13:52:07 +0100 Subject: [PATCH 04/14] zml/platform: fix `zml$print` leak (#1060) --- zml/platform.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zml/platform.zig b/zml/platform.zig index 6c6081aa..54fb90e4 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1015,12 +1015,15 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; + const buffers = call_frame.args.buffers(); + if (buffers.len <= 0) return null; + const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = call_frame.args.buffers()[0]; + const buffer = buffers[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -1044,9 +1047,13 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); + defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); + + const event = pjrt_buffer.readyEvent(pjrt_api); + defer event.deinit(pjrt_api); + try event.awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); From 8c41b898019a2511d05da9d90e54188569ef962d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 13 Nov 2026 13:52:58 +0100 Subject: [PATCH 05/14] zml/ops: manualComputation v2 (#1045) Last PR for manualComputation v2 which promote as default and only one API for manual computation. Custom calls now require to be wrapped in a manual computation too. --- dsl/tcl/README.md | 3 +- examples/sharding/main.zig | 4 +- zml/BUILD.bazel | 1 - zml/attention/attention.zig | 14 +- zml/kernel.zig | 9 +- zml/kernels/moe/moe.zig | 4 +- .../paged_attention/cute/benchmark.zig | 2 +- .../paged_attention/paged_attention.zig | 7 +- zml/kernels/paged_attention/tcl/tcl.zig | 13 - zml/nn.zig | 26 +- zml/ops.zig | 94 +--- zml/ops/manual_computation.zig | 494 +++++++++++------- zml/ops/manual_computation2.zig | 465 ----------------- zml/tensor.zig | 2 +- 14 files changed, 376 insertions(+), 762 deletions(-) delete mode 100644 zml/ops/manual_computation2.zig diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md index ec1ae61f..b01b7189 100644 --- a/dsl/tcl/README.md +++ b/dsl/tcl/README.md @@ -104,8 +104,7 @@ def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16 across the partitions; other kernels stay per chip. Its arguments and results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis `C`, sized to the chip count, across chips (each partition holds its share), - the default `.broadcast` keeps a copy per chip. Call it inside - `zml.ops.manualComputation` so it sees each partition's shards. + the default `.broadcast` keeps a copy per chip. * **Convolutions** pad and slide the input in its read: a slide replaces its axis by `window_axis, frame_axis`, then `contract` with the filter. On its own (without a slide) the padding is dropped. diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 7e519bf0..34a78bfe 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -42,7 +42,7 @@ const DemoModel = struct { const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); - return zml.ops.manualComputation2( + return zml.ops.manualComputation( (struct { y: zml.Tensor, gate: zml.Tensor, @@ -70,7 +70,7 @@ fn usage() void { \\Simple sharding demo: \\ - explicit partitioning on data/model axes \\ - tensor print - \\ - manualComputation2 with multiple inputs and one output + \\ - manualComputation with multiple inputs and one output \\ \\Mesh modes: \\ - auto: physical_mesh=auto, inferred cpu device_count=2 diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index b47d4f59..2b14c79f 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -103,7 +103,6 @@ zig_library( "nn.zig", "ops.zig", "ops/manual_computation.zig", - "ops/manual_computation2.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 0ac2a95c..edd9b4ca 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -97,7 +97,6 @@ pub const Options = @import("Options.zig"); pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; - const InputSpecs = @FieldType(zml.ops.ManualPartitioning(Attention.forward), "in_specs"); in_specs: struct { q: Logical, @@ -105,7 +104,7 @@ pub const Partitioning = struct { v: Logical, token_index: Logical = .replicated, /// Omit to replicate backend metadata tensors. - metadata: ?@FieldType(InputSpecs, "metadata") = null, + metadata: ?zml.ops.PartitionSpecs(Metadata) = null, }, out_specs: Logical, @@ -135,20 +134,13 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T /// `attention`, causal or not depending on `opts`. pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; - return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, .{ + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ .in_specs = .{ .q = partitioning.in_specs.q, .k = partitioning.in_specs.k, .v = partitioning.in_specs.v, .token_index = partitioning.in_specs.token_index, - .metadata = partitioning.in_specs.metadata orelse switch (metadata) { - .stablehlo => .{ .stablehlo = {} }, - .flash_attention => |metadata_| .{ .flash_attention = switch (metadata_) { - .cute => .{ .cute = {} }, - .nki => .{ .nki = {} }, - .metal => .{ .metal = .{ .num_tokens = .replicated } }, - } }, - }, + .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), }, .out_specs = partitioning.out_specs, }); diff --git a/zml/kernel.zig b/zml/kernel.zig index d85b6a56..f111f721 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1534,13 +1534,16 @@ test "tcl multi-chip kernels run on two furiosa chips" { const tp = zml.testing.tpMesh(); const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); const Mod = struct { - pub fn forward(x: Tensor) [2]Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); + pub fn forward(x: Tensor, mesh: *const zml.Sharding.Mesh) [2]Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); return .{ r[0], r[1] }; } }; const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{tp} }); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); defer exe.deinit(); var hx: [2 * 1024]f32 = undefined; for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 45f41b2a..eb697f36 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -266,7 +266,7 @@ pub fn forwardMoe( .global_experts = @intCast(gate_up.weight.dim(.expert)), }; const Logical = zml.Sharding.Partitioning.Logical; - var projection_specs: [2]zml.meta.MapRestrict(zml.Tensor, Logical).map(zml.nn.Linear) = undefined; + var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { zml.meta.mapAlloc(struct { fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { @@ -285,7 +285,7 @@ pub fn forwardMoe( .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, }, linear, specs) catch @panic("OOM"); } - return zml.ops.manualComputation2(Call.forward, args, output_shape, mesh, .{ + return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ .in_specs = .{ .input = partitioning.in_specs.input, .ids = partitioning.in_specs.ids, diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 3e05fed8..587af69f 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -125,7 +125,7 @@ fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation2(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index aaa42b20..4a97a38e 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -207,7 +207,7 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z _ = k; _ = v; const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; - return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, partitioning); + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); } const Attention = struct { @@ -435,12 +435,9 @@ test pagedAttention { const parameters: Parameters = .init(.fromBackend(backend_options_args)); const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - const ParameterSpecs = zml.meta.MapRestrict(zml.Tensor, zml.Sharding.Partitioning.Logical).map(Parameters); const partitioning: Partitioning = .{ .in_specs = .{ - .parameters = switch (parameters) { - inline else => |_, tag| @unionInit(ParameterSpecs, @tagName(tag), .{ .block_table = .replicated, .seq_lens = .replicated, .query_start_len = .replicated }), - }, + .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), .q = head_spec, .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, .opts = .{ .mask = .{ .standard = {} }, .sink = null }, diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index 881593aa..d04b684e 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -67,22 +67,9 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); - if (compiler.meshes[0].numDevices() > 1) return zml.ops.manualComputation(Shard.body, .{ .parameters = parameters, .q = q, .k_cache = k_cache, .v_cache = v_cache, .opts = opts }, q.shape()); return attend(parameters, q, k_cache, v_cache, opts); } - const Shard = struct { - parameters: Parameters, - q: Tensor, - k_cache: Tensor, - v_cache: Tensor, - opts: AttentionOptions, - - fn body(self: Shard, _: zml.Shape) Tensor { - return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); - } - }; - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); diff --git a/zml/nn.zig b/zml/nn.zig index a2b582ee..eceb316b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,11 +2,13 @@ const std = @import("std"); const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; @@ -320,13 +322,23 @@ fn onesGrid(shape: Shape, dt: DataType) Shape { } fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { - return ops.customCall( - "zml$scaled_dot_unmatched", - .{ in[0], in[1], in[2], in[3] }, - out_shape, - {}, - .{ .has_side_effect = false }, - ); + // Keep the unmatched-composite marker in the graph for backend lowering. + const ctx = zml.Compiler.current(); + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, + &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, + .{ + .call_target_name = "zml$scaled_dot_unmatched", + .has_side_effect = false, + .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); + } + return Tensor._result(out_shape, op.result(0)); } fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { diff --git a/zml/ops.zig b/zml/ops.zig index a9ae0f8d..f8fdba7e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -18,9 +18,10 @@ const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const manual_computation = @import("ops/manual_computation.zig"); pub const manualComputation = manual_computation.manualComputation; -const manual_computation2 = @import("ops/manual_computation2.zig"); -pub const manualComputation2 = manual_computation2.manualComputation2; -pub const ManualPartitioning = manual_computation2.ManualPartitioning; +pub const ManualPartitioning = manual_computation.ManualPartitioning; +pub const PartitionSpecs = manual_computation.PartitionSpecs; +pub const partitionSpecs = manual_computation.partitionSpecs; +pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; @@ -30,7 +31,6 @@ const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; test { _ = manual_computation; - _ = manual_computation2; } const log = std.log.scoped(.@"zml/Compiler"); @@ -89,7 +89,7 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ /// ``` /// /// Each expert shard now has the complete output for its data shard. To expose -/// this result through `manualComputation2`, an output spec such as +/// this result through `manualComputation`, an output spec such as /// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the /// result replicated along `.experts`. pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { @@ -2760,11 +2760,6 @@ pub fn CustomCall( comptime params: struct { /// Name of the custom call, must be unique. name: [:0]const u8, - /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies - /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, - /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output - /// sharding, this setting will avoid the shuffling of the buffer's data. - sharding_aware: bool, /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. has_side_effect: bool, /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. @@ -2798,23 +2793,7 @@ pub fn CustomCall( .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), .compute_on_host = params.compute_on_host, }; - if (params.sharding_aware) { - return shardingAwareTypedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } else { - return typedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } + return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); } fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { @@ -2842,44 +2821,6 @@ pub fn CustomCallOutputOperandAliases(I: type, O: type) type { ); } -pub fn shardingAwareTypedCustomCall( - comptime target_name: [:0]const u8, - comptime opts: CustomCallOptions, - input: anytype, - output: anytype, - attributes: anytype, -) ShapeToTensor(@TypeOf(output)) { - const Input = @TypeOf(input); - const Output = @TypeOf(output); - const Attributes = @TypeOf(attributes); - - var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - output_shapes[i] = @field(output, field_name); - } - - const Handler = struct { - input: Input, - attributes: Attributes, - - fn body(self: @This(), sharded_output_shapes: []const Shape) []const Tensor { - return typedCustomCall(target_name, opts, self.input, sharded_output_shapes, self.attributes); - } - }; - const output_tensors = manualComputation( - Handler.body, - .{ .input = input, .attributes = attributes }, - @as([]const Shape, &output_shapes), - ); - - // Convert the slice back to a struct - var out: ShapeToTensor(Output) = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - @field(out, field_name) = output_tensors[i]; - } - return out; -} - pub fn typedCustomCall( target_name: [:0]const u8, opts: CustomCallOptions, @@ -2894,7 +2835,7 @@ pub fn typedCustomCall( const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); - stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); + stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { @@ -2904,7 +2845,6 @@ pub fn typedCustomCall( } break :b &input_tensors; }, - // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -2920,7 +2860,6 @@ pub fn typedCustomCall( } break :b &output_shapes; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -2985,7 +2924,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + if (ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -2997,7 +2936,6 @@ pub fn typedCustomCall( } return out; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -3050,7 +2988,7 @@ test customCall { const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3061,9 +2999,17 @@ test customCall { const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); - const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + const Local = struct { + fn body(x: Tensor, output_shape: Shape) Tensor { + return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + }); + } + }; + const output = manualComputation(Local.body, input, shape, tp, .{ + .in_specs = .parse(.{ ._0 = .model }), + .out_specs = .parse(.{ ._0 = .model }), }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index 45400206..a0ec0037 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); @@ -7,44 +6,126 @@ const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("../Compiler.zig"); -const DataType = @import("../dtype.zig").DataType; +const meta = @import("../meta.zig"); const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; -const zml = struct { - // Nested to disambiguate from stdx.meta - const meta = @import("../meta.zig"); - const testing = @import("../testing.zig"); -}; - const log = std.log.scoped(.@"zml/Compiler"); -/// ZML operator describes computation across several devices, -/// but `manualComputation` allows to partition the devices in small group, -/// split the `Tensor` on those partitions, -/// and describes sub-computation that work inside a partition. +/// Partitioning declarations for the tensors in `T`. +/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. +/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes +/// `struct { x: Logical, bias: ?Logical }`. /// -/// See: -/// https://github.com/openxla/shardy/blob/main/docs/sdy_dialect.md#sdymanual_computation-sdymanualcomputationop +/// Initialize individual fields to declare different shardings, or use +/// `partitionSpecs` to apply one spec to every tensor in a value. +pub fn PartitionSpecs(comptime T: type) type { + return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); +} + +/// Declares the same sharding for a group of tensors at a `manualComputation` +/// boundary. /// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R` where: -/// - `I` is user defined and contains inputs tensors and any metadata required by the computation -/// - `O` contains the partionned output shapes, types must be `void`, `Shape` or `[]const Shape` for 0, 1 or many outputs -/// - `R` must be `void`, `Tensor, or `[]const Tensor` for 0, 1 or many outputs -/// * `inputs` must be using the same mesh (using .withPartitioning), or be fully replicated -/// * `outputs` contains the global output shapes, -/// and `manualComputation` will pass to `body_fn` the shapes of each local output shard. +/// ```zig +/// .in_specs = .{ +/// .query = .parse(.{ .h = .model }), +/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), +/// .metadata = partitionSpecs(metadata, .replicated), +/// } +/// ``` /// -/// This depart from the rest of ZML where output shapes are generally inferred, -/// but in this case since `body_fn` explains how to compute parts of the result, -/// it's not clear how to put them back together, -/// hence why the user is asked to explicitly provided the expected output shape. +/// Requires an active compiler and uses its arena for allocations. Use +/// `partitionSpecsAlloc` outside graph construction. Use named struct fields +/// when mixing tensors with non-tensor metadata. +pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { + return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); +} + +/// Builds the same declarations as `partitionSpecs` using an explicit allocator. +pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { + var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; + try meta.mapAlloc(struct { + fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { + return spec_; + } + }.assign, allocator, spec, tensors, &specs); + return specs; +} + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = PartitionSpecs(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), ) stdx.meta.FnReturn(body_fn) { const Inputs = stdx.meta.FnParam(body_fn, 0); const OutputShapes = stdx.meta.FnParam(body_fn, 1); @@ -55,13 +136,42 @@ pub fn manualComputation( defer arena_state.deinit(); const arena = arena_state.allocator(); - var owned_inputs: Inputs = inputs; - const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - defer arena.free(flat_input_tensors); + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); - var owned_output_shapes: OutputShapes = output_shapes; - const flat_output_shapes: []*Shape = zml.meta.collectMutPtrs(Shape, arena, &owned_output_shapes) catch @panic("OOM"); - defer arena.free(flat_output_shapes); + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } const WrappedInputs = struct { inputs: *const Inputs, @@ -73,15 +183,17 @@ pub fn manualComputation( outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); } }; var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &owned_output_shapes }; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + _ = manualComputationInternal( compiler, arena, + mesh, @typeName(@TypeOf(body_fn)), &wrapped_inputs, flat_input_tensors, @@ -91,6 +203,7 @@ pub fn manualComputation( ) catch |err| switch (err) { error.OutOfMemory => @panic("OOM"), }; + return outputs_struct; } @@ -101,27 +214,21 @@ const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyo noinline fn manualComputationInternal( compiler: *Compiler, arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, functor_name: []const u8, input_struct: *anyopaque, input_tensors: []*Tensor, output_struct: *anyopaque, - output_shapes: []*Shape, + global_output_shapes: []const Shape, local_fn: LocalComputationFn, ) error{OutOfMemory}![]*Tensor { const global_scope = compiler.currentScope(); const mlir_ctx = compiler.mlir_ctx; - const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { - .mesh => |m| m, - .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {d}: {f} and input {d}: {f}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), - }; - const manual_axes = collectManualAxes(mesh, input_tensors); - log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); + log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); const global_input_shapes = try arena.alloc(Shape, input_tensors.len); for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - const global_output_shapes = try arena.alloc(Shape, output_shapes.len); - for (global_output_shapes, output_shapes) |*shape, shape_ptr| shape.* = shape_ptr.*; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -135,11 +242,17 @@ noinline fn manualComputationInternal( const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } input_val.* = global_scope.value(tensor_ptr.*); } - const manual_block = mlir.Block.init(input_types, input_locs); - errdefer manual_block.deinit(); + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); const local_scope = compiler.pushBlock(manual_block); defer local_scope.pop(); @@ -151,17 +264,14 @@ noinline fn manualComputationInternal( }; } - for (output_shapes, global_output_shapes) |shape_ptr, global| { - shape_ptr.* = Sharding.shardedShape(global) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - compiler.manual_computation_depth += 1; defer compiler.manual_computation_depth -= 1; const local_outputs = try local_fn(arena, input_struct, output_struct); stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, output_shapes) |i, local_tensor, local_shape| { - stdx.debug.assert(local_shape.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, local_shape.*, local_tensor._shape }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); } const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); @@ -218,95 +328,11 @@ noinline fn manualComputationInternal( // Returned tensors describe the operation's global output shapes and sharding. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape = output_shape; + output_tensor_ptr._shape._sharding = output_shape._sharding; } return local_outputs; } -const ChosenMesh = union(enum) { - mesh: *const Sharding.Mesh, - incompatible: [2]u32, -}; - -fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ChosenMesh { - var mesh: ?*const Sharding.Mesh = null; - var mesh_source_id: u32 = @intCast(inputs.len); - for (0.., inputs) |i, input| { - if (input.shape()._sharding.mesh) |shape_mesh| { - if (mesh) |m| { - if (m != shape_mesh) - return .{ .incompatible = .{ mesh_source_id, @intCast(i) } }; - } else { - mesh = shape_mesh; - mesh_source_id = @intCast(i); - } - } - } - return .{ .mesh = mesh orelse fallback }; -} - -test chooseCommonMesh { - const fallback: Sharding.Mesh = .initTest(.fallback, .{.all}); - const first: Sharding.Mesh = .initTest(.first, .{ .data, .model }); - const second: Sharding.Mesh = .initTest(.second, .{ .experts, .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); - const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); - - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); -} - -fn collectManualAxes(mesh: *const Sharding.Mesh, inputs: []const *const Tensor) stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) { - var used: [Sharding.MAX_MESH_RANK]bool = @splat(false); - for (inputs) |input| { - const sharding = input.shape()._sharding; - if (sharding.mesh == null) continue; - std.debug.assert(sharding.mesh.? == mesh); - for (0..input.rank()) |dim| { - if (sharding.partition.get(dim).meshAxis()) |axis| used[axis] = true; - } - } - - var axes: stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) = .empty; - for (mesh.logical.axes.constSlice(), 0..) |axis, i| { - if (used[i]) axes.appendAssumeCapacity(axis); - } - return axes; -} - -test collectManualAxes { - const mesh: Sharding.Mesh = .initTest(.manual, .{ .data, .model, .unused }); - var data = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .data }); - var model = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const open = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .open }); - - const axes = collectManualAxes(&mesh, &.{ &model, &replicated, &data, &model, &open }); - data = replicated; - model = replicated; - try std.testing.expectEqual(@as(usize, 2), axes.len); - try std.testing.expectEqualStrings("data", std.mem.span(axes.get(0))); - try std.testing.expectEqualStrings("model", std.mem.span(axes.get(1))); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{ &replicated, &open }).len); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{}).len); - - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const expected = shardy.ManualAxesAttr.get(ctx, &.{ - .init(ctx, "data"), - .init(ctx, "model"), - }).attribute(); - try std.testing.expect(shardy_.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); -} - const gspmd = struct { fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); @@ -349,54 +375,172 @@ const gspmd = struct { } }; -const shardy_ = struct { - pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { - var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; - for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); - return shardy.ManualAxesAttr.get(mlir_ctx, attrs.constSlice()).attribute(); - } -}; - -test "manualComputation passes local output shapes to the callback" { +test manualComputation { + const zml = @import("../zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const mesh = zml.testing.tpMesh(); - const partitions = mesh.numPartitionsForLogicalAxis(.model); - if (partitions == 1) return error.SkipZigTest; - var compiler: Compiler = .init(allocator, io, platform, .{ .meshes = &.{mesh} }); - defer compiler.deinit(); - compiler.activate(); - defer compiler.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - defer block.deinit(); - const scope = compiler.pushBlock(block); - defer scope.pop(); - - const global_shape = Shape.init(.{ .b = 4, .h = 3 * partitions, .hd = 128 }, .bf16) - .withPartitioning(mesh, .{ .h = .model }); - const local_shape = Shape.init(.{ .b = 4, .h = 3, .hd = 128 }, .bf16); - const input = Tensor.zeroes(global_shape).withPartitioning(mesh, .{ .h = .model }); - var received_shape: ?Shape = null; - const Context = struct { - input: Tensor, - received_shape: *?Shape, + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, fn body(self: @This(), output_shape: Shape) Tensor { - self.received_shape.* = output_shape; - return self.input; + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); } }; - const output = manualComputation( - Context.body, - .{ .input = input, .received_shape = &received_shape }, - global_shape, - ); - try std.testing.expect(received_shape != null); - try std.testing.expect(received_shape.?.eql(local_shape)); - try std.testing.expect(output.shape().eql(global_shape)); - try std.testing.expectEqualDeep(global_shape._sharding, output.shape()._sharding); + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} + +test "partitionSpecs preserves optional tensors and active cache variants" { + const zml = @import("../zml.zig"); + const Logical = Sharding.Partitioning.Logical; + const Inputs = struct { + layers: []const Tensor, + bias: ?Tensor, + cache: union(enum) { disabled, values: [2]Tensor }, + scale: f32, + }; + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); + var inputs: Inputs = .{ + .layers = &.{ tensor, tensor }, + .bias = null, + .cache = .{ .values = .{ tensor, tensor } }, + .scale = 2, + }; + const heads: Logical = .parse(.{ .h = .model }); + const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); + const expected: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{ heads, heads }, + .bias = null, + .cache = .{ .values = .{ heads, heads } }, + }; + try std.testing.expectEqualDeep(expected, specs); + + inputs.layers = &.{}; + inputs.bias = tensor; + inputs.cache = .disabled; + const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); + const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{}, + .bias = .replicated, + .cache = .disabled, + }; + try std.testing.expectEqualDeep(expected_replicated, replicated); } diff --git a/zml/ops/manual_computation2.zig b/zml/ops/manual_computation2.zig deleted file mode 100644 index 2d3f0aec..00000000 --- a/zml/ops/manual_computation2.zig +++ /dev/null @@ -1,465 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const shardy = @import("mlir/dialects/shardy"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../Compiler.zig"); -const meta = @import("../meta.zig"); -const mlirx = @import("../mlirx.zig"); -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const Tensor = @import("../tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/Compiler"); - -/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. -/// Returns the type of the `partitioning` argument to `manualComputation`. -/// -/// * `in_specs` specifies the sharding of each tensor in the body's first argument. -/// It follows the argument's structure, omitting non-tensor metadata. -/// * `out_specs` specifies the sharding of each shape in the body's second argument. -/// It follows the output shapes' structure and can be omitted for a `void` argument. -/// -/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as -/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across -/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` -/// replicates the entire tensor. The specs must not contain open dimensions. -/// Slice lengths, optional presence and active union tags must match the arguments. -/// -/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single -/// output `Shape`, the partitioning can be declared as: -/// -/// ```zig -/// const partitioning: ManualPartitioning(body) = .{ -/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, -/// .out_specs = .parse(.{ .hidden = .model }), -/// }; -/// ``` -pub fn ManualPartitioning(comptime body_fn: anytype) type { - const Logical = Sharding.Partitioning.Logical; - - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - - const InputSpecs = meta.MapRestrict(Tensor, Logical).map(Inputs); - const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); - - if (OutputSpecs == void) return struct { - in_specs: InputSpecs, - out_specs: void = {}, - }; - - return struct { - in_specs: InputSpecs, - out_specs: OutputSpecs, - }; -} - -/// Describes a computation on local shards of tensors distributed across `mesh`. -/// ZML operations normally describe work on global tensors. The partitioner -/// distributes the inputs according to `in_specs`, and `body_fn` describes the work -/// performed on each shard. The results are exposed to the surrounding graph as -/// global tensors with the sharding declared by `out_specs`. -/// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: -/// - `I` contains the local input tensors and any metadata needed by the body. -/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. -/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes -/// in traversal order. -/// * `inputs` contains the global tensors and metadata. The partitioner reshards -/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. -/// * `output_shapes` contains the global output shapes. The body receives their -/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. -/// * `mesh` is the logical mesh whose axis names are referenced by the specs. -/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. -/// Each partitioned dimension must be divisible by the size of its mesh axis. -/// -/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives -/// the body a dimension of size 8. A replicated dimension keeps its global size. -/// The caller supplies the global output shapes because the body only describes -/// local results: their shapes alone do not determine the global tensor shapes. -pub fn manualComputation2( - comptime body_fn: anytype, - inputs: stdx.meta.FnParam(body_fn, 0), - output_shapes: stdx.meta.FnParam(body_fn, 1), - mesh: *const Sharding.Mesh, - partitioning: ManualPartitioning(body_fn), -) stdx.meta.FnReturn(body_fn) { - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - const Outputs = stdx.meta.FnReturn(body_fn); - - const compiler = Compiler.current(); - var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // The body may return containers borrowed from its inputs or output shapes. - const body_arena = compiler.arena.allocator(); - - var owned_inputs: Inputs = undefined; - meta.mapAlloc((struct { - fn copy(_: void, tensor: Tensor) Tensor { - return tensor; - } - }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); - - const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); - stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); - - for (flat_input_tensors, in_specs) |tensor, spec| { - tensor._shape = tensor.shape().withPartitioning(mesh, spec); - stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); - } - - var local_output_shapes: OutputShapes = undefined; - meta.mapAlloc((struct { - fn copy(_: void, shape: Shape) Shape { - return shape; - } - }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); - - const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); - const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); - stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); - - const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); - for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { - global.* = local.withPartitioning(mesh, spec); - stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); - local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - - const WrappedInputs = struct { - inputs: *const Inputs, - output_shapes: *const OutputShapes, - - fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { - const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); - const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); - - outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - - return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); - } - }; - - var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; - - _ = manualComputationInternal( - compiler, - arena, - mesh, - @typeName(@TypeOf(body_fn)), - &wrapped_inputs, - flat_input_tensors, - @ptrCast(&outputs_struct), - flat_output_shapes, - WrappedInputs.wrap, - ) catch |err| switch (err) { - error.OutOfMemory => @panic("OOM"), - }; - - return outputs_struct; -} - -const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; - -/// Reified version of manualComputation2. -/// This one is compiled exactly once, user code just need to compile the `collectPtrs` -noinline fn manualComputationInternal( - compiler: *Compiler, - arena: std.mem.Allocator, - mesh: *const Sharding.Mesh, - functor_name: []const u8, - input_struct: *anyopaque, - input_tensors: []*Tensor, - output_struct: *anyopaque, - global_output_shapes: []const Shape, - local_fn: LocalComputationFn, -) error{OutOfMemory}![]*Tensor { - const global_scope = compiler.currentScope(); - const mlir_ctx = compiler.mlir_ctx; - - log.debug("manualComputation2({s}) will use mesh {s}", .{ functor_name, mesh.name }); - - const global_input_shapes = try arena.alloc(Shape, input_tensors.len); - for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - - const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); - for (input_types, input_tensors) |*input_ty, tensor_ptr| { - const global = tensor_ptr._shape; - const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); - log.debug("\t- {f} -> {f}", .{ global, local }); - input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); - } - const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); - @memset(input_locs, compiler.unknown_location); - - const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); - for (input_values, input_tensors) |*input_val, tensor_ptr| { - if (compiler.partitioner == .gspmd) { - tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); - } - input_val.* = global_scope.value(tensor_ptr.*); - } - - const manual_block = switch (compiler.partitioner) { - .shardy => mlir.Block.init(input_types, input_locs), - .gspmd => global_scope.block, - }; - errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); - - const local_scope = compiler.pushBlock(manual_block); - defer local_scope.pop(); - for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { - const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above - tensor_ptr.* = switch (compiler.partitioner) { - .shardy => Tensor._result(local, manual_block.argument(i)), - .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), - }; - } - - compiler.manual_computation_depth += 1; - defer compiler.manual_computation_depth -= 1; - - const local_outputs = try local_fn(arena, input_struct, output_struct); - stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation2 body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { - const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); - stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation2 body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); - } - - const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |output_shape, i| { - global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); - } - - const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); - for (local_output_values, local_outputs) |*output_val, tensor_ptr| { - output_val.* = local_scope.value(tensor_ptr.*); - } - - const op: *const mlir.Operation = switch (compiler.partitioner) { - .shardy => op: { - // Shardy creates a sdy.manual_computation block. - _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); - - const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); - for (out_mesh_attrs, global_output_shapes) |*attr, shape| { - attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); - } - - break :op shardy.manual_computation( - mlir_ctx, - input_values, - global_output_types, - manual_block, - try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), - .get(mlir_ctx, out_mesh_attrs), - try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), - compiler.location, - ).appendTo(global_scope.block); - }, - .gspmd => op: { - // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier - // Shortcut to avoid empty optimization barrier - if (global_output_shapes.len == 0) return &.{}; - - const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); - const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |global_shape, i| { - global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); - } - - break :op stablehlo.optimizationBarrier( - mlir_ctx, - global_values, - global_types, - compiler.location, - ).appendTo(global_scope.block); - }, - }; - - // Returned tensors describe the operation's global output shapes and sharding. - for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { - output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape._sharding = output_shape._sharding; - } - return local_outputs; -} - -const gspmd = struct { - fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { - const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); - const full_to_shard = stablehlo.custom_call( - mlir_ctx, - &.{scope.value(local_tensor)}, - &.{local_type}, - .{ - .call_target_name = "SPMDFullToShardShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), - }, - }, - location, - ).appendTo(scope.block); - return Tensor._result(local_shape, full_to_shard.result(0)); - } - - fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { - const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); - const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); - - const shard_to_full = stablehlo.custom_call( - compiler.mlir_ctx, - &.{local_value}, - &.{global_type}, - .{ - .call_target_name = "SPMDShardToFullShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), - }, - }, - compiler.location, - ).appendTo(scope.block); - return .{ shard_to_full.result(0), global_type }; - } -}; - -test manualComputation2 { - const zml = @import("../zml.zig"); - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const mesh = zml.testing.tpMesh(); - - const LocalSum = struct { - x: Tensor, - bias: Tensor, - scale: i32, - - fn body(self: @This(), output_shape: Shape) Tensor { - return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); - } - - fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { - return manualComputation2( - body, - .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, - x.shape(), - mesh_, - .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, - ); - } - }; - - // Each shard sums two elements and broadcasts the result to its local output shape. - const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); - const values = try allocator.alloc(i32, shape.count()); - defer allocator.free(values); - - for (values, 0..) |*value, i| value.* = @intCast(i + 1); - - var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - const actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - - for (actual.constItems(i32), 0..) |value, i| { - const pair = values[i / 2 * 2 ..][0..2]; - try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); - } - - try std.testing.expect(output.shape().eqlWithTags(shape)); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } -} - -test "manualComputation2 composes across logical and physical meshes" { - const zml = @import("../zml.zig"); - if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - - const Topologies = enum { flat, torus }; - - const Model = struct { - fn increment(x: Tensor, _: Shape) Tensor { - return x.addConstant(1); - } - - fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { - const partitioning: ManualPartitioning(increment) = .{ - .in_specs = .parse(.{ .x = .model }), - .out_specs = .parse(.{ .x = .model }), - }; - - const first = manualComputation2(increment, x, x.shape(), mesh, partitioning); - const converted = first.convert(.f32); - return manualComputation2(increment, converted, converted.shape(), mesh, partitioning); - } - }; - - const shape: Shape = .init(.{ .x = 16 }, .i32); - - var values: [16]i32 = undefined; - var expected: [16]f32 = undefined; - for (&values, &expected, 0..) |*value, *result, i| { - value.* = @intCast(i); - result.* = @floatFromInt(i + 2); - } - - for ([_]Topologies{ .flat, .torus }) |topology| { - const platform = try zml.Platform.init(allocator, io, .cpu, .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, - }); - defer platform.deinit(allocator, io); - - // Fold three physical axes into one logical axis, or split one into three. - const logical: Sharding.LogicalMesh = if (topology == .torus) - try .init(.{.model}, &.{.balanced}, &.{8}) - else - try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); - - const mesh = try platform.registerMesh("manual", logical); - - var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); - try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } - } -} diff --git a/zml/tensor.zig b/zml/tensor.zig index a7782844..a09a5d38 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4813,7 +4813,7 @@ pub const Tensor = struct { defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation2((struct { + ops.manualComputation((struct { input: Tensor, name: []const u8, From a0533da7b87c83ff57f24faae015d0140ffc777b Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 13 Nov 2026 14:13:10 +0100 Subject: [PATCH 06/14] zml/kernel: introduce packages per DSL (#1061) Split `kernel.zig` into one package per DSL --- dsl/fly/README.md | 2 +- mlir/dialects/fly/REGENERATING.md | 2 +- platforms/tpu/ragged_paged.zig | 2 +- zml/BUILD.bazel | 2 +- zml/kernel.zig | 1630 ----------------- zml/kernel/BUILD.bazel | 15 + zml/kernel/cuda_tile/BUILD.bazel | 9 + zml/kernel/cuda_tile/cuda_tile.zig | 163 ++ zml/kernel/cuda_tile/test.zig | 28 + zml/kernel/cute/BUILD.bazel | 9 + zml/kernel/cute/cute.zig | 264 +++ zml/kernel/cute/test.zig | 34 + zml/kernel/fly/BUILD.bazel | 9 + zml/kernel/fly/fly.zig | 248 +++ zml/kernel/fly/test.zig | 149 ++ zml/kernel/kernel.zig | 12 + zml/kernel/metal/BUILD.bazel | 9 + zml/kernel/metal/metal.zig | 237 +++ zml/kernel/metal/test.zig | 71 + zml/kernel/mosaic_tpu/BUILD.bazel | 6 + zml/kernel/mosaic_tpu/mosaic_tpu.zig | 284 +++ zml/kernel/tcl/BUILD.bazel | 9 + zml/kernel/tcl/tcl.zig | 206 +++ zml/kernel/tcl/test.zig | 123 ++ zml/kernel/triton/BUILD.bazel | 6 + zml/kernel/triton/triton.zig | 165 ++ .../flash_attention/metal/flash_attn.zig | 3 +- .../paged_attention/metal/paged_attn.zig | 3 +- zml/zml.zig | 2 +- 29 files changed, 2063 insertions(+), 1639 deletions(-) delete mode 100644 zml/kernel.zig create mode 100644 zml/kernel/BUILD.bazel create mode 100644 zml/kernel/cuda_tile/BUILD.bazel create mode 100644 zml/kernel/cuda_tile/cuda_tile.zig create mode 100644 zml/kernel/cuda_tile/test.zig create mode 100644 zml/kernel/cute/BUILD.bazel create mode 100644 zml/kernel/cute/cute.zig create mode 100644 zml/kernel/cute/test.zig create mode 100644 zml/kernel/fly/BUILD.bazel create mode 100644 zml/kernel/fly/fly.zig create mode 100644 zml/kernel/fly/test.zig create mode 100644 zml/kernel/kernel.zig create mode 100644 zml/kernel/metal/BUILD.bazel create mode 100644 zml/kernel/metal/metal.zig create mode 100644 zml/kernel/metal/test.zig create mode 100644 zml/kernel/mosaic_tpu/BUILD.bazel create mode 100644 zml/kernel/mosaic_tpu/mosaic_tpu.zig create mode 100644 zml/kernel/tcl/BUILD.bazel create mode 100644 zml/kernel/tcl/tcl.zig create mode 100644 zml/kernel/tcl/test.zig create mode 100644 zml/kernel/triton/BUILD.bazel create mode 100644 zml/kernel/triton/triton.zig diff --git a/dsl/fly/README.md b/dsl/fly/README.md index b886988b..687c99dc 100644 --- a/dsl/fly/README.md +++ b/dsl/fly/README.md @@ -81,7 +81,7 @@ const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ ``` Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` -and `zml/kernel.zig`. +and `zml/kernel/fly/test.zig`. Model kernels live beside their consumers: diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md index b66b7abe..9673870f 100644 --- a/mlir/dialects/fly/REGENERATING.md +++ b/mlir/dialects/fly/REGENERATING.md @@ -71,7 +71,7 @@ dialect today; when one appears, extend the generator and this file. `test.zig`. - Callers that no longer compile are the point of generating: an operation whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`dsl/fly/builder.zig`, `zml/kernel.zig`, `zml/kernels/*/fly/`, + the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, and downstream llmd's `llmd/kernels/*/fly/`). - Emit the IR of the fly kernels before and after (for instance llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index 4c1c50ed..6a0fdfae 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -482,7 +482,7 @@ fn flashAttention( } /// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in -/// `zml/kernel.zig` opens the builder, calls this `run`, and finalizes the +/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the /// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 2b14c79f..f52ecdd5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -87,7 +87,6 @@ zig_library( "exe.zig", "floats.zig", "io.zig", - "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", "kernels/moe/api.zig", @@ -114,6 +113,7 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", + "//zml/kernel", "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", diff --git a/zml/kernel.zig b/zml/kernel.zig deleted file mode 100644 index f111f721..00000000 --- a/zml/kernel.zig +++ /dev/null @@ -1,1630 +0,0 @@ -const std = @import("std"); - -const cuda_tile_builder = @import("dsl/cuda_tile/builder"); -const cute_builder = @import("dsl/cute/builder"); -const fly_builder = @import("dsl/fly/builder"); -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); -const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const tcl_builder = @import("dsl/tcl/builder"); -const triton_builder = @import("dsl/triton/builder"); - -const Compiler = @import("Compiler.zig"); -const DataType = @import("dtype.zig").DataType; -const mlirx = @import("mlirx.zig"); -const ops = @import("ops.zig"); -const platform_ = @import("platform.zig"); -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub const triton = struct { - pub const Builder = triton_builder.Builder; - pub const Value = triton_builder.Value; - pub const DType = triton_builder.DType; - pub const FinishError = triton_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(triton_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - debug: bool = false, - /// Bytes for Triton's implicit global scratch argument, across all CTAs. - global_scratch_memory_size: i32 = 0, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try triton_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ttir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ttir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const call_opts: ops.TritonOps = .{ - .debug = opts.debug, - .name = name, - .ir = ttir, - .grid = opts.grid, - .num_stages = opts.num_stages, - .num_warps = opts.num_warps, - .output_operand_aliases = aliases.constSlice(), - .is_tma_allowed = opts.global_scratch_memory_size > 0, - .global_scratch_memory_size = opts.global_scratch_memory_size, - }; - - const triton_results = if (opts.global_scratch_memory_size > 0) blk: { - // XLA maps the final custom-call result to Triton's implicit - // scratch pointer; it must not appear in the TTIR signature. - const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; - const res = ops.triton(inputs_arr, extended, call_opts); - break :blk res[0..spec.outputs.len].*; - } else ops.triton(inputs_arr, outputs_arr, call_opts); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; - return results; - } - }; - } -}; - -pub const mosaic_tpu = struct { - const Builder = mosaic_tpu_builder.Builder; - const DType = mosaic_tpu_builder.DType; - const FinishError = mosaic_tpu_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(mosaic_tpu_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - dynamic_grid_bounds: []const Tensor = &.{}, - }; - - pub const CallOpts = struct { - cfg: ConfigT, - extras: CallExtras = .{}, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finishOpts(&.{}, .{ .canonicalize = true }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const backend_config = buildBackendConfig(cur.allocator, ir, .{ - .vmem_limit_bytes = opts.extras.vmem_limit_bytes, - .disable_bounds_checks = opts.extras.disable_bounds_checks, - .disable_semaphore_checks = opts.extras.disable_semaphore_checks, - .has_communication = opts.extras.has_communication, - }) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); - defer cur.allocator.free(backend_config); - - var values: [spec.inputs.len]*const mlir.Value = undefined; - inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); - - var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (&res_types, spec.outputs) |*r, fname| { - r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); - } - - const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); - - const op = callTpuCustomCall(.{ - .inputs = &values, - .result_types = &res_types, - .backend_config = backend_config, - .aliases = aliases.constSlice(), - .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, - .additional_attributes = opts.extras.additional_attributes, - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - @field(results, fname) = Tensor._result(out, op.result(i)); - } - return results; - } - }; - } - - const CustomCallConfig = struct { - body: []const u8, - has_communication: ?bool = null, - serialization_format: i64 = 1, - needs_layout_passes: bool = true, - shape_invariant_numerics: bool = false, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - }; - - const ScopedMemoryConfig = struct { - memory_space: i64 = 1, - offset: i64 = 0, - size: i64, - }; - - const BackendConfig = struct { - custom_call_config: CustomCallConfig, - scoped_memory_configs: ?[]const ScopedMemoryConfig = null, - }; - - const BackendConfigOpts = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - }; - - fn buildBackendConfig( - allocator: std.mem.Allocator, - ir: [:0]const u8, - opts: BackendConfigOpts, - ) ![]u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - const module = try mlir.Module.parse(ctx, ir); - defer module.deinit(); - - mosaic_tpu_dialect.registerMosaicSerdePass(); - const prev_allow_unregistered = ctx.allowUnregisteredDialects(); - ctx.setAllowUnregisteredDialects(true); - defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); - - const pm = mlir.PassManager.init(ctx); - defer pm.deinit(); - try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); - try pm.runOnOp(module.operation()); - - const Encoder = std.base64.standard.Encoder; - const b64 = b64: { - var bytecode: std.Io.Writer.Allocating = .init(allocator); - defer bytecode.deinit(); - try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); - const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); - break :b64 Encoder.encode(buf, bytecode.written()); - }; - defer allocator.free(b64); - - const config = BackendConfig{ - .custom_call_config = .{ - .body = b64, - .has_communication = opts.has_communication, - .disable_bounds_checks = opts.disable_bounds_checks, - .disable_semaphore_checks = opts.disable_semaphore_checks, - }, - .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| - &.{.{ .size = vmem }} - else - null, - }; - - var json: std.Io.Writer.Allocating = .init(allocator); - errdefer json.deinit(); - - try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - return try json.toOwnedSlice(); - } - - const TpuCustomCallArgs = struct { - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - backend_config: []const u8, - aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, - dynamic_grid_bounds: []const Tensor, - additional_attributes: []const mlir.NamedAttribute, - }; - - fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = Compiler.current(); - - var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; - for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); - all_inputs.appendSliceAssumeCapacity(args.inputs); - - return stablehlo.custom_call( - cur.mlir_ctx, - all_inputs.constSlice(), - args.result_types, - .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = args.backend_config }, - .has_side_effect = false, - .api_version = .original, - .additional_attributes = args.additional_attributes, - .output_operand_aliases = args.aliases, - }, - cur.location, - ).appendTo(cur.currentScope().block); - } -}; - -pub const cuda_tile = struct { - pub const Builder = cuda_tile_builder.Builder; - pub const Value = cuda_tile_builder.Value; - pub const DType = cuda_tile_builder.DType; - pub const FinishError = cuda_tile_builder.FinishError; - pub const EntryHint = cuda_tile_builder.EntryHint; - pub const Arch = cuda_tile_builder.Arch; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cuda_tile_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i4, .u4 => .i4, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - .f4e2m1 => .f4e2m1fn, - else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// No `num_warps`/`num_stages`: warp and CTA tuning is - /// `optimization_hints` inside the IR (`Opts.hints`). - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Outputs XLA zeroes before the launch, by position in - /// `spec.outputs`, ascending. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(&.{}); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const cute = struct { - pub const AlgebraExpr = cute_builder.AlgebraExpr; - pub const Builder = cute_builder.Builder; - pub const ArgSpec = cute_builder.ArgSpec; - pub const Value = cute_builder.Value; - pub const View = cute_builder.View; - pub const Atom = cute_builder.Atom; - pub const DType = cute_builder.DType; - pub const AlgebraToken = cute_builder.AlgebraToken; - pub const LayoutSpec = cute_builder.LayoutSpec; - pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; - pub const FinishError = cute_builder.FinishError; - pub const TmaConfig = cute_builder.TmaConfig; - pub const TmaFormat = cute_builder.TmaFormat; - pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; - pub const CtaGroup = cute_builder.CtaGroup; - pub const ctaCount = cute_builder.ctaCount; - pub const MBarrierScope = cute_builder.MBarrierScope; - pub const OperandMajorMode = cute_builder.OperandMajorMode; - pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; - pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; - pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; - pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; - pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; - pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cute_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4e2m1fn, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared - /// memory is what the kernel declares. - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - block: [3]i32, - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - // XLA passes one pointer per input then per output; it does - // not check the count against the kernel. - const expected = spec.inputs.len + spec.outputs.len; - if (b.args.len != expected) { - std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); - } - - return b.finish(block); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| - std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .block = opts.block, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } - - fn ProgramSpec(comptime Config: type) type { - return struct { - /// Public host entry selected by XLA. The callback may give the - /// nested CUDA kernel a different (private) symbol. - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A complete CuTe program: one or more `cuda.kernel` operations inside - /// a `gpu.module`, followed by a public `func.func` which constructs the - /// launch-time CuTe objects and calls `cuda.launch_ex`. - /// - /// This is the source-level equivalent of invoking a Python `@cute.jit` - /// object whose `__call__` builds TMA descriptors and launches an - /// `@cute.kernel`. XLA executes the generated host function on its CUDA - /// stream, so the custom call deliberately carries no external grid or - /// block dimensions. - pub fn Program( - comptime ConfigT: type, - comptime spec: ProgramSpec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.openProgram(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - return b.finishProgram(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .zeroed_outputs = opts.zeroed_outputs, - .scalars = opts.scalars, - .symmetric_workspace = opts.symmetric_workspace, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const tcl = struct { - pub const Builder = tcl_builder.Builder; - pub const TensorOperation = tcl_builder.TensorOperation; - pub const Value = tcl_builder.Value; - pub const Axis = tcl_builder.Axis; - pub const Expr = tcl_builder.Expr; - pub const Dram = tcl_builder.Dram; - pub const Context = tcl_builder.Context; - pub const DType = tcl_builder.DType; - pub const VeOp = tcl_builder.VeOp; - pub const ReduceMode = tcl_builder.ReduceMode; - pub const Predicate = tcl_builder.Predicate; - pub const Tactic = tcl_builder.Tactic; - pub const FinishError = tcl_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(tcl_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .bool, - .i4 => .i4, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4_e2, - .f8e4m3fn => .f8_e4, - .f8e5m2 => .f8_e5, - else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A TCL function compiled by the Furiosa plugin as one native unit: - /// `declareArgs` declares the inputs in order, `ret` the outputs. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - return build(allocator, cfg, null, null, null); - } - - /// `updated`, when given, receives the input each output updates - /// in place (`Builder.update`), or null. - fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try Builder.open(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - - const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); - if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { - std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); - } - - if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); - if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); - if (updated) |u| for (u, results) |*dst, r| { - dst.* = b.updatedArgument(r); - }; - - return b.finish(); - } - - fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { - var same = from(s.dtype()) == t.dtype; - if (t.shards > 1) { - // A partition holds one chip's share of a chip-split DRAM tensor. - var n: i64 = 1; - for (t.axes) |a| n *= a.size; - same = same and n == @as(i64, @intCast(s.count())) * t.shards; - } else { - same = same and s.rank() == t.axes.len; - if (same) for (t.axes, 0..) |a, i| { - same = same and a.size == s.dim(i); - }; - } - - if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - var input_shapes: [spec.inputs.len]Shape = undefined; - inline for (spec.inputs, 0..) |fname, i| { - inputs_arr[i] = @field(inputs, fname); - input_shapes[i] = inputs_arr[i].shape(); - } - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var updated: [spec.outputs.len]?usize = undefined; - const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| - std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ - .output_index = @intCast(output), - .operand_index = @intCast(i), - }); - const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - @field(results, fname) = tensor_results[i]; - } - - return results; - } - }; - } -}; - -pub const metal = struct { - pub const Builder = metal_builder.Builder; - pub const Value = metal_builder.Value; - pub const Type = metal_builder.Type; - pub const Struct = metal_builder.Struct; - pub const ArgSpec = metal_builder.ArgSpec; - pub const LoopOpts = metal_builder.LoopOpts; - pub const FinishError = metal_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(metal_builder.registerDialects); - } - - pub fn from(dt: DataType) Type { - return switch (dt) { - .bool => .bool, - .i8 => .char, - .i16 => .short, - .i32 => .int, - .i64 => .long, - .u8 => .uchar, - .u16 => .ushort, - .u32 => .uint, - .u64 => .ulong, - .f16 => .half, - .bf16 => .bfloat, - .f32 => .float, - else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), - }; - } - - /// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host - /// `extern struct` laid out like a `constant S &` argument. - pub const Constant = struct { - buffer: [256]u8 = undefined, - len: usize = 0, - - pub fn of(value: anytype) Constant { - const raw = std.mem.asBytes(&value); - var c: Constant = .{ .len = raw.len }; - @memcpy(c.buffer[0..raw.len], raw); - return c; - } - - pub fn bytes(self: *const Constant) []const u8 { - return self.buffer[0..self.len]; - } - }; - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - /// The arguments bound to input tensors. - inputs: []const [:0]const u8, - /// The arguments bound to output tensors; one named like an - /// input updates it in place. - outputs: []const [:0]const u8, - /// The `constant` arguments given by value at the call. - constants: []const [:0]const u8 = &.{}, - run: *const fn (*Builder, Config) FinishError!void, - fast_math: bool = true, - }; - } - - /// A Metal kernel: `run` declares its arguments with `declareArgs`, by - /// name, and builds its body. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const Constants = StructOf(spec.constants, Constant); - - pub const CallOpts = struct { - cfg: ConfigT, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - constants: Constants = std.mem.zeroes(Constants), - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - return build(allocator, cfg, &arguments); - } - - fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); - defer b.deinit(); - try spec.run(&b, cfg); - const ir = try b.finish(); - errdefer allocator.free(ir); - try arguments.appendSlice(allocator, b.arguments.items); - return ir; - } - - fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { - for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; - return null; - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const allocator = cur.allocator; - - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - const ir = build(allocator, opts.cfg, &arguments) catch |err| - std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); - var constants: [spec.constants.len][]const u8 = undefined; - inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); - - var bound: [spec.outputs.len]bool = @splat(false); - var bound_inputs: [spec.inputs.len]bool = @splat(false); - var bound_constants: [spec.constants.len]bool = @splat(false); - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - var bindings: std.ArrayList(i32) = .empty; - defer bindings.deinit(allocator); - for (arguments.items) |arg| { - if (arg.kind == .threadgroup) { - // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. - if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); - continue; - } - std.debug.assert(arg.index == bindings.items.len); - const input = indexOf(spec.inputs, arg.name); - const output = indexOf(spec.outputs, arg.name); - if (input) |i| bound_inputs[i] = true; - if (output) |o| bound[o] = true; - const binding: i32 = if (input) |i| blk: { - if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); - break :blk @intCast(i); - } else if (output) |o| - @intCast(spec.inputs.len + o) - else if (indexOf(spec.constants, arg.name)) |k| blk: { - if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); - bound_constants[k] = true; - break :blk -1 - @as(i32, @intCast(k)); - } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); - bindings.append(allocator, binding) catch @panic("OOM"); - } - for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); - for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); - for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); - - const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .threadgroup = opts.threadgroup, - .threadgroup_memory = opts.threadgroup_memory, - .arguments = bindings.items, - .constants = &constants, - .output_operand_aliases = aliases.constSlice(), - }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const fly = struct { - /// Threads per wavefront: 64 on CDNA, 32 on RDNA. - pub fn waveSize(p: *const platform_.Platform) i32 { - const cc = platform_.rocm.computeCapability(p) orelse return 64; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => 64, - .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, - }; - } - - /// The plugin sizes a block in wavefronts, so a thread count has to divide - /// evenly: a kernel's layouts are built for an exact number of threads and - /// spare ones would partition out of range. - fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { - const wave = waveSize(p); - if (@rem(threads, wave) != 0) { - std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); - } - return @divExact(threads, wave); - } - - pub const Builder = fly_builder.Builder; - pub const Value = fly_builder.Value; - pub const DType = fly_builder.DType; - pub const FinishError = fly_builder.FinishError; - pub const TiledCopy = fly_builder.TiledCopy; - pub const TiledMma = fly_builder.TiledMma; - pub const Arch = fly_builder.Arch; - pub const MmaFlavor = fly_builder.MmaFlavor; - pub const fly_rocdl = fly_builder.fly_rocdl; - - /// The matrix atom this device provides, or null when it has none: WMMA - /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. - pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { - const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, - .rdna3, .rdna3_5 => .gfx11_wmma, - .rdna4 => .gfx120x_wmma, - .rdna2 => null, - }; - } - pub const layout = fly_builder.layout; - pub const Layout = fly_builder.Layout; - pub const Tile = fly_builder.Tile; - pub const IntTuple = fly_builder.IntTuple; - pub const L = fly_builder.L; - pub const rowMajor = fly_builder.rowMajor; - pub const colMajor = fly_builder.colMajor; - pub const ordered = fly_builder.ordered; - pub const tile = fly_builder.tile; - pub const it = fly_builder.it; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(fly_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - .f8e5m2fnuz => .f8e5m2fnuz, - else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const arg_names = spec.inputs ++ spec.outputs; - /// By name, as `!fly.memref` views. - pub const Args = StructOf(arg_names, Value); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Threads per block. Converted to the wavefronts the plugin - /// wants using the device's wave size, which is 64 on CDNA and - /// 32 on RDNA, so a kernel states the thread count its layouts - /// were built for and runs on both. - threads: i32, - /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. - waves_per_eu: i32 = 0, - /// Required when the body uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Inputs first, then outputs; see `zeroedOutput`. A no-op - /// inside command buffers. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - /// The arguments `run` was declared with. - pub fn args(b: *Builder) Args { - var out: Args = undefined; - inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); - return out; - } - - /// `zeroed_args` index of an output. - pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); - } - - /// `shapes` is inputs, then outputs. - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { - std.debug.assert(shapes.len == arg_names.len); - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try fly_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - var specs: [arg_names.len]Builder.ArgSpec = undefined; - inline for (arg_names, 0..) |fname, i| { - const shape = shapes[i]; - specs[i] = .{ - .name = fname, - .dtype = from(shape.dtype()), - .dims = if (shape.rank() > 0) shape.dims() else null, - }; - } - try b.declareArgsLow(&specs); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var shapes: [arg_names.len]Shape = undefined; - inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); - inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; - - const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| - std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .num_warps = warpsFor(cur.platform, opts.threads), - .waves_per_eu = opts.waves_per_eu, - .shared_mem_bytes = opts.shared_mem_bytes, - .zeroed_args = opts.zeroed_args, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -const FlyVectorAdd = struct { - const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), - }; - - const K = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); - } -}; - -test "fly kernel emits a module XLA can parse" { - const shapes = [_]Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; - const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); -} - -/// C = A @ B^T with one MFMA-tiled block. -const FlyTiledMma = struct { - const Cfg = struct { flavor: fly.MmaFlavor }; - const K = fly.Kernel(Cfg, .{ - .name = "tiled_mma", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); - } -}; - -test "fly kernels run on rocm" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward`, upload both inputs, run, and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); - defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); - defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); - defer bb.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // vector add: elementwise over a ragged (100, 1000), predicated at the edge. - { - const M, const N = .{ 100, 1000 }; - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, - .threads = 8 * 16, - }).c; - } - }; - - const ha = try allocator.alloc(f32, M * N); - defer allocator.free(ha); - const hb = try allocator.alloc(f32, M * N); - defer allocator.free(hb); - for (ha, hb, 0..) |*va, *vb, i| { - va.* = @floatFromInt(i % 977); - vb.* = @floatFromInt(1000 + (i % 313)); - } - - var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); - } - - // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix - // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, - // whose verifier rejects f32 operands, so the operand type and K come from - // the flavour rather than being written into the test. - if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { - inline else => |f| { - const Elem = switch (comptime f.operandDType()) { - .f32 => f32, - .f16 => f16, - else => @compileError("unhandled MMA operand type"), - }; - const M, const N = .{ 64, 64 }; - const Kd = comptime f.blockK(); - const operand_dtype: DataType = switch (comptime f.operandDType()) { - .f32 => .f32, - .f16 => .f16, - else => unreachable, - }; - - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = Shape.init(.{ M, N }, .f32) }, .{ - .cfg = .{ .flavor = f }, - .grid = .{ 1, 1, 1 }, - .threads = comptime f.blockThreads(), - }).c; - } - }; - - var ha: [M * Kd]Elem = undefined; - var hb: [N * Kd]Elem = undefined; - for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; - for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; - - var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); - defer host.free(allocator); - - // f16 operands accumulate in f32, so the error is set by the input - // rounding rather than the sum. - const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; - const out = host.items(f32); - for (0..M) |m| for (0..N) |n| { - var acc: f32 = 0; - for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); - try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); - }; - }, - }; -} - -/// `furiosa.tcl.examples.softmax` over f32. -const TclSoftmax = struct { - const Cfg = struct { b: i64, s: i64, d: i64 }; - const K = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"x"}, - .outputs = &.{"y"}, - .run = run, - }); - - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.x, .{}); - const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); - b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); - } -}; - -test "tcl kernels run on furiosa" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; - - const ir = try TclSoftmax.K.emit(allocator, cfg); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); - - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - - const Mod = struct { - pub fn forward(x: Tensor) Tensor { - return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; - } - }; - const x: Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); - defer exe.deinit(); - - var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); - defer x_buffer.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - - const out = host.items(f32); - var row: usize = 0; - while (row < cfg.b * cfg.s) : (row += 1) { - const in = input[row * cfg.d ..][0..cfg.d]; - var max: f32 = in[0]; - for (in) |v| max = @max(max, v); - var sum: f32 = 0; - for (in) |v| sum += @exp(v - max); - for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); - } -} - -test "cute kernel emits the module the CuTe compiler takes" { - const Cfg = struct { n: i64 }; - const AddOne = cute.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - }); - const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); - var guard = b.openIf(i.lt(cfg.n)); - { - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.close(); - } - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); -} - -test "cuda_tile kernel emits a module XLA can parse" { - const Cfg = struct { n: i64 }; - const AddOne = cuda_tile.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); -} - -/// exp over a tensor split across two chips in one launch, returning the -/// split result and its all_gather. -const TclSplitExp = struct { - const Cfg = struct {}; - const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); - fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - } - fn body(ctx: struct { x: Tensor }, shapes: [2]Shape) [2]Tensor { - const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); - return .{ r.y, r.all }; - } -}; - -// Needs FURIOSA_VISIBLE_DEVICES=0,1. -test "tcl multi-chip kernels run on two furiosa chips" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const tp = zml.testing.tpMesh(); - const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); - const Mod = struct { - pub fn forward(x: Tensor, mesh: *const zml.Sharding.Mesh) [2]Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ - .in_specs = .{ .x = .parse(.{ .c = .model }) }, - .out_specs = .{ .parse(.{ .c = .model }), .replicated }, - }); - return .{ r[0], r[1] }; - } - }; - const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); - defer exe.deinit(); - var hx: [2 * 1024]f32 = undefined; - for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var res: [2]zml.Buffer = undefined; - runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); - defer for (&res) |*r| r.deinit(); - for (res) |r| { - var host = try r.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); - } -} - -/// `out[i] = a[i] * scale + b[i]` over `n` floats. -const MetalAxpy = struct { - const Cfg = struct {}; - const K = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -test "metal kernels run on metal" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - - const ir = try MetalAxpy.K.emit(allocator, .{}); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); - - const platform = zml.testing.env(); - if (platform.target != .metal) return error.SkipZigTest; - - const n = 1000; - const Mod = struct { - pub fn forward(x: Tensor, y: Tensor) Tensor { - return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ (n + 255) / 256, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, - }).out; - } - }; - const x: Tensor = .init(.{n}, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, Tensor.init(.{n}, .f32) }, platform, .{}); - defer exe.deinit(); - - var hx: [n]f32 = undefined; - var hy: [n]f32 = undefined; - for (&hx, &hy, 0..) |*a, *b, i| { - a.* = @floatFromInt(i); - b.* = 1; - } - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); - defer by.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); -} diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel new file mode 100644 index 00000000..61ab44f8 --- /dev/null +++ b/zml/kernel/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "kernel", + srcs = [ + "kernel.zig", + "//zml/kernel/cuda_tile", + "//zml/kernel/cute", + "//zml/kernel/fly", + "//zml/kernel/metal", + "//zml/kernel/mosaic_tpu", + "//zml/kernel/tcl", + "//zml/kernel/triton", + ], +) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..89b7977e --- /dev/null +++ b/zml/kernel/cuda_tile/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cuda_tile", + srcs = [ + "cuda_tile.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..3f6d8734 --- /dev/null +++ b/zml/kernel/cuda_tile/cuda_tile.zig @@ -0,0 +1,163 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +pub const Builder = cuda_tile_builder.Builder; +pub const Value = cuda_tile_builder.Value; +pub const DType = cuda_tile_builder.DType; +pub const FinishError = cuda_tile_builder.FinishError; +pub const EntryHint = cuda_tile_builder.EntryHint; +pub const Arch = cuda_tile_builder.Arch; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig new file mode 100644 index 00000000..c929b0b9 --- /dev/null +++ b/zml/kernel/cuda_tile/test.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +const cuda_tile = @import("cuda_tile.zig"); + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel new file mode 100644 index 00000000..7a613072 --- /dev/null +++ b/zml/kernel/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig new file mode 100644 index 00000000..2e8c919f --- /dev/null +++ b/zml/kernel/cute/cute.zig @@ -0,0 +1,264 @@ +const std = @import("std"); + +const cute_builder = @import("dsl/cute/builder"); +pub const AlgebraExpr = cute_builder.AlgebraExpr; +pub const Builder = cute_builder.Builder; +pub const ArgSpec = cute_builder.ArgSpec; +pub const Value = cute_builder.Value; +pub const View = cute_builder.View; +pub const Atom = cute_builder.Atom; +pub const DType = cute_builder.DType; +pub const AlgebraToken = cute_builder.AlgebraToken; +pub const LayoutSpec = cute_builder.LayoutSpec; +pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; +pub const FinishError = cute_builder.FinishError; +pub const TmaConfig = cute_builder.TmaConfig; +pub const TmaFormat = cute_builder.TmaFormat; +pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; +pub const CtaGroup = cute_builder.CtaGroup; +pub const ctaCount = cute_builder.ctaCount; +pub const MBarrierScope = cute_builder.MBarrierScope; +pub const OperandMajorMode = cute_builder.OperandMajorMode; +pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; +pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; +pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; +pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; +pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; +pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} + +fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A complete CuTe program: one or more `cuda.kernel` operations inside +/// a `gpu.module`, followed by a public `func.func` which constructs the +/// launch-time CuTe objects and calls `cuda.launch_ex`. +/// +/// This is the source-level equivalent of invoking a Python `@cute.jit` +/// object whose `__call__` builds TMA descriptors and launches an +/// `@cute.kernel`. XLA executes the generated host function on its CUDA +/// stream, so the custom call deliberately carries no external grid or +/// block dimensions. +pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig new file mode 100644 index 00000000..f4d457a3 --- /dev/null +++ b/zml/kernel/cute/test.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const cute = @import("cute.zig"); + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel new file mode 100644 index 00000000..57203521 --- /dev/null +++ b/zml/kernel/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "test.zig", + ], +) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig new file mode 100644 index 00000000..b7d6c943 --- /dev/null +++ b/zml/kernel/fly/fly.zig @@ -0,0 +1,248 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); +pub const Builder = fly_builder.Builder; +pub const Value = fly_builder.Value; +pub const DType = fly_builder.DType; +pub const FinishError = fly_builder.FinishError; +pub const TiledCopy = fly_builder.TiledCopy; +pub const TiledMma = fly_builder.TiledMma; +pub const Arch = fly_builder.Arch; +pub const MmaFlavor = fly_builder.MmaFlavor; +pub const fly_rocdl = fly_builder.fly_rocdl; +pub const layout = fly_builder.layout; +pub const Layout = fly_builder.Layout; +pub const Tile = fly_builder.Tile; +pub const IntTuple = fly_builder.IntTuple; +pub const L = fly_builder.L; +pub const rowMajor = fly_builder.rowMajor; +pub const colMajor = fly_builder.colMajor; +pub const ordered = fly_builder.ordered; +pub const tile = fly_builder.tile; +pub const it = fly_builder.it; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +/// Threads per wavefront: 64 on CDNA, 32 on RDNA. +pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; +} + +/// The plugin sizes a block in wavefronts, so a thread count has to divide +/// evenly: a kernel's layouts are built for an exact number of threads and +/// spare ones would partition out of range. +fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); +} + +/// The matrix atom this device provides, or null when it has none: WMMA +/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. +pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig new file mode 100644 index 00000000..ae788d96 --- /dev/null +++ b/zml/kernel/fly/test.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); + +const fly = @import("fly.zig"); +const zml = @import("../../zml.zig"); + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +test "fly kernels run on rocm" { + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig new file mode 100644 index 00000000..204c6204 --- /dev/null +++ b/zml/kernel/kernel.zig @@ -0,0 +1,12 @@ +pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); +pub const cute = @import("cute/cute.zig"); +pub const fly = @import("fly/fly.zig"); +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const tcl = @import("tcl/tcl.zig"); +pub const triton = @import("triton/triton.zig"); + +test { + const std = @import("std"); + std.testing.refAllDecls(@This()); +} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel new file mode 100644 index 00000000..4007beb4 --- /dev/null +++ b/zml/kernel/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + "test.zig", + ], +) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig new file mode 100644 index 00000000..9626a1de --- /dev/null +++ b/zml/kernel/metal/metal.zig @@ -0,0 +1,237 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +pub const Builder = metal_builder.Builder; +pub const Value = metal_builder.Value; +pub const Type = metal_builder.Type; +pub const Struct = metal_builder.Struct; +pub const ArgSpec = metal_builder.ArgSpec; +pub const LoopOpts = metal_builder.LoopOpts; +pub const FinishError = metal_builder.FinishError; +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); +} + +pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; +} + +/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host +/// `extern struct` laid out like a `constant S &` argument. +pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } +}; + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; +} + +/// A Metal kernel: `run` declares its arguments with `declareArgs`, by +/// name, and builds its body. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig new file mode 100644 index 00000000..0a342af0 --- /dev/null +++ b/zml/kernel/metal/test.zig @@ -0,0 +1,71 @@ +const std = @import("std"); + +const metal = @import("metal.zig"); +const zml = @import("../../zml.zig"); + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: zml.Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ba3b6b66 --- /dev/null +++ b/zml/kernel/mosaic_tpu/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = ["mosaic_tpu.zig"], +) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..a09e8368 --- /dev/null +++ b/zml/kernel/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,284 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const Builder = mosaic_tpu_builder.Builder; +const DType = mosaic_tpu_builder.DType; +const FinishError = mosaic_tpu_builder.FinishError; +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; +} + +const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, +}; + +const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, +}; + +const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, +}; + +const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, +}; + +fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, +) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); +} + +const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, +}; + +fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); +} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel new file mode 100644 index 00000000..bc7a0aa2 --- /dev/null +++ b/zml/kernel/tcl/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "tcl", + srcs = [ + "tcl.zig", + "test.zig", + ], +) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig new file mode 100644 index 00000000..023e862f --- /dev/null +++ b/zml/kernel/tcl/tcl.zig @@ -0,0 +1,206 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +pub const Builder = tcl_builder.Builder; +pub const TensorOperation = tcl_builder.TensorOperation; +pub const Value = tcl_builder.Value; +pub const Axis = tcl_builder.Axis; +pub const Expr = tcl_builder.Expr; +pub const Dram = tcl_builder.Dram; +pub const Context = tcl_builder.Context; +pub const DType = tcl_builder.DType; +pub const VeOp = tcl_builder.VeOp; +pub const ReduceMode = tcl_builder.ReduceMode; +pub const Predicate = tcl_builder.Predicate; +pub const Tactic = tcl_builder.Tactic; +pub const FinishError = tcl_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A TCL function compiled by the Furiosa plugin as one native unit: +/// `declareArgs` declares the inputs in order, `ret` the outputs. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; +} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig new file mode 100644 index 00000000..b9ae1875 --- /dev/null +++ b/zml/kernel/tcl/test.zig @@ -0,0 +1,123 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const tcl = @import("tcl.zig"); + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: zml.Tensor) zml.Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const tp = zml.testing.tpMesh(); + const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); + return .{ r[0], r[1] }; + } + }; + const x: zml.Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel new file mode 100644 index 00000000..1e992352 --- /dev/null +++ b/zml/kernel/triton/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "triton", + srcs = ["triton.zig"], +) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig new file mode 100644 index 00000000..6a91a03f --- /dev/null +++ b/zml/kernel/triton/triton.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const triton_builder = @import("dsl/triton/builder"); +pub const Builder = triton_builder.Builder; +pub const Value = triton_builder.Value; +pub const DType = triton_builder.DType; +pub const FinishError = triton_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const ops = @import("../../ops.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; +} diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig index 4faa1e82..de4b68da 100644 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -4,9 +4,8 @@ const std = @import("std"); -const kernel = @import("../../../kernel.zig"); +const metal = @import("../../../kernel/kernel.zig").metal; -const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 337432e7..04f4c1ac 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,9 +3,8 @@ const std = @import("std"); -const kernel = @import("../../../kernel.zig"); +const metal = @import("../../../kernel/kernel.zig").metal; -const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/zml.zig b/zml/zml.zig index 35c2f9df..dafc3a0f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -26,7 +26,7 @@ pub const Exe = exe.Exe; pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); -pub const kernel = @import("kernel.zig"); +pub const kernel = @import("kernel/kernel.zig"); pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; From 4af281a0bed2bb9c6f4de262b36ed635f5663f0a Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:38:46 +0100 Subject: [PATCH 07/14] stdx/flags: cleanup flag_name using mem.replaceScalar (#1068) Just porting 5253ea1362d04d976734e7fdc6e417422e6e693d from #1065 --- stdx/flags.zig | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/stdx/flags.zig b/stdx/flags.zig index 52a9e6a4..d0c4352d 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -591,25 +591,19 @@ fn fields_to_comma_list(comptime E: type) []const u8 { } } +/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. pub fn flag_name(comptime field_name: []const u8) []const u8 { - // TODO(Zig): Cleanup when this is fixed after Zig 0.11. - // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. - return comptime blk: { - assert(!std.mem.eql(u8, field_name, "positional")); - - var result: []const u8 = "--"; - var index = 0; - while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { - result = result ++ field_name[index..][0..i] ++ "-"; - index = index + i + 1; - } - result = result ++ field_name[index..]; - break :blk result; - }; + comptime assert(!std.mem.eql(u8, field_name, "positional")); + comptime var name = field_name[0..field_name.len].*; + comptime std.mem.replaceScalar(u8, &name, '_', '-'); + const dashed = name; + return "--" ++ dashed; } test flag_name { - try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); + try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); + try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); + try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); } fn flag_name_positional(comptime field_name: []const u8) []const u8 { From 3ecf1ff68be97b79519a096719666c0d4fcbb54e Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 13 Nov 2026 17:34:08 +0100 Subject: [PATCH 08/14] zml/moe: mask nonlocal BF16 routes on Furiosa (#1062) Replace nonlocal expert IDs with an in-range placeholder and zero their routing weights before dispatch. This prevents the activation-stationary and blockwise kernels from using -1 in gathers and scatters. Verified BF16 outputs against the CPU reference and coherent Qwen3.5-35B-A3B generation on two and four RNGD cards. --- zml/kernels/moe/tcl/tcl.zig | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 6238d5f6..c26fff1b 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -89,10 +89,14 @@ fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Te const x = input.reshape(.{ cfg.t, cfg.d }); const ids = topk_ids.reshape(.{ cfg.t, cfg.k }); const weights = topk_weights.reshape(.{ cfg.t, cfg.k }); + // These kernels gather every route, and blockwise also scatters each route + // into an expert slot. Give nonlocal routes a valid slot and zero weight. + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); + const safe_ids = valid.select(ids, Tensor.scalar(0, .i32)); + const safe_weights = valid.select(weights, Tensor.scalar(0, .f32)); const out_shape = x.shape(); if (scales) |s| { - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); - const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = valid.select(ids, Tensor.scalar(0, .i32)), .w = valid.select(weights, Tensor.scalar(0, .f32)), .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; + const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; const y = if (cfg.t * cfg.k <= cfg.e) ActivationStationaryFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y else @@ -100,9 +104,9 @@ fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Te return y.reshape(input.shape()); } const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y + ActivationStationary.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y else - Blockwise.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; + Blockwise.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; return y.reshape(input.shape()); } From 4c8f520251d587889d88123604a505c2b5644791 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 13 Nov 2026 17:34:09 +0100 Subject: [PATCH 09/14] zml/moe: support BF16 GEGLU-tanh on Furiosa (#1063) Pass the requested activation to both BF16 TCL MoE kernels and compute tanh GELU on the gate before multiplying by the up projection. This lets Gemma 4 compile and run with its required activation. Verified both kernel paths against a CPU reference and confirmed coherent Gemma 4 generation on two and four RNGD cards. --- zml/kernels/moe/tcl/tcl.zig | 62 ++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index c26fff1b..71653f8e 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -27,22 +27,26 @@ const tcl = zml.kernel.tcl; const Options = @import("../api.zig").Options; const LocalRouting = @import("../api.zig").LocalRouting; +const Activation = enum { swiglu, geglu_tanh }; + /// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. -/// SwiGLU, routing weights applied after the down projection. +/// Supports plain SwiGLU and, for BF16 experts, GEGLU with tanh GELU. pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { const topk_ids = routing.localIds(); stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); const gate_up = gate_up_linear.weight; const down = down_linear.weight; const quantized = gate_up_linear.quantization != null or down_linear.quantization != null; + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (!quantized and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + const activation: Activation = if (swiglu) .swiglu else .geglu_tanh; const scales: ?Scales = if (quantized) .{ .gate_up = fp8Scales(gate_up_linear), .down = fp8Scales(down_linear) } else null; stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32 and (quantized or (gate_up.dtype() == .bf16 and down.dtype() == .bf16)), "TCL MoE takes BF16 activations, BF16/block-128 FP8 experts and i32 ids", .{}); const weights = routing.weights.convert(.f32); const local_weights = if (scales != null) topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)) else weights; - return runExperts(input, topk_ids, local_weights, gate_up, down, scales); + return runExperts(input, topk_ids, local_weights, gate_up, down, scales, activation); } const Scales = struct { gate_up: Tensor, down: Tensor }; @@ -60,13 +64,14 @@ fn fp8Scales(linear: zml.nn.Linear) Tensor { /// Tokens up to which a device runs its experts stationary. pub const expert_stationary_tokens = 64; -fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales) Tensor { +fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales, activation: Activation) Tensor { var cfg: Cfg = .{ .t = input.dim(.t), .k = topk_ids.dim(.topk), .e = gate_up.dim(.expert), .d = input.dim(.d), .i = down.dim(.dout), + .activation = activation, }; stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); if (scales) |s| { @@ -115,7 +120,15 @@ fn scaleDtype(scales: Scales) tcl.DType { return if (scales.gate_up.dtype() == .bf16) .bf16 else .f32; } -pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64, scale_dtype: tcl.DType = .bf16 }; +pub const Cfg = struct { + t: i64, + k: i64, + e: i64, + d: i64, + i: i64, + scale_dtype: tcl.DType = .bf16, + activation: Activation = .swiglu, +}; const Axes = struct { T: tcl.Axis, @@ -246,7 +259,7 @@ fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?b /// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` /// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, comptime fp8: bool) tcl.FinishError!builder.Tensor { +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: Activation, comptime fp8: bool) tcl.FinishError!builder.Tensor { var sliced: [4]tcl.Axis = undefined; var flat: [3]tcl.Axis = undefined; @memcpy(sliced[0..batch.len], batch); @@ -258,7 +271,7 @@ fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, w const f = flat[0 .. batch.len + 1]; const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // Interleaving requires one binary instruction; SwiGLU chains several. + // Interleaving requires one binary instruction; these activations chain several. var op = b.tensorOperation(.{ .tactic = .elementwise }); const g = op.fetch(gate, .{}); const activated = if (fp8) blk: { @@ -273,12 +286,21 @@ fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, w const silu = try op.commit(rounded, .{ .dtype = .bf16 }); op = b.tensorOperation(.{ .tactic = .elementwise }); break :blk op.fetch(silu, .{}); - } else g.sigmoid().mulf(g); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + }; var v = activated.mulf(op.fetch(up, .{})); if (weights) |w| v = v.mulf(op.fetch(w, .{})); return op.commit(v, .{ .dtype = .bf16 }); } +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + /// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): /// IO-bound tactics for activation stationary, compute-bound for blockwise. /// Keep the compiler's default lowering mode and omit the adaptive-einsum and @@ -335,7 +357,7 @@ fn activationStationaryKernel(comptime fp8: bool) type { const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }); const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, fp8); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, cfg.activation, fp8); if (fp8) { const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32); const op = b.tensorOperation(.{}); @@ -423,7 +445,7 @@ fn blockwiseKernel(comptime fp8: bool) type { const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); const scales = try gatherScales(b, a, in, ids, &.{BGi}); const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ BGi, BB }, null, fp8); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, fp8); const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (fp8) .f32 else .bf16); try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); } @@ -506,7 +528,7 @@ pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); const scales = try gatherScales(b, a, in, ids, &.{C}); const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16); - const act = try gated(b, a, h, &.{ C, T }, null, true); + const act = try gated(b, a, h, &.{ C, T }, null, .swiglu, true); const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32); op = b.tensorOperation(.{}); const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); @@ -519,6 +541,11 @@ pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ }); test "tcl fused experts run on furiosa" { + try testFusedExperts(.{ .swiglu = .{} }); + try testFusedExperts(.geglu_tanh); +} + +fn testFusedExperts(activation: zml.moe.Activation) !void { const platform = zml.testing.env(); if (platform.target != .furiosa) return error.SkipZigTest; const allocator = std.testing.allocator; @@ -537,15 +564,15 @@ test "tcl fused experts run on furiosa" { const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ - .activation = .{ .swiglu = .{} }, + .activation = act, .quantize_input = false, .routing_weight_placement = .after_down, }); } }; - var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, .{}); + var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); @@ -597,7 +624,12 @@ test "tcl fused experts run on furiosa" { g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); } - act[n] = g / (1 + @exp(-g)) * u; + const gate = switch (activation) { + .swiglu => g / (1 + @exp(-g)), + .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), + else => unreachable, + }; + act[n] = gate * u; } for (0..d) |m| { var y: f32 = 0; From d2a69b9718e5127d9247491c4b5816d279d78095 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 13 Nov 2026 17:34:09 +0100 Subject: [PATCH 10/14] zml/moe: [refacto] consolidate Furiosa TCL kernel setup (#1066) Separate dispatch from kernel generation. Use one factory to select BF16 and block-128 FP8 kernels by format and schedule, and reuse the shared ActivationKind. --- platforms/furiosa/furiosa.bzl | 4 +- zml/kernels/moe/tcl/BUILD.bazel | 5 +- zml/kernels/moe/tcl/moe.zig | 457 +++++++++++++++++++++++ zml/kernels/moe/tcl/tcl.zig | 623 ++++++-------------------------- 4 files changed, 578 insertions(+), 511 deletions(-) create mode 100644 zml/kernels/moe/tcl/moe.zig diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 623c4f0d..afb26864 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611121233.371.1.9ff7ddf1e19f/zml-furiosa-linux-amd64.tar.zst", - sha256 = "d732b56832b1f009cbe4d808c70c6f8d1dc3857a83eb06804fc53b39bef9dd6b", + url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", + sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", ) return mctx.extension_metadata( diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel index 4057eea3..44b56e08 100644 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -2,5 +2,8 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "tcl", - srcs = ["tcl.zig"], + srcs = [ + "moe.zig", + "tcl.zig", + ], ) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig new file mode 100644 index 00000000..23fb2274 --- /dev/null +++ b/zml/kernels/moe/tcl/moe.zig @@ -0,0 +1,457 @@ +//! TCL kernel schedules for Furiosa RNGD fused experts. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); + +const zml = @import("../../../zml.zig"); +const tcl = zml.kernel.tcl; +const ActivationKind = @import("../api.zig").ActivationKind; + +pub const Format = enum { bf16, fp8_block128 }; + +/// Dispatch in `tcl.zig` uses the first matching schedule. +/// +/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected +/// expert on the whole batch to reuse its weights, with zero routing weight +/// for unrouted pairs. +/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather +/// weights for each route to avoid the overhead of grouping routes. +/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to +/// reuse weights within each block, then gather results back by route. +pub const Schedule = enum { per_route, blockwise, small_batch }; + +pub const Cfg = struct { + num_tokens: i64, + top_k: i64, + local_num_experts: i64, + hidden_size: i64, + expert_intermediate_size: i64, + weight_scale_dtype: tcl.DType = .bf16, + activation: ActivationKind = .swiglu, +}; + +const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; +const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; + +pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { + if (schedule == .small_batch and format != .fp8_block128) { + @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); + } + return tcl.Kernel(Cfg, .{ + .name = "moe_" ++ @tagName(schedule) ++ switch (format) { + .bf16 => "", + .fp8_block128 => "_fp8", + }, + .inputs = switch (format) { + .bf16 => base_inputs, + .fp8_block128 => base_inputs ++ scale_inputs, + }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, schedule); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, format); + switch (schedule) { + .per_route => try perRoute(b, a, in, cfg.activation, format), + .blockwise => try blockwise(b, a, in, cfg, format), + .small_batch => try fp8SmallBatch(b, a, in, cfg), + } + } + }.run, + }); +} + +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.num_tokens), + .K = b.axis("K", cfg.top_k), + .E = b.axis("E", cfg.local_num_experts), + .D = b.axis("D", cfg.hidden_size), + .I = b.axis("I", cfg.expert_intermediate_size), + .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } +}; + +fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { + const weight_dtype: tcl.DType = switch (format) { + .bf16 => .bf16, + .fp8_block128 => .f8_e4, + }; + const spec = .{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, + }; + if (format == .bf16) { + const t = try b.declareArgs(spec); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + } + const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); + const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); + const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); + const t = try b.declareArgs(.{ + .x = spec.x, + .ids = spec.ids, + .w = spec.w, + .gate_up = spec.gate_up, + .down = spec.down, + .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, + .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; +} + +const Args = struct { + x: builder.Tensor, + ids: builder.Tensor, + w: builder.Tensor, + gate_up: builder.Tensor, + down: builder.Tensor, + gate_up_scale: ?builder.Tensor = null, + down_scale: ?builder.Tensor = null, +}; + +const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; + +fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { + if (format == .bf16) return .{}; + const IG = b.axis("IG", @divExact(a.I.size, 128)); + const DG = b.axis("DG", @divExact(a.D.size, 128)); + const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); + var gu_axes: [8]tcl.Axis = undefined; + var dn_axes: [8]tcl.Axis = undefined; + @memcpy(gu_axes[0..batch.len], batch); + @memcpy(dn_axes[0..batch.len], batch); + @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); + @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); + return .{ + .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), + .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), + }; +} + +fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { + var n: usize = 0; + for (axes) |a| { + if (std.mem.eql(u8, a.name, axis.name)) { + storage[n] = group; + n += 1; + storage[n] = lane; + } else storage[n] = a; + n += 1; + } + return storage[0..n]; +} + +/// Block scales multiply each K-block partial before K-block reduction. +/// Only the requested experts are gathered; their bank remains FP8 in HBM. +fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { + if (format == .bf16) { + const op = b.tensorOperation(.{}); + return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); + } + const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); + const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); + const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); + const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); + var x_axes: [16]tcl.Axis = undefined; + var w_axes: [16]tcl.Axis = undefined; + var w2_axes: [16]tcl.Axis = undefined; + var out_axes: [16]tcl.Axis = undefined; + var partial_axes: [16]tcl.Axis = undefined; + const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); + const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); + const out = blockedAxes(to, output, OG, OL, &out_axes); + @memcpy(partial_axes[0..out.len], out); + partial_axes[out.len] = CG; + + // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. + var op = b.tensorOperation(.{}); + const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); + op = b.tensorOperation(.{}); + // Native F32 reciprocal accuracy is qualified separately (within one ULP). + const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); + const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); + op = b.tensorOperation(.{}); + const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); + return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); +} + +/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` +/// (gate first), times `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // The builder can infer the TCL `Interleaving` tactic for two inputs with + // matching axes, such as `gate` and `up`. That tactic allows exactly one + // binary instruction, so force `.elementwise` for this activation chain. + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + const activated = if (format == .fp8_block128) blk: { + if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); + + // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal + // BF16 SiLU write, even through as_dram/as_logical. Make its RNE + // rounding observable as integer bits before multiplying by up. + // Remove once the native reproducer preserves the BF16 boundary. + const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const v = op.fetch(bits, .{}); + const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); + const silu = try op.commit(rounded, .{ .dtype = .bf16 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + break :blk op.fetch(silu, .{}); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), + }; + var v = activated.mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); +} + +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// Use IO-bound tactics for per-route and FP8 small-batch schedules and +/// compute-bound tactics for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { + const config = comptime blk: { + const common = .{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "None", + }; + const tactics = switch (schedule) { + .blockwise => .{ + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + }, + .per_route, .small_batch => .{ + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }, + }; + // The transaction-simulation setting is only emitted for IO-bound schedules. + const common_info = @typeInfo(@TypeOf(common)).@"struct"; + const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; + const Config = @Struct( + .auto, + null, + common_info.field_names ++ tactics_info.field_names, + common_info.field_types ++ tactics_info.field_types, + common_info.field_attrs ++ tactics_info.field_attrs, + ); + var merged: Config = undefined; + for (common_info.field_names) |field| @field(merged, field) = @field(common, field); + for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); + break :blk merged; + }; + b.compilerConfig(config); +} + +/// Gather the selected experts for every token. +fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); + const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); + if (format == .fp8_block128) { + const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); + } else { + // The BF16 recipe weights activations and contracts experts together. + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } +} + +/// Rows per expert block: the next power of two of the routes per expert. +fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); +} + +const blocks_per_chunk = 32; + +/// Group routes into expert-major blocks, then run both projections in a +/// device loop over chunks of blocks. +fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); + const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); + const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); +} + +/// Number of experts processed per device-loop iteration, based on the +/// batch's route count (`cfg.num_tokens * cfg.top_k`). +fn expertsPerChunk(cfg: Cfg) i64 { + const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); + return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); +} + +/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. +/// Load each selected expert once and run it for every token. Routing weights +/// scale the F32 down-projection results before summing the experts and steps. +fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { + const chunk = expertsPerChunk(cfg); + const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; + const C = b.axis("C", chunk); + const Go = b.axis("Go", steps); + const J = b.axis("J", steps * chunk); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Each token's routing weight per expert, and the used experts. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); + op = b.tensorOperation(.{}); + const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); + // A used expert's rank among them, -1 for the others. + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); + + // Slot j holds the used expert of rank j and its tokens' weights; + // slots past the last hold expert 0 with no weight. + const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); + op = b.tensorOperation(.{}); + const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); + // A reduction's condition is read plainly, so broadcast it first. + op = b.tensorOperation(.{}); + const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); + op = b.tensorOperation(.{}); + const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); + const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); + const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); + { + const ids = try b.indexRead(step_experts, loop.iv); + const weights = try b.indexRead(step_gate, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); + const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); + const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); + const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); + op = b.tensorOperation(.{}); + const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); +} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 71653f8e..f05727ce 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -1,59 +1,88 @@ //! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. -//! -//! Few tokens (at most `experts / top_k`): activation stationary, each token -//! gathers the weights of its experts and the down projection contracts over -//! the experts and the intermediate axis at once. -//! -//! More tokens: blockwise. The kernel routes on the device: routes are placed -//! in expert-major blocks of `BB` rows (one-hot, cumulative sums), then a -//! device loop over the used chunks of `BGi` blocks gathers each block's -//! expert weights and runs both projections; outputs are gathered back per -//! route and weighted. -//! -//! FP8 experts split across devices (expert parallel): each device runs its own -//! experts for the routes that reach them and an all-reduce sums the devices' -//! outputs. Up to `expert_stationary_tokens` tokens, expert stationary: a -//! device loop over the used experts, a few at a time, reads each once and -//! runs it for every token, scaled by the token's routing weight (zero for -//! the tokens it does not serve); more tokens take blockwise. const std = @import("std"); -const builder = @import("dsl/tcl/builder"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; +const kernels = @import("moe.zig"); +const api = @import("../api.zig"); -const Activation = enum { swiglu, geglu_tanh }; - -/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` -/// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. -/// Supports plain SwiGLU and, for BF16 experts, GEGLU with tanh GELU. -pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { +/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` +/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is +/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with +/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. +pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { const topk_ids = routing.localIds(); - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); + stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); + const format = expertFormat(gate_up_linear, down_linear); + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + const gate_up = gate_up_linear.weight; const down = down_linear.weight; - const quantized = gate_up_linear.quantization != null or down_linear.quantization != null; - const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); - stdx.debug.assert(swiglu or (!quantized and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); - const activation: Activation = if (swiglu) .swiglu else .geglu_tanh; - const scales: ?Scales = if (quantized) .{ .gate_up = fp8Scales(gate_up_linear), .down = fp8Scales(down_linear) } else null; - stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32 and (quantized or (gate_up.dtype() == .bf16 and down.dtype() == .bf16)), "TCL MoE takes BF16 activations, BF16/block-128 FP8 experts and i32 ids", .{}); + const cfg: kernels.Cfg = .{ + .num_tokens = input.dim(.t), + .top_k = topk_ids.dim(.topk), + .local_num_experts = gate_up.dim(.expert), + .hidden_size = input.dim(.d), + .expert_intermediate_size = down.dim(.dout), + .weight_scale_dtype = switch (format) { + .bf16 => .bf16, + .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), + }, + .activation = std.meta.activeTag(opts.activation), + }; + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); + const weights = routing.weights.convert(.f32); - const local_weights = if (scales != null) topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)) else weights; - return runExperts(input, topk_ids, local_weights, gate_up, down, scales, activation); + const local_weights = switch (format) { + .bf16 => weights, + .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), + }; + return switch (format) { + .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), + .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ + .x = input, + .ids = topk_ids, + .w = local_weights, + .gate_up = gate_up, + .down = down, + .gate_up_scale = s.gate_up, + .down_scale = s.down, + }), + }; } const Scales = struct { gate_up: Tensor, down: Tensor }; +const Format = union(kernels.Format) { + bf16: void, + fp8_block128: Scales, +}; + +fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { + const scheme = gate_up.quantizationScheme(); + stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); + return switch (scheme orelse { + stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); + return .bf16; + }) { + .fp8_block128 => blk: { + const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; + stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); + break :blk .{ .fp8_block128 = scales }; + }, + else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), + }; +} + fn fp8Scales(linear: zml.nn.Linear) Tensor { - const q = linear.quantization orelse std.debug.panic("TCL MoE requires both FP8 projections", .{}); - stdx.debug.assert(q.scheme == .fp8_block128 and linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); + const q = linear.quantization.?; + stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); const w = linear.weight; const scale = q.scales; @@ -61,485 +90,62 @@ fn fp8Scales(linear: zml.nn.Linear) Tensor { return scale; } -/// Tokens up to which a device runs its experts stationary. -pub const expert_stationary_tokens = 64; +// Maximum total input tokens per MoE call before padding for the FP8 +// small-batch schedule. It loads each used expert's weights once and reuses +// them across the batch. +// Larger batches fall back to per-route or blockwise dispatch. +const fp8_small_batch_max_tokens = 64; -fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales, activation: Activation) Tensor { - var cfg: Cfg = .{ - .t = input.dim(.t), - .k = topk_ids.dim(.topk), - .e = gate_up.dim(.expert), - .d = input.dim(.d), - .i = down.dim(.dout), - .activation = activation, - }; - stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); - if (scales) |s| { - cfg.scale_dtype = scaleDtype(s); - if (cfg.t <= expert_stationary_tokens) { - // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two - // panics its adder tree ("Fail to cover valid range", 5 tokens padded - // to 8): pad with unrouted tokens. - var padded = cfg; - padded.t = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.t)) catch unreachable); - const pad = padded.t - cfg.t; - const x = input.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.d }); - const ids = topk_ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); - const weights = topk_weights.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); - const y = ExpertStationaryFp8.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }, .{ .y = x.shape() }, .{ .cfg = padded }).y; - return y.slice(0, .{ .end = cfg.t }).reshape(input.shape()); - } +fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { + return run(format, .small_batch, cfg, inputs); } + if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { + return run(format, .per_route, cfg, inputs); + } + return run(format, .blockwise, cfg, inputs); +} - const x = input.reshape(.{ cfg.t, cfg.d }); - const ids = topk_ids.reshape(.{ cfg.t, cfg.k }); - const weights = topk_weights.reshape(.{ cfg.t, cfg.k }); +fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + const call_cfg, const prepared = if (schedule == .small_batch) + prepareFp8SmallBatch(cfg, inputs) + else + .{ cfg, prepareRoutedInputs(cfg, inputs) }; + + const shape = prepared.x.shape(); + var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; + + if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); + return y.reshape(inputs.x.shape()); +} + +fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { + var prepared = inputs; + prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); + const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); + const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); // These kernels gather every route, and blockwise also scatters each route // into an expert slot. Give nonlocal routes a valid slot and zero weight. - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); - const safe_ids = valid.select(ids, Tensor.scalar(0, .i32)); - const safe_weights = valid.select(weights, Tensor.scalar(0, .f32)); - const out_shape = x.shape(); - if (scales) |s| { - const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; - const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationaryFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y - else - BlockwiseFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y; - return y.reshape(input.shape()); - } - const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationary.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y - else - Blockwise.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; - return y.reshape(input.shape()); + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); + prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); + prepared.w = valid.select(w, Tensor.scalar(0, .f32)); + return prepared; } -fn scaleDtype(scales: Scales) tcl.DType { - stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); - return if (scales.gate_up.dtype() == .bf16) .bf16 else .f32; +fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { + // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two + // panics its adder tree ("Fail to cover valid range", 5 tokens padded + // to 8): pad with unrouted tokens. + var padded = cfg; + padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); + const pad = padded.num_tokens - cfg.num_tokens; + var prepared = inputs; + prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); + prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + return .{ padded, prepared }; } -pub const Cfg = struct { - t: i64, - k: i64, - e: i64, - d: i64, - i: i64, - scale_dtype: tcl.DType = .bf16, - activation: Activation = .swiglu, -}; - -const Axes = struct { - T: tcl.Axis, - K: tcl.Axis, - E: tcl.Axis, - D: tcl.Axis, - I: tcl.Axis, - I2: tcl.Axis, - S2: tcl.Axis, - S1: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.t), - .K = b.axis("K", cfg.k), - .E = b.axis("E", cfg.e), - .D = b.axis("D", cfg.d), - .I = b.axis("I", cfg.i), - .I2 = b.axis("I2", 2 * cfg.i), - .S2 = b.axis("S2", 2), - .S1 = b.axis("S1", 1), - }; - } -}; - -fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime fp8: bool) tcl.FinishError!Args { - const spec = .{ - .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, - .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, - .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.D, a.I } }, - }; - if (!fp8) { - const t = try b.declareArgs(spec); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; - } - const IG = b.axis("IG", @divExact(cfg.i, 128)); - const DG = b.axis("DG", @divExact(cfg.d, 128)); - const I2G = b.axis("I2G", 2 * @divExact(cfg.i, 128)); - const t = try b.declareArgs(.{ - .x = spec.x, - .ids = spec.ids, - .w = spec.w, - .gate_up = spec.gate_up, - .down = spec.down, - .gate_up_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, I2G, DG } }, - .down_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, DG, IG } }, - }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; -} - -const Args = struct { - x: builder.Tensor, - ids: builder.Tensor, - w: builder.Tensor, - gate_up: builder.Tensor, - down: builder.Tensor, - gate_up_scale: ?builder.Tensor = null, - down_scale: ?builder.Tensor = null, -}; - -const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; - -fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis) tcl.FinishError!GatheredScales { - const gu = in.gate_up_scale orelse return .{}; - const IG = b.axis("IG", @divExact(a.I.size, 128)); - const DG = b.axis("DG", @divExact(a.D.size, 128)); - const gate_up = try b.reshape(gu, &.{ a.E, a.S2, IG, DG }, .{}); - var gu_axes: [8]tcl.Axis = undefined; - var dn_axes: [8]tcl.Axis = undefined; - @memcpy(gu_axes[0..batch.len], batch); - @memcpy(dn_axes[0..batch.len], batch); - @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); - @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); - return .{ - .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), - .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), - }; -} - -fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { - var n: usize = 0; - for (axes) |a| { - if (std.mem.eql(u8, a.name, axis.name)) { - storage[n] = group; - n += 1; - storage[n] = lane; - } else storage[n] = a; - n += 1; - } - return storage[0..n]; -} - -/// Block scales multiply each K-block partial before K-block reduction. -/// Only the requested experts are gathered; their bank remains FP8 in HBM. -fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType) tcl.FinishError!builder.Tensor { - if (scale == null) { - const op = b.tensorOperation(.{}); - return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); - } - const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); - const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); - const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); - const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); - var x_axes: [16]tcl.Axis = undefined; - var w_axes: [16]tcl.Axis = undefined; - var w2_axes: [16]tcl.Axis = undefined; - var out_axes: [16]tcl.Axis = undefined; - var partial_axes: [16]tcl.Axis = undefined; - const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); - const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); - const out = blockedAxes(to, output, OG, OL, &out_axes); - @memcpy(partial_axes[0..out.len], out); - partial_axes[out.len] = CG; - - // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. - var op = b.tensorOperation(.{}); - const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); - op = b.tensorOperation(.{}); - // Native F32 reciprocal accuracy is qualified separately (within one ULP). - const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); - const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); - op = b.tensorOperation(.{}); - const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); - return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); -} - -/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` -/// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: Activation, comptime fp8: bool) tcl.FinishError!builder.Tensor { - var sliced: [4]tcl.Axis = undefined; - var flat: [3]tcl.Axis = undefined; - @memcpy(sliced[0..batch.len], batch); - @memcpy(flat[0..batch.len], batch); - sliced[batch.len] = a.S1; - sliced[batch.len + 1] = a.I; - flat[batch.len] = a.I; - const s = sliced[0 .. batch.len + 2]; - const f = flat[0 .. batch.len + 1]; - const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); - const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // Interleaving requires one binary instruction; these activations chain several. - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const g = op.fetch(gate, .{}); - const activated = if (fp8) blk: { - // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal - // BF16 SiLU write, even through as_dram/as_logical. Make its RNE - // rounding observable as integer bits before multiplying by up. - // Remove once the native reproducer preserves the BF16 boundary. - const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const v = op.fetch(bits, .{}); - const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); - const silu = try op.commit(rounded, .{ .dtype = .bf16 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - break :blk op.fetch(silu, .{}); - } else switch (activation) { - .swiglu => g.sigmoid().mulf(g), - .geglu_tanh => geluTanh(g), - }; - var v = activated.mulf(op.fetch(up, .{})); - if (weights) |w| v = v.mulf(op.fetch(w, .{})); - return op.commit(v, .{ .dtype = .bf16 }); -} - -fn geluTanh(g: builder.Value) builder.Value { - const cube = g.mulf(g).mulf(g); - const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); - return inner.tanh().addf(1.0).mulf(g).mulf(0.5); -} - -/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): -/// IO-bound tactics for activation stationary, compute-bound for blockwise. -/// Keep the compiler's default lowering mode and omit the adaptive-einsum and -/// half-VRF settings removed in TCC 2026.5. -fn configure(b: *tcl.Builder, blockwise: bool) void { - if (blockwise) b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, - .dma_preference = 1.2, - .propagate_sparse_axis_from_op = "None", - }) else b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - .propagate_sparse_axis_from_op = "None", - }); -} - -pub const ActivationStationary = activationStationaryKernel(false); -pub const ActivationStationaryFp8 = activationStationaryKernel(true); - -fn activationStationaryKernel(comptime fp8: bool) type { - return tcl.Kernel(Cfg, .{ - .name = if (fp8) "moe_activation_stationary_fp8" else "moe_activation_stationary", - .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, fp8); - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - - const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }); - const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, cfg.activation, fp8); - if (fp8) { - const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32); - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); - } else { - // The BF16 recipe weights activations and contracts experts together. - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } - } - }.run, - }); -} - -/// Rows per expert block: the next power of two of the routes per expert. -pub fn blockRows(cfg: Cfg) i64 { - const per_expert = std.math.divCeil(i64, cfg.t * cfg.k, cfg.e) catch unreachable; - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); -} - -const blocks_per_chunk = 32; - -pub const Blockwise = blockwiseKernel(false); -pub const BlockwiseFp8 = blockwiseKernel(true); - -fn blockwiseKernel(comptime fp8: bool) type { - return tcl.Kernel(Cfg, .{ - .name = if (fp8) "moe_blockwise_fp8" else "moe_blockwise", - .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, true); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, fp8); - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. - const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); - op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); - op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); - - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(used, .{try b.scratchpad(if (fp8) .f32 else .bf16, &.{ Go, BGi, BB, D })}); - { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{BGi}); - const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, fp8); - const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (fp8) .f32 else .bf16); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); - } - }.run, - }); -} - -/// Experts per `ExpertStationary` loop step: more as the routes, so the -/// experts they use, grow. -pub fn stationaryChunk(cfg: Cfg) i64 { - const routes: u64 = @intCast(std.math.divCeil(i64, cfg.t * cfg.k, 16) catch unreachable); - return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); -} - -/// The experts that `ids` use (-1 for none), `stationaryChunk` at a time in a -/// device loop: each is read once and runs for every token. Routing weights -/// scale the F32 down-projection results before summing the experts and steps. -pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ - .name = "moe_expert_stationary_fp8", - .inputs = &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, true); - const chunk = stationaryChunk(cfg); - const steps = std.math.divCeil(i64, @min(cfg.e, cfg.t * cfg.k), chunk) catch unreachable; - const C = b.axis("C", chunk); - const Go = b.axis("Go", steps); - const J = b.axis("J", steps * chunk); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Each token's routing weight per expert, and the used experts. - const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); - op = b.tensorOperation(.{}); - const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); - // A used expert's rank among them, -1 for the others. - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - - // Slot j holds the used expert of rank j and its tokens' weights; - // slots past the last hold expert 0 with no weight. - const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); - op = b.tensorOperation(.{}); - const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); - // A reduction's condition is read plainly, so broadcast it first. - op = b.tensorOperation(.{}); - const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); - op = b.tensorOperation(.{}); - const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); - const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); - const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); - { - const ids = try b.indexRead(step_experts, loop.iv); - const weights = try b.indexRead(step_gate, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{C}); - const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16); - const act = try gated(b, a, h, &.{ C, T }, null, .swiglu, true); - const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32); - op = b.tensorOperation(.{}); - const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); - } - }.run, -}); - test "tcl fused experts run on furiosa" { try testFusedExperts(.{ .swiglu = .{} }); try testFusedExperts(.geglu_tanh); @@ -552,7 +158,8 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { const io = std.testing.io; const bf16 = zml.floats.BFloat16; - // 2 * 4 <= 8 experts is activation stationary, 16 tokens is blockwise. + // With two routes per token and eight experts, four tokens use per-route + // dispatch and sixteen use blockwise dispatch. for ([_]i64{ 4, 16 }) |t| { const k = 2; const e = 8; From 9c9161bac6fd8b6dbbfd285120c6a68b69af2844 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 13 Nov 2026 17:59:02 +0100 Subject: [PATCH 11/14] platforms/oneapi: update pjrt plugin and bundle libmkl_def (#1070) Update PJRT OneAPI plugin with fixes for Triton --- platforms/oneapi/oneapi.bzl | 5 +++-- platforms/oneapi/packages.lock.json | 6 +++--- platforms/oneapi/packages.yaml | 10 ++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 6016e389..5484bec8 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611131511.374.1.f393d1632b4f/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "374ebcad90493fb311a65d8954120b7f2e8a85d52b8641329ee712ef1cb5b23a" ONEAPI_VERSION = "2026.1" @@ -248,6 +248,7 @@ filegroup( name = "mkl_core_runtime", srcs = [ "{ONEAPI_MKL_LIB}/libmkl_core.so.3", + "{ONEAPI_MKL_LIB}/libmkl_def.so.3", "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index a3e69ad4..140ce102 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -216,7 +216,7 @@ "name": "libigc2", "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -227,7 +227,7 @@ "name": "libigdfcl2", "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -249,7 +249,7 @@ "name": "libze1", "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" ], "version": "1.32.0-1~24.04~ppa1" }, diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 272bc02c..5a829ac5 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -15,9 +15,11 @@ sources: - channel: noble main url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z - # IGC 2.41.15 is newer than the graphics snapshot above. + # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics + # packages are pinned below to keep their versions from it. Use snapshots, + # not the live PPA: the PPA deletes superseded packages from its pool. - channel: noble main - url: https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z archs: - "amd64" @@ -43,9 +45,9 @@ packages: - "libnl-genl-3-200" - "libnl-route-3-200" - "libnuma1" - - "libigdgmm12" + - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" - "libigc2 (= 2.41.15-1~24.04)" - "libigdfcl2 (= 2.41.15-1~24.04)" - - "libze-intel-gpu1" + - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" - "libze1" - "zlib1g" From f65072a740fee2b415649f0c41e58935b55410e6 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 22:09:27 +0100 Subject: [PATCH 12/14] platforms/furiosa: use all RNGD chips by default (#1083) --- platforms/furiosa/furiosa.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index ebbd9e3c..ee33fd58 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -50,7 +50,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); - try setEnv("FURIOSA_VISIBLE_DEVICES", "0", 0); const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); log.info("Loading Furiosa plugin: {s}", .{library}); From 8fcff8c462070d06056f8a85ba0c8a81f0f3a636 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 22:27:25 +0100 Subject: [PATCH 13/14] workspace: bump ZML plugins (#1082) Bump plugins to [202611132003.375.1.7d765f57a407](https://github.com/zml/xla-private/tree/202611132003.375.1.7d765f57a407) oneAPI: weight-only FP8 and FP4 (NVFP4, MXFP4) scaled dots now run in a Triton kernel that reads the weights in their stored format. --- platforms/oneapi/oneapi.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 5484bec8..a3c968d3 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611131511.374.1.f393d1632b4f/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "374ebcad90493fb311a65d8954120b7f2e8a85d52b8641329ee712ef1cb5b23a" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" ONEAPI_VERSION = "2026.1" From 1a26a051a50ed8293e8c9edb539a93ef0858ca7a Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:17:19 +0100 Subject: [PATCH 14/14] zml/tensor: return a distinct tensor from toMemory on single-memory platforms (#1087) On cpu, neuron and metal, `toMemory` returned self. Donating a result to a device buffer and its `toMemory` copy to a host buffer then called `reuseBuffer` twice on the same tensor id: the second donation replaced the first, and compilation panicked because both outputs aliased the host buffer. This broke llmd async scheduling on Metal and CPU. Return the same value under a new tensor id instead. --- zml/tensor.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index a09a5d38..ee084835 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -282,7 +282,13 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { - .cpu, .neuron, .metal => return self, + .cpu, .neuron, .metal => { + // A single memory needs no copy, but the result must still be a distinct + // tensor: `reuseBuffer` donations are keyed by tensor id. + var res = _result(self._shape, self.value()); + res._shape._sharding = self._shape._sharding; + return res; + }, .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, }