From 86ab457d0b24d238564e51bc9a057ef1d034b1b1 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:26:31 +0100 Subject: [PATCH 01/47] dsl/cute: add SM100 primitives and CTA-group support (#1007) Add helpers for TMA gather/prefetch, weight-stationary MMA, shared-memory descriptors, packed FP32 arithmetic, async cluster stores, and register limits. Make CTA groups and barrier scopes explicit across TMA, block-scaled MMA, TMEM, and synchronization, including two-CTA operations and multicast completion. Consolidate named CTA barriers and update the MoE and paged-attention callers. Cover instruction operands, descriptor encoding, and MLIR round-trips in builder tests. --- devenv.lock | 18 +- dsl/cute/README.md | 19 +- dsl/cute/builder.zig | 1259 +++++++++++++++-- mlir/dialects/cute_ir/capi.cc | 4 + mlir/dialects/cute_ir/capi.h | 1 + mlir/dialects/cute_ir/cute_ir.zig | 4 + mlir/dialects/llvm/llvm.zig | 1 + mlir/dialects/llvm/ops.zig | 9 + mlir/dialects/llvm/test.zig | 1 + mlir/dialects/nvvm/nvvm.zig | 34 +- zml/kernel.zig | 11 + zml/kernels/moe/cute/sm10x/common.zig | 2 +- .../moe/cute/sm10x/persistent_mxfp4.zig | 58 +- .../moe/cute/sm10x/persistent_nvfp4.zig | 54 +- .../paged_attention/cute/sm10x/attention.zig | 20 +- 15 files changed, 1295 insertions(+), 200 deletions(-) diff --git a/devenv.lock b/devenv.lock index bb3ef660..6c3be332 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,11 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1779636362, - "narHash": "sha256-Q/hy/ybvbe91fjA1MbulUwPjJH9yKaFZnSaTkwLE8S0=", + "lastModified": 1793199252, + "narHash": "sha256-LRgdAdMHL3uIX+kix/oyMzG+gMxzSYOzLrrRaoZthog=", "owner": "cachix", "repo": "devenv", - "rev": "9d6e68dac81fab1f7f2ec6784220151b4fba4b59", + "rev": "0d6c556e5f862c14a75a543e2ee4fdb7acc005ad", "type": "github" }, "original": { @@ -22,11 +22,11 @@ "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1778507786, - "narHash": "sha256-HzSQCKMsMr8r55LwM1JuzIOB+8bzk0FEv6sItKvsfoY=", + "lastModified": 1793043970, + "narHash": "sha256-m14odYTdkBUEkJOiEYs2zjGjn8fcjvdCJsm45+k084U=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "8f24a228a782e24576b155d1e39f0d914b380691", + "rev": "d97234e58d4e90e3bf77c082fc3b546e51200938", "type": "github" }, "original": { @@ -39,11 +39,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1778274207, - "narHash": "sha256-I4puXmX1iovcCHZlRmztO3vW0mAbbRvq4F8wgIMQ1MM=", + "lastModified": 1792812153, + "narHash": "sha256-T2+XBfNbf9pecLUWS4+RtuVNZ3W8K5taG5QdV71mWlk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b3da656039dc7a6240f27b2ef8cc6a3ef3bccae7", + "rev": "674a2028661e550a97400c1cc992b9cca9a668a1", "type": "github" }, "original": { diff --git a/dsl/cute/README.md b/dsl/cute/README.md index c7828abe..946c542d 100644 --- a/dsl/cute/README.md +++ b/dsl/cute/README.md @@ -88,19 +88,20 @@ the actual IR regions. | `cute.arch.griddepcontrol_launch_dependents()` | `b.launchDependents()` | | `x.to(Float32)` | `x.to(.f32)` | | `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(0, n, 1, .{})` | -| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaLoadAtom(...)` | +| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaAtom(...)` | | `cpasync.tma_partition(...)` | `b.tmaPartition(...)` | -| `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, pred)` | +| `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, .{})` | | `cute.make_tiled_mma(atom)` | `b.makeTiledMma(atom, result_type)` | | `tiled_mma.make_fragment_A/B/C(...)` | `b.mmaMakeFragment(...)` | | `cute.gemm(...)` | `b.gemm(...)` | | `cute.arch.alloc_tmem(...)` | `b.allocTmem(...)` | | `cute.arch.retrieve_tmem_ptr(...)` | `b.retrieveTmemPtr(...)` | -* **Layouts are static.** `makeLayout` takes Zig slices and emits one - `cute.static`; the tensor keeps its shape and stride in Zig, so `dim`, - `size` and coordinates need no MLIR queries. Coordinates are tuples of - `Value`s and ints; ints stay static in the `!cute.coord` type. +* **Layouts accept static and dynamic extents.** `makeLayout` takes Zig slices + for a tracked static `Layout`, or nested tuples of integers and SSA `Value`s + for a `View`. Dynamic shape and stride leaves retain their integer widths. + The tracked `Tensor` keeps its shape and stride in Zig; use `makeTensorView` + with dynamic layouts. Integers stay static in coordinate types. * **Stride defaults differ by origin.** `makeLayout` defaults to CuTe's compact left-most stride like `cute.make_layout`; a `tensor` argument defaults to XLA's row-major layout, which is what the buffer holds. @@ -122,7 +123,11 @@ the actual IR regions. copies, tensor-memory fragments, and SM100 block-scaled MMA. CuTe derives their layouts in the compiler, so callers spell out the resulting CuTe type with `View` instead of relying on Python's dynamic type objects. -* **Not covered yet:** dynamic tensor shapes and automatic CuTe layout algebra. +* **TMA tilers follow Python.** `makeTiledTmaAtom` accepts a shape tuple, tile, + or layout, composes it with the GMEM identity layout, and removes the SMEM + stage mode when present. The lower-level `makeTiledTmaLoadAtom` and + `makeTiledTmaStoreAtom` take an already composed coordinate map. `copy` + makes store atoms executable automatically; loads require `tma_bar_ptr`. * **Typed result types.** Operations whose result type CuTe derives (`localTile`, `slice`, `tmaPartition`, ...) take an `*mlir.Type` assembled with `layoutSpec`, `layoutType`, `memrefType`, `coordTensorType`, diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig index d19819fa..926b8419 100644 --- a/dsl/cute/builder.zig +++ b/dsl/cute/builder.zig @@ -51,6 +51,8 @@ pub const AlgebraExpr = union(enum) { static: i64, /// A dynamic leaf known to be a multiple of this, 1 when nothing is known. dynamic: i64, + /// A dynamic integer with an explicit bit width and divisibility. + dynamic_int: struct { width: u8, divisible_by: i64 = 1 }, all, basis: ScaledBasis, tuple: []const AlgebraExpr, @@ -466,8 +468,122 @@ pub const TmaFormat = enum { u16, }; -/// Typed description of a tiled TMA atom: the element type, the bits moved -/// per copy, the basis of one box, and the coordinate layout of the tensor. +/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM90. +pub const CacheHintSm90 = enum(u64) { + evict_normal = 0x1000000000000000, + evict_first = 0x12F0000000000000, + evict_last = 0x14F0000000000000, +}; + +/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM100. +pub const CacheHintSm100 = enum(u64) { + evict_normal = 0x1000000000000000, + evict_first = 0x12F0000000000000, + evict_last = 0x14F0000000000000, +}; + +pub const CtaGroup = enum(u8) { + one = 1, + two = 2, + + fn toNvvm(self: CtaGroup) nvvm.CtaGroup { + return switch (self) { + .one => .cta_1, + .two => .cta_2, + }; + } +}; +pub const MBarrierScope = nvvm.MBarrierScope; +pub const OperandMajorMode = enum(u1) { k = 0, mn = 1 }; + +/// Number of CTAs cooperating on one instruction, as in CuTe's CtaGroup.value. +pub fn ctaCount(group: CtaGroup) u8 { + return @backingInt(group); +} + +pub const Tcgen05SmemSwizzle = enum(u3) { + none = 0, + swizzle_128b_atom_32b = 1, + swizzle_128b = 2, + swizzle_64b = 4, + swizzle_32b = 6, +}; + +/// CuTe's packed SM100 shared-memory descriptor. All offsets are in bytes. +pub const Tcgen05SmemDesc = struct { + pub const Options = struct { + leading_byte_offset: u32 = 0, + stride_byte_offset: u32 = 0, + version: u2 = 1, + base_offset: u3 = 0, + layout: Tcgen05SmemSwizzle = .none, + }; + + pub fn build(b: *Builder, start_address: Value, opts: Options) Value { + std.debug.assert(opts.leading_byte_offset % 16 == 0 and opts.leading_byte_offset < 0x40000); + std.debug.assert(opts.stride_byte_offset % 16 == 0 and opts.stride_byte_offset < 0x40000); + const fields = (@as(u64, opts.leading_byte_offset >> 4) << 16) | + (@as(u64, opts.stride_byte_offset >> 4) << 32) | + (@as(u64, opts.version) << 46) | + (@as(u64, opts.base_offset) << 49) | + (@as(u64, @backingInt(opts.layout)) << 61); + const address = if (start_address.isInt()) start_address else b.ptrToInt(start_address, .i32); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(fields)))); + } +}; + +/// Dense SM100 instruction descriptor; the MMA kind disambiguates dtype encodings. +pub const Tcgen05InstrDesc = struct { + pub const Options = struct { + a_dtype: DType, + b_dtype: DType, + c_dtype: DType = .f32, + m_dim: u16, + n_dim: u16, + a_major: OperandMajorMode = .k, + b_major: OperandMajorMode = .k, + a_negate: bool = false, + b_negate: bool = false, + }; + + pub fn build(comptime opts: Options) u32 { + comptime std.debug.assert(opts.m_dim > 0 and opts.m_dim <= 256 and opts.m_dim % 16 == 0); + comptime std.debug.assert(opts.n_dim > 0 and opts.n_dim <= 256 and opts.n_dim % 8 == 0); + const half_inputs = (opts.a_dtype == .f16 or opts.a_dtype == .bf16) and (opts.b_dtype == .f16 or opts.b_dtype == .bf16); + const fp8_inputs = (opts.a_dtype == .f8e4m3fn or opts.a_dtype == .f8e5m2) and (opts.b_dtype == .f8e4m3fn or opts.b_dtype == .f8e5m2); + const valid = (half_inputs and opts.c_dtype == .f32) or + (opts.a_dtype == .f16 and opts.b_dtype == .f16 and opts.c_dtype == .f16) or + (opts.a_dtype == .f32 and opts.b_dtype == .f32 and opts.c_dtype == .f32) or + (fp8_inputs and (opts.c_dtype == .f16 or opts.c_dtype == .f32)) or + (opts.a_dtype == .i8 and opts.b_dtype == .i8 and opts.c_dtype == .i32); + if (!valid) @compileError("unsupported dense tcgen05 dtype combination"); + const c: u32 = switch (opts.c_dtype) { + .f16 => 0, + .f32 => 1, + .i32 => 2, + else => @compileError("unsupported tcgen05 accumulator dtype"), + }; + return (c << 4) | (inputFormat(opts.a_dtype) << 7) | (inputFormat(opts.b_dtype) << 10) | + (@as(u32, @intFromBool(opts.a_negate)) << 13) | + (@as(u32, @intFromBool(opts.b_negate)) << 14) | + (@as(u32, @backingInt(opts.a_major)) << 15) | + (@as(u32, @backingInt(opts.b_major)) << 16) | + (@as(u32, opts.n_dim >> 3) << 17) | (@as(u32, opts.m_dim >> 4) << 24); + } + + fn inputFormat(comptime dtype: DType) u32 { + return switch (dtype) { + .f16, .f8e4m3fn => 0, + .bf16, .f8e5m2, .i8 => 1, + .f32 => 2, + .f4e2m1fn => 5, + else => @compileError("unsupported tcgen05 input dtype"), + }; + } +}; + +/// Typed description of a non-multicast tiled TMA atom: the element type, +/// bits moved per copy, basis of one box, and coordinate layout of the tensor. pub const TmaConfig = struct { dtype: DType, copy_bits: u32, @@ -475,18 +591,29 @@ pub const TmaConfig = struct { coordinate_layout: LayoutSpec, coordinate_rank: u8, format: TmaFormat = .default, + /// Load completion group: with `.two`, each CTA's load completes on + /// the mbarrier of the pair's even CTA. Ignored by TMA stores. + cta_group: CtaGroup = .one, + mode: enum { tiled, gather4 } = .tiled, }; +pub const CopyBulkTensorTileG2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensor2DGather4G2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensorTileS2GOp = struct {}; + pub const BlockScaledMma = struct { atom: Atom, tiled: Atom, }; -/// SM100 block-scaled MMA instruction: one CTA computes a 128 x `n` tile, -/// FP4 weights as A. The instruction `kind` fixes B, the scales and K. +/// SS, K-major SM100 block-scaled MMA instruction with FP4 weights as A. +/// The instruction `kind` fixes B, the scales and K. pub const BlockScaledMmaConfig = struct { kind: Kind = .mxfp4_mxfp8, + /// M is 128 for one CTA, or 128 or 256 for a CTA pair. + m: u16 = 128, n: u16, + cta_group: CtaGroup = .one, pub const Kind = enum { /// MXFP4 weights x MXFP8 activations: E8M0 scales every 32 @@ -524,8 +651,23 @@ pub const BlockScaledMmaConfig = struct { .nvfp4_nvfp4 => 16, }; } + + fn validate(self: BlockScaledMmaConfig) void { + const m_ok = self.m == 128 or (self.cta_group == .two and self.m == 256); + const n_multiple: u16 = if (self.cta_group == .two) 16 else 8; + if (!m_ok or self.n < n_multiple or self.n > 256 or self.n % n_multiple != 0) { + std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d} on {d} CTAs", .{ self.m, self.n, self.k(), ctaCount(self.cta_group) }); + } + } }; +fn tmaLoadKind(config: TmaConfig) cute_nvgpu.TiledTmaLoad { + return switch (config.cta_group) { + .one => .sm_90, + .two => .sm_100_2sm, + }; +} + fn tmaDataFormat(format: TmaFormat, dtype: DType) cute_nvgpu.TmaDataFormat { return switch (format) { .u4 => .U4, @@ -1019,7 +1161,8 @@ pub const Builder = struct { /// `cute.ptrtoint`: the integer address of a pointer (`i32` for shared /// memory, `i64` for global memory), for hand-written PTX operands. pub fn ptrToInt(self: *Builder, pointer: anytype, dt: DType) Value { - return self.emit(cute_ir.ptrtoint(self.ctx, self.asValue(pointer).inner, dt.toMlir(self.ctx), self.loc())); + const ptr = self.asValue(pointer); + return self.emit(if (ptr.isPtr()) cute_ir.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc()) else llvm.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc())); } /// Reinterpret a pointer without changing its address. This is the typed @@ -1074,6 +1217,7 @@ pub const Builder = struct { const result = switch (e) { .static => |v| if (@hasDecl(A, "getStatic")) A.getStatic(self.ctx, v) else std.debug.panic("{s} has no integer leaf", .{@typeName(A)}), .dynamic => |divisibility| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, 32, divisibility) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), + .dynamic_int => |v| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, v.width, v.divisible_by) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), .all => if (@hasDecl(A, "getUnderscore")) A.getUnderscore(self.ctx) else std.debug.panic("{s} has no `_` leaf", .{@typeName(A)}), .basis => |b| if (@hasDecl(A, "getBasis")) blk: { const modes = self.arena.allocator().alloc(i32, b.mode_count) catch @panic("OOM"); @@ -1238,8 +1382,59 @@ pub const Builder = struct { // ==================== cute.core ==================== - /// `cute.make_layout(shape, stride=...)`, static only. - pub fn makeLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { + fn layoutResult(comptime Shape: type) type { + return switch (@typeInfo(Shape)) { + .pointer => Layout, + else => View, + }; + } + + /// `cute.make_layout(shape, stride=...)`. Slices keep the tracked static + /// Layout API; nested tuples accept mixed integer and SSA shape/stride leaves. + pub fn makeLayout(self: *Builder, shape: anytype, opts: anytype) layoutResult(@TypeOf(shape)) { + if (comptime layoutResult(@TypeOf(shape)) == Layout) return self.makeStaticLayout(shape, .{ + .stride = if (@hasField(@TypeOf(opts), "stride")) opts.stride else null, + }); + var shape_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + var stride_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + const shape_expr = self.coordExpr(&shape_values, shape); + const stride_expr = if (@hasField(@TypeOf(opts), "stride") and @TypeOf(opts.stride) != @TypeOf(null)) self.coordExpr(&stride_values, opts.stride) else blk: { + var stride: AlgebraExpr = .{ .static = 1 }; + var stride_value: ?Value = null; + var index: usize = 0; + break :blk self.compactLayoutStride(shape_expr, shape_values.constSlice(), &index, &stride, &stride_value, &stride_values); + }; + const shape_value = self.emit(cute_ir.make_shape(self.ctx, shape_values.constSlice(), self.shapeType(shape_expr), self.loc())); + const stride_type = (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, stride_expr).attribute() }) catch @panic("invalid stride")).type_(); + const stride_value = self.emit(cute_ir.make_stride(self.ctx, stride_values.constSlice(), stride_type, self.loc())); + const result = self.emit(cute_ir.make_layout(self.ctx, self.layoutType(.{ .shape = shape_expr, .stride = stride_expr }), .{ .shape = shape_value.inner, .stride = stride_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + fn compactLayoutStride(self: *Builder, shape: AlgebraExpr, values: []const *const mlir.Value, index: *usize, stride: *AlgebraExpr, stride_value: *?Value, dynamic: *stdx.BoundedArray(*const mlir.Value, MAX_RANK)) AlgebraExpr { + if (shape == .tuple) { + const elements = self.arena.allocator().alloc(AlgebraExpr, shape.tuple.len) catch @panic("OOM"); + for (shape.tuple, elements) |extent, *element| element.* = self.compactLayoutStride(extent, values, index, stride, stride_value, dynamic); + return .{ .tuple = elements }; + } + if (shape == .static and shape.static == 1) return .{ .static = 0 }; + const result = stride.*; + if (stride_value.*) |v| dynamic.append(v.inner) catch @panic("too many stride leaves"); + if (shape == .static and stride.* == .static) { + stride.static *= shape.static; + } else { + const extent = if (shape == .dynamic or shape == .dynamic_int) blk: { + const v: Value = .{ .inner = values[index.*], .kernel = self }; + index.* += 1; + break :blk v; + } else self.lift(shape.static); + stride_value.* = (stride_value.* orelse self.lift(stride.static)).mul(extent); + stride.* = dynamicExpr(stride_value.*.?); + } + return result; + } + + fn makeStaticLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); const stride = opts.stride orelse self.leftMost(shape); if (stride.len != shape.len) std.debug.panic("makeLayout: stride rank {d} != shape rank {d}", .{ stride.len, shape.len }); @@ -1262,8 +1457,8 @@ pub const Builder = struct { fn coordExpr(self: *Builder, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) AlgebraExpr { const T = @TypeOf(coord); if (T == Value) { - dyn.append(self.lift(coord).to(.i32).inner) catch @panic("makeCoord: too many dynamic leaves"); - return .{ .dynamic = 1 }; + dyn.append(coord.inner) catch @panic("makeCoord: too many dynamic leaves"); + return dynamicExpr(coord); } if (T == AlgebraToken) return switch (coord) { .all => .all, @@ -1282,6 +1477,12 @@ pub const Builder = struct { } } + fn dynamicExpr(value: Value) AlgebraExpr { + std.debug.assert(value.isInt()); + const width = dtypeBitwidth(value.dtype()); + return if (width == 32) .{ .dynamic = 1 } else .{ .dynamic_int = .{ .width = @intCast(width) } }; + } + /// A `!cute.coord` from nested tuples of `Value`s, integers, and `.all` /// placeholders. Integer leaves stay static in the dependent CuTe type. pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { @@ -1730,8 +1931,55 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, pred: ?Value) void { - self.emitVoid(cute_ir.copy(self.ctx, atom.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (pred) |p| p.inner else null }, self.loc())); + pub const CopyOptions = struct { pred: ?Value = null, tma_bar_ptr: ?Value = null, cache_policy: ?Value = null }; + + pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, opts: CopyOptions) void { + var exec = atom; + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaStoreType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA store atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + if (opts.tma_bar_ptr) |mbar| { + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .gather4, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable gather atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } else if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + exec = self.setTmaBarrier(exec, mbar); + } + if (opts.cache_policy) |policy| { + const field = if (exec.type_().isA(cute_nvgpu.CopyAtomTmaLoadType) != null) + (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else if (exec.type_().isA(cute_nvgpu.CopyAtomTmaStoreType) != null) + (cute_nvgpu.AtomCopyFieldTmaStoreAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else + @panic("cache_policy requires an executable TMA copy atom"); + const updated = self.emit(cute_nvgpu.atom_set_value(self.ctx, exec.inner, policy.to(.i64).inner, exec.type_(), field, self.loc())); + exec = .{ .inner = updated.inner, .kernel = self }; + } + self.emitVoid(cute_ir.copy(self.ctx, exec.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); } pub fn gemm(self: *Builder, tiled: Atom, d: anytype, a: anytype, rhs: anytype, c: anytype) void { @@ -1762,9 +2010,67 @@ pub const Builder = struct { // the kernel receives them as grid-constant arguments and makes them // executable before issuing copies. + pub const TmaAtomOptions = struct { gmem_coord_tensor: ?View = null }; + + fn inferValue(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attributes: []const mlir.NamedAttribute) Value { + return self.emit(mlir.Operation.make(self.ctx, name, .{ + .operands = .{ .flat = operands }, + .attributes = attributes, + .result_type_inference = true, + .location = self.loc(), + })); + } + + fn valueShape(self: *Builder, value: Value) Value { + if (value.type_().isA(cute_ir.ShapeType) != null) return value; + return self.inferValue("cute.get_shape", &.{value.inner}, &.{}); + } + + fn shapeRank(shape: Value) usize { + return shape.type_().isA(cute_ir.ShapeType).?.attr().isA(cute_ir.ShapeAttr).?.rank(); + } + + /// Common CuTe TMA atom constructor. Gather uses the index tensor's layout + /// at construction and passes data/index tensors together to partition/copy. + pub fn makeTiledTmaAtom(self: *Builder, op: anytype, gmem: anytype, smem_layout: anytype, cta_tiler: anytype, config: TmaConfig, opts: TmaAtomOptions) TmaDescriptor { + const Tiler = @TypeOf(cta_tiler); + const tiler = if (Tiler == Value or Tiler == View or Tiler == Layout) + self.asValue(cta_tiler) + else if (@typeInfo(Tiler) == .pointer) + self.makeShape(self.dimsExpr(cta_tiler)).value() + else + self.makeShape(cta_tiler).value(); + var smem = self.asValue(smem_layout); + const tiler_rank = shapeRank(self.valueShape(tiler)); + const smem_rank = shapeRank(self.valueShape(smem)); + std.debug.assert(smem_rank == tiler_rank or smem_rank == tiler_rank + 1); + if (smem_rank == tiler_rank + 1) { + const modes = self.arena.allocator().alloc(i32, tiler_rank) catch @panic("OOM"); + for (modes, 0..) |*mode, i| mode.* = @intCast(i); + smem = self.inferValue("cute.select", &.{smem.inner}, &.{.named(self.ctx, "mode", .denseArray(self.ctx, .i32, modes))}); + } + const shape = self.valueShape(self.asValue(gmem)); + const identity = self.inferValue("cute.make_identity_layout", &.{shape.inner}, &.{}); + const cta_map = self.inferValue("cute.composition", &.{ identity.inner, tiler.inner }, &.{}); + var cfg = config; + cfg.mode = .tiled; + if (@TypeOf(op) == CopyBulkTensorTileS2GOp) return self.makeTiledTmaStoreAtom(gmem, smem, cta_map, cfg); + cfg.cta_group = op.cta_group; + if (@TypeOf(op) == CopyBulkTensorTileG2SOp) return self.makeTiledTmaLoadAtom(gmem, smem, cta_map, cfg); + if (@TypeOf(op) != CopyBulkTensor2DGather4G2SOp) @compileError("unsupported TMA copy operation"); + cfg.mode = .gather4; + const indices = opts.gmem_coord_tensor orelse @panic("gather4 requires gmem_coord_tensor"); + const index_type = indices.type_().isA(cute_ir.MemRefType) orelse @panic("gather4 indices must be a tensor"); + const layout = self.emit(cute_ir.get_layout(self.ctx, indices.inner, index_type.layout(), self.loc())); + const kind: cute_nvgpu.GatherScatterTmaLoad = if (cfg.cta_group == .two) .sm_100_2sm else .sm_100; + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_2d_gather4_tma_load(self.ctx, self.asValue(gmem).inner, layout.inner, smem.inner, cta_map.inner, self.tmaLoadAtomType(cfg), self.coordTensorType(cfg.coordinate_rank, cfg.coordinate_layout), (cute_nvgpu.GatherScatterTmaLoadAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, cfg) }, self.loc())); + return .{ .atom = .{ .inner = results[0].inner, .kernel = self }, .tensor = .{ .inner = results[1].inner, .kernel = self } }; + } + /// Build the host-visible TMA descriptor and its coordinate tensor, whose /// types are assembled from `config`. pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + std.debug.assert(config.mode == .tiled); const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_load( self.ctx, self.asValue(gmem).inner, @@ -1772,7 +2078,7 @@ pub const Builder = struct { self.asValue(cta_map).inner, self.tmaLoadAtomType(config), self.coordTensorType(config.coordinate_rank, config.coordinate_layout), - (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = .sm_90 }) catch @panic("invalid tiled TMA load")).attribute(), + (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = tmaLoadKind(config) }) catch @panic("invalid tiled TMA load")).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, config) }, self.loc(), )); @@ -1802,8 +2108,15 @@ pub const Builder = struct { /// Host-side tiled TMA load atom type described by `config`. pub fn tmaLoadAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + if (config.mode == .gather4) return (cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType.get(self.ctx, .{ + .kind = if (config.cta_group == .two) .sm_100_2sm else .sm_100, + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = if (config.format == .default) null else tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid gather TMA atom")).type_(); return (cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(self.ctx, .{ - .kind = .sm_90, + .kind = tmaLoadKind(config), .valType = config.dtype.toMlir(self.ctx), .copyBits = @intCast(config.copy_bits), .tmaGbasis = self.layoutType(config.global_basis), @@ -1818,8 +2131,8 @@ pub const Builder = struct { return (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ .valType = config.dtype.toMlir(self.ctx), .copyBits = @intCast(config.copy_bits), - .mode = .tiled, - .numCta = 1, + .mode = if (config.mode == .gather4) .gather4 else .tiled, + .numCta = ctaCount(config.cta_group), .gStride = self.emptyStrideType(), .tmaGbasis = self.layoutType(config.global_basis), }) catch @panic("invalid TMA load atom")).type_(); @@ -1851,6 +2164,12 @@ pub const Builder = struct { return (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, .{ .tuple = &.{} }).attribute() }) catch @panic("invalid CuTe stride")).type_(); } + /// Generic address (i64) of the tensor map in an executable tiled TMA atom. + pub fn tmaDescriptorAddress(self: *Builder, descriptor: Atom) Value { + const ptr = self.emit(cute_nvgpu.get_tma_desc_addr(self.ctx, descriptor.inner, (cute_ir.PtrType.get(self.ctx, .{ .valueType = (cute_nvgpu.TmaDescriptorTiledType.get(self.ctx, .{}) catch unreachable).type_(), .addressSpace = .generic, .alignment = 64 }) catch unreachable).type_(), self.loc())); + return self.ptrToInt(ptr, .i64); + } + pub fn prefetchTmaDesc(self: *Builder, descriptor: Atom) void { self.emitVoid(cute_nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); } @@ -1861,11 +2180,11 @@ pub const Builder = struct { } /// Attach the mbarrier that an executable TMA load atom completes. - pub fn setTmaBarrier(self: *Builder, atom: Atom, barrier: Value) Atom { + pub fn setTmaBarrier(self: *Builder, atom: Atom, mbar: Value) Atom { const result = self.emit(cute_nvgpu.atom_set_value( self.ctx, atom.inner, - barrier.inner, + mbar.inner, atom.type_(), (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .tma_bar }) catch @panic("invalid TMA load field")).attribute(), self.loc(), @@ -1915,10 +2234,10 @@ pub const Builder = struct { } fn blockScaledAtomType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { - if (cfg.n < 8 or cfg.n > 256 or cfg.n % 8 != 0) std.debug.panic("unsupported block-scaled MMA N {d}", .{cfg.n}); + cfg.validate(); return (cute_nvgpu.MmaAtomSM100UMMABlockScaledType.get(self.ctx, .{ - .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ 128, cfg.n, cfg.k() })).attribute(), - .numCta = 1, + .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ cfg.m, cfg.n, cfg.k() })).attribute(), + .numCta = ctaCount(cfg.cta_group), .aMajor = .k, .bMajor = .k, .aType = DType.f4e2m1fn.toMlir(self.ctx), @@ -1952,10 +2271,27 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - /// Commit the preceding tcgen05 MMA operations to an mbarrier. One elected - /// lane of the MMA warp issues this after each K-stage. - pub fn tcgen05Commit(self: *Builder, barrier: Value) void { - self.emitVoid(nvvm.tcgen05_commit(self.ctx, self.sharedBarrierPtr(barrier).inner, .cta_1, self.loc())); + pub const Tcgen05CommitOptions = struct { + /// Must match the group of the operations being tracked. + cta_group: CtaGroup = .one, + /// Arrive at the same barrier offset in each CTA whose rank bit is set. + /// Without a mask, arrive only on the issuing CTA's barrier. + multicast_mask: ?Value = null, + }; + + /// Track this thread's preceding tcgen05 MMA/copy operations of the + /// selected group, signaling one arrival per destination on completion. + /// The same elected thread that issues those operations must commit them. + /// For `.two`, either CTA in the pair can be the issuer. All tcgen05 + /// operations in a kernel must use the same group. + pub fn tcgen05Commit(self: *Builder, mbar: Value, opts: Tcgen05CommitOptions) void { + self.emitVoid(nvvm.tcgen05_commit( + self.ctx, + self.sharedBarrierPtr(mbar).inner, + opts.cta_group.toNvvm(), + if (opts.multicast_mask) |mask| mask.to(.i16).inner else null, + self.loc(), + )); } /// Wait until asynchronous tensor-memory loads have reached registers. @@ -1963,14 +2299,14 @@ pub const Builder = struct { self.emitVoid(nvvm.tcgen05_wait(self.ctx, .load, self.loc())); } - pub const TmemOptions = struct { two_cta: bool = false, exclusive: bool = false }; + pub const TmemOptions = struct { cta_group: CtaGroup = .one, exclusive: bool = false }; pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { self.emitVoid(cute_nvgpu.arch_sm100_alloc_tmem( self.ctx, self.constant(.i32, columns).inner, self.asValue(holding_ptr).inner, - .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } @@ -1981,7 +2317,7 @@ pub const Builder = struct { } pub fn relinquishTmemAllocPermit(self: *Builder, opts: TmemOptions) void { - self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null }, self.loc())); + self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null }, self.loc())); } pub fn deallocTmem(self: *Builder, pointer: anytype, columns: u16, opts: TmemOptions) void { @@ -1989,7 +2325,7 @@ pub const Builder = struct { self.ctx, self.asValue(pointer).inner, self.constant(.i32, columns).inner, - .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } @@ -2182,22 +2518,37 @@ pub const Builder = struct { _ = nvvm.barrier(self.ctx, self.loc()).appendTo(self.currentBlock()); } - /// Synchronize a fixed subset of the CTA on a named hardware barrier. - /// `participants` must be a warp multiple and every participating thread - /// must execute the same barrier id. - pub fn namedBarrier(self: *Builder, id: i32, participants: i32) void { - // The embedded CuTe compiler currently reads NVVM through a different - // MLIR revision. A generic `nvvm.barrier` with optional operands is - // accepted, but those operands are discarded and it lowers to - // `bar.sync 0`. Constant inline PTX preserves the named barrier ABI - // used by CUTLASS (`2,160` for TMEM allocation and `1,128` for the - // epilogue pipeline). - const assembly = std.fmt.allocPrint( - self.arena.allocator(), - "bar.sync {d}, {d};", - .{ id, participants }, - ) catch @panic("namedBarrier OOM"); - _ = llvm.inline_asm(self.ctx, assembly, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + /// `bar.sync id, participants` (`barrier.cta.sync.aligned`). + /// `id` is an integer or an i32 Value in [0, 15]. `participants` must be + /// a positive warp multiple, and every participating thread must execute + /// the same barrier id. The aligned form requires CTA-uniform control flow. + pub fn barrierCtaSyncAligned(self: *Builder, id: anytype, participants: anytype) void { + if (@TypeOf(participants) == ?Value) { + if (participants) |count| return self.barrierCtaSyncAligned(id, count); + return self.barrierCtaSyncAligned(id, null); + } + if (@TypeOf(participants) != Value and @TypeOf(participants) != @TypeOf(null)) { + std.debug.assert(participants > 0 and participants <= 1024 and @mod(participants, 32) == 0); + } + if (@TypeOf(id) == Value) { + std.debug.assert(id.dtype() == .i32); + } else { + const immediate: i32 = id; + std.debug.assert(immediate >= 0 and immediate < 16); + } + // The LLVM intrinsic preserves convergence and both operands across + // the embedded compiler's NVVM dialect version boundary. + if (@TypeOf(participants) == @TypeOf(null)) { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned", &.{self.lift(id).to(.i32)}, null); + } else { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.lift(id).to(.i32), self.lift(participants).to(.i32) }, null); + } + } + + pub const BarrierOptions = struct { barrier_id: ?Value = null, number_of_threads: ?Value = null }; + + pub fn barrier(self: *Builder, opts: BarrierOptions) void { + self.barrierCtaSyncAligned(opts.barrier_id orelse self.constant(.i32, 0), opts.number_of_threads); } /// Build the CuTe first-class divisor used by CUTLASS's persistent tile @@ -2368,48 +2719,101 @@ pub const Builder = struct { /// mbarrier operands are `!llvm.ptr<3>` for NVVM. A barrier is either a CuTe /// shared-memory pointer or a raw `i32` shared-memory address. - fn sharedBarrierPtr(self: *Builder, barrier: Value) Value { - if (barrier.isInt()) return self.addressPtr(barrier, .shared); - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{barrier.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); + fn sharedBarrierPtr(self: *Builder, mbar: Value) Value { + if (mbar.isInt()) return self.addressPtr(mbar, .shared); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{mbar.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); } /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. - pub fn mbarrierInit(self: *Builder, barrier: Value, arrivals: u32) void { - self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, arrivals).inner, self.loc())); + pub fn mbarrierInit(self: *Builder, mbar: Value, arrivals: u32) void { + self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, arrivals).inner, self.loc())); } /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a /// TMA producer does before issuing its copies. - pub fn mbarrierArriveExpectTx(self: *Builder, barrier: Value, bytes: u32) void { - self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); + pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: u32) void { + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); } - pub fn mbarrierArrive(self: *Builder, barrier: Value, count: u32) void { - self.emitVoid(nvvm.mbarrier_arrive(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, count).inner, .cta, self.loc())); + pub const MBarrierArriveOptions = struct { + peer_cta_rank_in_cluster: ?Value = null, + arrive_count: ?Value = null, + scope: MBarrierScope = .cta, + relaxed: bool = false, + }; + + /// Arrive locally or at the same offset in a peer CTA. Local arrivals + /// return the opaque state token; remote arrivals have no state result. + pub fn mbarrierArrive(self: *Builder, mbar: Value, opts: MBarrierArriveOptions) ?Value { + var ptr = self.sharedBarrierPtr(mbar); + if (opts.peer_cta_rank_in_cluster) |rank| { + ptr = self.emit(nvvm.mapa(self.ctx, ptr.inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + } + const op = nvvm.ops.mbarrier_arrive(self.ctx, ptr.inner, .{ + .count = if (opts.arrive_count) |count| count.to(.i32).inner else null, + .scope = (nvvm.ops.MemScopeKindAttr.get(self.ctx, .{ .value = switch (opts.scope) { + .cta => .cta, + .cluster => .cluster, + } }) catch unreachable).attribute(), + .relaxed = .boolean(self.ctx, opts.relaxed), + .res_type = if (opts.peer_cta_rank_in_cluster == null) mlir.Type.int(self.ctx, .i64) else null, + }, self.loc()); + if (opts.peer_cta_rank_in_cluster == null) return self.emit(op); + self.emitVoid(op); + return null; } - /// Test whether phase `phase` has completed, without suspending (SM90+). - pub fn mbarrierWaitParity(self: *Builder, barrier: Value, phase: Value) Value { - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, .@"try", .cta, self.loc())); + pub const MBarrierTryWaitOptions = struct { + scope: MBarrierScope = .cta, + time_limit: ?Value = null, + }; + + /// Try an acquire wait for phase `phase` (i32 parity, 0 or 1). May suspend + /// briefly and return false before completion. `scope` selects which + /// threads this wait can synchronize with; the barrier remains local. + pub fn mbarrierTryWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierTryWaitOptions) Value { + const ptr = self.sharedBarrierPtr(mbar); + if (opts.time_limit) |ticks| { + return self.emit(nvvm.mbarrier_try_wait_parity_timelimit(self.ctx, ptr.inner, phase.inner, ticks.to(.i32).inner, opts.scope, self.loc())); + } + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, ptr.inner, phase.inner, .@"try", opts.scope, self.loc())); } - /// Wait for phase `phase`, letting the warp sleep for up to - /// `suspend_time` cycles per attempt (SM90+). - pub fn mbarrierTryWaitParity(self: *Builder, barrier: Value, phase: Value, suspend_time: u32) void { - self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, self.constant(.i32, suspend_time).inner, self.loc())); + pub const MBarrierWaitOptions = struct { + scope: MBarrierScope = .cta, + /// Hardware suspend hint in nanoseconds per attempt, not a total timeout. + time_limit: u32 = 10_000_000, + /// Probe before suspending, as in CuTe's pipeline consumer wait. + test_first: bool = true, + }; + + fn mbarrierWaitLoop(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + const ptr = self.sharedBarrierPtr(mbar); + const ticks = self.constant(.i32, opts.time_limit); + switch (opts.scope) { + // Preserve the operation recognized by CuTe's pipeline compiler. + .cta => self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, ptr.inner, phase.inner, ticks.inner, self.loc())), + .cluster => { + var pending = self.openWhile(.{}, .{}); + const ready = self.mbarrierTryWait(mbar, phase, .{ .scope = opts.scope, .time_limit = ticks }); + pending.yieldBefore(ready.eq(false), .{}); + pending.yieldAfter(.{}); + }, + } } - /// Wait for a pipeline phase with the two-step sequence of CuTe's - /// `PipelineTmaUmma`: test the barrier without suspending, then suspend - /// only when the phase is still outstanding. The initial test matters when + /// Acquire a pipeline phase at `scope`: test without suspending, then + /// block only when the phase is still outstanding. The initial test matters when /// a producer reaches a full ring and must yield to the consumer that /// recycles its next stage. - pub fn mbarrierWait(self: *Builder, barrier: Value, phase: Value) void { - const ready = self.mbarrierWaitParity(barrier, phase); - var pending = self.openIf(ready.eq(false)); - { - self.mbarrierTryWaitParity(barrier, phase, 10_000_000); + pub fn mbarrierWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + if (opts.test_first) { + const ready = self.mbarrierTestWait(mbar, phase, .{ .scope = opts.scope }); + var pending = self.openIf(ready.eq(false)); + self.mbarrierWaitLoop(mbar, phase, opts); pending.close(); + } else { + self.mbarrierWaitLoop(mbar, phase, opts); } } @@ -2438,7 +2842,7 @@ pub const Builder = struct { // Hand-scheduled kernels address shared memory with `i32` offsets in the // shared window, global memory with `i64` addresses and tensor memory with // `i32` TMEM addresses (lane << 16 | column). These helpers emit the NVVM / - // LLVM operations for them directly (no inline PTX). + // LLVM operations for them directly, using inline PTX where needed. pub const NVPTXAddressSpace = llvm.NVPTXAddressSpace; @@ -2500,11 +2904,6 @@ pub const Builder = struct { self.emitVoid(llvm.store(self.ctx, word.to(.i16).inner, self.addressPtr(address, space).inner, .{ .alignment = 2 }, self.loc())); } - /// `bar.sync id, threads`: named barrier over `threads` threads. - pub fn barrierSync(self: *Builder, id: i32, threads: i32) void { - _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.constant(.i32, id), self.constant(.i32, threads) }, null); - } - /// `bar.red.or.pred`: named barrier over `threads` threads that also returns whether /// `predicate` (i1) is true in any of them. pub fn barrierReduceOr(self: *Builder, id: i32, threads: i32, predicate: Value) Value { @@ -2516,9 +2915,10 @@ pub const Builder = struct { self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.constant(.i32, -1).inner, self.loc())); } - /// Non-blocking test of mbarrier phase `parity` (i1). - pub fn mbarrierTestParity(self: *Builder, barrier: Value, parity: Value) Value { - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, parity.inner, .@"test", .cta, self.loc())); + /// Non-blocking acquire test of mbarrier phase `parity` (i32, 0 or 1). + pub fn mbarrierTestWait(self: *Builder, mbar: Value, parity: Value, opts: MBarrierTryWaitOptions) Value { + std.debug.assert(opts.time_limit == null); + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(mbar).inner, parity.inner, .@"test", opts.scope, self.loc())); } /// Call an LLVM intrinsic by name (e.g. an `llvm.nvvm.*` intrinsic the NVVM dialect of @@ -2549,8 +2949,8 @@ pub const Builder = struct { /// Arrive on `barrier` once this thread's prior cp.async copies have landed; with /// `noinc`, the arrival is counted in the barrier's expected count. - pub fn cpAsyncMbarrierArrive(self: *Builder, barrier: Value, comptime noinc: bool) void { - _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(barrier)}, null); + pub fn cpAsyncMbarrierArrive(self: *Builder, mbar: Value, comptime noinc: bool) void { + _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(mbar)}, null); } /// Wait until all of this thread's cp.async copies have landed. @@ -2705,13 +3105,271 @@ pub const Builder = struct { return self.gridDim().z.mul(value); } + /// SM100 `tile::gather4` TMA load from a tiled 2-D descriptor address `desc` (i64). + /// Gather four rows `r` (i32) at column `col` into 128-byte-aligned shared `dst` (i32), + /// completing their bytes on an 8-byte-aligned local mbarrier `bar` (i32). + /// The descriptor must have row box extent 1 and no interleaving. `col` and `r` + /// are element coordinates; row length and out-of-bounds fill come from the + /// descriptor (zero or OOB-NaN). Arm the barrier for all four rows' bytes before waiting. + pub fn tmaGather4(self: *Builder, desc: Value, dst: Value, bar: Value, col: Value, r: [4]Value, policy: CacheHintSm100) void { + std.debug.assert(desc.dtype() == .i64 and dst.dtype() == .i32 and bar.dtype() == .i32 and col.dtype() == .i32); + for (r) |row| std.debug.assert(row.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [$0], [$1, {$2, $3, $4, $5, $6}], [$7], $8;", &.{ dst.inner, desc.inner, col.inner, r[0].inner, r[1].inner, r[2].inner, r[3].inner, bar.inner, self.constant(.i64, policy).inner }, .{ .constraints = "r,l,r,r,r,r,r,r,l,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const BulkPrefetchOptions = struct { l2_cache_hint: ?Value = null }; + + /// Best-effort byte-range prefetch into L2; size must be a multiple of 16. + pub fn cpAsyncBulkPrefetch(self: *Builder, addr: Value, bytes: anytype, opts: BulkPrefetchOptions) void { + if (@TypeOf(bytes) != Value) std.debug.assert(bytes > 0 and bytes % 16 == 0); + const address = if (addr.isInt()) self.addressPtr(addr, .global) else self.llvmPointer(addr, .global); + self.emitVoid(nvvm.ops.cp_async_bulk_prefetch(self.ctx, address.inner, self.lift(bytes).to(.i32).inner, .{ + .l2CacheHint = if (opts.l2_cache_hint) |hint| hint.to(.i64).inner else null, + }, self.loc())); + } + + pub fn prefetchL1(self: *Builder, addr: Value) void { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.global.L1 [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn prefetchTensorMap(self: *Builder, desc: Value) void { + const address = if (desc.isInt()) desc else self.ptrToInt(desc, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.tensormap [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const OperandSource = union(enum) { smem: Value, tmem: Value }; + pub const Tcgen05MmaWsOptions = struct { + collector_b_buffer: ?nvvm.ops.Tcgen05MMACollectorBBuffer = null, + collector_op: ?nvvm.ops.Tcgen05MMACollectorOp = null, + col_b_zero_mask: ?Value = null, + }; + + /// Single-thread, asynchronous weight-stationary MMA, as in Python primitives. + /// The caller elects the issuer and commits/waits before consuming D. + pub fn tcgen05MmaWs(self: *Builder, kind: nvvm.Tcgen05MmaKind, d: Value, a: OperandSource, bdesc: Value, idesc: anytype, enable_input_d: Value, opts: Tcgen05MmaWsOptions) void { + std.debug.assert(kind == .f16 or kind == .tf32 or kind == .f8f6f4 or kind == .i8); + const adesc = switch (a) { + .smem => |desc| self.smemDescriptorToInt(desc), + .tmem => |ptr| if (ptr.isInt()) self.addressPtr(ptr, .tensor) else self.llvmPointer(ptr, .tensor), + }; + const dest = if (d.isInt()) self.addressPtr(d, .tensor) else self.llvmPointer(d, .tensor); + self.emitVoid(nvvm.ops.tcgen05_mma_ws(self.ctx, dest.inner, adesc.inner, self.smemDescriptorToInt(bdesc).inner, self.lift(idesc).to(.i32).inner, enable_input_d.to(.i1).inner, (nvvm.ops.Tcgen05MMAKindAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ + .collectorBBuffer = if (opts.collector_b_buffer) |v| (nvvm.ops.Tcgen05MMACollectorBBufferAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .collectorOp = if (opts.collector_op) |v| (nvvm.ops.Tcgen05MMACollectorOpAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .zeroColMask = if (opts.col_b_zero_mask) |v| v.to(.i64).inner else null, + }, self.loc())); + } + + pub fn makeUmmaSmemDesc(self: *Builder, src: Value, layout: LayoutSpec, major: OperandMajorMode, next_src: ?Value) Value { + return self.emit(cute_nvgpu.make_umma_smem_desc(self.ctx, src.inner, self.smemDescType(), self.layoutAttr(layout).attribute(), (cute_nvgpu.MajorModeAttr.get(self.ctx, .{ .value = switch (major) { + .k => .k, + .mn => .mn, + } }) catch unreachable).attribute(), .{ .nextSrc = if (next_src) |v| v.inner else null }, self.loc())); + } + + pub fn intToSmemDescriptor(self: *Builder, value: anytype) Value { + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{self.lift(value).to(.i64).inner}, &.{self.smemDescType()}, self.loc())); + } + + pub fn smemDescriptorToInt(self: *Builder, desc: Value) Value { + if (desc.isInt()) return desc.to(.i64); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{desc.inner}, &.{mlir.Type.int(self.ctx, .i64)}, self.loc())); + } + + pub fn setMaxRegisterIncrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .increase); + } + + pub fn setMaxRegisterDecrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .decrease); + } + + fn setMaxRegister(self: *Builder, comptime count: u32, comptime action: enum { increase, decrease }) void { + comptime std.debug.assert(count >= 24 and count <= 256 and count % 8 == 0); + const text = std.fmt.comptimePrint("setmaxnreg.{s}.sync.aligned.u32 {d};", .{ if (action == .increase) "inc" else "dec", count }); + _ = llvm.inline_asm(self.ctx, text, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Specialized busy wait for hand-scheduled kernels; never hardware-suspends. + pub fn mbarrierSpin(self: *Builder, bar: Value, parity: Value, scope: MBarrierScope) void { + std.debug.assert(bar.dtype() == .i32 and parity.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, switch (scope) { + .cta => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + .cluster => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + }, &.{ bar.inner, parity.inner }, .{ .constraints = "r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Convert a CuTe or LLVM pointer to the LLVM pointer required by NVVM. + fn llvmPointer(self: *Builder, ptr: Value, comptime space: NVPTXAddressSpace) Value { + if (!ptr.isPtr()) return ptr; + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{ptr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } + + /// Map a local shared pointer/address to a typed pointer in peer CTA memory. + pub fn mapDsmemPtr(self: *Builder, smem: Value, peer_cta_rank: Value) Value { + const ptr = if (smem.isInt()) self.addressPtr(smem, .shared) else self.llvmPointer(smem, .shared); + const remote = self.emit(nvvm.mapa(self.ctx, ptr.inner, peer_cta_rank.to(.i32).inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem); + const result_type = (cute_ir.PtrType.get(self.ctx, .{ + .valueType = ty.valueType(), + .addressSpace = .dsmem, + .alignment = ty.alignment(), + .swizzle = ty.swizzle(), + .bitlayout = ty.bitlayout(), + }) catch @panic("invalid DSMEM pointer")).type_(); + return self.emit(cute_ir.inttoptr(self.ctx, self.ptrToInt(remote, .i32).inner, result_type, self.loc())); + } + return remote; + } + + /// Store one, two, or four i32 words in a peer CTA, completing bytes on its + /// mbarrier. Both inputs name local shared memory; both are mapped to the peer. + pub fn storeAsyncDsmem(self: *Builder, smem: Value, value: anytype, mbar: Value, peer_cta_rank: Value) void { + const n = if (@TypeOf(value) == Value) 1 else common.arity(@TypeOf(value), "storeAsyncDsmem: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % (n * 4) == 0); + } + if (mbar.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % 8 == 0); + } + const packed_value = if (@TypeOf(value) == Value) value else blk: { + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + break :blk self.joinWords(n, words); + }; + self.emitVoid(nvvm.ops.store_async(self.ctx, self.llvmPointer(self.mapDsmemPtr(smem, peer_cta_rank), .shared_cluster).inner, packed_value.inner, self.llvmPointer(self.mapDsmemPtr(mbar, peer_cta_rank), .shared_cluster).inner, self.loc())); + } + + /// Explicit aligned variants retain the convergence requirement of raw PTX. + pub fn barrierClusterArriveRelaxedAligned(self: *Builder) void { + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed.aligned;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn barrierClusterWaitAligned(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); + } + + pub const ClusterArriveOptions = struct { aligned: bool = false }; + + pub fn clusterArriveRelaxed(self: *Builder, opts: ClusterArriveOptions) void { + if (opts.aligned) return self.barrierClusterArriveRelaxedAligned(); + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn clusterWait(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, false, self.loc())); + } + + /// A null mask selects CuTe's FULL_MASK; integers and SSA masks also work. + pub fn voteAnySync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.any.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i1)).?; + } + + /// A null mask selects CuTe's FULL_MASK. + pub fn voteBallotSync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.ballot.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i32)).?; + } + + pub const FmaxOptions = struct { abs: bool = false, nan: bool = false, ftz: bool = false }; + + pub fn fmax(self: *Builder, x: Value, y: Value, opts: FmaxOptions) Value { + if (!opts.abs and !opts.nan and !opts.ftz) return self.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, mlir.Type.float(self.ctx, .f32)).?; + return self.emit(nvvm.ops.fmax(self.ctx, x.inner, y.inner, mlir.Type.float(self.ctx, .f32), .{ + .abs = if (opts.abs) .unit(self.ctx) else null, + .nan = if (opts.nan) .unit(self.ctx) else null, + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }, self.loc())); + } + + fn vec2(self: *Builder, x: [2]Value) Value { + const ty = mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)); + return self.emit(vector.from_elements(self.ctx, &.{ x[0].inner, x[1].inner }, ty, self.loc())); + } + + fn unvec2(self: *Builder, v: Value) [2]Value { + return .{ self.vectorExtract(v, 0, .f32), self.vectorExtract(v, 1, .f32) }; + } + + pub const PackedF32x2Options = struct { rnd: nvvm.ops.FPRoundingMode = .rn, ftz: bool = false }; + + fn packedF32x2Attrs(self: *Builder, opts: PackedF32x2Options) struct { rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute } { + return .{ + .rnd = (nvvm.ops.FPRoundingModeAttr.get(self.ctx, .{ .value = opts.rnd }) catch unreachable).attribute(), + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }; + } + + pub fn fmaPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, c: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.fma_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, self.vec2(c).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn addPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.add_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn mulPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.mul_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub const StoreOptions = struct { + /// Address space for raw integer addresses; typed pointers retain theirs. + space: NVPTXAddressSpace = .global, + level1_eviction_priority: nvvm.ops.EvictKind = .evict_normal, + }; + + /// Scalar/vector store. Eight i32 words with no_allocate keep STG.E.256. + pub fn store(self: *Builder, addr: Value, value: anytype, comptime opts: StoreOptions) void { + if (@TypeOf(value) != Value) { + const n = common.arity(@TypeOf(value), "store: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4 or n == 8); + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + const is_global = if (addr.type_().isA(cute_ir.PtrType)) |ty| ty.addressSpace() == .gmem else addr.isInt() and opts.space == .global; + if (n == 8 and is_global and opts.level1_eviction_priority == .evict_no_allocate) { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + var args: [9]*const mlir.Value = undefined; + args[0] = address.inner; + for (words, 0..) |word, i| args[i + 1] = word.inner; + _ = llvm.inline_asm(self.ctx, "st.global.L1::no_allocate.v8.b32 [$0], {$1, $2, $3, $4, $5, $6, $7, $8};", &args, .{ .constraints = "l,r,r,r,r,r,r,r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + return; + } + self.store(addr, self.joinWords(n, words), opts); + return; + } + const ptr = if (addr.isInt()) self.addressPtr(addr, opts.space) else if (addr.type_().isA(cute_ir.PtrType)) |ty| blk: { + const space: NVPTXAddressSpace = @fromBackingInt(@intCast(@backingInt(ty.addressSpace()))); + break :blk self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{addr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } else addr; + self.emitVoid(nvvm.ops.store_ext(self.ctx, value.inner, ptr.inner, .{ + .evict = if (opts.level1_eviction_priority == .evict_normal) null else (nvvm.ops.EvictKindAttr.get(self.ctx, .{ .value = opts.level1_eviction_priority }) catch unreachable).attribute(), + }, self.loc())); + } + + pub fn fma(self: *Builder, x: Value, y: Value, z: Value) Value { + return self.callIntrinsic("llvm.fma.f32", &.{ x, y, z }, mlir.Type.float(self.ctx, .f32)).?; + } + + pub fn clock(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u32 $0, %clock;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i32), .constraints = "=r", .has_side_effects = true }, self.loc())); + } + + pub fn globalTimer(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u64 $0, %globaltimer;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i64), .constraints = "=l", .has_side_effects = true }, self.loc())); + } + // ==================== scalars ==================== - /// `value` as a constant of `dt`; a Value passes through. An integer is - /// signed or an unsigned bit pattern of `dt`'s width; anything else - /// panics. + /// `value` as a constant of `dt`; a Value passes through and enums use + /// their backing integers. An integer is signed or an unsigned bit + /// pattern of `dt`'s width; anything else panics. pub fn constant(self: *Builder, dt: DType, value: anytype) Value { if (@TypeOf(value) == Value) return value; + if (@typeInfo(@TypeOf(value)) == .@"enum") return self.constant(dt, @backingInt(value)); const ctx = self.ctx; const T = @TypeOf(value); if (isFloatDtype(dt)) { @@ -2957,7 +3615,7 @@ test "guard, shared memory, sync, layouts, for loop" { const i = b.blockIdx().x.mul(128).add(tid); smem.set(.{tid}, gx.get(.{i})); b.syncThreads(); - b.namedBarrier(2, 128); + b.barrierCtaSyncAligned(2, 128); b.launchDependents(); b.waitForDependency(); const j = b.constant(.i32, 127).sub(tid); @@ -2976,7 +3634,9 @@ test "guard, shared memory, sync, layouts, for loop" { try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); try expectContains(kernel, "!cute.ptr>"); try expectContains(kernel, "\"nvvm.barrier\"() <{operandSegmentSizes = array}> : () -> ()"); - try expectContains(kernel, "bar.sync 2, 128;"); + try expectContains(kernel, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(kernel, "value = 2 : i32"); + try expectContains(kernel, "value = 128 : i32"); try expectContains(kernel, "griddepcontrol.launch_dependents;"); try expectContains(kernel, "griddepcontrol.wait;"); try expectContains(kernel, "scf.if"); @@ -3113,30 +3773,433 @@ test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { var b = try Builder.open(std.testing.allocator, ctx, "sm100_blockscaled_atoms"); defer b.deinit(); - const descriptor = try cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(ctx, .{ - .kind = .sm_90, - .valType = DType.f8e4m3fn.toMlir(ctx), - .copyBits = 16384, - .tmaGbasis = b.layoutType(b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) })), - .tmaFormat = .U8, + const tma: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 16384, + .global_basis = b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }), + .coordinate_layout = b.layoutSpec(.{ 16, 128, 8 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}), b.basis(1, 1, .{2}) }), + .coordinate_rank = 3, + }; + var pair_tma = tma; + pair_tma.cta_group = .two; + const args = try b.declareArgs(.{ + .single = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(tma), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair_tma), .grid_constant = true } }, }); - _ = try b.declareArgs(.{ .descriptor = .{ .mlir_type = .{ .type_ = descriptor.type_(), .grid_constant = true } } }); + _ = b.makeExecTma(.{ .inner = args.single.inner, .kernel = &b }, b.tmaLoadExecAtomType(tma)); + _ = b.makeExecTma(.{ .inner = args.pair.inner, .kernel = &b }, b.tmaLoadExecAtomType(pair_tma)); const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); - _ = b.makeBlockScaledMma(.{ .n = 16 }, sf_ptr); - _ = b.makeBlockScaledMma(.{ .n = 128 }, sf_ptr); + for ([_]CtaGroup{ .one, .two }) |group| { + _ = b.makeBlockScaledMma(.{ .n = 16, .cta_group = group }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .n = 128, .cta_group = group }, sf_ptr); + } + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 16, .cta_group = .two }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 128, .cta_group = .two }, sf_ptr); const nvfp4_sf_ptr = b.intToPtr(0, .f8e4m3fn, .tmem, 1); _ = b.makeBlockScaledMma(.{ .kind = .nvfp4_nvfp4, .n = 16 }, nvfp4_sf_ptr); const ir = try b.finish(.{ 192, 1, 1 }); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x16x32") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x128x32") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.make_tiled_mma") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute_nvgpu.grid_constant") != null); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>"); + try expectContains(ir, "non_exec_tiled_tma_load"); + try expectContains(ir, "value = 3 : i32"); + try expectContains(ir, "value = 32 : i32"); + try expectContains(ir, "!llvm.ptr<7>"); + try expectContains(ir, "#nvvm.mem_scope"); + try expectContains(ir, "#nvvm.mem_scope"); + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "mbarrier waits preserve acquire scope through polling and blocking retries" { + const ctx = try testContext(); + defer ctx.deinit(); + for ([_]MBarrierScope{ .cta, .cluster }) |scope| { + var b = try Builder.open(std.testing.allocator, ctx, "barrier_waits"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + + const barrier = b.allocSmem(.i64, 1, 8); + const phase = b.constant(.i32, 0); + _ = b.mbarrierTestWait(barrier, phase, .{ .scope = scope }); + _ = b.mbarrierTryWait(barrier, phase, .{ .scope = scope }); + b.mbarrierWait(barrier, phase, .{ .scope = scope, .time_limit = 1234, .test_first = false }); + b.mbarrierWait(barrier, phase, .{ .scope = scope }); + + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try expectContains(ir, "value = 1234 : i32"); + try expectContains(ir, "\"scf.if\""); + switch (scope) { + .cta => { + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "\"scf.while\"")); + }, + .cluster => { + try std.testing.expectEqual(@as(usize, 5), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mem_order")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity.timelimit\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.while\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.condition\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + }, + } + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); + } +} + +test "SM100 raw primitives preserve operand types and round-trip through MLIR" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "sm100_raw_primitives"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 1024, + .global_basis = b.layoutSpec(.{ 128, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 1024, 16 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const a = try b.declareArgs(.{ + .gmem = .{ .ptr = DType.i32 }, + .tma = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + }); + const atom = b.makeExecTma(.{ .inner = a.tma.inner, .kernel = &b }, b.tmaLoadExecAtomType(cfg)); + const desc = b.tmaDescriptorAddress(atom); + const dst = b.constant(.i32, 1024); + const bar = b.constant(.i32, 0); + const zero = b.constant(.i32, 0); + const gmem = b.ptrToInt(a.gmem, .i64); + const pred = b.threadIdx().x.eq(0); + b.tmaGather4(desc, dst, bar, zero, .{ zero, zero.add(1), zero.add(2), zero.add(3) }, .evict_last); + b.cpAsyncBulkPrefetch(gmem, b.constant(.i32, 128), .{}); + b.prefetchL1(gmem); + b.prefetchTensorMap(desc); + const swizzled = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .layout = .swizzle_128b }); + const interleaved = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 1024, .stride_byte_offset = 128 }); + b.tcgen05MmaWs(.f16, zero, .{ .tmem = dst }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k }), pred, .{}); + b.tcgen05MmaWs(.f16, zero, .{ .smem = interleaved }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn }), pred, .{}); + b.setMaxRegisterIncrease(128); + b.barrierCtaSyncAligned(0, 512); + b.setMaxRegisterDecrease(128); + b.barrierCtaSyncAligned(b.threadIdx().x.div(128), 128); + b.mbarrierSpin(bar, zero, .cta); + b.mbarrierSpin(bar, zero, .cluster); + const state = b.mbarrierArrive(bar, .{}).?; + const remote = b.mapDsmemPtr(dst, zero); + _ = remote; + _ = b.mbarrierArrive(bar, .{ .peer_cta_rank_in_cluster = zero, .arrive_count = b.constant(.i32, 1), .scope = .cluster }); + b.storeAsyncDsmem(dst, .{ zero, zero }, bar, zero); + b.storeAsyncDsmem(dst, .{ zero, zero, zero, zero }, bar, zero); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + const any = b.voteAnySync(pred, null); + const mask = b.voteBallotSync(pred, null); + const x = b.constant(.f32, 1.0); + const y = b.constant(.f32, 2.0); + const maximum = b.fmax(x, y, .{}); + const sum = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{}); + const product = b.mulPackedF32x2(sum, .{ x, y }, .{}); + const fused = b.fmaPackedF32x2(product, sum, .{ x, y }, .{}); + const scalar = b.fma(maximum, fused[0], fused[1]); + b.store(gmem, .{ scalar.bitCast(.i32), maximum.bitCast(.i32), mask, any.to(.i32), state.to(.i32), b.clock(), b.globalTimer().to(.i32), zero }, .{ .level1_eviction_priority = .evict_no_allocate }); + + const ir = try b.finish(.{ 512, 1, 1 }); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "cute_nvgpu.get_tma_desc_addr", "tile::gather4", "nvvm.cp.async.bulk.prefetch", + "prefetch.global.L1", "prefetch.tensormap", "nvvm.tcgen05.mma.ws", + "nvvm.tcgen05.mma.ws", "setmaxnreg.inc", "setmaxnreg.dec", + "llvm.nvvm.barrier.cta.sync.aligned.count", "mbarrier.test_wait.parity.acquire.cta", "mbarrier.test_wait.parity.acquire.cluster", + "nvvm.mapa", "#nvvm.mem_scope", "nvvm.store.async", + "vector<4xi32>", "barrier.cluster.arrive.relaxed.aligned", "nvvm.cluster.wait", + "llvm.nvvm.vote.any.sync", "llvm.nvvm.vote.ballot.sync", "llvm.nvvm.fmax.f", + "nvvm.add.packed.f32x2", "nvvm.mul.packed.f32x2", "nvvm.fma.packed.f32x2", + "llvm.fma.f32", "st.global.L1::no_allocate.v8.b32", "%clock", + "%globaltimer", + }) |op| try expectContains(ir, op); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "SM100 BF16 weight-stationary descriptor encodes supported shapes and major modes" { + try std.testing.expectEqual(@as(u32, 0x04200490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k })); + try std.testing.expectEqual(@as(u32, 0x04410490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x08118490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 128, .n_dim = 64, .a_major = .mn, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x02100490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 32, .n_dim = 64, .a_major = .k, .b_major = .k })); +} + +test "SM100 shared descriptors preserve nonzero swizzle bases and boundary offsets" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "smem_descriptors"); + defer b.deinit(); + const args = try b.declareArgs(.{ .out = .{ .ptr = DType.i64 } }); + const addr = b.constant(.i32, 128); + const fields = [_]u64{ + 0x4000404000010000, // sw128, base 0, leading 16 bytes, stride 1024 bytes + 0x4002404000010000, // sw128, pattern at byte 128: base 1 + 0x400e404000010000, // sw128, pattern at byte 896: base 7 + 0x00007fff3fff0000, // unswizzled, largest representable offsets + }; + const descs = [_]Value{ + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 0, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 1, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 7, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 0x3fff0, .stride_byte_offset = 0x3fff0 }), + }; + const out = b.ptrToInt(args.out, .i64); + for (descs, 0..) |desc, i| b.storeWords(2, out.add(@as(i64, @intCast(i * 8))), .global, .{ desc.to(.i32), desc.shrLogical(32).to(.i32) }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + for (fields) |bits| { + const expected = try std.fmt.allocPrint(std.testing.allocator, "value = {d} : i64", .{bits}); + defer std.testing.allocator.free(expected); + try expectContains(ir, expected); + } + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves dynamic singleton and nested shape and stride leaves" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "mixed_layout"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const n = b.blockIdx().x.add(1); + const singleton = b.constant(.i32, 1); + const compact = b.makeLayout(.{ .{ 4, n }, singleton }, .{}); + const explicit = b.makeLayout(.{ n, 16 }, .{ .stride = .{ 1, n } }); + const static_nested = b.makeLayout(.{ .{ 4, 8 }, 16 }, .{}); + _ = compact; + _ = explicit; + _ = static_nested; + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "((4,?),?):((1,4),?)"); + try expectContains(ir, "(?,16):(1,?)"); + try expectContains(ir, "((4,8),16):((1,4),32)"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves 64-bit shape and stride values" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "wide_layout"); + defer b.deinit(); + const args = try b.declareArgs(.{ .extent = .{ .mlir_type = .{ .type_ = DType.i64.toMlir(ctx) } } }); + const wide = args.extent.add(@as(i64, 1) << 32); + const explicit = b.makeLayout(.{ 8, 2 }, .{ .stride = .{ 1, wide } }); + const wide_leaf: AlgebraExpr = .{ .dynamic_int = .{ .width = 64 } }; + try std.testing.expect(explicit.type_().eql(b.layoutType(.{ + .shape = b.expr(.{ 8, 2 }), + .stride = .{ .tuple = &.{ .{ .static = 1 }, wide_leaf } }, + }))); + _ = b.makeLayout(.{ .{ 4, wide }, 1, 8 }, .{ .stride = null }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") == null); + try expectContains(ir, "((4,?{i64}),1,8):((1,4),0,?{i64})"); + try expectContains(ir, "value = 4294967296 : i64"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "TMA constructors normalize tilers and staged layouts and copy executes store atoms" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "tma_stores"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 4 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const args = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + }); + const staged = b.makeLayout(.{ 16, 4, 2 }, .{ .stride = .{ 1, 16, 64 } }); + const store = b.makeTiledTmaAtom(CopyBulkTensorTileS2GOp{}, args.data, staged, .{ 16, 4 }, cfg, .{}); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), b.makeLayout(.{ 16, 4 }, .{})); + const tile = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.global_basis)); + b.copy(store.atom, .{smem}, .{tile}, .{}); + b.copy(store.atom, .{smem}, .{tile}, .{ .cache_policy = b.constant(.i64, CacheHintSm90.evict_last) }); + const swizzled = b.makeComposedLayout(.{ .bits = 1, .base = 4, .shift = 3 }, 0, b.layoutSpec(.{ 16, 4, 2 }, .{ 1, 16, 64 })); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, swizzled, b.makeTile(&.{ 16, 4 }), cfg, .{}); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, staged, b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 128 } }), cfg, .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"cute_nvgpu.atom.make_exec_tma\"")); + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "\"cute.select\"")); + try expectContains(ir, "(16,4):(1@0,1@1)"); + try expectContains(ir, "tma_store"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "CuTe options preserve runtime masks, timed try waits, descriptors and store widths" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "cute_options"); + defer b.deinit(); + const a = try b.declareArgs(.{ .out = .{ .ptr = DType.i32 } }); + const tid = b.threadIdx().x; + const mbar = b.allocSmem(.i64, 1, 8); + b.barrier(.{}); + b.barrier(.{ .barrier_id = tid.div(128), .number_of_threads = b.blockDim().x }); + b.tcgen05Commit(mbar, .{ .cta_group = .two, .multicast_mask = b.clusterCtaRank().shl(1).bitOr(1) }); + _ = b.mbarrierTryWait(mbar, b.constant(.i32, 0), .{ .time_limit = tid.add(100), .scope = .cluster }); + const state = b.mbarrierArrive(mbar, .{ .relaxed = true }).?; + b.storeAsyncDsmem(b.constant(.i32, 128), tid, mbar, b.clusterCtaRank().bitXor(1)); + b.clusterArriveRelaxed(.{}); + b.clusterWait(); + b.clusterArriveRelaxed(.{ .aligned = true }); + b.barrierClusterWaitAligned(); + _ = b.voteAnySync(tid.eq(0), tid.bitOr(1)); + _ = b.voteBallotSync(tid.eq(0), 0xff); + const x = tid.to(.f32); + const y = b.fmax(x, b.constant(.f32, 0), .{ .nan = true, .ftz = true, .abs = true }); + const pair = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{ .rnd = .rz, .ftz = true }); + b.store(a.out, state.to(.i32), .{}); + b.store(a.out, .{ pair[0].bitCast(.i32), pair[1].bitCast(.i32) }, .{}); + const desc = b.makeUmmaSmemDesc(b.intToPtr(0, .bf16, .smem, 16), b.layoutSpec(.{ 64, 16 }, .{ 16, 1 }), .k, null); + const raw_desc = b.smemDescriptorToInt(desc); + _ = b.intToSmemDescriptor(raw_desc); + const mapped = b.mapDsmemPtr(b.intToPtr(128, .i32, .smem, 16), b.clusterCtaRank()); + const mapped_ty = mapped.type_().isA(cute_ir.PtrType).?; + try std.testing.expectEqual(MemorySpace.dsmem, mapped_ty.addressSpace()); + try std.testing.expectEqual(@as(u64, 16), mapped_ty.alignment()); + b.store(mapped, tid, .{}); + const swizzles = [_]Tcgen05SmemSwizzle{ .none, .swizzle_32b, .swizzle_64b, .swizzle_128b, .swizzle_128b_atom_32b }; + for (swizzles) |layout| _ = Tcgen05SmemDesc.build(&b, tid.mul(16), .{ .layout = layout }); + const idesc = b.constant(.i32, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128 })); + b.tcgen05MmaWs(.f16, tid, .{ .tmem = tid }, desc, idesc, tid.ne(0), .{ .collector_b_buffer = .b2, .collector_op = .use, .col_b_zero_mask = state }); + b.cpAsyncBulkPrefetch(a.out, 128, .{ .l2_cache_hint = state }); + const ir = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "nvvm.mbarrier.try_wait.parity.timelimit"); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "scf.while")); + try expectContains(ir, "nvvm.store.ext"); + try expectContains(ir, "vector<2xi32>"); + try expectContains(ir, "#nvvm.fp_rnd_mode"); + try expectContains(ir, "cute_nvgpu.make_umma_smem_desc"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.openProgram(std.testing.allocator, ctx, "gather4_copy"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + .mode = .gather4, + }; + var pair = cfg; + pair.cta_group = .two; + const args = try b.declareArgs(.{ + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + .atom = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair), .grid_constant = true } }, + }); + const smem_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), smem_layout); + const coords = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.coordinate_layout)); + const atom: Atom = .{ .inner = args.atom.inner, .kernel = &b }; + const partitioned = b.tmaPartition(atom, 0, b.makeLayout(1, .{}), smem, .{ coords, args.indices.view() }, smem.type_(), &.{ coords.type_(), args.indices.inner.type_() }); + const mbar = b.allocSmem(.i64, 1, 8); + b.copy(atom, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ + .tma_bar_ptr = mbar, + .cache_policy = b.constant(.i64, CacheHintSm100.evict_last), + }); + b.copy(.{ .inner = args.pair.inner, .kernel = &b }, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ .tma_bar_ptr = mbar }); + b.endFunction(.{ 32, 1, 1 }); + + b.beginFunction("gather4_copy", .host); + const host = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + }); + const host_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const tiler = b.makeTile(&.{ 16, 4 }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{}, host.data, host_layout, tiler, cfg, .{ .gmem_coord_tensor = host.indices.view() }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{ .cta_group = .two }, host.data, host_layout, tiler, pair, .{ .gmem_coord_tensor = host.indices.view() }); + b.returnHostStatus(b.constant(.i32, 0)); + b.endFunction(null); + const ir = try b.finishProgram(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "atom.make_non_exec_2d_gather4_tma_load"); + try expectContains(ir, "atom.tma_partition"); + try expectContains(ir, "mode = gather4, num_cta = 1"); + try expectContains(ir, "mode = gather4, num_cta = 2"); + try expectContains(ir, "non_exec_2d_gather4_tma_load(attr); } +intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr) { + return cutegen::rank(llvm::cast(unwrap(attr)).getRef()); +} + MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value) { return getStaticAttr(context, value); } diff --git a/mlir/dialects/cute_ir/capi.h b/mlir/dialects/cute_ir/capi.h index e9e285d2..002f4885 100644 --- a/mlir/dialects/cute_ir/capi.h +++ b/mlir/dialects/cute_ir/capi.h @@ -54,6 +54,7 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrTupleGet( MlirContext context, intptr_t numElements, MlirAttribute const *elements); MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value); diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index 8a031d0b..b277d061 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -145,6 +145,10 @@ pub const ShapeAttr = opaque { return mlir.string(c.mlirCuteShapeAttrGetValue(self.ptr())); } + pub fn rank(self: *const ShapeAttr) usize { + return @intCast(c.mlirCuteShapeAttrGetRank(self.ptr())); + } + pub fn attribute(self: *const ShapeAttr) *const mlir.Attribute { return @ptrCast(self); } diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig index 78524e0a..d1197317 100644 --- a/mlir/dialects/llvm/llvm.zig +++ b/mlir/dialects/llvm/llvm.zig @@ -22,6 +22,7 @@ pub const elementtype = ops.elementtype; pub const AtomicBinOp = ops.AtomicBinOp; pub const AtomicOrdering = ops.AtomicOrdering; pub const inttoptr = ops.inttoptr; +pub const ptrtoint = ops.ptrtoint; pub const bitcast = ops.bitcast; pub const fmul = ops.fmul; pub const extractvalue = ops.extractvalue; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig index 2102f68a..d1fb5771 100644 --- a/mlir/dialects/llvm/ops.zig +++ b/mlir/dialects/llvm/ops.zig @@ -234,6 +234,15 @@ pub fn inttoptr(ctx: *mlir.Context, address: *const mlir.Value, result_type: *co }); } +/// llvm.ptrtoint — the integer address of an LLVM pointer. +pub fn ptrtoint(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{pointer} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// llvm.bitcast — reinterpret `value` as `result_type` (same bit width). pub fn bitcast(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "llvm.bitcast", .{ diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig index 28541d44..6ba84b4d 100644 --- a/mlir/dialects/llvm/test.zig +++ b/mlir/dialects/llvm/test.zig @@ -20,6 +20,7 @@ const operation_names: []const []const u8 = &.{ "llvm.intr.lifetime.end", "llvm.intr.lifetime.start", "llvm.inttoptr", + "llvm.ptrtoint", "llvm.load", "llvm.mlir.poison", "llvm.mlir.undef", diff --git a/mlir/dialects/nvvm/nvvm.zig b/mlir/dialects/nvvm/nvvm.zig index d46f3a4f..14a0e8e2 100644 --- a/mlir/dialects/nvvm/nvvm.zig +++ b/mlir/dialects/nvvm/nvvm.zig @@ -136,7 +136,8 @@ pub fn mbarrier_wait_parity( } /// nvvm.mbarrier.try_wait.parity — wait (looping in PTX) until phase `parity` (i32) -/// of the mbarrier completes, suspending up to `suspend_time` (i32) cycles per try. +/// of the mbarrier completes, with CTA acquire semantics. `suspend_time` (i32) +/// is a suspend-time hint in nanoseconds per try. pub fn mbarrier_try_wait_parity( ctx: *mlir.Context, ptr: *const mlir.Value, @@ -147,6 +148,27 @@ pub fn mbarrier_try_wait_parity( return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .{ .useIntrinsic = .boolean(ctx, false) }, location); } +/// One potentially suspending parity test with acquire semantics. A false +/// result means the timeout expired; callers that need completion must retry. +pub fn mbarrier_try_wait_parity_timelimit( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + suspend_time: *const mlir.Value, + scope: MBarrierScope, + location: *const mlir.Location, +) *mlir.Operation { + return ops.mbarrier_try_wait_parity_timelimit( + ctx, + ptr, + parity, + suspend_time, + .int(ctx, .i1), + .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope), .order = enumAttribute(ops.MemOrderKindAttr, ctx, .acquire) }, + location, + ); +} + /// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the cluster. pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return ops.fence_mbarrier_init(ctx, location); @@ -208,10 +230,12 @@ pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const return ops.tcgen05_wait(ctx, enumAttribute(ops.Tcgen05WaitKindAttr, ctx, kind), location); } -/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the issued -/// tcgen05 operations complete. -pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_commit(ctx, barrier_ptr, .{ .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); +/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the +/// tcgen05 MMA/copy operations issued by the executing thread with `group` +/// complete. An optional mask selects CTAs in the cluster whose barriers at +/// the same shared-memory offset receive one arrival each. +pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, multicast_mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_commit(ctx, barrier_ptr, .{ .multicastMask = multicast_mask, .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); } /// nvvm.tcgen05.mma — D (tensor memory, `!llvm.ptr<6>`) = A (shared-memory diff --git a/zml/kernel.zig b/zml/kernel.zig index 03a64d70..36c9d8d3 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -513,6 +513,7 @@ pub const cuda_tile = struct { }; pub const cute = struct { + pub const AlgebraExpr = cute_builder.AlgebraExpr; pub const Builder = cute_builder.Builder; pub const ArgSpec = cute_builder.ArgSpec; pub const Value = cute_builder.Value; @@ -526,6 +527,16 @@ pub const cute = struct { pub const TmaConfig = cute_builder.TmaConfig; pub const TmaFormat = cute_builder.TmaFormat; pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; + pub const CtaGroup = cute_builder.CtaGroup; + pub const ctaCount = cute_builder.ctaCount; + pub const MBarrierScope = cute_builder.MBarrierScope; + pub const OperandMajorMode = cute_builder.OperandMajorMode; + pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; + pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; + pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; + pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; + pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; + pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; pub fn newContext() std.mem.Allocator.Error!*mlir.Context { return makeKernelContext(cute_builder.registerDialects); diff --git a/zml/kernels/moe/cute/sm10x/common.zig b/zml/kernels/moe/cute/sm10x/common.zig index 47e05d37..da639c75 100644 --- a/zml/kernels/moe/cute/sm10x/common.zig +++ b/zml/kernels/moe/cute/sm10x/common.zig @@ -138,7 +138,7 @@ pub fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) Pipeli pub fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { var peek = b.openIfElse(count.lt(limit), .{.i1}); { - peek.yieldThen(.{b.mbarrierWaitParity(barrierAt(b, base, state.index), state.phase)}); + peek.yieldThen(.{b.mbarrierTryWait(barrierAt(b, base, state.index), state.phase, .{ .scope = .cta })}); } { peek.yieldElse(.{b.constant(.i1, true)}); diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig index e8901fb1..21a76436 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -762,10 +762,10 @@ fn issueTmaStage( b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); elected.close(); } - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); } fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { @@ -826,10 +826,7 @@ fn runTmaProducer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.mma_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); var stages = b.openFor( 0, k_tiles, @@ -845,7 +842,7 @@ fn runTmaProducer( const buffers = stageBuffersAt(b, tile, storage, state.index); var pending = b.openIf(stages.carried[0].eq(false)); { - b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); } issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); @@ -893,11 +890,7 @@ fn runTmaProducer( .phase = tiles.results[6], }; inline for (0..tile.stages) |stage| { - b.mbarrierTryWaitParity( - barrierAt(b, storage.mma_barriers, tail_state.index), - tail_state.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); if (stage + 1 < tile.stages) { tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); } @@ -922,8 +915,8 @@ const Mxfp4Mainloop = struct { sfb_target: V, pub fn copyScales(self: Mxfp4Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); } }; @@ -1164,17 +1157,14 @@ fn runMmaConsumer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.load_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); const accumulator_state: PipelinePosition = .{ .index = tiles.after_carried[9], .phase = tiles.after_carried[10], }; const accumulator_stage = accumulator_state.index; const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); @@ -1192,7 +1182,7 @@ fn runMmaConsumer( const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; var pending = b.openIf(stages.carried[0].eq(false)); { - b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); } const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); @@ -1209,7 +1199,7 @@ fn runMmaConsumer( ); var commit_lane = b.openIf(b.electSync()); { - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); commit_lane.close(); } const next_count = stages.carried[2].add(1); @@ -1220,7 +1210,7 @@ fn runMmaConsumer( var full_lane = b.openIf(b.electSync()); { - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); full_lane.close(); } const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); @@ -1271,11 +1261,7 @@ fn runMmaConsumer( inline for (0..accumulator_stages - 1) |_| { accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); } - b.mbarrierTryWaitParity( - barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), - accumulator_tail.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); } /// Epilogue operands; which fields are set depends on the `Epilogue`. @@ -1349,7 +1335,7 @@ fn runEpilogue( .phase = tiles.after_carried[5], }; const stage = accumulator_state.index; - b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); const warp_tmem = b.addOffset( @@ -1375,7 +1361,7 @@ fn runEpilogue( if (chunk + 1 == tile_chunks) { // Registers now own the accumulator values, so all 128 epilogue // threads release this TMEM stage before the stores. - b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); } var columns: [@intCast(@min(n, 32))]V = undefined; inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); @@ -1405,7 +1391,7 @@ fn runEpilogue( b.relinquishTmemAllocPermit(.{}); release_warp.close(); } - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); var dealloc_warp = b.openIf(warp.eq(0)); { b.deallocTmem(tmem, tile.tmem_columns, .{}); @@ -1545,14 +1531,14 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo scratch.set(.{ warp, lane }, lane_maximum); owner.close(); } - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); // Lane `c` turns the two half-block maxima of column `c` into its scale. const pair = warp.bitAnd(~@as(i32, 1)); const lane_column = b.select(lane.lt(n), lane, b.constant(.i32, 0)); const scale = mxScale(b, scratch.get(.{ pair, lane_column }, .f32).maximum(scratch.get(.{ pair.add(1), lane_column }, .f32))); // The next tile reuses the scratch maxima. - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); const activation_index = tile_m.mul(64).add(tid.div(2)); for (0..row_chunks) |row_chunk| { @@ -1767,7 +1753,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E var mma_warp = b.openIf(warp.eq(4)); { // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); switch (k_tiles) { inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), else => unreachable, @@ -1784,7 +1770,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E b.allocTmem(tile.tmem_columns, tmem_holding, .{}); alloc_warp.close(); } - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); switch (epilogue) { .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ .group_sizes = view(b, d.group_sizes), diff --git a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig index ac59735c..0de45007 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig @@ -699,10 +699,10 @@ fn issueTmaStage( var elected = b.openIf(b.electSync()); b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); elected.close(); - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); } fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { @@ -756,10 +756,7 @@ fn runTmaProducer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.mma_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); var stages = b.openFor( 0, @as(i32, @intCast(k_tiles)), @@ -773,7 +770,7 @@ fn runTmaProducer( }; const buffers = stageBuffersAt(b, tile, storage, state.index); var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); const next_count = stages.carried[1].add(1); @@ -817,11 +814,7 @@ fn runTmaProducer( .phase = tiles.results[6], }; inline for (0..tile.stages) |stage| { - b.mbarrierTryWaitParity( - barrierAt(b, storage.mma_barriers, tail_state.index), - tail_state.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); if (stage + 1 < tile.stages) { tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); } @@ -843,8 +836,8 @@ const Mainloop = struct { sfb_target: V, pub fn copyScales(self: Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); } }; @@ -1020,17 +1013,14 @@ fn runMmaConsumer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.load_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); const accumulator_state: PipelinePosition = .{ .index = tiles.after_carried[9], .phase = tiles.after_carried[10], }; const accumulator_stage = accumulator_state.index; const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); @@ -1043,7 +1033,7 @@ fn runMmaConsumer( const k = stages.carried[2]; const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); @@ -1058,7 +1048,7 @@ fn runMmaConsumer( mainloop.accumulator, ); var commit_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); commit_lane.close(); const next_count = stages.carried[2].add(1); const next_state = advancePipeline(b, state.index, state.phase, tile.stages); @@ -1066,7 +1056,7 @@ fn runMmaConsumer( stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); var full_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); full_lane.close(); const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); const next_acc_count = tiles.after_carried[8].add(1); @@ -1113,11 +1103,7 @@ fn runMmaConsumer( inline for (0..accumulator_stages - 1) |_| { accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); } - b.mbarrierTryWaitParity( - barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), - accumulator_tail.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); } /// Epilogue operands; which fields are set depends on the `Epilogue`. @@ -1193,7 +1179,7 @@ fn runEpilogue( const expert = operands.expert_ids.get(.{group}, .i32); const alpha = operands.alpha.get(.{expert}, .f32); const act_scale = if (operands.act_scale) |scales| scales.get(.{expert}, .f32) else null; - b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); const warp_tmem = b.addOffset( @@ -1213,7 +1199,7 @@ fn runEpilogue( const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); b.fenceTmemLoad(); if (chunk + 1 == tile_chunks) { - b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); } var columns: [@intCast(@min(n, 32))]V = undefined; inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32).mul(alpha); @@ -1239,7 +1225,7 @@ fn runEpilogue( var release_warp = b.openIf(warp.eq(0)); b.relinquishTmemAllocPermit(.{}); release_warp.close(); - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); var dealloc_warp = b.openIf(warp.eq(0)); b.deallocTmem(tmem, tile.tmem_columns, .{}); dealloc_warp.close(); @@ -1543,7 +1529,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E tma_warp.close(); var mma_warp = b.openIf(warp.eq(4)); - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); runMmaConsumer(b, tile, reduction_tiles, storage, tiled_mma, tmem_holding, active, d.tiles_m); mma_warp.close(); @@ -1551,7 +1537,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E var alloc_warp = b.openIf(warp.eq(0)); b.allocTmem(tile.tmem_columns, tmem_holding, .{}); alloc_warp.close(); - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); const shared_operands: EpilogueOperands = .{ .expert_ids = expert_ids, .alpha = view(b, d.alpha), diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig index 89761bdd..12d6273c 100644 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -317,7 +317,7 @@ fn produce(t: Tile) void { const id = tiles.iv.mul(c.features() * 2).add(stage); var reuse = b.openIf(id.ge(c.buffers())); { - b.mbarrierWait(t.released(id), t.phase(id).bitXor(1)); + b.mbarrierWait(t.released(id), t.phase(id).bitXor(1), .{ .scope = .cta }); reuse.close(); } const source = if (stage < c.features()) t.k else t.v; @@ -339,7 +339,7 @@ fn multiply(t: Tile, tile: V, comptime qk: bool) void { { for (0..@as(usize, @intCast(c.features()))) |part| { const id = tile.mul(c.features() * 2).add(@as(i64, @intCast(part)) + if (qk) @as(i64, 0) else c.features()); - b.mbarrierWait(t.ready(id), t.phase(id)); + b.mbarrierWait(t.ready(id), t.phase(id), .{ .scope = .cta }); b.tcgen05Fence(.after); const kv = descriptor(b, t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())), if (qk) 16 else c.keys() * 128); const qp = descriptor(b, t.shared.add(if (qk) c.qBase() else c.pBase()), 16); @@ -353,22 +353,22 @@ fn multiply(t: Tile, tile: V, comptime qk: bool) void { const accumulate = if (qk) b.constant(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.constant(.i32, @intFromBool(step > 0)).ne(0)); b.tcgen05MmaF16(t.tmem.add(column), left, right, b.constant(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); } - b.tcgen05Commit(t.released(id)); + b.tcgen05Commit(t.released(id), .{}); } - b.tcgen05Commit(t.shared.add(c.completion())); + b.tcgen05Commit(t.shared.add(c.completion()), .{}); leader.close(); } - b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1)); + b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1), .{ .scope = .cta }); b.tcgen05Fence(.after); // A fast warp must not wrap completion parity before every consumer has // observed the preceding PV completion. - if (!qk) b.barrierSync(1, 128); + if (!qk) b.barrierCtaSyncAligned(1, 128); } fn publishProbabilities(t: Tile) void { t.b.fenceProxyAsyncShared(); t.b.tcgen05Fence(.before); - t.b.barrierSync(1, 128); + t.b.barrierCtaSyncAligned(1, 128); t.b.tcgen05Fence(.after); } @@ -393,7 +393,7 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu leader.close(); } } - b.barrierSync(1, 128); + b.barrierCtaSyncAligned(1, 128); var result: [heads]V = undefined; for (0..heads) |h| { var value = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); @@ -403,7 +403,7 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu } result[h] = value; } - b.barrierSync(1, 128); + b.barrierCtaSyncAligned(1, 128); return result; } @@ -514,7 +514,7 @@ fn prefill(t: Tile) void { preserve.yieldElse(@as([64]V, @splat(zero))); } @memcpy(&saved, &preserve.results); - b.barrierSync(1, 128); + b.barrierCtaSyncAligned(1, 128); } multiply(t, tiles.iv, true); var scores: [64]V = undefined; From 439ba6c0a61990f4a8d4a08217eaa17b751112a7 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 3 Nov 2026 22:45:23 +0900 Subject: [PATCH 02/47] bazel: register runfiles initialization from C (#1005) ZML initializes runfiles before `main` so its loaders can locate packaged libraries and data. When using `-fno-llvm -fincremental` and linking with the incremental ELF linker, declaring the registration pointer in Zig produces incorrect `.init_array` data. See https://codeberg.org/ziglang/zig/issues/37453 Use a C constructor on Linux and macOS so the C compiler emits the platform-specific startup registration, while Zig exports the existing runfiles initialization function directly. We can revert when https://codeberg.org/ziglang/zig/issues/37453 is fixed. With this fix, we bypass the wrong behavior in incremental ELF linker and get closer to ZML compliance with zig incremental using their incremental linker. The registration is shared by the `use_cc_common_link=true` and `false` paths. Keep the explicit linker retention so the constructor is included even when its object is in a static archive. --- bazel/BUILD.bazel | 4 +++- bazel/bazel.zig | 41 ++++++++++++++++------------------------- bazel/runfiles_init.c | 15 +++++++++++++++ 3 files changed, 34 insertions(+), 26 deletions(-) create mode 100644 bazel/runfiles_init.c diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 434ecfa9..eb2b2ab2 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -3,7 +3,9 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "retain_runfiles_init", - # Mach-O adds an underscore to the exported Zig symbol. + srcs = ["runfiles_init.c"], + # Retain the constructor even when its object is in a static archive. + # Mach-O adds an underscore to the exported C symbol. linkopts = select({ "@platforms//os:macos": ["-Wl,-u,__bazel_runfiles_init"], "//conditions:default": ["-Wl,-u,_bazel_runfiles_init"], diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 2ae3324a..8af28b00 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -1,35 +1,26 @@ const std = @import("std"); -const builtin = @import("builtin"); const bazel_runfiles = @import("runfiles"); const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -const init_array_section = switch (builtin.object_format) { - .macho => "__DATA,__mod_init_func", - .elf => ".init_array", - else => "", -}; - -export const _bazel_runfiles_init align(@alignOf(usize)) linksection(init_array_section) = &struct { - fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { - _ = argc; - _ = envp; - var threaded: std.Io.Threaded = .init_single_threaded; - runfiles_global = bazel_runfiles.Runfiles.create(.{ - .allocator = std.heap.c_allocator, - .io = threaded.io(), - .argv0 = std.mem.span(argv[0]), - .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, - .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, - }) catch { - std.debug.panic("Unable to find runfiles", .{}); - } orelse { - std.debug.panic("Unable to initialize runfiles", .{}); - }; - } -}.call; +export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = threaded.io(), + .argv0 = std.mem.span(argv[0]), + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; +} pub fn runfiles(source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { return runfiles_global.withSourceRepo(source_repository); diff --git a/bazel/runfiles_init.c b/bazel/runfiles_init.c new file mode 100644 index 00000000..770a4b91 --- /dev/null +++ b/bazel/runfiles_init.c @@ -0,0 +1,15 @@ +extern void _bazel_runfiles_init_impl(int argc, char **argv, char **envp); + +// Preserve the existing startup ABI. Registration happens in C; +// runfiles discovery and initialization remain in Zig. +// glibc and Darwin's dyld pass argc/argv/envp to constructors; musl does not. +// The original Zig initializer already made this assumption to read argv[0]. +// +// Let C emit the platform's constructor registration. Zig's incremental ELF +// linker does not yet handle .init_array's section type, flags, and size correctly. +// TODO: Revert this workaround once Zig issue #37453 is fixed: +// https://codeberg.org/ziglang/zig/issues/37453 +__attribute__((constructor)) +void _bazel_runfiles_init(int argc, char **argv, char **envp) { + _bazel_runfiles_init_impl(argc, argv, envp); +} From 4c355c61b792017469e96904973bfdfeb0244e09 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 3 Nov 2026 18:11:15 +0100 Subject: [PATCH 03/47] zml/sharding: keep unconstrained graph results open (#1004) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ordinary graph results should have an unspecified layout until constrained or inferred by the partitioner. Previously, we could report replication simply because a tensor had no mesh, even when its stored partition specs were open. Other paths preserved selected operations’ sharding metadata, making the frontend’s interpretation depend on how a tensor was produced. This change gives those results a consistent open state while requiring concrete layouts at executable boundaries. Shardy explicitly defines the intended semantics: [“no sharding attribute on a tensor is equivalent to a fully open tensor sharding.”](https://openxla.org/shardy/sharding_representation#open) Open dimensions remain available for further partitioning; closed dimensions fix the specified layout. Replication is therefore a constraint, whereas an open layout lets the partitioner choose either sharding or replication: This distinction also applies to unused mesh axes. Marking an axis explicitly replicated prevents propagation from using it, even when a tensor dimension is open. For example, `[{?}, {}]` must leave unused axes available; adding replicated={"data", "model"} removes those choices. Keeping intermediates open lets the partitioner use constraints from producers and consumers, with operation-specific rules for transformations such as dot and reshape. --- examples/mnist/mnist_data_parallel.zig | 18 +++- zml/Compiler.zig | 39 +++++---- zml/Sharding.zig | 84 +++++++++++++++++-- zml/kernel.zig | 6 +- zml/kernels/flash_attention/cute/cute.zig | 2 +- zml/kernels/flash_attention/nki/nki.zig | 4 +- .../paged_attention/paged_attention.zig | 19 +++-- zml/ops.zig | 3 +- zml/ops/manual_computation.zig | 18 ++-- zml/shape.zig | 3 + zml/tensor.zig | 76 ++++++++++------- 11 files changed, 190 insertions(+), 82 deletions(-) diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index a539ae5a..a45c0534 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -2,9 +2,11 @@ //! Separated to kept `mnist.zig` minimal const std = @import("std"); + const zml = @import("zml"); const base = @import("mnist.zig"); + const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ @@ -53,14 +55,20 @@ const Mnist = struct { /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); + const layers: []const Layer = &.{ self.fc1, self.fc2 }; + const compiler = zml.Compiler.current(); + for (0.., layers) |i, layer| { compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); defer compiler.popLocation(); x = layer.forward(x); } - return x.argMax(.d).indices.convert(.u8).withPartitioning(input.shape()._sharding.mesh.?, .{ .b = .data }).toMemory(.host_pinned); + + return x.argMax(.d).indices.convert(.u8) + .withPartitioning(.dp_mp, .{ .b = .data, .d = .replicated }) + .toMemory(.host_pinned); } const Layer = struct { @@ -68,7 +76,7 @@ const Mnist = struct { bias: zml.Tensor, pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - log.warn("layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); + log.info("Layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); const x = input.dot(self.weight, .d); return x.add(self.bias.broad(x.shape())).relu().rename(.{ .d_out = .d }); } @@ -100,18 +108,20 @@ pub fn main(init: std.process.Init) !void { &.{ .low_bandwidth, .high_bandwidth }, &.{ 2, -1 }, )); - log.info("topology used: {f}", .{dp_mp}); + log.info("{f}", .{dp_mp}); var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); defer profiler.deinit(); var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); defer store.deinit(); + const mnist_model: Mnist = .init(store.view()); - // // Compile model + // Compile model const bs: u32 = 4; const input: zml.Tensor = .withPartitioning(.init(.{ .b = bs, .x = 28, .y = 28 }, .u8), dp_mp, .{ .b = .data }); + var exe = exe: { log.info("Compiling model....", .{}); const start: std.Io.Timestamp = .now(io, .awake); diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 3daba2ec..5c6e3933 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -100,7 +100,7 @@ pub const Error = std.mem.Allocator.Error || mlir.Error || upb.SerializeError || pjrtx.Client.CompileError || - error{MissingDeviceInTile}; + error{ MissingDeviceInTile, OpenShardingAtBoundaries }; pub const Donation = union(enum) { implicit: u32, @@ -462,10 +462,7 @@ pub fn compileInternal( var compiler: Compiler = .init(allocator, io, platform, opts); defer compiler.deinit(); - var result = compiler.emitMlir(func_, args) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => unreachable, - }; + var result = try compiler.emitMlir(func_, args); defer result.output_info.deinit(compiler.allocator); defer result.input_info.deinit(compiler.allocator); @@ -593,20 +590,21 @@ fn emitMlir(compiler: *Compiler, comptime func_: anytype, args: std.meta.ArgsTup var input_info = try createBlockArguments(compiler, fn_scope, &args); errdefer input_info.deinit(compiler.allocator); - var result = blk: { + var output_info = blk: { compiler.activate(); defer compiler.deactivate(); - break :blk @call(.auto, func_, args); - }; - var output_info = try collectOutputInfo(compiler, fn_scope, &result); + var result = @call(.auto, func_, args); + // Output collection can emit replication constraints, so it is part of tracing. + break :blk try collectOutputInfo(compiler, fn_scope, &result); + }; errdefer output_info.deinit(compiler.allocator); // Finalize in a separate function that doesn't depend on `func` type. return try finalizeMlirFunc(compiler, fn_scope, input_info, output_info); } -fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { +fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { const CreateBlockArgumentsCb = struct { compiler: *Compiler, scope: *Scope, @@ -614,9 +612,13 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou current_argument_id: u32 = 0, infos: std.MultiArrayList(TensorInfo) = .empty, - fn cb(ctx: *@This(), tensor: *const Tensor) error{OutOfMemory}!void { + fn cb(ctx: *@This(), tensor: *const Tensor) error{ OutOfMemory, OpenShardingAtBoundaries }!void { // Declare the program argument as a packed u8 const og_shape = tensor._shape; + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable input {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.current_argument_id, og_shape }); + return error.OpenShardingAtBoundaries; + } const packed_shape = og_shape.packedShape(); const mlir_type = mlirx.Type.rankedTensor(ctx.compiler.mlir_ctx, packed_shape); var value = ctx.scope.block.addArgument(mlir_type, .unknown(ctx.compiler.mlir_ctx)); @@ -650,7 +652,7 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou return context.infos; } -fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { +fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { const CollectOutputInfoCb = struct { compiler: *Compiler, scope: *Scope, @@ -658,9 +660,16 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf infos: std.MultiArrayList(TensorInfo) = .empty, fn cb(ctx: *@This(), tensor: *const Tensor) !void { - const og_shape = tensor.shape(); - var value = ctx.scope.id_to_argument.get(tensor.id) orelse - tensor._value orelse + // Outputs with no declared sharding default to replication. Constrain the + // returned value so the partitioner produces a replicated output buffer. + const output = if (tensor.shape()._sharding.mesh != null or tensor.shape().isFullyReplicated()) tensor.* else tensor.replicate(); + const og_shape = output.shape(); + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable output {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.infos.len, og_shape }); + return error.OpenShardingAtBoundaries; + } + var value = ctx.scope.id_to_argument.get(output.id) orelse + output._value orelse @panic("no value found for output tensor"); if (tensor.dtype().bitSizeOf() < 8) { diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 2637f209..c03dd07e 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -21,10 +21,22 @@ mesh: ?*const Mesh, partition: Partitioning, pub const MAX_MESH_RANK = 4; + +/// Full replication across participating devices. +/// For buffers, each device holds a complete copy. Applied as a graph constraint, +/// this closes every dimension to partitioning. `Shape.init` uses this sharding. +/// Unconstrained graph results use `open` instead. pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; +/// Unspecified sharding for graph results. During compilation, the partitioner +/// may infer sharding from surrounding constraints or leave the value replicated. +/// Executable inputs require closed sharding. An executable output with this +/// unspecified sharding defaults to replication. An explicitly declared output +/// sharding containing open dimensions is rejected. +pub const open: Sharding = .{ .mesh = null, .partition = .open(Shape.MAX_RANK) }; + pub fn get(sharding: Sharding, ax: usize) PartitionSpec { - return if (sharding.mesh == null) .replicated else sharding.partition.get(ax); + return sharding.partition.get(ax); } pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { @@ -52,7 +64,6 @@ pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { } pub fn isFullyReplicated(self: Sharding) bool { - if (self.mesh == null) return true; for (self.partition.toArray()) |spec| { if (spec != .replicated and spec != .out_of_bound) return false; } @@ -98,8 +109,18 @@ test isCompatible { } } +/// Formats the declared sharding, not the partitioner's inferred sharding. +/// Without a mesh, prints `{replicated}` for full replication or `{open}` for +/// unspecified graph sharding. +/// +/// With a mesh, entries follow tensor dimension order, e.g. `{data,replicated,?}@dp`: +/// - `data`: split this dimension over the logical mesh axis named `data`. +/// - `replicated`: keep this dimension whole on each shard. +/// - `?`: leave this dimension to partitioner inference during compilation. +/// - `@dp`: the mesh name. +/// Trailing `.out_of_bound` entries are omitted. Any remaining ones print as `!`. pub fn format(self: Sharding, writer: *std.Io.Writer) !void { - const mesh = self.mesh orelse return writer.writeAll("{replicated}"); + const mesh = self.mesh orelse return writer.writeAll(if (self.isFullyReplicated()) "{replicated}" else "{open}"); const specs = self.partition.toArray(); var rank = specs.len; while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; @@ -117,8 +138,10 @@ pub fn format(self: Sharding, writer: *std.Io.Writer) !void { try writer.print("}}@{s}", .{mesh.name}); } -/// Describes how a given Shape axis behaves inside a mesh. -/// Is it replicated ? sharded along a specific logical axis ? or open to replication ? +/// Describes how one tensor dimension is distributed across a logical mesh. +/// A mesh axis splits the dimension. `.replicated` keeps it whole on each shard. +/// In a graph constraint, `.open` lets the partitioner choose either behavior. +/// Executable buffers require closed sharding specs: a mesh axis or `.replicated`. pub const PartitionSpec = enum(u4) { mesh_axis_0 = 0, mesh_axis_1 = 1, @@ -1676,6 +1699,50 @@ pub const Mesh = struct { unreachable; } + /// Converts `shape`'s declared partitioning into a `#sdy.sharding` attribute + /// referencing this mesh. Entries in `[...]` follow tensor dimension order. + /// Mesh axis indices in the specs resolve to this mesh's logical axis names. + /// The tensor rank and logical mesh rank are independent. + /// + /// Each dimension spec is encoded as follows: + /// - `PartitionSpec.sharded(i)` -> `{"axis_name"}`: split the dimension over + /// logical mesh axis `i`. The dimension is closed to further partitioning. + /// - `.replicated` -> `{}`: keep the dimension whole on each shard. It is + /// closed, so the partitioner cannot assign a mesh axis to it. + /// - `.open` -> `{?}`: let the partitioner infer sharding from the graph. + /// It may acquire available mesh axes or remain replicated. + /// Closed therefore means no additional mesh axes, not necessarily replicated. + /// Every dimension within `shape.rank()` must have a spec other than `.out_of_bound`. + /// + /// Dimension replication (`{}`) applies to one tensor dimension. The optional + /// tensor-wide `replicated={...}` instead names mesh axes: each listed axis + /// must carry identical tensor data across its coordinates and cannot be used + /// to split any dimension, including an open one. + /// + /// For example, take a mesh `@dp_mp` with `data=2`, `model=2`, no extra replicas, + /// and a shape whose dimensions are ordered `batch`, `hidden`: + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .open } + /// #sdy.sharding<@dp_mp, [{"data"}, {?}]> + /// ``` + /// `batch` is split over `data`. The partitioner may assign `model` to `hidden`. + /// The unused `model` axis stays available because an open dimension exists. + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{"data"}, {}], replicated={"model"}> + /// ``` + /// Both dimensions are closed. Unused logical axes are explicitly replicated, + /// so devices differing only in their `model` coordinate hold the same shard. + /// + /// ``` + /// Partitioning: .{ .batch = .open, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{?}, {}]> + /// ``` + /// Only `batch` may acquire axes, and both `data` and `model` are available. + /// Adding `replicated={"data", "model"}` would prevent either axis from being + /// inferred for `batch`, despite its `{?}` entry. pub fn sdyShardingAttrForShape( data: *const Mesh, parent_allocator: std.mem.Allocator, @@ -1687,6 +1754,7 @@ pub const Mesh = struct { const allocator = arena.allocator(); const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); var used: [MAX_MESH_RANK]bool = @splat(false); + const has_open_dimension = shape.containsPartitionSpec(.open); for (0.., dimensions) |ax, *d| { d.* = switch (shape.partition(ax)) { .out_of_bound => @panic("undefined"), @@ -1704,7 +1772,9 @@ pub const Mesh = struct { var tile_count: usize = 1; for (data.logical.axes.slice(), 0..) |axis, i| { tile_count *= data.grid.axis_sizes[i]; - if (!used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(axis))); + // Open dimensions can acquire unused logical axes during inference. + // Only reserve those axes for replication when every dimension is closed. + if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(axis))); } if (tile_count < data.grid.device_count) { replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); @@ -2741,7 +2811,7 @@ test "mesh: open and replicated dimension mix" { .expected_num_replicas = 1, .mesh = &mesh, .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), - .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"batch\", \"model\"}>", + .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, .{ .device_id = 1, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, diff --git a/zml/kernel.zig b/zml/kernel.zig index 36c9d8d3..94d6046a 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1529,8 +1529,8 @@ test "tcl multi-chip kernels run on two furiosa chips" { const allocator = std.testing.allocator; const platform = zml.testing.env(); if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const model = platform.meshes.get("model").?; - const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(model, .{ .c = .model }); + const tp = zml.testing.tpMesh(); + const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); const Mod = struct { pub fn forward(x: Tensor) [2]Tensor { const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); @@ -1538,7 +1538,7 @@ test "tcl multi-chip kernels run on two furiosa chips" { } }; const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{model} }); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{tp} }); defer exe.deinit(); var hx: [2 * 1024]f32 = undefined; for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig index 2958888d..1136dc9a 100644 --- a/zml/kernels/flash_attention/cute/cute.zig +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -41,7 +41,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) else token_index.convert(.i32); - return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions }, q_sharded.shape()); + return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions }, q.shape().withPartitioning(mesh, .{ .h = .model })); } const Context = struct { diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig index 282da132..905f3aa0 100644 --- a/zml/kernels/flash_attention/nki/nki.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -82,7 +82,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T .token_index = token_index_2d, .parameters = parameters, }, - q_sharded.shape(), + q.shape().withPartitioning(tp, .{ .h = .model }), ).convert(q.dtype()); } @@ -108,6 +108,6 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T } }).body, .{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .parameters = parameters }, - q_sharded.shape(), + q.shape().withPartitioning(tp, .{ .h = .model }), ).convert(q.dtype()); } diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 99065140..65cf6cea 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -6,9 +6,9 @@ const zml = @import("../../zml.zig"); const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const triton = @import("triton/triton.zig"); -pub const tcl = @import("tcl/tcl.zig"); const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); +const triton = @import("triton/triton.zig"); const PagedAttention = @This(); @@ -335,21 +335,22 @@ test pagedAttention { const batch_size = active_batch_size + 1; const query_token_count = prefill_token_count + num_decode; const dt: zml.DataType = .bf16; - const model_meshe = platform.meshes.get("model").?; + const tp = zml.testing.tpMesh(); + const num_kv_heads = tp.numPartitionsForLogicalAxis(.model); const partition = .{ .hkv = .model }; const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hg = 4, .hd = 32 }, dt)).withPartitioning(tp, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), .kv_cache = .{ .split = .{ - .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), }, }, }; - const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, platform.meshes.get("model").? }; + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, tp }; const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); defer rng_q.deinit(); const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); diff --git a/zml/ops.zig b/zml/ops.zig index b1a26d23..e598d99c 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -830,10 +830,9 @@ fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(b fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); - // copy tags and mesh info over + // Copy tags; the partitioner infers result sharding. // some ops can change dims eg reduceWindow, so we trust mlir here. new._shape._tags = tensor._shape._tags; - new._shape._sharding = tensor._shape._sharding; tensor.* = new; inner_ctx.index += 1; } diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index c9596f98..170f96b5 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -6,18 +6,19 @@ const shardy = @import("mlir/dialects/shardy"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const zml = struct { - // Nested to disambiguate from stdx.meta - const meta = @import("../meta.zig"); - const testing = @import("../testing.zig"); -}; - const Compiler = @import("../Compiler.zig"); const DataType = @import("../dtype.zig").DataType; const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; + +const zml = struct { + // Nested to disambiguate from stdx.meta + const meta = @import("../meta.zig"); + const testing = @import("../testing.zig"); +}; + const log = std.log.scoped(.@"zml/Compiler"); /// ZML operator describes computation across several devices, @@ -214,9 +215,10 @@ noinline fn manualComputationInternal( }, }; - // reuse local_outputs buffer to store the final result. + // Returned tensors describe the operation's global output shapes and sharding. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { - output_tensor_ptr.* = Tensor._resultPropagateSharding(output_shape, op.result(o)); + output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); + output_tensor_ptr._shape = output_shape; } return local_outputs; } diff --git a/zml/shape.zig b/zml/shape.zig index b0863367..f866d2e2 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -827,6 +827,9 @@ pub const Shape = struct { .grid = undefined, }; + /// Returns shape metadata with the requested sharding on `mesh`. + /// Omitted dimensions in a mapping are replicated, including during compilation. + /// Use `Tensor.withPartitioning` to emit a constraint on a graph value. pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { if (@TypeOf(partitioning) != Sharding.Partitioning) { return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); diff --git a/zml/tensor.zig b/zml/tensor.zig index 8c148867..59d32cbd 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -75,14 +75,11 @@ pub const Tensor = struct { return self._shape.byteSize(); } - /// Internal use - /// - /// Creates a tensor from a Shape and an mlir.Value. - /// Note: sharding information is discarded, the output tensor is considered open for replication. - /// Use _resultPropagateSharding when the shape sharding should be propagated. + /// Internal: creates a graph tensor with dimensions, dtype and tags from `sh`, + /// and open sharding for the partitioner to infer from graph constraints. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; - res._shape._sharding.partition = .open(sh.rank()); + res._shape._sharding = .open; if (builtin.mode == .debug) { // Check that the MLIR value actually have the same shape. @@ -95,16 +92,16 @@ pub const Tensor = struct { return res; } + /// Internal: creates a constructor result and honors `sh`'s partitioning when + /// it specifies a mesh. pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { - var res = _result(sh, val); - res._shape._sharding = sh._sharding; - return res; + const result = _result(sh, val); + const mesh = sh._sharding.mesh orelse return result; + return result.withPartitioning(mesh, sh._sharding.partition); } - /// Creates a Tensor from a mlir.Value - /// - /// The shape is derived from the type of the mlir.Value. - /// mesh is `unknown` + /// Wraps an MLIR tensor value, deriving dimensions and dtype from its type. + /// Dimension tags are unknown and sharding is open for partitioner inference. pub fn fromMlirValue(val: *const mlir.Value) Tensor { const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; @@ -116,7 +113,7 @@ pub const Tensor = struct { ._dtype = ctx.dtype(ranked_tensor.elementType()), ._dims = .empty, ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, - ._sharding = .replicated, + ._sharding = .open, }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); @@ -166,12 +163,22 @@ pub const Tensor = struct { return res; } - /// Specify the mesh of the input tensor. - /// * mesh: *const zml.Mesh, but during compilation the `.meshe_name` syntax can used to get - /// a known mesh from the compilation options. - /// * partition spec: a struct where the field names match the axis of the given mesh + /// Declares this tensor's sharding on `mesh_`. + /// Pass a `*const Mesh`, or a mesh name such as `.tp` during compilation. + /// `partitioning` accepts a `Sharding.Partitioning` or a mapping from dimension + /// tags to logical mesh axes, `.replicated`, or `.open`. /// - /// eg `x.withPartitioning(tp, .{ .h = .model }))` or `x.withPartitioning(.tp, .{ .h = .model }))` + /// During compilation, emits a graph constraint. For example, + /// `x.withPartitioning(.tp, .{ .h = .model })` splits dimension `h` over `model` + /// and leaves omitted dimensions open for inference. Use `.replicated` to + /// keep a dimension whole on each shard. + /// + /// Outside compilation, updates the tensor's shape to describe an executable + /// input's sharding. Omitted dimensions are replicated and no graph constraint is + /// emitted. An explicit `Sharding.Partitioning` uses its supplied specs in both contexts. + /// + /// Executable inputs and explicitly annotated outputs must have no open + /// dimensions. An output with no declared sharding defaults to replication. pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { if (@TypeOf(mesh_) == @EnumLiteral()) { const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); @@ -179,7 +186,8 @@ pub const Tensor = struct { } if (@TypeOf(partitioning) != Sharding.Partitioning) { - const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, .open, partitioning); + const default: Sharding.PartitionSpec = if (Compiler.currentOrNull() == null) .replicated else .open; + const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, default, partitioning); return self.withPartitioning(mesh_, parsed); } @@ -222,11 +230,14 @@ pub const Tensor = struct { }, }; - return _resultPropagateSharding(partitioned_shape, op_result); + var res = _result(partitioned_shape, op_result); + res._shape = partitioned_shape; + return res; } - /// Force the input tensor to be fully replicated. - /// This can be detrimental for performance. + /// Emits a graph constraint requiring a complete copy on every device. + /// Requires active compilation. For a replicated input or buffer description, + /// use the default sharding from `Tensor.init` or `Shape.init`. pub fn replicate(self: Tensor) Tensor { const compiler = Compiler.current(); const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); @@ -267,7 +278,7 @@ pub const Tensor = struct { try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); } - /// Copy the given tensor to the specified memory. + /// Copy the given tensor to the specified memory, preserving its sharding. pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { @@ -298,7 +309,8 @@ pub const Tensor = struct { ctx.location, ).appendTo(currentBlock()); - const res = _result(self._shape, op.result(0)); + var res = _result(self._shape, op.result(0)); + res._shape._sharding = self._shape._sharding; ctx.currentScope().id_to_memory.putNoClobber(ctx.currentScope().arena.allocator(), res.id, kind) catch @panic("OOM"); return res; } @@ -845,7 +857,7 @@ pub const Tensor = struct { fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ - ._state = _resultPropagateSharding(self._state._shape, new_state).reuseBuffer(self._state), + ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, }; } @@ -886,7 +898,8 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - return .{ rng, _resultPropagateSharding(shape_, floats.convert(shape_.dtype()).value()) }; + const result = floats.convert(shape_.dtype()); + return .{ rng, _resultPropagateSharding(shape_, result.value()) }; } test uniform { @@ -2705,7 +2718,7 @@ pub const Tensor = struct { mlirx.Type.rankedTensor(mlirCtx(), res_shape), currentLoc(), ).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); + return _resultPropagateSharding(res_shape, op.result(0)); } pub const LinspaceArgs = struct { @@ -2780,7 +2793,8 @@ pub const Tensor = struct { } pub fn zeroes(sh: Shape) Tensor { - return Tensor.constant(sh.dtype().zero()).broad(sh); + const result = Tensor.constant(sh.dtype().zero()).broad(sh); + return _resultPropagateSharding(sh, result.value()); } pub fn uninitialized(sh: Shape) Tensor { @@ -2795,7 +2809,7 @@ pub const Tensor = struct { .appendTo(currentBlock()); const tensor = stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) .appendTo(currentBlock()); - return _result(sh, tensor.result(0)); + return _resultPropagateSharding(sh, tensor.result(0)); } /// Embeds a buffer with concrete values into an Mlir program. @@ -2803,7 +2817,7 @@ pub const Tensor = struct { const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); const constant_op = stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); - return _result(sh, constant_op.result(0)); + return _resultPropagateSharding(sh, constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. From 6a1698f27066bec5e00a13cbd88f72ffcc88a844 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 3 Nov 2026 18:22:59 +0100 Subject: [PATCH 04/47] zml/ops: add manualComputation2 with explicit partitioning (#1013) Accept a logical mesh and typed input/output partition specs at manual computation boundaries. Will replace `zml.ops.manualComputation` in a future PR. --- examples/sharding/main.zig | 19 +- zml/BUILD.bazel | 1 + zml/Sharding.zig | 50 ++-- zml/meta.zig | 18 +- zml/ops.zig | 5 + zml/ops/manual_computation2.zig | 465 ++++++++++++++++++++++++++++++++ zml/tensor.zig | 4 +- 7 files changed, 538 insertions(+), 24 deletions(-) create mode 100644 zml/ops/manual_computation2.zig diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index cc46a1bd..7e519bf0 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -30,15 +30,19 @@ const DemoModel = struct { } pub fn forward(self: DemoModel, input: zml.Tensor) zml.Tensor { - const x = input.convert(.f32); + const compiler = zml.Compiler.current(); + const x = input.convert(.f32); var y = x.dot(self.w, .feature); y = y.add(self.b.broad(y.shape())); y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); y.print("dense_out"); - const gate = y.scale(0.01).sigmoid().withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); - return zml.ops.manualComputation( + const gate = y.scale(0.01).sigmoid(); + + const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); + + return zml.ops.manualComputation2( (struct { y: zml.Tensor, gate: zml.Tensor, @@ -50,6 +54,11 @@ const DemoModel = struct { }).body, .{ .y = y, .gate = gate }, y.shape(), + compiler.mesh(.dp_mp), + .{ + .in_specs = .{ .y = spec, .gate = spec }, + .out_specs = spec, + }, ); } }; @@ -60,8 +69,8 @@ fn usage() void { \\ \\Simple sharding demo: \\ - explicit partitioning on data/model axes - \\ - shard-local tensor print - \\ - manualComputation with multiple inputs and one output + \\ - tensor print + \\ - manualComputation2 with multiple inputs and one output \\ \\Mesh modes: \\ - auto: physical_mesh=auto, inferred cpu device_count=2 diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 8188c743..dbb2ad33 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -98,6 +98,7 @@ zig_library( "nn.zig", "ops.zig", "ops/manual_computation.zig", + "ops/manual_computation2.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", diff --git a/zml/Sharding.zig b/zml/Sharding.zig index c03dd07e..2f887a8a 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -437,18 +437,22 @@ pub const Partitioning = packed struct { const logical = if (@TypeOf(partitioning) == Logical) partitioning else - // capture all the comptime logic in the Logical struct. + // capture all the comptime logic in Logical. comptime Logical.parse(partitioning); return logical.applyUseAll(tags, mesh, default_value); } - pub const Logical = struct { - src: Shape.TagsArray, - tgt: Shape.TagsArray, + pub const Logical = union(enum) { + /// Replicates every dimension, including when used as a graph constraint. + replicated, + mapping: struct { + src: Shape.TagsArray, + tgt: Shape.TagsArray, + }, - const replicated: Shape.Tag = Shape.toTag(.replicated); - const open: Shape.Tag = Shape.toTag(.open); + const replicated_tag: Shape.Tag = Shape.toTag(.replicated); + const open_tag: Shape.Tag = Shape.toTag(.open); pub fn parse(v: anytype) Logical { const V = @TypeOf(v); @@ -457,7 +461,7 @@ pub const Partitioning = packed struct { var tgt_: Shape.TagsArray = .empty; if (comptime stdx.meta.isStruct(V)) { - const field_names = std.meta.fieldNames(V); + const field_names = comptime std.meta.fieldNames(V); stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); inline for (field_names) |field_name| { const fv = @field(v, field_name); @@ -467,7 +471,7 @@ pub const Partitioning = packed struct { src_.appendAssumeCapacity(field_name); } } - return .{ .src = src_, .tgt = tgt_ }; + return .{ .mapping = .{ .src = src_, .tgt = tgt_ } }; } stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); @@ -476,23 +480,39 @@ pub const Partitioning = packed struct { test "Logical.parse" { const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); - try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.src.constSlice()); - try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.tgt.constSlice()); + try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.mapping.src.constSlice()); + try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.mapping.tgt.constSlice()); + } + + test "Logical.replicated" { + const mesh: Mesh = .initTest(.logical, .{.model}); + const tags: []const Shape.Tag = &.{ "a", "b" }; + const logical: Logical = .replicated; + for ([_]PartitionSpec{ .open, .replicated }) |default| { + try std.testing.expectEqual(Partitioning.replicated(tags.len), logical.applyUseAll(tags, &mesh, default)); + } + const unspecified = Logical.parse(.{}); + try std.testing.expectEqual(Partitioning.open(tags.len), unspecified.applyUseAll(tags, &mesh, .open)); } pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { + const mapping = switch (logical.*) { + .replicated => return .{ .replicated(tags.len), null }, + .mapping => |*mapping_| mapping_, + }; + var partition_: Partitioning = .repeat(default_value, tags.len); var missing_src_axis: ?u8 = null; - for (0.., logical.src.slice(), logical.tgt.slice()) |i, shape_tag, mesh_tag| { + for (0.., mapping.src.slice(), mapping.tgt.slice()) |i, shape_tag, mesh_tag| { const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { missing_src_axis = @intCast(i); continue; }; - const spec: PartitionSpec = if (mesh_tag == Logical.replicated) + const spec: PartitionSpec = if (mesh_tag == Logical.replicated_tag) .replicated - else if (mesh_tag == Logical.open) + else if (mesh_tag == Logical.open_tag) .open else spec: { const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { @@ -509,7 +529,7 @@ pub const Partitioning = packed struct { const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); if (missing_shape_axis) |ax| { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.src.get(ax) }); + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.mapping.src.get(ax) }); } return p; } @@ -2068,7 +2088,7 @@ pub const DeviceGrid = struct { for (strategy.foldSources(part.axis) orelse &.{part.axis}) |source| { if (used.contains(source)) continue; used.insert(source); - const info = mesh_.physical.axisInfo(source) orelse continue; + const info = mesh_.physical.axisInfo(source) orelse return error.InvalidPhysicalAxis; try grid.appendAxis(mesh_, logical_axis, source, 1, @intCast(info.size)); } } diff --git a/zml/meta.zig b/zml/meta.zig index 1eefea11..84fb9d36 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -143,7 +143,7 @@ test MapType { /// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. /// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. /// For example it can convert from a comptime array to a runtime slice. -/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. +/// `mapAlloc` allocates slices and single-item pointers needed to hold mapped values. /// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. /// /// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. @@ -180,6 +180,11 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam return; } + if (FromStruct == ToStruct and !comptime Contains(FromStruct, From)) { + to.* = from; + return; + } + if (@sizeOf(ToStruct) == 0) return; switch (type_info_to) { @@ -226,7 +231,11 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } to.* = items; }, - else => try mapAlloc(cb, allocator, ctx, from.*, to.*), + else => { + const item = try allocator.create(ptr_info.child); + try mapAlloc(cb, allocator, ctx, from.*, item); + to.* = item; + }, }, .slice => { const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); @@ -267,6 +276,7 @@ test mapAlloc { field: A, array: [2]A, slice: []const A, + pointer: *const A, other: u8, tag: [*:0]const u8, // We want to allow conversion from comptime to runtime, because Zig type inference works like this. @@ -278,6 +288,7 @@ test mapAlloc { field: B, array: [2]B, slice: []const B, + pointer: *const B, other: u8, tag: [*:0]const u8, static_val: u8, @@ -291,6 +302,7 @@ test mapAlloc { .other = 7, .tag = "feature", .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, + .pointer = &.{ .a = 13 }, .field_with_empty = .{ .{ .a = 9 }, .{} }, }; var bb: BB = undefined; @@ -298,6 +310,7 @@ test mapAlloc { try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); defer testing.allocator.free(bb.slice); defer testing.allocator.free(bb.static_slice); + defer testing.allocator.destroy(bb.pointer); try testing.expectEqual(4, bb.field.b); try testing.expectEqual(5, bb.array[0].b); @@ -309,6 +322,7 @@ test mapAlloc { try testing.expectEqual(10, bb.slice[1].b); try testing.expectEqual(11, bb.static_slice[0].b); try testing.expectEqual(12, bb.static_slice[1].b); + try testing.expectEqual(13, bb.pointer.b); } /// Visit a given type `T` and: diff --git a/zml/ops.zig b/zml/ops.zig index e598d99c..19d15ec3 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -11,8 +11,13 @@ const stdx = @import("stdx"); const manual_computation = @import("ops/manual_computation.zig"); pub const manualComputation = manual_computation.manualComputation; +const manual_computation2 = @import("ops/manual_computation2.zig"); +pub const manualComputation2 = manual_computation2.manualComputation2; +pub const ManualPartitioning = manual_computation2.ManualPartitioning; + test { _ = manual_computation; + _ = manual_computation2; } const Buffer = @import("buffer.zig").Buffer; diff --git a/zml/ops/manual_computation2.zig b/zml/ops/manual_computation2.zig new file mode 100644 index 00000000..2d3f0aec --- /dev/null +++ b/zml/ops/manual_computation2.zig @@ -0,0 +1,465 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../Compiler.zig"); +const meta = @import("../meta.zig"); +const mlirx = @import("../mlirx.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/Compiler"); + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = meta.MapRestrict(Tensor, Logical).map(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. +pub fn manualComputation2( + comptime body_fn: anytype, + inputs: stdx.meta.FnParam(body_fn, 0), + output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), +) stdx.meta.FnReturn(body_fn) { + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + const Outputs = stdx.meta.FnReturn(body_fn); + + const compiler = Compiler.current(); + var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); + + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + + const WrappedInputs = struct { + inputs: *const Inputs, + output_shapes: *const OutputShapes, + + fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { + const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); + const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); + + outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); + + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + } + }; + + var outputs_struct: Outputs = undefined; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + + _ = manualComputationInternal( + compiler, + arena, + mesh, + @typeName(@TypeOf(body_fn)), + &wrapped_inputs, + flat_input_tensors, + @ptrCast(&outputs_struct), + flat_output_shapes, + WrappedInputs.wrap, + ) catch |err| switch (err) { + error.OutOfMemory => @panic("OOM"), + }; + + return outputs_struct; +} + +const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; + +/// Reified version of manualComputation2. +/// This one is compiled exactly once, user code just need to compile the `collectPtrs` +noinline fn manualComputationInternal( + compiler: *Compiler, + arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, + functor_name: []const u8, + input_struct: *anyopaque, + input_tensors: []*Tensor, + output_struct: *anyopaque, + global_output_shapes: []const Shape, + local_fn: LocalComputationFn, +) error{OutOfMemory}![]*Tensor { + const global_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + + log.debug("manualComputation2({s}) will use mesh {s}", .{ functor_name, mesh.name }); + + const global_input_shapes = try arena.alloc(Shape, input_tensors.len); + for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + + const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); + for (input_types, input_tensors) |*input_ty, tensor_ptr| { + const global = tensor_ptr._shape; + const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); + log.debug("\t- {f} -> {f}", .{ global, local }); + input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); + } + const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); + @memset(input_locs, compiler.unknown_location); + + const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); + for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } + input_val.* = global_scope.value(tensor_ptr.*); + } + + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); + + const local_scope = compiler.pushBlock(manual_block); + defer local_scope.pop(); + for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { + const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above + tensor_ptr.* = switch (compiler.partitioner) { + .shardy => Tensor._result(local, manual_block.argument(i)), + .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), + }; + } + + compiler.manual_computation_depth += 1; + defer compiler.manual_computation_depth -= 1; + + const local_outputs = try local_fn(arena, input_struct, output_struct); + stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation2 body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation2 body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + } + + const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |output_shape, i| { + global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); + } + + const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); + for (local_output_values, local_outputs) |*output_val, tensor_ptr| { + output_val.* = local_scope.value(tensor_ptr.*); + } + + const op: *const mlir.Operation = switch (compiler.partitioner) { + .shardy => op: { + // Shardy creates a sdy.manual_computation block. + _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); + + const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); + for (out_mesh_attrs, global_output_shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); + } + + break :op shardy.manual_computation( + mlir_ctx, + input_values, + global_output_types, + manual_block, + try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), + .get(mlir_ctx, out_mesh_attrs), + try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), + compiler.location, + ).appendTo(global_scope.block); + }, + .gspmd => op: { + // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier + // Shortcut to avoid empty optimization barrier + if (global_output_shapes.len == 0) return &.{}; + + const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); + const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |global_shape, i| { + global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); + } + + break :op stablehlo.optimizationBarrier( + mlir_ctx, + global_values, + global_types, + compiler.location, + ).appendTo(global_scope.block); + }, + }; + + // Returned tensors describe the operation's global output shapes and sharding. + for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { + output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); + output_tensor_ptr._shape._sharding = output_shape._sharding; + } + return local_outputs; +} + +const gspmd = struct { + fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { + const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); + const full_to_shard = stablehlo.custom_call( + mlir_ctx, + &.{scope.value(local_tensor)}, + &.{local_type}, + .{ + .call_target_name = "SPMDFullToShardShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), + }, + }, + location, + ).appendTo(scope.block); + return Tensor._result(local_shape, full_to_shard.result(0)); + } + + fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { + const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); + const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); + + const shard_to_full = stablehlo.custom_call( + compiler.mlir_ctx, + &.{local_value}, + &.{global_type}, + .{ + .call_target_name = "SPMDShardToFullShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), + }, + }, + compiler.location, + ).appendTo(scope.block); + return .{ shard_to_full.result(0), global_type }; + } +}; + +test manualComputation2 { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const mesh = zml.testing.tpMesh(); + + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, + + fn body(self: @This(), output_shape: Shape) Tensor { + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation2( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); + } + }; + + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation2 composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = .parse(.{ .x = .model }), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation2(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation2(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 59d32cbd..890b5c63 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4813,14 +4813,14 @@ pub const Tensor = struct { defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation((struct { + ops.manualComputation2((struct { input: Tensor, name: []const u8, fn body(body_ctx: @This(), _: void) void { ops.customCall("zml$print", body_ctx.input, {}, .{ .name = body_ctx.name }, .{ .has_side_effect = true }); } - }).body, .{ .input = input, .name = full_name }, {}); + }).body, .{ .input = input, .name = full_name }, {}, ctx.platform.replicated_mesh, .{ .in_specs = .{ .input = .replicated } }); }, .oneapi, .neuron, .furiosa => {}, } From 194175b36d826baf8255ab1c4df2d79d811d625a Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:35:55 +0100 Subject: [PATCH 05/47] zml/kernels/flash_attn: add prefill flash_attention kernel and add Causal (#1000) Prefill kernel is based on libflashattn.so from zml/flash-attention --- zml/attention/attention.zig | 22 +- .../flash_attention/cute/attention.zig | 504 +++++++++++++++++- zml/kernels/flash_attention/cute/cute.zig | 13 +- .../flash_attention/flash_attention.zig | 111 +++- 4 files changed, 626 insertions(+), 24 deletions(-) diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 49fe947b..4cfad309 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -93,6 +93,12 @@ pub const Metadata = union(BackendType) { } }; +pub const Options = struct { + /// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every + /// key, as in an encoder or a diffusion transformer, and token_index is ignored. + causal: bool = true, +}; + /// Causal attention as used in transformers. /// /// **Shapes**: @@ -105,18 +111,26 @@ pub const Metadata = union(BackendType) { /// - .k is the number of keys /// - .hd is the head dimension pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + return attentionOpts(q, k, v, token_index, metadata, parameters, .{}); +} + +/// `attention`, causal or not depending on `opts`. +pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { return switch (parameters) { .stablehlo => b: { // Generate the attention mask. const seq_len = k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.appendAxes(.{.coord}), .{}); + const attn_mask: ?zml.Tensor = if (opts.causal) + zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null) + .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, q.dtype()), token_index.appendAxes(.{.coord}), .{}) + else + null; + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); break :b attn_output; }, - .flash_attention => flash_attention.attention(q, k, v, token_index, metadata.flash_attention, parameters.flash_attention), + .flash_attention => flash_attention.attention(q, k, v, token_index, metadata.flash_attention, parameters.flash_attention, opts), }; } diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig index 0d01c357..ec0fce16 100644 --- a/zml/kernels/flash_attention/cute/attention.zig +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -1,4 +1,7 @@ const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); +const vector = @import("mlir/dialects/vector"); const zml = @import("../../../zml.zig"); const cute = zml.kernel.cute; @@ -14,6 +17,8 @@ pub const Config = struct { head_dim: i64, dtype: cute.DType, splits: i64 = 1, + /// Query j sees the keys up to positions + j. Otherwise every query sees every key and positions is ignored. + causal: bool = true, }; /// Planning uses local (post-sharding) dimensions. XLA owns the compiled CUDA @@ -144,13 +149,16 @@ fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void const query = row.div(group).minimum(cfg.query_length - 1); const head = kv_head.mul(group).add(row.rem(group)); const start = a.positions.get(.{batch}); - const position = start.add(query); - const causal_end = start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1); + const position = if (cfg.causal) start.add(query) else b.constant(.i32, cfg.key_length - 1); + const live_end = if (cfg.causal) + start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1) + else + b.constant(.i32, cfg.key_length); // Partition the live cache, rather than its capacity, so decode uses all // blocks even when only a prefix of the cache is populated. - const chunk = causal_end.add(cfg.splits - 1).div(cfg.splits); + const chunk = live_end.add(cfg.splits - 1).div(cfg.splits); const begin = if (split) b.blockIdx().z.mul(chunk) else b.constant(.i32, 0); - const end = if (split) begin.add(chunk).minimum(causal_end) else causal_end; + const end = if (split) begin.add(chunk).minimum(live_end) else live_end; const zero = b.constant(.f32, 0); const neg_inf = b.constant(.f32, -std.math.inf(f32)); @@ -305,6 +313,494 @@ fn merge(b: *B, cfg: Config) cute.FinishError!void { } } +// ==================== Prefill ==================== +// +// FlashAttention-2 forward for `query_length > 1`, with the tiling FA2's sm80 kernel uses on sm_120 +// (`Flash_fwd_kernel_traits<128, 128, 64, 4>`): a 128-thread block owns `prefillRows` queries of one +// head and every warp owns whole rows (all the keys of a tile, the whole head dimension). Scores and +// output stay in `mma.sync` accumulators, so P is reused from registers as the A operand of P @ V, +// without cross-warp reductions. Q, K and V tiles go through swizzled shared memory: V_j is copied +// while S_j = Q K_j^T is computed, K_{j+1} while O += P_j V_j is. +// +// It is a program, not a bare kernel: its host function launches the kernel with the dynamic shared +// memory the kernel allocates (up to 96 KiB). A bare kernel's shared memory is static, which ptxas +// caps at 48 KiB before sm_120. + +pub const Prefill = cute.Program(Config, .{ + .name = "flash_attention_cute_prefill", + .inputs = &.{ "q", "k", "v", "positions" }, + .outputs = &.{"out"}, + .run = prefillProgram, +}); + +const prefill_kernel_name = "flash_attention_cute_prefill_kernel"; + +/// Keys per tile (FA2's kBlockN). +const prefill_keys = 64; + +/// Whether `Prefill` handles `cfg`; the other configurations use `Kernel`. +/// Same as FA2 we only support 16-bit inputs +pub fn prefillSupported(cfg: Config) bool { + const dtype_ok = cfg.dtype == .bf16 or cfg.dtype == .f16; + const head_dim_ok = cfg.head_dim == 32 or cfg.head_dim == 64 or cfg.head_dim == 128 or cfg.head_dim == 256; + return cfg.query_length > 1 and cfg.splits == 1 and dtype_ok and head_dim_ok; +} + +/// m16 row tiles per warp: two up to head dimension 128, one above so the output accumulators fit in registers. +fn prefillRowTiles(head_dim: i64) i64 { + return if (head_dim <= 128) 2 else 1; +} + +/// Queries per block (FA2's kBlockM). +fn prefillRows(cfg: Config) i64 { + return 4 * 16 * prefillRowTiles(cfg.head_dim); +} + +/// Query tiles on x, so the blocks resident at the same time share the K/V of one head in L2. +fn prefillGrid(cfg: Config) [3]i32 { + return .{ @intCast(@divTrunc(cfg.query_length + prefillRows(cfg) - 1, prefillRows(cfg))), @intCast(cfg.query_heads), @intCast(cfg.batch) }; +} + +fn prefillProgram(b: *B, cfg: Config) cute.FinishError!void { + std.debug.assert(prefillSupported(cfg)); + switch (cfg.head_dim) { + inline 32, 64, 128, 256 => |d| switch (cfg.dtype) { + inline .bf16, .f16 => |dt| { + b.beginFunction(prefill_kernel_name, .cuda_kernel); + try prefillSized(b, cfg, d, dt); + b.endFunction(block); + + b.beginFunction(Prefill.name, .host); + const host = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + const grid = prefillGrid(cfg); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, + .block = .{ b.constant(.i32, block[0]), b.constant(.i32, block[1]), b.constant(.i32, block[2]) }, + .dynamic_smem = b.kernelSmemSize(prefill_kernel_name), + .stream = b.cudaStream(), + }); + const launch = b.launchEx(prefill_kernel_name, config, .{ host.q, host.k, host.v, host.positions, host.out }); + b.returnHostStatus(b.cudaResultStatus(launch)); + b.endFunction(null); + }, + else => unreachable, + }, + else => unreachable, + } +} + +/// PTX `max.f32` (FMNMX). `Value.maximum` and `Value.maxnum` both lower to NaN-handling compare and +/// select sequences (three instructions); prefill scores are never NaN, masked ones are -inf. +fn fmax(b: *B, x: Value, y: Value) Value { + return b.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, .float(b.ctx, .f32)).?; +} + +fn unpackStruct(b: *B, comptime n: usize, value: Value, dtype: cute.DType) [n]Value { + var result: [n]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + } + return result; +} + +/// `ldmatrix.x4`: lanes 8i..8i+7 address the rows of matrix i, the result holds the four 8x8 fragments. +fn matrixLoad(b: *B, comptime transpose: bool, address: Value) [4]Value { + const assembly = "ldmatrix.sync.aligned.m8n8.x4" ++ (if (transpose) ".trans" else "") ++ ".shared.b16 {$0, $1, $2, $3}, [$4];"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.int(b.ctx, .i32)))), + .constraints = "=r,=r,=r,=r,r", + .has_side_effects = true, + }, b.loc())); + return unpackStruct(b, 4, result, .i32); +} + +/// `mma.sync.m16n8k16` with f32 accumulators: acc += a (16x16, row) @ b (16x8, col). +fn mma(b: *B, comptime dtype: cute.DType, a: [4]Value, rhs: [2]Value, acc: [4]Value) [4]Value { + const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, rhs[0].inner, rhs[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), + .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + }, b.loc())); + return unpackStruct(b, 4, result, .f32); +} + +/// The four f32 accumulators of one mma.sync, carried across loop iterations as one value (the loop +/// carries at most 64 values). +fn packAcc(b: *B, values: [4]Value) Value { + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); +} + +fn unpackAcc(b: *B, value: Value) [4]Value { + var result: [4]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.vectorExtract(value, @intCast(i), .f32); + } + return result; +} + +/// Comptime layout of one prefill block, shared by the setup and the per-tile steps. +/// +/// Tiles are row major with the 16-byte vector index of every row XORed with row % swizzle_rows, so +/// 16-byte cp.async stores and 8-row ldmatrix reads are free of bank conflicts. Shared memory addresses +/// are folded the way CuTe's static layouts let nvcc fold them: the XORed row bits of every address a +/// thread uses are a per-thread constant, and the k-step part of a column offset is bit-disjoint from +/// the per-thread part, so each ldmatrix / cp.async address is a per-thread base plus a constant +/// (XORed into the 16-byte vector index). +fn PrefillTile(comptime d: i64, comptime dt: cute.DType) type { + return struct { + const row_tiles = prefillRowTiles(d); + const rows = 4 * 16 * row_tiles; + const n = prefill_keys; + const score_tiles = n / 8; + const out_tiles = d / 8; + const row_bytes = d * 2; + /// The XORed vector index must stay in its row: rows of d=32 only have 4 vectors. + const swizzle_rows = @min(8, d / 8); + const q_bytes = rows * row_bytes; + const kv_bytes = n * row_bytes; + /// Rows covered by one 16-byte cp.async per thread of the block. + const copy_rows = @divExact(128, d / 8); + /// Running max and per-thread partial sum of the two rows of every row tile. + const stats = 2 * row_tiles; + /// Carried by the key loop: maxima, sums, then the packed output accumulators of every + /// (row tile, 8 output features). + const carried = 2 * stats + row_tiles * out_tiles; + const scale: f32 = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(d))); + + comptime { + // copy_rows rows starting at a multiple of copy_rows never carry into the XORed bits. + std.debug.assert(std.math.isPowerOfTwo(copy_rows)); + std.debug.assert(std.math.isPowerOfTwo(swizzle_rows)); + } + + b: *B, + cfg: Config, + lane: Value, + warp: Value, + first_query: Value, + start: Value, + q_smem: Value, + k_smem: Value, + v_smem: Value, + k_ptr: Value, + v_ptr: Value, + kv_base: Value, + tiles: Value, + /// cp.async: this thread copies the 16-byte vector `copy_col` of rows `copy_row + i * copy_rows`. + copy_row: Value, + copy_col: Value, + /// Byte offset in a tile of this thread's first cp.async destination, without the XORed part. + copy_row_offset: Value, + copy_vector_offset: Value, + /// ldmatrix rows of this thread, as byte offsets, and their XORed 16-byte vector offsets. + q_frag_row: Value, + k_frag_row: Value, + v_frag_row: Value, + qv_frag_vector: Value, + k_frag_vector: Value, + + const Self = @This(); + + fn init(b: *B, cfg: Config, tid: Value, first_query: Value, start: Value, shared: Value, k_ptr: Value, v_ptr: Value, kv_base: Value, tiles: Value) Self { + const lane = tid.rem(32); + const warp = tid.div(32); + const copy_row = tid.div(d / 8); + const copy_col = tid.rem(d / 8); + const lane8 = lane.rem(8); + // The fragment rows a thread reads are lane % 8 plus multiples of 8. + const lane_xor = lane.rem(swizzle_rows); + return .{ + .b = b, + .cfg = cfg, + .lane = lane, + .warp = warp, + .first_query = first_query, + .start = start, + .q_smem = shared, + .k_smem = shared.add(q_bytes), + .v_smem = shared.add(q_bytes + kv_bytes), + .k_ptr = k_ptr, + .v_ptr = v_ptr, + .kv_base = kv_base, + .tiles = tiles, + .copy_row = copy_row, + .copy_col = copy_col, + .copy_row_offset = copy_row.mul(row_bytes), + .copy_vector_offset = copy_col.bitXor(copy_row.rem(swizzle_rows)).mul(16), + // Q (A operand): rows lane % 16, vectors lane / 16 (+ 2 per k-step). + .q_frag_row = shared.add(warp.mul(16 * row_tiles).add(lane.rem(16)).mul(row_bytes)), + // K (B operand): keys lane % 8 + 8 * (lane / 16), vectors (lane / 8) % 2 (+ 2 per k-step). + .k_frag_row = shared.add(q_bytes).add(lane8.add(lane.div(16).mul(8)).mul(row_bytes)), + // V (B operand, transposed): keys lane % 8 + 8 * ((lane / 8) % 2), vectors lane / 16 (+ 2 per 16 features). + .v_frag_row = shared.add(q_bytes + kv_bytes).add(lane8.add(lane.div(8).rem(2).mul(8)).mul(row_bytes)), + .qv_frag_vector = lane.div(16).bitXor(lane_xor).mul(16), + .k_frag_vector = lane.div(8).rem(2).bitXor(lane_xor).mul(16), + }; + } + + /// Destination of the i-th cp.async of this thread in a tile starting at `base`. + fn copyDst(t: Self, base: Value, comptime i: usize) Value { + const xor = ((copy_rows * i) % swizzle_rows) * 16; + const vector_offset = if (xor == 0) t.copy_vector_offset else t.copy_vector_offset.bitXor(xor); + return base.add(t.copy_row_offset).add(vector_offset).add(copy_rows * i * row_bytes); + } + + /// Copy keys `key_base..key_base + n` of `src` into `dst`; keys past the end are zero-filled. + fn loadTile(t: Self, src: Value, dst: Value, key_base: Value) void { + const b = t.b; + const stride = t.cfg.kv_heads * d; + const first_key = key_base.add(t.copy_row); + const safe = src.add(t.kv_base.mul(2)); + const first = safe.add(first_key.to(.i64).mul(stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(n, copy_rows)) |i| { + const valid = first_key.add(copy_rows * i).lt(t.cfg.key_length); + const source = b.select(valid, first.add(copy_rows * i * stride * 2), safe); + b.cpAsync16(t.copyDst(dst, i), source, valid); + } + } + + fn queryOf(t: Self, mi: usize, h: usize) Value { + return t.first_query.add(t.warp.mul(16 * row_tiles)).add(mi * 16).add(t.lane.div(4)).add(h * 8); + } + + fn accIndex(comptime mi: usize, comptime di: usize) usize { + return @intCast(2 * stats + @as(i64, mi) * out_tiles + @as(i64, di)); + } + + /// One key tile. `masked` tiles apply the causal and key length masks; the other ones are + /// fully visible to every query of the block. + fn step(t: Self, comptime masked: bool, tile: Value, carried_in: [carried]Value) [carried]Value { + const b = t.b; + const lane = t.lane; + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const key_base = tile.mul(n); + b.cpAsyncWaitAll(); + b.syncThreads(); + t.loadTile(t.v_ptr, t.v_smem, key_base); + + // S = Q K^T, k-steps of 16 features. + var s: [row_tiles][score_tiles][4]Value = @splat(@splat(@splat(zero))); + inline for (0..d / 16) |ks| { + var qa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + qa[mi] = matrixLoad(b, false, t.q_frag_row.add(mi * 16 * row_bytes).add(t.qv_frag_vector.bitXor(ks * 32))); + } + const k_vector = t.k_frag_vector.bitXor(ks * 32); + inline for (0..score_tiles / 2) |np| { + const kb = matrixLoad(b, false, t.k_frag_row.add(np * 16 * row_bytes).add(k_vector)); + inline for (0..row_tiles) |mi| { + s[mi][2 * np] = mma(b, dt, qa[mi], .{ kb[0], kb[1] }, s[mi][2 * np]); + s[mi][2 * np + 1] = mma(b, dt, qa[mi], .{ kb[2], kb[3] }, s[mi][2 * np + 1]); + } + } + } + + // Online softmax in base 2 on the raw scores, the scale is folded into the exponent: + // p = exp2(s * scale - m * scale). Thread rows: lane / 4 (accumulators 0, 1) and lane / 4 + 8 (2, 3). + var next: [carried]Value = undefined; + var alpha: [row_tiles][2]Value = undefined; + inline for (0..row_tiles) |mi| { + if (masked) { + inline for (0..score_tiles) |ni| { + inline for (0..4) |j| { + const key = key_base.add(ni * 8).add(lane.rem(4).mul(2)).add(j % 2); + const in_range = key.lt(t.cfg.key_length); + const visible = if (t.cfg.causal) in_range.bitAnd(key.le(t.start.add(t.queryOf(mi, j / 2)))) else in_range; + s[mi][ni][j] = b.select(visible, s[mi][ni][j], neg_inf); + } + } + } + inline for (0..2) |h| { + var row_max = s[mi][0][h * 2]; + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + row_max = fmax(b, row_max, s[mi][ni][h * 2 + j]); + } + } + inline for (.{ 2, 1 }) |offset| { + row_max = fmax(b, row_max, b.shuffleXor(row_max, offset)); + } + const old_max = carried_in[mi * 2 + h]; + const new_max = fmax(b, old_max, row_max); + // A row without any visible key so far keeps m = -inf: exponentiate against a finite value. + const neg_scaled_max = fmax(b, new_max, b.constant(.f32, -std.math.floatMax(f32))).mul(-scale); + alpha[mi][h] = b.exp2Approx(old_max.mul(scale).add(neg_scaled_max)); + next[mi * 2 + h] = new_max; + var sum = carried_in[stats + mi * 2 + h].mul(alpha[mi][h]); + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + s[mi][ni][h * 2 + j] = b.exp2Approx(s[mi][ni][h * 2 + j].mul(scale).add(neg_scaled_max)); + sum = sum.add(s[mi][ni][h * 2 + j]); + } + } + next[stats + mi * 2 + h] = sum; + } + } + + // All warps are done with K_j once V_j has landed: start copying K_{j+1}. + b.cpAsyncWaitAll(); + b.syncThreads(); + var prefetch = b.openIf(tile.add(1).lt(t.tiles)); + { + t.loadTile(t.k_ptr, t.k_smem, key_base.add(n)); + prefetch.close(); + } + + // O = O * alpha + P V. The accumulators of two adjacent score tiles are the A operand of one k-step. + var o: [row_tiles][out_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + const prev = unpackAcc(b, carried_in[accIndex(mi, di)]); + inline for (0..4) |j| { + o[mi][di][j] = prev[j].mul(alpha[mi][j / 2]); + } + } + } + inline for (0..n / 16) |ks| { + var pa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + pa[mi] = .{ + b.packHalf2(s[mi][2 * ks][0], s[mi][2 * ks][1], dt), + b.packHalf2(s[mi][2 * ks][2], s[mi][2 * ks][3], dt), + b.packHalf2(s[mi][2 * ks + 1][0], s[mi][2 * ks + 1][1], dt), + b.packHalf2(s[mi][2 * ks + 1][2], s[mi][2 * ks + 1][3], dt), + }; + } + const v_row = t.v_frag_row.add(ks * 16 * row_bytes); + inline for (0..out_tiles / 2) |dp| { + const vb = matrixLoad(b, true, v_row.add(t.qv_frag_vector.bitXor(dp * 32))); + inline for (0..row_tiles) |mi| { + o[mi][2 * dp] = mma(b, dt, pa[mi], .{ vb[0], vb[1] }, o[mi][2 * dp]); + o[mi][2 * dp + 1] = mma(b, dt, pa[mi], .{ vb[2], vb[3] }, o[mi][2 * dp + 1]); + } + } + } + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + next[accIndex(mi, di)] = packAcc(b, o[mi][di]); + } + } + return next; + } + }; +} + +fn prefillSized(b: *B, cfg: Config, comptime d: i64, comptime dt: cute.DType) cute.FinishError!void { + const T = PrefillTile(d, dt); + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + + const tid = b.threadIdx().x; + const first_query = b.blockIdx().x.mul(T.rows); + const head = b.blockIdx().y; + const batch = b.blockIdx().z; + const kv_head = head.div(@divExact(cfg.query_heads, cfg.kv_heads)); + // One shared storage object (like a Python CuTe `SharedStorage`): the host compiler sizes the + // launch's dynamic shared memory from it, which `kernelSmemSize` returns. `allocSmem` and + // `allocSmemView` both become static shared memory instead. + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 1))); + const fields = [_][]const u8{ + std.fmt.comptimePrint("sQ:{d}:0", .{T.q_bytes}), + std.fmt.comptimePrint("sK:{d}:{d}", .{ T.kv_bytes, T.q_bytes }), + std.fmt.comptimePrint("sV:{d}:{d}", .{ T.kv_bytes, T.q_bytes + T.kv_bytes }), + }; + const smem = b.allocSmemStorage(.i8, 128, b.layoutType(b.layoutSpec(.{T.q_bytes + 2 * T.kv_bytes}, .{1})), 0, &fields); + const shared = b.ptrToInt(b.getIter(smem, b.ptrTy(.i8, .smem, 128) catch @panic("bad shared memory pointer")).value(), .i32); + const q_ptr = b.ptrToInt(a.q, .i64); + const out_ptr = b.ptrToInt(a.out, .i64); + const start = b.loadWords(1, b.ptrToInt(a.positions, .i64).add(batch.to(.i64).mul(4)), .global, .{})[0]; + + // Element offsets of this (batch, head) in q/out and (batch, kv head) in k/v, and the q/out row stride. + const q_stride = cfg.query_heads * d; + const q_base = batch.to(.i64).mul(cfg.query_length * q_stride).add(head.to(.i64).mul(d)); + const kv_base = batch.to(.i64).mul(cfg.key_length * cfg.kv_heads * d).add(kv_head.to(.i64).mul(d)); + + // Causal: the last query of the block sees keys up to start + last_query, and every query of the + // block sees the keys up to start + first_query. Otherwise only the last tile, past the end of the + // keys, is masked. + const last_query = first_query.add(T.rows).minimum(cfg.query_length).sub(1); + const tiles = if (cfg.causal) + start.add(last_query).add(1).minimum(cfg.key_length).add(T.n - 1).div(T.n) + else + b.constant(.i32, @divTrunc(cfg.key_length + T.n - 1, T.n)); + const full_tiles = if (cfg.causal) + start.add(first_query).add(1).minimum(cfg.key_length).div(T.n).minimum(tiles) + else + b.constant(.i32, @divTrunc(cfg.key_length, T.n)); + + const t: T = .init(b, cfg, tid, first_query, start, shared, b.ptrToInt(a.k, .i64), b.ptrToInt(a.v, .i64), kv_base, tiles); + + // Out of range rows are zero-filled and never read from global memory. + { + const first_row = first_query.add(t.copy_row); + const safe = q_ptr.add(q_base.mul(2)); + const first = safe.add(first_row.to(.i64).mul(q_stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(T.rows, T.copy_rows)) |i| { + const valid = first_row.add(T.copy_rows * i).lt(cfg.query_length); + b.cpAsync16(t.copyDst(t.q_smem, i), b.select(valid, first.add(T.copy_rows * i * q_stride * 2), safe), valid); + } + } + t.loadTile(t.k_ptr, t.k_smem, b.constant(.i32, 0)); + + var initial: [T.carried]Value = undefined; + const zero = b.constant(.f32, 0); + for (initial[0..T.stats]) |*v| { + v.* = b.constant(.f32, -std.math.inf(f32)); + } + for (initial[T.stats .. 2 * T.stats]) |*v| { + v.* = zero; + } + for (initial[2 * T.stats ..]) |*v| { + v.* = packAcc(b, .{ zero, zero, zero, zero }); + } + + var full = b.openFor(0, full_tiles, 1, initial); + { + full.yield(t.step(false, full.iv, full.carried[0..T.carried].*)); + } + var diagonal = b.openFor(full_tiles, tiles, 1, full.results[0..T.carried].*); + { + diagonal.yield(t.step(true, diagonal.iv, diagonal.carried[0..T.carried].*)); + } + const results = diagonal.results[0..T.carried].*; + + // Normalize by the row sums (reduced over the quad) and store pairs of features as one 32-bit word. + inline for (0..T.row_tiles) |mi| { + inline for (0..2) |h| { + var sum = results[T.stats + mi * 2 + h]; + inline for (.{ 2, 1 }) |offset| { + sum = sum.add(b.shuffleXor(sum, offset)); + } + const norm = b.select(sum.gt(0), b.rcpApprox(sum), zero); + const query = t.queryOf(mi, h); + var valid = b.openIf(query.lt(cfg.query_length)); + { + const row_offset = q_base.add(query.to(.i64).mul(q_stride)).add(t.lane.rem(4).mul(2).to(.i64)); + inline for (0..T.out_tiles) |di| { + const acc = unpackAcc(b, results[T.accIndex(mi, di)]); + const word = b.packHalf2(acc[h * 2].mul(norm), acc[h * 2 + 1].mul(norm), dt); + b.storeWords(1, out_ptr.add(row_offset.add(di * 8).mul(2)), .global, .{word}); + } + valid.close(); + } + } + } +} + test "CuTe attention emits tiled online softmax through head dimension 512" { for ([_]i64{ 64, 80, 128, 256, 512 }) |head_dim| { const ir = try Kernel.emit(std.testing.allocator, .{ diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig index 1136dc9a..c1ffa58f 100644 --- a/zml/kernels/flash_attention/cute/cute.zig +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -15,7 +15,8 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major >= 8; } -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor) zml.Tensor { +/// `causal`: query j sees the keys up to token_index + j. Otherwise every key, and token_index is ignored. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, causal: bool) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }) and k.shape().hasTags(.{ .k, .h, .hd }) and v.shape().hasTags(.{ .k, .h, .hd }), "CuTe attention requires q/h/hd and k/h/hd tags", .{}); const batched = q.shape().hasTag(.b) != null; const rank: usize = if (batched) 4 else 3; @@ -24,7 +25,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T stdx.debug.assert(q.dtype() == .bf16 or q.dtype() == .f16 or q.dtype() == .f32, "CuTe attention supports bf16, f16, and f32", .{}); stdx.debug.assert(q.dim(.hd) > 0 and q.dim(.hd) <= 512 and q.dim(.hd) == k.dim(.hd) and k.dim(.hd) == v.dim(.hd), "CuTe attention requires matching head dimensions in [1, 512]", .{}); stdx.debug.assert(k.dim(.h) > 0 and q.dim(.h) > 0 and @rem(q.dim(.h), k.dim(.h)) == 0 and k.dim(.h) == v.dim(.h), "CuTe attention requires Hq divisible by Hkv and matching K/V heads", .{}); - stdx.debug.assert(q.dim(.q) > 0 and q.dim(.q) <= k.dim(.k) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq <= Lk and matching K/V lengths", .{}); + stdx.debug.assert(q.dim(.q) > 0 and (!causal or q.dim(.q) <= k.dim(.k)) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq, Lq <= Lk when causal, and matching K/V lengths", .{}); stdx.debug.assert(token_index.dtype().isInteger(), "CuTe attention requires integer token positions", .{}); if (batched) { stdx.debug.assert(k.shape().hasTag(.b) != null and v.shape().hasTag(.b) != null and q.dim(.b) == k.dim(.b) and q.dim(.b) == v.dim(.b), "CuTe attention requires matching batch dimensions", .{}); @@ -41,7 +42,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) else token_index.convert(.i32); - return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions }, q.shape().withPartitioning(mesh, .{ .h = .model })); + return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions, .causal = causal }, q.shape().withPartitioning(mesh, .{ .h = .model })); } const Context = struct { @@ -49,6 +50,7 @@ const Context = struct { k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, + causal: bool, fn body(self: Context, output_shape: zml.Shape) zml.Tensor { const batched = self.q.shape().hasTag(.b) != null; @@ -67,9 +69,12 @@ const Context = struct { .head_dim = q.dim(.hd), .dtype = zml.kernel.cute.from(q.dtype()), .splits = splits, + .causal = self.causal, }; - const out = if (cfg.splits > 1) b: { + const out = if (kernel.prefillSupported(cfg)) + kernel.Prefill.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = q.shape() }, .{ .cfg = cfg }).out + else if (cfg.splits > 1) b: { const partials = kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), }, .{ diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index c1a27a36..b0a6106b 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -126,11 +126,12 @@ pub const Metadata = union(Backend) { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: zml.attention.Options) zml.Tensor { return switch (parameters) { - .cute => cute.attention(q, k, v, token_index), - .nki => nki.attention(q, k, v, token_index, parameters.nki), - .metal => metal.attention(q, k, v, token_index, metadata.metal), + .cute => cute.attention(q, k, v, token_index, opts.causal), + // The NKI and Metal kernels are causal only. + .nki => if (opts.causal) nki.attention(q, k, v, token_index, parameters.nki) else zml.nn.sdpa(q, k, v, .{}), + .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), }; } @@ -185,11 +186,95 @@ test "attention: q=8,qh=64,kh=8" { ); } -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - return testAttentionBackends(q_shape, k_shape, token_index_h, std.enums.values(Backend)); +test "attention: prefill, q=k=300,qh=8,kh=4" { + // Neither length is a multiple of the query (128) or key (64) tiles. + try testAttention( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + ); } -fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, backends: []const Backend) !void { +test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries start at 800: every query tile sees full key tiles before the diagonal. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{800}, + ); +} + +test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { + // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), + .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { + try testAttention( + .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), + .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { + try testAttention( + .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), + .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), + &.{ 150, 100 }, + ); +} + +test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. + try testAttentionOpts( + .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{ 0, 0 }, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), + .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { + try testAttentionOpts( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), + &.{5}, + .{ .causal = false }, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); +} + +fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const Backend) !void { const io = std.testing.io; const allocator = std.testing.allocator; @@ -206,9 +291,11 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: const max_k_idx: i64 = k_shape.dim(.k) - 1; const max_q_idx: i64 = q_shape.dim(.q) - 1; - for (token_index_h) |index| { - // Check for out of bound reads - std.debug.assert(index + max_q_idx <= max_k_idx); + if (opts.causal) { + // Check for out of bound reads, token_index is ignored by non-causal attention. + for (token_index_h) |index| { + std.debug.assert(index + max_q_idx <= max_k_idx); + } } const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ @@ -230,7 +317,7 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: const meshes = platform.meshes.values(); const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, zml.attention.attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters }, .{ + const reference_exe = try platform.compileFn(allocator, io, zml.attention.attentionOpts, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts }, .{ .program_name = "attention_stablehlo", .meshes = meshes, }); @@ -251,7 +338,7 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: allocator, io, attention, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters, opts }, .{ .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), .meshes = meshes, From 4707e58774409811d10b0aae95edfebab5441fef Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:17 +0100 Subject: [PATCH 06/47] zml.Compiler: allow to propagate tensor store names (#1018) The name can appear in the IR and thanks to #1003 it shows in the web view of the model making all compute blocks pretty obvious. The screenshot below without names would be quite confusing, but now we can see this is the QKV of first layer of a Llama model. image --- examples/mnist/mnist_data_parallel.zig | 2 +- zml/Compiler.zig | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index a45c0534..b7805e49 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -135,7 +135,7 @@ pub fn main(init: std.process.Init) !void { .{ .program_name = "mnist", .meshes = &.{dp_mp}, - .xla_dump_to = "/tmp/zml/mnist", + .xla_dump_to = "/tmp/zml/mnist_dp", }, ); }; diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 5c6e3933..654d9cba 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -23,6 +23,7 @@ const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; +const TensorStore = @import("io.zig").TensorStore; const Compiler = @This(); const log = std.log.scoped(.@"zml/Compiler"); @@ -31,6 +32,7 @@ allocator: std.mem.Allocator, io: std.Io, arena: std.heap.ArenaAllocator, +options: Options, mlir_registry: *mlir.DialectRegistry, mlir_ctx: *mlir.Context, mlir_pass_manager: *mlir.PassManager, @@ -77,6 +79,7 @@ pub const Options = struct { /// Incremented once after successful compilation. progress: std.Progress.Node = .none, meshes: []const *const Sharding.Mesh = &.{}, + store: ?*const TensorStore = null, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, /// HACK: Extra bytes added to the device memory size reported to XLA. @@ -194,6 +197,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, .mlir_known_types = mlir_known_types, .module = module, .platform = platform, + .options = opts, .partitioner = opts.partitioner orelse .fromTarget(platform.target), .meshes = meshes, .location = unknown_location, @@ -288,6 +292,7 @@ test pushLocation { .mlir_pass_manager = undefined, .mlir_known_types = undefined, .module = undefined, + .options = .{}, .platform = undefined, .partitioner = undefined, .meshes = undefined, @@ -620,8 +625,15 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ O return error.OpenShardingAtBoundaries; } const packed_shape = og_shape.packedShape(); - const mlir_type = mlirx.Type.rankedTensor(ctx.compiler.mlir_ctx, packed_shape); - var value = ctx.scope.block.addArgument(mlir_type, .unknown(ctx.compiler.mlir_ctx)); + const mlir_ctx = ctx.compiler.mlir_ctx; + const mlir_type = mlirx.Type.rankedTensor(mlir_ctx, packed_shape); + var location: *const mlir.Location = ctx.compiler.unknown_location; + if (ctx.compiler.options.store) |store| { + if (store.id_to_sources.get(tensor.id)) |sources| { + location = location.named(mlir_ctx, sources.tensors[0].name); + } + } + var value = ctx.scope.block.addArgument(mlir_type, location); // But immediately create the MLIR value corresponding to the fp4 data. // That way the user code only sees a proper fp4 tensor. From 350c6c1606398424e554660c52a580e80470e95f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:17 +0100 Subject: [PATCH 07/47] zml.Mesh: axis/hasAxis helpers (#1019) --- zml/Sharding.zig | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 2f887a8a..ddb23cf9 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -1515,6 +1515,10 @@ pub const LogicalMesh = struct { return .{ .axes = axes, .intents = intents, .dimensions = @splat(-1) }; } + pub fn hasAxis(m: *const LogicalMesh, logical_ax: Shape.Tag) ?u3 { + return @truncate(std.mem.findScalar(Shape.Tag, m.axes.slice(), logical_ax) orelse return null); + } + pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { const target = std.mem.span(tag); for (self.axes.slice(), self.intents.slice()) |t, i| { @@ -1680,13 +1684,17 @@ pub const Mesh = struct { return @divExact(self.numDevices(), self.numPartitions()); } + pub fn axis(mesh: *const Mesh, ax: @EnumLiteral()) u3 { + return mesh.logical.hasAxis(Shape.toTag(ax)) orelse std.debug.panic("Mesh(name={s}) doesn't have axis {}: {f}", .{ mesh.name, ax, mesh.logical }); + } + pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const shardy.MeshAttr { var axes: stdx.BoundedArray(*const shardy.MeshAxisAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; - for (self.logical.axes.slice(), 0..) |axis, i| { + for (self.logical.axes.slice(), 0..) |ax, i| { const size = self.grid.axis_sizes[i]; tile_count *= size; - axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(axis), .size = @intCast(size) })); + axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(ax), .size = @intCast(size) })); } const replicas = @divExact(self.grid.device_count, tile_count); if (replicas > 1) { @@ -1697,8 +1705,8 @@ pub const Mesh = struct { for (0..self.grid.device_count) |partition_index| { const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; var tile_index: usize = 0; - for (0..self.logical.axes.len) |axis| { - tile_index = tile_index * self.grid.axis_sizes[axis] + coords[axis]; + for (0..self.logical.axes.len) |ax| { + tile_index = tile_index * self.grid.axis_sizes[ax] + coords[ax]; } // Device IDs here are executable partition indices, not PJRT device IDs. ids[tile_index * replicas + replica_indices[tile_index]] = @intCast(partition_index); @@ -1711,8 +1719,8 @@ pub const Mesh = struct { // There are at most MAX_MESH_RANK logical names, so one candidate is free. inline for (0..MAX_MESH_RANK + 1) |i| { const name = std.fmt.comptimePrint("__replica_{d}", .{i}); - const taken = for (self.logical.axes.slice()) |axis| { - if (std.mem.eql(u8, std.mem.span(axis), name)) break true; + const taken = for (self.logical.axes.slice()) |ax| { + if (std.mem.eql(u8, std.mem.span(ax), name)) break true; } else false; if (!taken) return name; } @@ -1775,26 +1783,26 @@ pub const Mesh = struct { const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); var used: [MAX_MESH_RANK]bool = @splat(false); const has_open_dimension = shape.containsPartitionSpec(.open); - for (0.., dimensions) |ax, *d| { - d.* = switch (shape.partition(ax)) { + for (0.., dimensions) |i, *d| { + d.* = switch (shape.partition(i)) { .out_of_bound => @panic("undefined"), .replicated => .replicated(ctx), .open => .open(ctx, &.{}), else => d: { - const axis = shape.partition(ax).meshAxis().?; - used[axis] = true; - break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(axis)))}); + const ax = shape.partition(i).meshAxis().?; + used[ax] = true; + break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(ax)))}); }, }; } var replicated_axes: stdx.BoundedArray(*const shardy.AxisRefAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; - for (data.logical.axes.slice(), 0..) |axis, i| { + for (data.logical.axes.slice(), 0..) |ax, i| { tile_count *= data.grid.axis_sizes[i]; // Open dimensions can acquire unused logical axes during inference. // Only reserve those axes for replication when every dimension is closed. - if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(axis))); + if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(ax))); } if (tile_count < data.grid.device_count) { replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); @@ -1810,8 +1818,8 @@ pub const Mesh = struct { var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; var tile_count: usize = 1; - for (0..shape.rank()) |ax| { - const size = if (shape.partition(ax).meshAxis()) |axis| self.grid.axis_sizes[axis] else 1; + for (0..shape.rank()) |i| { + const size = if (shape.partition(i).meshAxis()) |ax| self.grid.axis_sizes[ax] else 1; tile_shape.appendAssumeCapacity(size); tile_count *= size; } From 17ba49cce828a260ca6e9f017e7818d6eafa6a95 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:18 +0100 Subject: [PATCH 08/47] zml.ops.manualComputation: fix error message (#1021) use `.format` instead of default Zig formatting --- zml/ops/manual_computation.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index 170f96b5..45400206 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -113,7 +113,7 @@ noinline fn manualComputationInternal( const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { .mesh => |m| m, - .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {}: {} and input {}: {}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), + .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {d}: {f} and input {d}: {f}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), }; const manual_axes = collectManualAxes(mesh, input_tensors); log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); From 8992fbe4343a5a1b241695920e494834d66a6554 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:18 +0100 Subject: [PATCH 09/47] zml.Shape.assertTags (#1022) we often use `withTags` to document the current shape, but it could also hide bug by overriding previous tags. I introduce `assertTags` that should be favored when possible --- zml/shape.zig | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/zml/shape.zig b/zml/shape.zig index f866d2e2..e793b520 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -808,6 +808,19 @@ pub const Shape = struct { } } + pub fn assertTags(self: Shape, tagz: anytype) void { + const T = @TypeOf(tagz); + if (T == Shape) { + return self.assertTags(tagz.tags()); + } else if (T != []const Shape.Tag) { + const parsed_tags: []const Shape.Tag = comptime parseTags(tagz).constSlice(); + return self.assertTags(parsed_tags); + } + + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, stdx.fmt.stringsZ(tagz) }); + stdx.debug.assert(std.mem.eql(Shape.Tag, tagz, self.tags()), "Expected tags {f}, got {f}", .{ stdx.fmt.stringsZ(tagz), self }); + } + pub fn withDtype(self: Shape, dt: DataType) Shape { var res = self; res._dtype = dt; From db119b2d3558a1e042ad0c217bedc5180a9adcc6 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:19 +0100 Subject: [PATCH 10/47] zml.Shape: improve parseTags (#1023) avoid `stdx.BoundedArray` for tags, use `[]const Tag`, this allows to simplify some logic and more explicitly separate comptime work from runtime work. It reduces binary sizes, preventing big functions to be duplicated because of different tags. --- zml/Compiler.zig | 7 ++----- zml/Sharding.zig | 6 +++--- zml/io.zig | 31 ++++++++++++++++++------------- zml/shape.zig | 32 ++++++++++++++++++++++++-------- zml/tensor.zig | 4 ++-- 5 files changed, 49 insertions(+), 31 deletions(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 654d9cba..93eafa8d 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -366,16 +366,13 @@ pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mes /// Find a mesh that covers all the given axes pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { if (@TypeOf(logical_axes) != []const Shape.Tag) { - const comp_tags = comptime Shape.parseTags(logical_axes); - const parsed_tags: []const Shape.Tag = comptime comp_tags.constSlice(); - return compiler.resolveMesh(parsed_tags); + return compiler.resolveMesh(comptime Shape.parseTags(logical_axes)); } var ok_meshe: ?*const Sharding.Mesh = null; for (compiler.meshes) |shd| { var covers_all: bool = true; - for (logical_axes) |ax| { - const input_axis = Shape.toTag(ax); + for (logical_axes) |input_axis| { var covers_this: bool = false; for (shd.logical.axes.slice()) |existing_axis| { if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { diff --git a/zml/Sharding.zig b/zml/Sharding.zig index ddb23cf9..040216bc 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -1478,8 +1478,8 @@ pub const LogicalMesh = struct { /// * dimensions: dimension for each axis, -1 infers the size from underlying topology pub fn init(names: anytype, intents: []const LogicalAxisIntent, dimensions: []const i32) !LogicalMesh { if (@TypeOf(names) != []const Shape.Tag) { - const parsed_names = comptime Shape.parseTags(names); - return .init(@as([]const Shape.Tag, parsed_names.slice()), intents, dimensions); + const parsed_names: []const Shape.Tag = comptime Shape.parseTags(names); + return .init(parsed_names, intents, dimensions); } if (names.len == 0 or names.len > MAX_MESH_RANK or names.len != intents.len or names.len != dimensions.len) @panic("invalid logical mesh"); @@ -1645,7 +1645,7 @@ pub const Mesh = struct { return .{ .name = @tagName(name_), .logical = .{ - .axes = .init(axes_tags.slice()), + .axes = .init(axes_tags), .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, .dimensions = @splat(-1), }, diff --git a/zml/io.zig b/zml/io.zig index 43bfdf23..1babdcfc 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -184,19 +184,22 @@ pub const TensorStore = struct { stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } - pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { - const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); - const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + pub fn maybeCreateTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const parsed_tags: ?[]const Shape.Tag = comptime parseTags(tags); if (mesh_ == .replicated) { - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .replicated); + return self.maybeCreateTensorInternal(subkey, parsed_tags, .replicated); } + const resolved_mesh = self.store.mesh(mesh_); + if (@TypeOf(partitioning) == Sharding.Partitioning) { + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partitioning }); + } // Unspecified tensor axes are treated as replicated - const partition: Sharding.Partitioning = if (has_tags) - .parse(parsed_tags.constSlice(), resolved_mesh, .replicated, partitioning) + const partition: Sharding.Partitioning = if (parsed_tags) |p_tags| + .parse(p_tags, resolved_mesh, .replicated, partitioning) else partitioning; - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .{ .mesh = resolved_mesh, .partition = partition }); + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partition }); } pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { @@ -204,8 +207,8 @@ pub const TensorStore = struct { stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { - const tensor = self.maybeCreateTensor(subkey, tags, mesh_, partitioning); + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { + const tensor = self.maybeCreateTensor(subkey, comptime parseTags(tags), mesh_, partitioning); if (tensor) |t| { const storage = self.store.id_to_sources.getPtr(t.id); storage.?.memory = .host_pinned; @@ -214,10 +217,7 @@ pub const TensorStore = struct { } pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { - const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); - const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; - - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.slice() else null, .replicated) orelse + return self.maybeCreateTensorInternal(subkey, comptime parseTags(tags), .replicated) orelse stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } @@ -244,6 +244,11 @@ pub const TensorStore = struct { return tensor; } + /// Accepts "null" as tags + fn parseTags(tags: anytype) ?[]const Shape.Tag { + return if (@TypeOf(tags) == @TypeOf(null)) null else Shape.parseTags(tags); + } + pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { const arena = self.store.arena.allocator(); diff --git a/zml/shape.zig b/zml/shape.zig index e793b520..386b594e 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -117,14 +117,31 @@ pub const Shape = struct { stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); } - pub fn parseTags(v: anytype) TagsArray { + pub fn parseTags(v: anytype) []const Shape.Tag { const T = @TypeOf(v); - stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); - var tags_: TagsArray = .empty; - inline for (v) |field| { - tags_.appendAssumeCapacity(toTag(field)); + if (T == []const Shape.Tag) return v; + comptime validateTags(T); + return comptime frozen: { + var tagz: [v.len]Shape.Tag = undefined; + for (v, 0..) |field, i| tagz[i] = @tagName(field).ptr; + // Force the creation of a comptime constant, so that we can return a pointer to it. + const frozen_tags = tagz; + break :frozen &frozen_tags; + }; + } + + fn validateTags(T: type) void { + const message = "Expected an enum-literal tuple such as .{ .b, .d }, got " ++ @typeName(T); + switch (@typeInfo(T)) { + .@"struct" => |info| { + if (!info.is_tuple) @compileError(message); + for (info.field_types) |Field| { + if (Field != @EnumLiteral()) @compileError(message); + } + if (info.field_types.len > Shape.MAX_RANK) @compileError("Too many shape tags (maximum 8)"); + }, + else => @compileError(message), } - return tags_; } /// Create a shape from a struct literal, eg: @@ -813,8 +830,7 @@ pub const Shape = struct { if (T == Shape) { return self.assertTags(tagz.tags()); } else if (T != []const Shape.Tag) { - const parsed_tags: []const Shape.Tag = comptime parseTags(tagz).constSlice(); - return self.assertTags(parsed_tags); + return self.assertTags(comptime parseTags(tagz)); } stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, stdx.fmt.stringsZ(tagz) }); diff --git a/zml/tensor.zig b/zml/tensor.zig index 890b5c63..a7782844 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1528,7 +1528,7 @@ pub const Tensor = struct { pub fn dotAxes(lhs: Tensor, rhs: Tensor, tags: anytype) DotAxes { const T = @TypeOf(tags); if (comptime T == Shape.Tag or T == @EnumLiteral()) { - return lhs.dotAxes(rhs, .{tags}); + return lhs.dotAxes(rhs, @as([]const Shape.Tag, &.{Shape.toTag(tags)})); } const lhs_contracting_axes, _ = lhs.shape().parseAxes(tags); @@ -2983,7 +2983,7 @@ pub const Tensor = struct { var res_shape = self._shape; const ones: [constants.MAX_RANK]i64 = @splat(1); res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; - res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + res_shape._tags.insertSlice(ax, tags_) catch unreachable; return self.reshape(res_shape); } From 5764202e5fe3e3af556c4e183fca21ee0e6cd452 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 4 Nov 2026 18:52:05 +0100 Subject: [PATCH 11/47] zml/kernels/paged_attention/tcl: follow sequence lengths and support several chips (#1026) - **Work follows sequence lengths:** decode's compute no longer runs over the block table's whole width, and flash prefill loops only over the chunks in use. - **One plan per call:** each call picks straight decode, the hybrid loop or flash attention, following a calibration sweep based on what compiles. The kernels might change once the compiler evolves. - **Several chips:** each chip runs the kernels over its own heads inside a manual computation. --- zml/kernels/paged_attention/tcl/tcl.zig | 759 +++++++++++++++++++----- 1 file changed, 624 insertions(+), 135 deletions(-) diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index ce8bdb4f..a7c2a5f4 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -1,19 +1,6 @@ -//! Furiosa RNGD paged attention in TCL, using the common/flash attention -//! algorithms from Furiosa's SDK (`furiosa-kernels 2026.4.0` reference). -//! This implementation retains FP32 normalization/accumulation and BF16 -//! contractions. The exact common SDK operation materializes scores in BF16; -//! it fails the required unit-scale tolerance and is not a selectable backend. -//! -//! Queries are cut in blocks of rows of one sequence (one token per block for -//! decode). A context of at most `chunk_keys` keys runs Furiosa's one-shot -//! attention: gather the keys by token slot, contract, mask, one softmax, -//! contract. Longer contexts run Furiosa's flash attention: a device loop over -//! chunks of `chunk_keys` keys up to the longest sequence, gathering whole -//! pages and accumulating the softmax statistics in f32. -//! -//! Decode without sinks or windows, up to `decode_keys` keys, instead runs one -//! fusible kernel per layer over the whole pages each token sees, with the -//! token metadata every layer shares computed once per forward pass. +//! Furiosa RNGD paged attention in TCL: f32 softmax and accumulation, BF16 contractions. +//! `defaultPlan` picks fusible BF16 decode kernels, else one-shot attention or a device loop over key chunks. +//! `Repro:` files are in the reproducer repository, platform/furiosa. const std = @import("std"); const stdx = @import("stdx"); @@ -25,17 +12,43 @@ const tcl = zml.kernel.tcl; const paged_attention = @import("../paged_attention.zig"); const triton = @import("../triton/triton.zig"); +/// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to +/// lower, so decode chunks hold 4096 keys. Repro: repro_flash_decode_8k_chunks.py. +/// Remove once 8192-key chunks compile. const chunk_keys = 4096; +const flash_chunk_pairs = 512 * 2048; +const prefill_chunk_keys = 2048; const prefill_block = 32; +/// WORKAROUND (TCC 2026.5.0b0): block counts that are not a power of two slow TCC's layout +/// search or panic its allocator. Repro: repro_flash_allocator_block_group_panic.py. +/// Remove once any count compiles. +fn prefillBlocks(tokens: i64, sequences: i64, block: i64) i64 { + const blocks = @divTrunc(tokens + sequences * (block - 1), block); + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable); +} + +fn prefillBlock(tokens: i64, sequences: i64) i64 { + const short = prefill_block / 2; + return if (short * prefillBlocks(tokens, sequences, short) < prefill_block * prefillBlocks(tokens, sequences, prefill_block)) short else prefill_block; +} + +/// A straight-line head over each token's first `head` keys, then a device loop over +/// `chunk`-key chunks (none when zero), with tokens in groups of `group`. +pub const Plan = struct { + head: i64, + chunk: i64, + group: i64, +}; + +pub var calibration_plan: ?Plan = null; + pub const paged = struct { pub const Options = triton.paged.Options; pub const Parameters = triton.paged.Parameters; - /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches - /// `{.page, .k_chunk, .hkv, .hd}`. Sinks are one logit per query head. - /// Unwritten cache entries are excluded before the contractions; callers - /// need only populate the tokens covered by their sequence lengths. + /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. + /// Sinks are one logit per query head. pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); @@ -46,7 +59,7 @@ pub const paged = struct { stdx.debug.assert(parameters.seq_lens.dim(.b) > 0 and parameters.block_table.dim(.p) > 0 and k_cache.dim(.page) > 0 and k_cache.dim(.k_chunk) > 0, "TCL attention requires nonempty sequence and cache capacities", .{}); stdx.debug.assert(k_cache.dim(.page) * k_cache.dim(.k_chunk) <= std.math.maxInt(i32) and q.dim(.b) <= std.math.maxInt(i32), "TCL attention indices must fit in I32", .{}); const compiler = zml.Compiler.current(); - stdx.debug.assert(compiler.platform.target == .furiosa and compiler.meshes[0].numDevices() == 1, "TCL attention requires one Furiosa chip", .{}); + stdx.debug.assert(compiler.platform.target == .furiosa, "TCL attention requires Furiosa chips", .{}); const attributes = compiler.platform.devices[0].pjrt_device.attributes(compiler.platform.pjrt_api) catch std.debug.panic("TCL attention could not verify the Furiosa device's PE range", .{}); defer attributes.deinit(); @@ -54,25 +67,46 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); + if (compiler.meshes[0].numDevices() > 1) return zml.ops.manualComputation(Shard.body, .{ .parameters = parameters, .q = q, .k_cache = k_cache, .v_cache = v_cache, .opts = opts }, q.shape()); + return attend(parameters, q, k_cache, v_cache, opts); + } + + const Shard = struct { + parameters: Parameters, + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + opts: paged_attention.AttentionOptions, + + fn body(self: Shard, _: zml.Shape) Tensor { + return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); + } + }; + + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); - // TCC 2026.5 has no DPE tactic for 32-key attention. Pad to - // at least 64 keys; the extra pages remain invalid and contribute zero. + // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to + // at least 64 keys with invalid pages. Repro: repro_decode_32_keys_no_tactic.py. + // Remove once 16- to 48-key decode compiles. const keys = @max(parameters.block_table.dim(.p), std.math.divCeil(i64, 64, page) catch unreachable) * page; - // Chunks hold whole pages and a multiple of 64 keys: TCC 2026.5 fails - // to allocate a chunk of 4080 keys (85 pages of 48). + // WORKAROUND (TCC 2026.5.0b0): TCC fails to allocate a 4080-key chunk (85 + // pages of 48), so chunks hold whole pages and a multiple of 64 keys. Repro: + // repro_flash_allocator_block_group_panic.py. Remove once such chunks compile. const key_alignment = @divExact(page, @as(i64, @intCast(std.math.gcd(@as(u64, @intCast(page)), 64)))) * 64; const alignment = if (key_alignment <= chunk_keys) key_alignment else page; - const chunk = if (keys <= chunk_keys) keys else @min(keys, @max(1, @divTrunc(chunk_keys, alignment)) * alignment); const decode = !parameters.options_.is_prefill and q.dim(.b) == parameters.seq_lens.dim(.b); - const blocks = (std.math.divCeil(i64, q.dim(.b), prefill_block) catch unreachable) + parameters.seq_lens.dim(.b); - // Regular block capacities limit layout-search cost for the fused graph. - // Extra blocks retain zero valid lengths and invalid query positions. - const cfg: Cfg = .{ + const block = if (decode) 1 else prefillBlock(q.dim(.b), parameters.seq_lens.dim(.b)); + const block_count = if (decode) q.dim(.b) else prefillBlocks(q.dim(.b), parameters.seq_lens.dim(.b), block); + const rows = block_count * block; + // A flash chunk's query-key pairs are bounded: the largest chunks do not finish compiling. + const chunk_limit = @max(1, @divTrunc(@min(@as(i64, if (decode) chunk_keys else prefill_chunk_keys), @divTrunc(flash_chunk_pairs, rows)), alignment)) * alignment; + const chunk = @min(keys, chunk_limit); + var cfg: Cfg = .{ .tokens = q.dim(.b), .sequences = parameters.seq_lens.dim(.b), - .b = if (decode) q.dim(.b) else @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable), - .i = if (decode) 1 else prefill_block, + .b = block_count, + .i = block, .a = (std.math.divCeil(i64, keys, chunk) catch unreachable) * chunk, .chunk = chunk, .hkv = q.dim(.hkv), @@ -86,10 +120,24 @@ pub const paged = struct { .is_causal = opts.mask.isCausal(), .window = @intCast(opts.mask.slidingWindow() orelse 0), }; - if (decode and !cfg.use_sink and cfg.window == 0 and keys <= decode_keys) { - return decodeAttention(parameters, q, k_cache, v_cache, .{ + const plain_decode = decode and !cfg.use_sink and cfg.window == 0; + // WORKAROUND (TCC 2026.5.0b0): an in-unit cache conversion and several tokens + // over a non-power-of-two page count fail to compile, so straight decode reads + // BF16 caches and pads the table. Repro: repro_sparse_propagation_transmute.py, + // repro_metadata_page_count_cover_valid_range.py. Remove once both compile. + const straight = decode and k_cache.dtype() == .bf16 and v_cache.dtype() == .bf16; + const straight_keys = if (straight and cfg.tokens > 1) @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@divExact(keys, page))) catch unreachable)) * page else keys; + const plan = calibration_plan orelse defaultPlan(cfg, straight_keys, parameters.block_table.dim(.p), plain_decode and straight, straight, chunk); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)); + if (plan.head > 0 and plan.chunk > 0) { + stdx.debug.assert(plain_decode and cfg.tokens == 1, "TCL hybrid attention takes one plain decode token", .{}); + return hybridDecodeAttention(parameters, q, k_cache, v_cache, cfg.scale, cfg.is_causal, plan).convert(q.dtype()); + } + if (plan.chunk == 0 and straight) { + return decodeAttention(parameters, q, k_cache, v_cache, sink, .{ .t = cfg.tokens, - .a = keys, + .a = straight_keys, + .group = plan.group, .hkv = cfg.hkv, .g = cfg.g, .d = cfg.d, @@ -97,8 +145,12 @@ pub const paged = struct { .pg = page, .scale = cfg.scale, .is_causal = cfg.is_causal, + .window = cfg.window, + .use_sink = cfg.use_sink, }).convert(q.dtype()); } + cfg.chunk = if (plan.chunk > 0) plan.chunk else keys; + cfg.a = (std.math.divCeil(i64, keys, cfg.chunk) catch unreachable) * cfg.chunk; const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); const inputs: OneShot.Inputs = .{ .q = q.convert(.bf16), @@ -107,7 +159,7 @@ pub const paged = struct { .query_offsets = parameters.query_start_len, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), - .sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)), + .sink = sink, }; const outputs: OneShot.Outputs = .{ .out = q.shape().withDtype(.bf16) }; const out = if (cfg.a == cfg.chunk) @@ -118,13 +170,55 @@ pub const paged = struct { } }; -/// Keys a decode kernel attends to in one pass: TCC 2026.5 fails on the -/// decode metadata for 131072 keys in pages of 16. +fn defaultPlan(cfg: Cfg, keys: i64, columns: i64, plain_decode: bool, straight: bool, chunk: i64) Plan { + // WORKAROUND (TCC 2026.5.0b0): the hybrid kernel's tail does not compile in + // reasonable time for 2 or 4 KV heads wider than 128, which run flash instead. + // Repro: repro_hybrid_decode_slow_compile.py. Remove once that tail compiles. + if (plain_decode and cfg.tokens == 1 and columns > std.math.divCeil(i64, straight_decode_keys, cfg.pg) catch unreachable) { + if (cfg.d <= 128 or cfg.hkv == 1 or cfg.hkv >= 8) return .{ .head = head_keys, .chunk = tail_chunk_keys, .group = 1 }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; + } + if (straight and cfg.window > 0) { + const window_chunk = @max(window_chunk_keys, @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.window)) catch unreachable))); + const chunks: i64 = if (cfg.tokens < chunked_decode_tokens) window_flash_chunks else batch_window_flash_chunks; + if (window_chunk <= chunk and @rem(window_chunk, cfg.pg) == 0 and keys >= chunks * window_chunk) + return .{ .head = 0, .chunk = window_chunk, .group = cfg.tokens }; + } + if (straight and keys <= decode_keys and (cfg.tokens < chunked_decode_tokens or keys <= straight_batch_decode_keys)) + return .{ .head = keys, .chunk = 0, .group = @divExact(cfg.tokens, decodeGroups(cfg.tokens, keys)) }; + if (chunk == keys) return .{ .head = keys, .chunk = 0, .group = cfg.tokens }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; +} + +const window_chunk_keys = 1024; +const window_flash_chunks = 16; +const batch_window_flash_chunks = 4; + +/// Widest block table the straight-line decode kernel takes for one token. +/// WORKAROUND (TCC 2026.5.0b0): several tokens do not use it, as TCC cannot +/// propagate gather bounds outside a loop over a token axis. Repro: +/// repro_sparse_propagation_loop.py. Remove once such kernels propagate. +const straight_decode_keys = 32768; + +/// Keys a straight-line decode kernel attends to in one pass. WORKAROUND (TCC +/// 2026.5.0b0): TCC fails on the decode metadata for 131072 keys in pages of 16. +/// Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once that compiles. const decode_keys = 65536; +const chunked_decode_tokens = 16; +const straight_batch_decode_keys = 8192; + +/// Decode contracts the gathered cache unmasked: the select gives unused entries (NaN included) zero +/// weight. WORKAROUND (TCC 2026.5.0b0): TCC panics ("slice out of bounds") on score contractions of more +/// tokens times keys than this. Repro: repro_gather_einsum_slice_panic.py. Remove once larger products compile. +const unmasked_decode_keys = 1 << 17; + +const masked_decode_keys = 4096; + const DecodeCfg = struct { t: i64, a: i64, + group: i64, hkv: i64, g: i64, d: i64, @@ -132,14 +226,12 @@ const DecodeCfg = struct { pg: i64, scale: f32, is_causal: bool, + window: i64, + use_sink: bool, }; -/// Decode attention in Furiosa's form: the kernel gathers whole pages, then -/// runs one contraction, a masked softmax and a second contraction. It reads -/// past each sequence's end, selecting zero for those unused cache entries. -/// The token metadata depends only on the attention parameters, which every -/// layer shares: one kernel computes it once per forward pass. -fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, cfg: DecodeCfg) Tensor { +/// The token metadata depends only on parameters every layer shares, so one kernel computes it per forward pass. +fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, sink: Tensor, cfg: DecodeCfg) Tensor { const columns = @divExact(cfg.a, cfg.pg); const table = parameters.block_table.pad(0, .{ .p = Tensor.Pad{ .high = columns - parameters.block_table.dim(.p) } }); const meta = DecodeMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ @@ -147,7 +239,7 @@ fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_c .bounds = zml.Shape.init(.{ .t = cfg.t }, .i32), .mask = zml.Shape.init(.{ .t = cfg.t, .a = cfg.a }, .i32), .keyed = zml.Shape.init(.{ .t = cfg.t }, .f32), - }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal } }); + }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal, .window = cfg.window } }); return Decode.call(.{ .q = q.convert(.bf16), .pages = meta.pages, @@ -156,19 +248,15 @@ fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_c .keyed = meta.keyed, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), + .sink = sink, }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; } -const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool }; +const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool, window: i64 }; -/// For each token: the cache page of each column, how many columns hold keys -/// it sees, the keys it sees (below its limit, in pages inside the cache) and -/// whether it sees any. Columns past the sequence's -/// pages read page `p`, so that the masked padding spreads over many pages -/// instead of hammering one. Rows are selected by one-hot products and -/// comparisons follow `attentionMask`'s clamp arithmetic: TCC 2026.5 has no -/// tactics for small integer gathers and fails on the plugin's lowering of the -/// same comparisons for several tokens. +/// WORKAROUND (TCC 2026.5.0b0): one-hot row selection and `attentionMask`'s clamp arithmetic, as TCC +/// fails on small integer gathers and the plugin's comparisons. Repro: repro_scalar_gather_no_axis_to_pad.py, +/// repro_row_broadcast_past_4096.py. Remove once both compile. const DecodeMeta = tcl.Kernel(MetaCfg, .{ .name = "paged_decode_metadata", .inputs = &.{ "seq_lens", "query_offsets", "table" }, @@ -188,14 +276,12 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ }); const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); - // Padding tokens follow every sequence's end and belong to none. const seq = try countBefore(b, ends, tokens, S); const own = try selector(b, seq, S, cfg.s); var op = b.tensorOperation(.{}); const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); var visible = try selectMetadata(b, t.seq_lens, own, S); if (cfg.is_causal) { - // Causal tokens see their sequence up to their own position. const end = try selectMetadata(b, ends, own, S); op = b.tensorOperation(.{}); visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); @@ -213,8 +299,6 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ const spread = try op.commit(op.fetch(try b.arange(cfg.ap, .i32, &.{AP}, .{}), .{}).binary(.mini, cfg.pages - 1), .{}); op = b.tensorOperation(.{}); const read = try op.commit(op.where(op.fetch(used, .{}), .ne, 0, op.fetch(pages.safe, .{}), op.fetch(spread, .{ .broadcast_to = &.{ T, AP } })), .{}); - // Tokens without keys have no defined attention: the attention - // kernel zeroes them. op = b.tensorOperation(.{}); const keyed = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{AP}, .maxi).toFloat(), .{}); op = b.tensorOperation(.{}); @@ -222,21 +306,19 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ op = b.tensorOperation(.{}); const position = try op.commit(op.fetch(try b.arange(cfg.ap * cfg.pg, .i32, &.{A}, .{}), .{ .broadcast_to = &.{ T, A } }), .{}); op = b.tensorOperation(.{ .tactic = .elementwise }); - const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); + var mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); + if (cfg.window > 0) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(position, .{}).subi(op.fetch(limit, .{})).addi(cfg.window + 1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } b.ret(&.{ read, bounds, mask, keyed }); } }.run, }); -/// Gathers the key and value pages a token sees, contracts with the query, -/// masks keys past the sequence, normalizes in one softmax times `keyed` (zero -/// for tokens without keys, whose softmax is uniform) and contracts with the -/// values. A token's pages are gathered up to its bound, and the operations -/// reading them skip the rest: the work follows the sequence's length, not the -/// table's. Fusible: it joins the layer around it, one launch per layer. const Decode = tcl.Kernel(DecodeCfg, .{ .name = "paged_decode_attention", - .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache" }, + .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache", "sink" }, .outputs = &.{"out"}, .run = struct { fn run(b: *tcl.Builder, cfg: DecodeCfg) tcl.FinishError!void { @@ -258,26 +340,360 @@ const Decode = tcl.Kernel(DecodeCfg, .{ .keyed = .{ .dtype = .f32, .axes = &.{T} }, .k_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, .v_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, + .sink = .{ .dtype = .f32, .axes = &.{ Hkv, G } }, }); - const mask = try b.reshape(t.mask, &.{ T, AP, PG }, .{}); - const k = try maskCacheValues(b, try b.gather(t.k_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); - const v = try maskCacheValues(b, try b.gather(t.v_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); - const keys: []const tcl.Axis = &.{ AP, PG }; - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(t.q, k, &.{ Hkv, G, T, AP, PG }).mulf(cfg.scale), .{}); - op = b.tensorOperation(.{}); - const s = op.fetch(scores, .{}); - const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, s, score_min), .{}); - op = b.tensorOperation(.{}); - const x = op.fetch(masked, .{}); - const e = x.subf(x.reduce(keys, .maxf)).exp(); - const p = try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(t.keyed, .{})), .{ .dtype = .bf16 }); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 })}); + const sink = if (cfg.use_sink) t.sink else null; + const groups = @divExact(cfg.t, cfg.group); + if (groups == 1) return b.ret(&.{try decodeTokens(b, cfg.scale, t.q, t.pages, t.bounds, t.mask, t.keyed, t.k_cache, t.v_cache, sink)}); + const NG = b.axis("NG", groups); + const TG = b.axis("TG", @divExact(cfg.t, groups)); + var loop = b.openFor(groups, .{try b.scratchpad(.bf16, &.{ NG, TG, Hkv, G, D })}); + { + const q = try b.indexRead(try b.reshape(t.q, &.{ NG, TG, Hkv, G, D }, .{}), loop.iv); + const pages = try b.indexRead(try b.reshape(t.pages, &.{ NG, TG, AP }, .{}), loop.iv); + const bounds = try b.indexRead(try b.reshape(t.bounds, &.{ NG, TG }, .{}), loop.iv); + const mask = try b.indexRead(try b.reshape(t.mask, &.{ NG, TG, A }, .{}), loop.iv); + const keyed = try b.indexRead(try b.reshape(t.keyed, &.{ NG, TG }, .{}), loop.iv); + const out = try decodeTokens(b, cfg.scale, q, pages, bounds, mask, keyed, t.k_cache, t.v_cache, sink); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, out)}); + } + b.ret(&.{try b.reshape(loop.results[0], &.{ T, Hkv, G, D }, .{})}); } }.run, }); +const unmasked_group_tokens = 4; + +/// Groups of at most `unmasked_group_tokens` tokens whose tokens times keys fit `unmasked_decode_keys`. +fn decodeGroups(tokens: i64, keys: i64) i64 { + var groups: i64 = 1; + while ((@divExact(tokens, groups) > unmasked_group_tokens or @divExact(tokens, groups) * keys > unmasked_decode_keys) and groups < tokens) { + groups += 1; + while (@rem(tokens, groups) != 0) groups += 1; + } + return groups; +} + +/// `keyed` scales the softmax: zero for tokens without keys. +fn decodeTokens(b: *tcl.Builder, scale: f32, q: builder.Tensor, pages: builder.Tensor, bounds: builder.Tensor, mask: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, sink: ?builder.Tensor) tcl.FinishError!builder.Tensor { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const AP = pages.axes[1]; + const P = k_cache.axes[0]; + const PG = k_cache.axes[1]; + const keys: []const tcl.Axis = &.{ AP, PG }; + const key_mask = try b.reshape(mask, &.{ T, AP, PG }, .{}); + var k = try b.gather(k_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + var v = try b.gather(v_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + if (T.size == 1 and AP.size * PG.size <= masked_decode_keys) { + k = try maskCacheValues(b, k, key_mask); + v = try maskCacheValues(b, v, key_mask); + } + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, AP, PG }).mulf(scale), .{}); + op = b.tensorOperation(.{}); + const s = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(key_mask, .{}), .ne, 0, s, score_min), .{}); + op = b.tensorOperation(.{}); + const p = if (sink) |logits| blk: { + const maximum = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(logits, .{ .broadcast_to = &.{ Hkv, G, T } })), .{}); + op = b.tensorOperation(.{}); + const e = try op.commit(op.fetch(masked, .{}).subf(op.fetch(maximum, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const sink_exp = try op.commit(op.fetch(maximum, .{}).mulf(-1).addf(op.fetch(logits, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(e, .{}).reduce(keys, .addf).addf(op.fetch(sink_exp, .{})), .{}); + op = b.tensorOperation(.{}); + break :blk try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + } else blk: { + const x = op.fetch(masked, .{}); + const e = x.subf(x.reduce(keys, .maxf)).exp(); + break :blk try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + }; + op = b.tensorOperation(.{}); + return op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 }); +} + +const head_keys = 4096; +const tail_chunk_keys = 8192; + +const HybridCfg = struct { + t: i64, + s: i64, + /// Head pages, tail chunks and pages per tail chunk. + hp: i64, + nc: i64, + cp: i64, + pg: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + scale: f32, + is_causal: bool, +}; + +fn hybridDecodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, scale: f32, is_causal: bool, plan: Plan) Tensor { + const pg = k_cache.dim(.k_chunk); + const t = q.dim(.b); + const s = parameters.seq_lens.dim(.b); + const hp = std.math.divCeil(i64, plan.head, pg) catch unreachable; + const cp = std.math.divCeil(i64, plan.chunk, pg) catch unreachable; + const nc = std.math.divCeil(i64, parameters.block_table.dim(.p) - hp, cp) catch unreachable; + const cfg: HybridCfg = .{ .t = t, .s = s, .hp = hp, .nc = nc, .cp = cp, .pg = pg, .hkv = q.dim(.hkv), .g = q.dim(.hg), .d = q.dim(.hd), .pages = k_cache.dim(.page), .scale = scale, .is_causal = is_causal }; + const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = hp + nc * cp - parameters.block_table.dim(.p) } }); + const meta = HybridMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ + .hpages = zml.Shape.init(.{ .t = t, .p = hp }, .i32), + .hmask = zml.Shape.init(.{ .t = t, .a = hp * pg }, .i32), + .hbounds = zml.Shape.init(.{ .t = t }, .i32), + .tpages = zml.Shape.init(.{ .c = nc, .t = t, .p = cp }, .i32), + .tmask = zml.Shape.init(.{ .c = nc, .t = t, .a = cp * pg }, .i32), + .tbounds = zml.Shape.init(.{ .c = nc, .t = t }, .i32), + .limits = zml.Shape.init(.{ .t = t }, .i32), + .keyed = zml.Shape.init(.{ .t = t }, .f32), + }, .{ .cfg = cfg }); + return HybridDecode.call(.{ + .q = q.convert(.bf16), + .hpages = meta.hpages, + .hmask = meta.hmask, + .hbounds = meta.hbounds, + .tpages = meta.tpages, + .tmask = meta.tmask, + .tbounds = meta.tbounds, + .limits = meta.limits, + .keyed = meta.keyed, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; +} + +/// Per token, once per forward pass. Tail chunks are chunk-major, only those holding keys of the +/// longest token; `limits` is the visible key count. +const HybridMeta = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_metadata", + .inputs = &.{ "seq_lens", "query_offsets", "table" }, + .outputs = &.{ "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed" }, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + const total = (cfg.hp + cfg.nc * cfg.cp) * cfg.pg; + const T = b.axis("T", cfg.t); + const S = b.axis("S", cfg.s); + const O = b.axis("O", cfg.s + 1); + const AP = b.axis("AP", cfg.hp + cfg.nc * cfg.cp); + const HP = b.axis("HP", cfg.hp); + const RP = b.axis("RP", cfg.nc * cfg.cp); + const NC = b.axis("NC", cfg.nc); + const CP = b.axis("CP", cfg.cp); + const HA = b.axis("HA", cfg.hp * cfg.pg); + const CK = b.axis("CK", cfg.cp * cfg.pg); + const t = try b.declareArgs(.{ + .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, + .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, + }); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); + const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); + const seq = try countBefore(b, ends, tokens, S); + const own = try selector(b, seq, S, cfg.s); + var op = b.tensorOperation(.{}); + const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); + var visible = try selectMetadata(b, t.seq_lens, own, S); + if (cfg.is_causal) { + const end = try selectMetadata(b, ends, own, S); + op = b.tensorOperation(.{}); + visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); + } + op = b.tensorOperation(.{}); + const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, total).muli(op.fetch(active, .{})), .{}); + + // WORKAROUND (TCC 2026.5.0b0): a 3-D one-hot product computes garbage when the one-hot is read + // first, so the selection is 2-D per chunk. Repro: repro_onehot_einsum_by_ve_wrong_values_run.py. + // Remove once it is exact on RNGD. + const rows1 = try b.reshape(try b.slice(t.table, AP, cfg.hp, &.{ S, RP }, .{}), &.{ S, NC, CP }, .{}); + op = b.tensorOperation(.{}); + const rows = try op.commit(op.fetch(rows1, .{}), .{ .axes = &.{ NC, S, CP } }); + // Trips never exceed nc: `limit` is at most the table's keys. + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limit), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.scratchpad(.i32, &.{ NC, T, CP }), try b.scratchpad(.i32, &.{ NC, T, CK }), try b.scratchpad(.i32, &.{ NC, T }), try b.full(0, .i32, &.{T}) }); + { + const reads, const masks, const counts, const seen = loop.carried; + const first = try b.symExpr(.add, try b.symExpr(.mul, loop.iv, cfg.cp * cfg.pg), cfg.hp * cfg.pg); + const chunk = try hybridColumns(b, cfg, own, try b.indexRead(rows, loop.iv), limit, first, CP, CK); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(seen, .{}).binary(.maxi, op.fetch(chunk.any, .{})), .{}); + try loop.yield(.{ + try b.indexWrite(reads, loop.iv, chunk.pages), + try b.indexWrite(masks, loop.iv, chunk.mask), + try b.indexWrite(counts, loop.iv, chunk.bounds), + any, + }); + } + const head = try hybridColumns(b, cfg, own, try b.slice(t.table, AP, 0, &.{ S, HP }, .{}), limit, 0, HP, HA); + op = b.tensorOperation(.{}); + const keyed = try op.commit(op.fetch(head.any, .{}).binary(.maxi, op.fetch(loop.results[3], .{})).toFloat(), .{}); + b.ret(&.{ head.pages, head.mask, head.bounds, loop.results[0], loop.results[1], loop.results[2], limit, keyed }); + } + }.run, +}); + +/// Columns start at key `first`; sparse gathers never read past a token's `bounds`. +fn hybridColumns(b: *tcl.Builder, cfg: HybridCfg, own: builder.Tensor, table: builder.Tensor, limit: builder.Tensor, first: anytype, cols: tcl.Axis, keys: tcl.Axis) tcl.FinishError!struct { pages: builder.Tensor, mask: builder.Tensor, bounds: builder.Tensor, any: builder.Tensor } { + const T = own.axes[0]; + const S = own.axes[1]; + const PG = b.axis("PG", cfg.pg); + const pages = try validIndices(b, try selectMetadata(b, table, own, S), cfg.pages); + var op = b.tensorOperation(.{}); + const firsts = try op.commit(op.fetch(try b.arange(cols.size * cfg.pg, .i32, &.{cols}, .{ .step = cfg.pg }), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{}); + const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{cols}, .addi), .{}); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{cols}, .maxi), .{}); + op = b.tensorOperation(.{}); + const key_ids = try op.commit(op.fetch(try b.arange(keys.size, .i32, &.{keys}, .{}), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + const to: []const tcl.Axis = &.{ T, cols, PG }; + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(try b.reshape(key_ids, &.{ cols, PG }, .{}), .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{}); + const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid, .{})), .{}); + return .{ .pages = pages.safe, .mask = try b.reshape(mask, &.{ T, keys }, .{}), .bounds = bounds, .any = any }; +} + +const SoftmaxState = struct { max: builder.Tensor, sum: builder.Tensor, out: builder.Tensor }; + +const HybridDecode = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_decode_attention", + .inputs = &.{ "q", "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed", "k_cache", "v_cache" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + b.fuse(); + const a = hybridAxes(b, cfg); + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, + .hpages = .{ .dtype = .i32, .axes = &.{ a.T, a.HP } }, + .hmask = .{ .dtype = .i32, .axes = &.{ a.T, a.HA } }, + .hbounds = .{ .dtype = .i32, .axes = &.{a.T} }, + .tpages = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CP } }, + .tmask = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CK } }, + .tbounds = .{ .dtype = .i32, .axes = &.{ a.NC, a.T } }, + .limits = .{ .dtype = .i32, .axes = &.{a.T} }, + .keyed = .{ .dtype = .f32, .axes = &.{a.T} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + }); + const tail = try hybridTail(b, cfg, a, t.q, t.tpages, t.tmask, t.tbounds, t.limits, t.k_cache, t.v_cache); + b.ret(&.{try hybridHead(b, cfg, a, t.q, t.hpages, t.hmask, t.hbounds, t.keyed, t.k_cache, t.v_cache, tail)}); + } + }.run, +}); + +const HybridAxes = struct { T: tcl.Axis, HP: tcl.Axis, HA: tcl.Axis, NC: tcl.Axis, CP: tcl.Axis, CK: tcl.Axis, Hkv: tcl.Axis, G: tcl.Axis, D: tcl.Axis, P: tcl.Axis, PG: tcl.Axis }; + +fn hybridAxes(b: *tcl.Builder, cfg: HybridCfg) HybridAxes { + return .{ + .T = b.axis("T", cfg.t), + .HP = b.axis("HP", cfg.hp), + .HA = b.axis("HA", cfg.hp * cfg.pg), + .NC = b.axis("NC", cfg.nc), + .CP = b.axis("CP", cfg.cp), + .CK = b.axis("CK", cfg.cp * cfg.pg), + .Hkv = b.axis("Hkv", cfg.hkv), + .G = b.axis("G", cfg.g), + .D = b.axis("D", cfg.d), + .P = b.axis("P", cfg.pages), + .PG = b.axis("PG", cfg.pg), + }; +} + +fn hybridTail(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, tpages: builder.Tensor, tmask: builder.Tensor, tbounds: builder.Tensor, limits: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor) tcl.FinishError!SoftmaxState { + const stats: []const tcl.Axis = &.{ a.Hkv, a.G, a.T }; + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limits), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.full(score_min, .f32, stats), try b.full(0, .f32, stats), try b.full(0, .f32, &.{ a.T, a.Hkv, a.G, a.D }) }); + { + const max, const sum, const out = loop.carried; + const mask = try b.reshape(try b.indexRead(tmask, loop.iv), &.{ a.T, a.CP, a.PG }, .{}); + const bounds = try b.indexRead(tbounds, loop.iv); + const pages = try b.indexRead(tpages, loop.iv); + const k = try b.gather(k_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const v = try b.gather(v_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const merged = try mergeSoftmax(b, .{ .max = max, .sum = sum, .out = out }, try chunkSoftmax(b, cfg, q, k, v, mask)); + try loop.yield(.{ merged.max, merged.sum, merged.out }); + } + return .{ .max = loop.results[0], .sum = loop.results[1], .out = loop.results[2] }; +} + +fn hybridHead(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, hpages: builder.Tensor, hmask: builder.Tensor, hbounds: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, tail: SoftmaxState) tcl.FinishError!builder.Tensor { + const head_mask = try b.reshape(hmask, &.{ a.T, a.HP, a.PG }, .{}); + const head_k = try b.gather(k_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const head_v = try b.gather(v_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const keys: []const tcl.Axis = &.{ a.HP, a.PG }; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, head_k, &.{ a.Hkv, a.G, a.T, a.HP, a.PG }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(head_mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(tail.max, .{})), .{}); + const alpha = try expDifference(b, tail.max, max); + op = b.tensorOperation(.{}); + const carried = try op.commit(op.fetch(tail.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().reduce(keys, .addf).addf(op.fetch(carried, .{})), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const head = try op.commit(op.contract(head_v, p, &.{ a.T, a.Hkv, a.G, a.D }), .{}); + op = b.tensorOperation(.{}); + const factor = try op.commit(op.fetch(alpha, .{}).divf(op.fetch(denominator, .{})), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.fetch(tail.out, .{}).mulf(op.fetch(factor, .{})).addf(op.fetch(head, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); +} + +fn chunkSoftmax(b: *tcl.Builder, cfg: HybridCfg, q: builder.Tensor, k: builder.Tensor, v: builder.Tensor, mask: builder.Tensor) tcl.FinishError!SoftmaxState { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const keys = mask.axes[1..]; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, keys[0], keys[1] }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf), .{}); + const exps = try maskedExp(b, masked, max, mask); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(exps, .{}).reduce(keys, .addf), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(exps, .{}).mulf(1.0), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const out = try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + +fn mergeSoftmax(b: *tcl.Builder, x: SoftmaxState, y: SoftmaxState) tcl.FinishError!SoftmaxState { + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const max = try op.commit(op.fetch(x.max, .{}).maxf(op.fetch(y.max, .{})), .{}); + const alpha = try expDifference(b, x.max, max); + const beta = try expDifference(b, y.max, max); + op = b.tensorOperation(.{}); + const xs = try op.commit(op.fetch(x.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(y.sum, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xs, .{})), .{}); + op = b.tensorOperation(.{}); + const xo = try op.commit(op.fetch(x.out, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const out = try op.commit(op.fetch(y.out, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xo, .{})), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + const Cfg = struct { tokens: i64, sequences: i64, @@ -299,11 +715,9 @@ const Cfg = struct { window: i32 = -1, }; -/// Attention tuning inherited from the original TCL backend, informed by -/// `furiosa.kernels.llama.config` and `common.attention`. Keep the compiler's -/// default lowering mode and omit the adaptive-einsum and half-VRF settings -/// removed in TCC 2026.5. +/// Prefill bounds the tactic shapes searched (fewer for narrow heads): the compute-bound hint compiles too slowly. fn configure(b: *tcl.Builder, cfg: Cfg) void { + const shapes: i64 = if (cfg.hkv * cfg.g * cfg.d >= 1024) 16 else 8; if (cfg.decode) b.compilerConfig(.{ .enable_einsum_fusion = true, .padding_policy = .{ .Small = 1.1 }, @@ -326,11 +740,10 @@ fn configure(b: *tcl.Builder, cfg: Cfg) void { .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, .tensor_unit_bridge_threshold_in_page = 12, .allow_external_operators = false, - .enable_tactic_pruning = false, .scheduler_beam_search = true, .allow_reduce_by_ve_cluster_chip_reduce = false, .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .tactic_hint = .{ .DefaultWithMaxNumUniqueShapes = shapes }, .dma_preference = 1.2, }); } @@ -390,11 +803,24 @@ fn declareInputs(b: *tcl.Builder, a: Axes) tcl.FinishError!NativeInputs { const Prepared = struct { q: builder.Tensor, + /// Page table rows: one per block for decode, one per sequence for prefill (blocks select theirs with `pick`). pages: builder.Tensor, + pick: ?builder.Tensor, qpos: builder.Tensor, kvlen: builder.Tensor, row: builder.Tensor, active: builder.Tensor, + /// Per page row: the first query position and the keys. + first: builder.Tensor, + keys: builder.Tensor, + + /// WORKAROUND (TCC 2026.5.0b0): TCC has no tactic for selecting whole rows of 8192 pages, so flash + /// attention selects per chunk. Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once such + /// products compile. + fn blockPages(self: Prepared, b: *tcl.Builder, a: Axes, pages: builder.Tensor) tcl.FinishError!builder.Tensor { + const pick = self.pick orelse return pages; + return selectMetadata(b, pages, pick, a.S); + } }; fn countBefore(b: *tcl.Builder, ends: builder.Tensor, positions: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { @@ -414,15 +840,16 @@ fn selector(b: *tcl.Builder, indices: builder.Tensor, axis: tcl.Axis, count: i64 return op.commit(op.where(distance, .eq, 0, 1, 0), .{}); } -// Keep metadata exact in I32. Scalar gathers and four-byte-offset slices do -// not have native tactics for these shapes in TCC 2026.5. +// Keep metadata exact in I32. WORKAROUND (TCC 2026.5.0b0): no native tactics for +// scalar gathers or four-byte-offset slices. Repro: repro_scalar_gather_no_axis_to_pad.py, +// repro_i32_slice_odd_offset.py. Remove once both compile. fn selectMetadata(b: *tcl.Builder, values: builder.Tensor, selected: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{ .tactic = if (values.axes.len > 1) .einsum_by_ve else .reduce_by_ve }); return op.commit(op.fetch(selected, .{}).muli(op.fetch(values, .{})).reduce(&.{axis}, .addi), .{}); } -/// Keep adjacent I32 reads inside the native unit. Exposing offsets[1:] as a -/// separate argument would require alignment staging at every invocation. +/// Reads adjacent I32 offsets inside the native unit: a separate `offsets[1:]` argument would +/// need alignment staging at every invocation. fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.FinishError!Prepared { const start_indices = try b.arange(cfg.sequences, .i32, &.{a.S}, .{}); const end_indices = try b.arange(cfg.sequences + 1, .i32, &.{a.S}, .{ .start = 1 }); @@ -432,8 +859,7 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const qlen = try op.commit(op.fetch(ends, .{}).subi(op.fetch(starts, .{})), .{}); const tokens = try b.arange(cfg.tokens, .i32, &.{a.T}, .{}); const seq_of_token = try countBefore(b, ends, tokens, a.S); - // All sequence ends precede a padding token. Reuse this vector instead of - // bringing a symbolic scalar reduction back into a vector operation. + // Padding tokens follow all sequence ends; this vector avoids a symbolic scalar reduction in a vector operation. op = b.tensorOperation(.{}); const token_tail = try op.commit(op.fetch(seq_of_token, .{}).subi(cfg.sequences), .{}); op = b.tensorOperation(.{}); @@ -453,16 +879,19 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis return .{ .q = try b.reshape(q, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}), .pages = try b.reshape(t.pages, &.{ a.B, a.AP }, .{}), + .pick = null, .qpos = try b.reshape(qpos, &.{ a.B, a.I }, .{}), .kvlen = try b.reshape(lengths, &.{a.B}, .{}), .row = token_seq, .active = active, + .first = try b.reshape(qpos, &.{a.B}, .{}), + .keys = try b.reshape(lengths, &.{a.B}, .{}), }; } - const shift = std.math.log2_int(u64, prefill_block); + const shift = std.math.log2_int(u64, @intCast(cfg.i)); op = b.tensorOperation(.{}); - const blocks = try op.commit(op.fetch(qlen, .{}).addi(prefill_block - 1).binary(.shr_arith, shift), .{}); + const blocks = try op.commit(op.fetch(qlen, .{}).addi(cfg.i - 1).binary(.shr_arith, shift), .{}); op = b.tensorOperation(.{}); const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{a.S}), .{}); op = b.tensorOperation(.{}); @@ -482,7 +911,7 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const key_length = try selectMetadata(b, t.seq_lens, pick, a.S); const local_rows = try b.arange(cfg.i, .i32, &.{a.I}, .{}); op = b.tensorOperation(.{}); - const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(prefill_block), .{}); + const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(cfg.i), .{}); op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const qi = try op.commit(op.fetch(first_row, .{}).addi(op.fetch(local_rows, .{})), .{}); op = b.tensorOperation(.{}); @@ -500,14 +929,15 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const qpos = try op.commit(op.where(live, .ne, 0, op.fetch(pos, .{}), -1), .{}); op = b.tensorOperation(.{}); const kvlen = try op.commit(op.fetch(key_length, .{}).muli(op.fetch(live_block, .{})), .{}); - const table = try selectMetadata(b, t.pages, pick, a.S); const token_pick = try selector(b, token_seq, a.S, cfg.sequences); const token_begin = try selectMetadata(b, starts, token_pick, a.S); const token_block = try selectMetadata(b, block_start, token_pick, a.S); op = b.tensorOperation(.{}); - const row = try op.commit(op.fetch(token_block, .{}).muli(prefill_block).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); - return .{ .q = q, .pages = table, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active }; + const row = try op.commit(op.fetch(token_block, .{}).muli(cfg.i).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); + op = b.tensorOperation(.{}); + const first = try op.commit(op.fetch(t.seq_lens, .{}).subi(op.fetch(qlen, .{})), .{}); + return .{ .q = q, .pages = t.pages, .pick = pick, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active, .first = first, .keys = t.seq_lens }; } fn restoreQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, prepared: Prepared, output: builder.Tensor) tcl.FinishError!builder.Tensor { @@ -533,8 +963,7 @@ fn validIndices(b: *tcl.Builder, indices: builder.Tensor, count: i64) tcl.Finish return .{ .safe = safe, .valid = valid }; } -/// Cache validity is per sequence, not per query: causal/window masking stays -/// on scores, without replicating K/V for every row of a prefill block. +/// Validity is per sequence, not per query, so K/V are not replicated per row of a prefill block. fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor) tcl.FinishError!builder.Tensor { var op = b.tensorOperation(.{}); const pos = try op.commit(op.fetch(position, .{ .broadcast_to = valid_pages.axes }), .{}); @@ -542,17 +971,15 @@ fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor return op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid_pages, .{})), .{}); } -/// A true select excludes NaN/Inf in unwritten slots; multiplying by zero -/// would still propagate them into the contraction. Materialize only BF16. +/// A select, not a multiply by zero: NaN/Inf in unwritten slots must not reach the contraction. fn maskCacheValues(b: *tcl.Builder, values: builder.Tensor, valid: builder.Tensor) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{ .tactic = .elementwise }); const data = op.fetch(values, .{}); return op.commit(op.where(op.fetch(valid, .{ .broadcast_to = values.axes }), .ne, 0, data, 0.0), .{ .dtype = .bf16 }); } -/// Build the sequence, query, page and window mask for the current keys. fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor, valid_keys: builder.Tensor) tcl.FinishError!builder.Tensor { - const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[0] }; + const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[position.axes.len - 1] }; const read: builder.TensorOperation.FetchOptions = .{ .broadcast_to = axes }; // Elementwise operations require their first read to cover the output // axes, and each VE instruction must consume the current accumulator. @@ -577,8 +1004,6 @@ fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder return mask; } -/// Keep the contraction accumulator in f32 through masking and softmax. -/// A finite sentinel also makes entirely masked chunks safe to rescale. fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builder.Tensor, mask: builder.Tensor, keys: tcl.Axis) tcl.FinishError!builder.Tensor { const scores_axes = &.{ a.Hkv, a.G, a.B, a.I, keys }; var op = b.tensorOperation(.{}); @@ -591,8 +1016,7 @@ fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builde fn maskedExp(b: *tcl.Builder, scores: builder.Tensor, maximum: builder.Tensor, mask: builder.Tensor) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{}); const e = op.fetch(scores, .{}).subf(op.fetch(maximum, .{})).exp(); - // Subtracting the sentinel from itself gives exp(0), so masking only - // the logits is insufficient for empty rows and fully masked chunks. + // A sentinel minus itself gives exp(0), so masking only the logits fails for empty rows and fully masked chunks. return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, e, 0.0), .{}); } @@ -622,7 +1046,7 @@ const OneShot = tcl.Kernel(Cfg, .{ const prepared = try prepareQueries(b, cfg, a, t); const kc = try b.reshape(t.k_cache, &.{ Nb, a.Hkv, a.D }, .{}); const vc = try b.reshape(t.v_cache, &.{ Nb, a.Hkv, a.D }, .{}); - const page_indices = try validIndices(b, prepared.pages, cfg.pages); + const page_indices = try validIndices(b, try prepared.blockPages(b, a, prepared.pages), cfg.pages); const within_page = try b.arange(cfg.pg, .i32, &.{a.PG}, .{}); var slot_op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const flat_slots = try slot_op.commit(slot_op.fetch(page_indices.safe, .{}).muli(cfg.pg).addi(slot_op.fetch(within_page, .{})), .{}); @@ -630,7 +1054,6 @@ const OneShot = tcl.Kernel(Cfg, .{ slot_op = b.tensorOperation(.{}); const valid = try slot_op.commit(slot_op.fetch(page_indices.valid, .{ .broadcast_to = &.{ a.B, a.AP, a.PG } }), .{}); const valid_keys = try b.reshape(valid, &.{ a.B, A }, .{}); - // Match the original slot sanitization before reading invalid pages. slot_op = b.tensorOperation(.{}); const slots = try slot_op.commit(slot_op.where(valid_keys, .ne, 0, slot_op.fetch(expanded_slots, .{}), 0), .{}); const valid_length = if (cfg.decode) prepared.kvlen else null; @@ -642,8 +1065,6 @@ const OneShot = tcl.Kernel(Cfg, .{ const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, A); var op = b.tensorOperation(.{}); const p = if (cfg.decode) compact: { - // Furiosa's common.softmax keeps normalization inside one VE - // operation. FP32 intermediates only narrow at the DPE input. const exp_sink = if (cfg.use_sink) sink: { const maximum = try op.commit(op.fetch(masked, .{}).reduce(&.{A}, .maxf).maxf(op.fetch(t.sink, .{})), .{}); break :sink try expDifference(b, t.sink, maximum); @@ -656,13 +1077,11 @@ const OneShot = tcl.Kernel(Cfg, .{ var sum = e.reduce(&.{A}, .addf); if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); const probabilities = try op.commit(e.divf(sum), .{ .dtype = .bf16 }); - // Empty rows have uniform probabilities from finite sentinels. - // Clear masked probabilities before contracting with values. + // Empty rows have uniform probabilities from finite sentinels: clear masked ones before the value contraction. op = b.tensorOperation(.{}); break :compact try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, op.fetch(probabilities, .{}), 0.0), .{ .dtype = .bf16 }); } else tiled: { - // Separate reductions schedule better for multi-query blocks - // with the current compiler; keep their intermediate state FP32. + // Separate reductions schedule better for multi-query blocks with the current compiler. var m = op.fetch(masked, .{}).reduce(&.{A}, .maxf); if (cfg.use_sink) m = m.maxf(op.fetch(t.sink, .{})); const maximum = try op.commit(m, .{}); @@ -695,28 +1114,66 @@ const Flash = tcl.Kernel(Cfg, .{ const Cp = b.axis("Cp", @divExact(cfg.chunk, cfg.pg)); const t = try declareInputs(b, a); const prepared = try prepareQueries(b, cfg, a, t); - const by_batch = try b.reshape(prepared.pages, &.{ a.B, C, Cp }, .{}); + const rows = prepared.pages.axes[0]; + const by_row = try b.reshape(prepared.pages, &.{ rows, C, Cp }, .{}); const page_op = b.tensorOperation(.{}); - const pages = try page_op.commit(page_op.fetch(by_batch, .{}), .{ .axes = &.{ C, a.B, Cp } }); + const pages = try page_op.commit(page_op.fetch(by_row, .{}), .{ .axes = &.{ C, rows, Cp } }); const stats = &.{ a.Hkv, a.G, a.B, a.I }; const out_axes = &.{ a.B, a.I, a.Hkv, a.G, a.D }; var op = b.tensorOperation(.{}); - const n = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); + const shift = std.math.log2_int(u64, @intCast(cfg.chunk)); + // With a window, each page row's loop starts at the chunk holding its first visible key. TCL divides + // by shifts, so chunks that are not a power of two run every chunk and the mask applies the window. + const window = if (cfg.window > 0 and std.math.isPowerOfTwo(cfg.chunk)) blk: { + op = b.tensorOperation(.{}); + const start = try op.commit(op.fetch(prepared.first, .{}).subi(cfg.window - 1).binary(.maxi, 0).binary(.shr_arith, shift), .{}); + op = b.tensorOperation(.{}); + const count = try op.commit(op.fetch(prepared.keys, .{}).addi(cfg.chunk - 1).binary(.shr_arith, shift).subi(op.fetch(start, .{})), .{}); + // WORKAROUND (TCC 2026.5.0b0): TCC reads the loop index only as an index, so each trip's chunk per + // row is built before the loop. Repro: repro_loop_index_as_data.py. Remove once the loop index + // compiles as data. + const trip = try b.arange(C.size, .i32, &.{C}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const chunks = try op.commit(op.fetch(trip, .{}).addi(op.fetch(start, .{})), .{ .axes = &.{ C, rows } }); + break :blk .{ .chunks = chunks, .trips = try b.reduceMaxI32(count) }; + } else null; + const n = if (window) |w| w.trips else try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); var loop = b.openFor(n, .{ try b.full(0, .f32, stats), try b.full(0, .f32, out_axes), try b.full(score_min, .f32, stats) }); { const sum_acc, const qkv_acc, const max_acc = loop.carried; - const indices = try validIndices(b, try b.indexRead(pages, loop.iv), cfg.pages); + const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); + const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); + var row_pages: builder.Tensor = undefined; + var position: builder.Tensor = undefined; + var first_keys: builder.Tensor = undefined; + if (window) |w| { + const chunk_index = try b.indexRead(w.chunks, loop.iv); + op = b.tensorOperation(.{}); + const read_index = try op.commit(op.fetch(chunk_index, .{}).binary(.mini, @divExact(cfg.a, cfg.chunk) - 1), .{}); + // WORKAROUND (TCC 2026.5.0b0): a batched gather of the chunk does not lower + // for some shapes. Repro: repro_batched_gather_all_tactics_failed.py. Remove + // once batched gathers of several rows compile. + row_pages = try selectMetadata(b, by_row, try selector(b, read_index, C, C.size), C); + op = b.tensorOperation(.{}); + const row_offset = try op.commit(op.fetch(chunk_index, .{}).muli(cfg.chunk), .{}); + const offset = if (prepared.pick) |pick| try selectMetadata(b, row_offset, pick, a.S) else row_offset; + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{ .broadcast_to = &.{ a.B, Ck } }).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{ .broadcast_to = &.{ a.B, Cp } }).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } else { + row_pages = try b.indexRead(pages, loop.iv); + const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } + const indices = try validIndices(b, try prepared.blockPages(b, a, row_pages), cfg.pages); const idx = indices.safe; op = b.tensorOperation(.{}); const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); - const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); - const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); - const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); - op = b.tensorOperation(.{}); - const first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); op = b.tensorOperation(.{}); @@ -724,9 +1181,8 @@ const Flash = tcl.Kernel(Cfg, .{ op = b.tensorOperation(.{}); const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); - const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); - const k = try maskCacheValues(b, try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); - const v = try maskCacheValues(b, try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); + const k = try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); + const v = try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); op = b.tensorOperation(.{}); const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); @@ -751,7 +1207,6 @@ const Flash = tcl.Kernel(Cfg, .{ var sum = loop.results[0]; var numerator = loop.results[1]; if (cfg.use_sink) { - // Merge the zero-valued sink once, after all key chunks. op = b.tensorOperation(.{}); const maximum = try op.commit(op.fetch(loop.results[2], .{}).maxf(op.fetch(t.sink, .{})), .{}); const alpha = try expDifference(b, loop.results[2], maximum); @@ -922,6 +1377,7 @@ const TestDims = struct { page: usize = 16, uniform: bool = false, uniform_scores: bool = false, + spikes: []const usize = &.{}, dtype: zml.DataType = .bf16, }; @@ -1030,7 +1486,7 @@ test "tcl paged attention cache masking emits a BF16 select without query replic try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.ve_opcode<\"mulf\">") == null); try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.symbol<\"I\">") == null); - const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true }); + const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .group = 2, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true, .window = 0, .use_sink = false }); defer std.testing.allocator.free(decode); try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, decode, "\"tcl.dpe\"")); try std.testing.expect(std.mem.indexOf(u8, decode, "tcl.fusible") != null); @@ -1143,6 +1599,32 @@ test "tcl paged attention chunked decode on furiosa" { try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); } +test "tcl paged attention hybrid decode on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(2049, 2101, &.{ + .{ .prefill = false, .seqs = &.{.{ 1, 300 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 4097 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both, .poison_value = std.math.inf(f32) }, + .{ .prefill = false, .seqs = &.{.{ 0, 0 }}, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 17 }}, .mask_all = true, .poison = .both }, + }, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2101, &.{.{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both }}, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32, .uniform_scores = true, .spikes = &.{ 4095, 4096, 12287, 12288, 32783 } }); +} + +test "tcl paged attention flash decode of many tokens on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const seqs = [_][2]i32{ .{ 1, 32784 }, .{ 1, 4097 }, .{ 1, 1 }, .{ 0, 0 } } ++ @as([12][2]i32, @splat(.{ 1, 300 })); + try runCases(2049, 2600, &.{ + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 5000, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .batch = 16, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2700, &.{ + .{ .prefill = true, .seqs = &.{ .{ 37, 32784 }, .{ 5, 9000 } }, .padding = 2, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = true, .seqs = &.{ .{ 70, 32784 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 3000 } }, + }, .{ .batch = 2, .hkv = 1, .g = 2, .d = 32 }); +} + test "tcl paged attention uniform probabilities on furiosa" { if (zml.testing.env().target != .furiosa) return error.SkipZigTest; try runCases(257, 514, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }}, .{ .hkv = 1, .g = 1, .d = 32, .uniform_scores = true }); @@ -1209,9 +1691,16 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const for (kc) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); for (vc, 0..) |*v, i| v.* = .fromF32(if (dims.uniform) @as(f32, @floatFromInt(i % d)) / d else r.floatNorm(f32)); + if (dims.spikes.len > 0) { + for (case.seqs[0..batch], 0..) |seq, row| { + for (0..@intCast(seq[1])) |key| { + const slot: usize = @intCast(table[row][key / page] * @as(i32, page) + @as(i32, @intCast(key % page))); + const spike = std.mem.indexOfScalar(usize, dims.spikes, key) != null; + @memset(vc[slot * hkv * d ..][0 .. hkv * d], bf16.fromF32(if (spike) 32768 else 0)); + } + } + } if (case.poison != .none) { - // Mark the union of live physical slots before poisoning: reordered - // or shared pages must retain every token a live sequence can read. const live = try allocator.alloc(bool, num_pages * page); defer allocator.free(live); @memset(live, false); From 8898051c7b6b8cd015cdce8e75afc5404396405f Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Wed, 4 Nov 2026 20:48:56 +0100 Subject: [PATCH 12/47] zml/kernels: fix Triton split-K (#1012) The split-K/3D path writes partial results only for active queries, but the reduction kernel processes inactive queries in the padded batch. These queries have no valid results and cause invalid computation/outputs. This fix zeroes the ouput for inactive queries and return before before accessing sequence metadata. Resolves #1006 --- .../triton/unified_attention.zig | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig index 39018f1b..7b2b9428 100644 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -1020,6 +1020,25 @@ fn reduceSegments( const query_token_idx = k.programId(.x); const query_head_idx = k.programId(.y); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const output_offset = query_token_idx.to(.i64).mul(output_stride_0) + .add(query_head_idx.to(.i64).mul(output_stride_1)) + .add(offs_d.to(.i64)); + + // The launch covers the padded query capacity, but the attention pass only + // writes scratch for active tokens. Zero the tail before looking up a + // sequence or dividing by its segment size (zero for inactive sequences). + const active_tokens = k.load(query_start_len_ptr.addPtr(num_seqs)); + const inactive = query_token_idx.ge(active_tokens); + k.storeOpts(output_ptr.addPtr(output_offset), k.zeros(&.{HEAD_SIZE_PADDED}, config.o_dtype), .{ + .mask = dim_mask.bitAnd(inactive), + }); + k.returnIf(inactive); + const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); @@ -1029,12 +1048,6 @@ fn reduceSegments( const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); const segm_mask = seg_range.lt(act_num_segments); - const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); - const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) - else - k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); - // segm_offset = query_token_idx*NQH*NSPS + query_head_idx*NSPS + arange(NSPS) const tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; const head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); @@ -1100,9 +1113,6 @@ fn reduceSegments( ); } - const output_offset = query_token_idx.to(.i64).mul(output_stride_0) - .add(query_head_idx.to(.i64).mul(output_stride_1)) - .add(offs_d.to(.i64)); k.storeOpts( output_ptr.addPtr(output_offset), acc.to(config.o_dtype), From 69a6b2bb227a881b63a53228e87957495a3887d4 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Thu, 5 Nov 2026 11:44:39 +0100 Subject: [PATCH 13/47] workspace: bump ZML plugins (#1031) --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 1105f776..b33ef4df 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index ced8ae98..a0c73948 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "77d2ddb19566604c827828fe3dcc53e938f3a1b16d431416fdd5c466266f5242", - "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-amd64.tar.zst", + "sha256": "b68b407efcd361bbe4b297a16feb080eabbae9132a7ee1a5b105e24a91d99de0", + "url": "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "2cd18393c44816d3d01aa22fbd890632ba46eca3939c644bddef00d590578a80", - "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-arm64.tar.zst", + "sha256": "9ac41221901eba84641aef35d1f53ff69ce471c5af3844550cd086abc6916f5c", + "url": "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 0aa3905d..fc2e5352 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610221403.283.1.d9f4f68759e1/zml-furiosa-linux-amd64.tar.zst", - sha256 = "f8d6bb03fb4cd5dad9b37856b2f8f12b28ec1a8d46f59ca03c401bea668215e9", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-furiosa-linux-amd64.tar.zst", + sha256 = "85edea12494278a015c1ad29cf57401217e4809bc3a0c6890d0279659bb6209d", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 03cdaf02..2d43a991 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "6eee91776397d472ac590413cf44d4ff3f7b31bebb163aa96339c6becaf2defd", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-metal-darwin-arm64.tar.zst", + sha256 = "bdd2de3ec8293faa25b358cdf35ee3e75d69be73a4acea87b6e007a18b8f10e8", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index babd73ab..ee875995 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "cd6c3c8673ce690a0fbf89dab5a16e6893f0f501bb2b41997c657cf6effa7549" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "02b86fa3a6599f6aa2b89806b8f6c0b16f11765ec04c94517ec1463a59d92175" def _rocm_package_name(name): return name + _ROCM_VERSION From 4baec259241a8293abef0b86a69711c00514e069 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 5 Nov 2026 11:53:31 +0100 Subject: [PATCH 14/47] kernels/moe: remove confusing bitcast (#1030) the bitcast discard partitioning info the API issue is tracked by https://github.com/zml/zml/issues/914 --- zml/kernels/moe/triton/triton.zig | 106 +++++++++++++++++++----------- 1 file changed, 67 insertions(+), 39 deletions(-) diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 908cd163..9d3a5529 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -897,14 +897,17 @@ pub fn fusedExpertsMxfp4( const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); + // Note: we directly forward input sharding to manualComputation + // TODO: improve sharding in attention and moe APIs + // https://github.com/zml/zml/issues/914 const context: Context = .{ .input = input, .ids = ids, .weights = weights, - .w1 = gate_up.weight.bitCast(.u8), - .s1 = gq.scales, - .w2 = down.weight.bitCast(.u8), - .s2 = dq.scales, + .gate_up = gate_up.weight, + .gate_up_scales = gq.scales, + .down = down.weight, + .down_scales = dq.scales, .global_experts = gate_up.weight.dim(.expert), .topk = ids.dim(.topk), .limit = options.activation.swiglu.limit.?, @@ -952,46 +955,71 @@ pub const Config = struct { } }; -pub fn validateInputs( +fn validateInputs( c: Config, input: zml.Tensor, ids: zml.Tensor, - weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, -) !void { - if (s1.rank() != 3 or s2.rank() != 3) return error.InvalidInputShape; - if (s1.dim(2) < @divTrunc(c.hidden, 32) or - s2.dim(2) < @divTrunc(c.intermediate, 32)) return error.InvalidInputShape; + expert_weights: zml.Tensor, + gate_up: zml.Tensor, + gate_up_scales: zml.Tensor, + down: zml.Tensor, + down_scales: zml.Tensor, +) void { + stdx.debug.assert(gate_up_scales.rank() == 3 and down_scales.rank() == 3, "Invalid MXFP4 tensor layout, expected gate_up_scales and down_scales to be rank 3, got {f} and {f}", .{ gate_up_scales, down_scales }); + const shapes = .{ - .x = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), + .input = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), - .scales = zml.Shape.init(.{ c.tokens, c.topk }, .f32), - .w1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divTrunc(c.hidden, 2) }, .u8), - .s1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, s1.dim(2) }, .f8e8m0), - .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), - .s2 = zml.Shape.init(.{ c.experts, c.hidden, s2.dim(2) }, .f8e8m0), + .expert_weights = zml.Shape.init(.{ c.tokens, c.topk }, .f32), + .gate_up = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divExact(c.hidden, 2) }, .i8), + .gate_up_scales = zml.Shape.init(.{ c.experts, 2 * c.intermediate, gate_up_scales.dim(2) }, .f8e8m0), + .down = zml.Shape.init(.{ c.experts, c.hidden, @divExact(c.intermediate, 2) }, .i8), + .down_scales = zml.Shape.init(.{ c.experts, c.hidden, down_scales.dim(2) }, .f8e8m0), }; - if (!input.shape().eql(shapes.x)) return error.InvalidInputShape; - if (!ids.shape().eql(shapes.ids)) return error.InvalidInputShape; - if (!weights.shape().eql(shapes.scales)) return error.InvalidInputShape; - if (!w1.shape().eql(shapes.w1)) return error.InvalidInputShape; - if (!s1.shape().eql(shapes.s1)) return error.InvalidInputShape; - if (!w2.shape().eql(shapes.w2)) return error.InvalidInputShape; - if (!s2.shape().eql(shapes.s2)) return error.InvalidInputShape; + stdx.debug.assert(input.shape().eql(shapes.input) and + ids.shape().eql(shapes.ids) and + expert_weights.shape().eql(shapes.expert_weights) and + // Note: we accept both i8 and u8 for gate_up and down + // TODO: we should instead use f4e2m1 + (gate_up.shape().eql(shapes.gate_up) or gate_up.shape().eql(shapes.gate_up.withDtype(.u8))) and + gate_up_scales.shape().eql(shapes.gate_up_scales) and + (down.shape().eql(shapes.down) or down.shape().eql(shapes.down.withDtype(.u8))) and + down_scales.shape().eql(shapes.down_scales), + \\Invalid MXFP4 tensor layout: + \\ - expected input={f}, got {f} + \\ - expected ids={f}, got {f} + \\ - expected expert_weights={f}, got {f} + \\ - expected gate_up={f}, got {f} + \\ - expected gate_up_scales={f}, got {f} + \\ - expected down={f}, got {f} + \\ - expected down_scales={f}, got {f} + , .{ + shapes.input, input, // + shapes.ids, ids, // + shapes.expert_weights, expert_weights, // + shapes.gate_up, gate_up, // + shapes.gate_up_scales, gate_up_scales, // + shapes.down, down, // + shapes.down_scales, down_scales, // + }); + + stdx.debug.assert( + gate_up_scales.dim(2) >= @divTrunc(c.hidden, 32) and + down_scales.dim(2) >= @divTrunc(c.intermediate, 32), + "Invalid MXFP4 tensor layout, not enough scales, expected a ratio of at most 32 for config {}, got {f} and {f}", + .{ c, gate_up_scales, down_scales }, + ); } const Context = struct { input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, + gate_up: zml.Tensor, + gate_up_scales: zml.Tensor, + down: zml.Tensor, + down_scales: zml.Tensor, global_experts: i64, topk: i64, limit: f32, @@ -999,9 +1027,9 @@ const Context = struct { expert_parallel: bool, fn body(self: Context, _: zml.Shape) zml.Tensor { - const experts = self.w1.dim(.expert); - const hidden = self.w2.dim(1); - const intermediate = self.w2.dim(2) * 2; + const experts = self.gate_up.dim(.expert); + const hidden = self.down.dim(1); + const intermediate = self.down.dim(2) * 2; var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); const tokens = ids.dim(.token); if (self.expert_parallel) { @@ -1023,7 +1051,7 @@ const Context = struct { }; cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); - validateInputs(cfg, x, ids, scales, self.w1, self.s1, self.w2, self.s2) catch @panic("Invalid MXFP4 tensor layout"); + validateInputs(cfg, x, ids, scales, self.gate_up, self.gate_up_scales, self.down, self.down_scales); const bn = 16; const bk = 512; @@ -1087,7 +1115,7 @@ const Context = struct { // Each narrow tile produces 64 activations: two complete scale groups. const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; const m = kernels.GateUp.Kernel.call( - .{ .q = q.q, .s = q.s, .w = self.w1, .ws = self.s1.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .q = q.q, .s = q.s, .w = self.gate_up.bitCast(.u8), .ws = self.gate_up_scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, .{ .cfg = .{ @@ -1098,7 +1126,7 @@ const Context = struct { .topk = cfg.topk, .expert_offset = cfg.expert_offset, .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.s1.dim(2), + .scale_stride = self.gate_up_scales.dim(2), .routing_weight_placement = cfg.routing_weight_placement, .bn = bn, .bk = bk, @@ -1111,7 +1139,7 @@ const Context = struct { ); const d = kernels.Down.Kernel.call( - .{ .mid = m.mid, .ms = m.ms, .w = self.w2, .ws = self.s2.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .mid = m.mid, .ms = m.ms, .w = self.down.bitCast(.u8), .ws = self.down_scales.bitCast(.u8), .ids = ids, .sched = sched }, .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, .{ .cfg = .{ @@ -1122,7 +1150,7 @@ const Context = struct { .topk = cfg.topk, .expert_offset = cfg.expert_offset, .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.s2.dim(2), + .scale_stride = self.down_scales.dim(2), .routing_weight_placement = cfg.routing_weight_placement, .bn = bn, .bk = bk, From 47f50667d879dff1f225ef35eb4878dd573bcea4 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 5 Nov 2026 13:00:00 +0100 Subject: [PATCH 15/47] zml/attention: declare manual partitioning in the shared API instead of kernels (#1016) Move manual regions out of flash attention kernels and use `manualComputation2` for every shared attention backend. Also fully replicate the embedding executable outputs in Qwen and Llama. Partial graph constraints leave their batch and sequence dimensions open, which is invalid at executable boundaries. Fix two further blockers exposed by the Qwen run that we forgot to move recently: softplus comparisons must use the input dtype and ROCm DNN runfiles must include the MIOpen kernel archives. --- examples/llm/models/lfm2/model.zig | 6 ++ examples/llm/models/llama/model.zig | 4 +- examples/llm/models/qwen3_5/model.zig | 8 +- examples/llm/models/qwen3_5_moe/model.zig | 8 +- platforms/rocm/rocm.bzl | 1 + zml/attention/attention.zig | 86 +++++++++++++++---- zml/kernels/flash_attention/cute/cute.zig | 85 ++++++++---------- .../flash_attention/flash_attention.zig | 18 +++- zml/kernels/flash_attention/nki/nki.zig | 80 +++-------------- zml/nn.zig | 32 +++---- 10 files changed, 170 insertions(+), 158 deletions(-) diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index dd21c7f9..9f7cb909 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -571,6 +571,8 @@ pub const Attention = struct { v = new_kv_cache.values(cache_index); stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); + const mesh = zml.Compiler.current().mesh(.model); + const partitioning: zml.attention.Partitioning = .head(.model); const attn = switch (attention_parameters) { .stablehlo => zml.attention.attention( q, @@ -579,6 +581,8 @@ pub const Attention = struct { tokens_position_offset, attention_metadata, attention_parameters, + mesh, + partitioning, ).merge(.{ .d = .{ .h, .hd } }), else => zml.attention.attention( q.squeeze(.batch), @@ -587,6 +591,8 @@ pub const Attention = struct { tokens_position_offset.squeeze(.batch), attention_metadata, attention_parameters, + mesh, + partitioning, ).merge(.{ .d = .{ .h, .hd } }) .rename(.{ .q = .seq }) .insertAxes(.seq, .{.batch}), diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 05610dc1..8a6a09e9 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -329,7 +329,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.model, .{ .d = .replicated }) }; + .replicate() }; } }; @@ -587,6 +587,8 @@ const SelfAttn = struct { token_index, attention_metadata, attention_parameters, + zml.Compiler.current().mesh(.model), + .head(.model), ); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 77d3884e..40174f5c 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -290,7 +290,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.model, .{ .d = .replicated }) }; + .replicate() }; } }; @@ -657,6 +657,10 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; const attn_output = zml.attention.attention( q, k, @@ -664,6 +668,8 @@ pub const SelfAttn = struct { token_index, zml.attention.Metadata.init(.fromBackend(.stablehlo)), zml.attention.Parameters.init(.fromBackend(.stablehlo)), + zml.Compiler.current().mesh(.model), + head_partitioning, ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 11ce8151..1375a1bd 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -641,6 +641,10 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; const attn_output = zml.attention.attention( q, k, @@ -648,7 +652,9 @@ pub const SelfAttn = struct { token_index, zml.attention.Metadata.init(.fromBackend(.stablehlo)), zml.attention.Parameters.init(.fromBackend(.stablehlo)), - ).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + zml.Compiler.current().mesh(.model), + head_partitioning, + ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index ee875995..0c1f7a92 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -73,6 +73,7 @@ def _rocm_runfile_build_files(loaded_packages): for pkg_name in _family_runfile_package_names(loaded_packages, "dnn"): rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ + ".kpack/**", "lib/libMIOpenCKGroupedConv_*.so", "share/miopen/db/**", ]) diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 4cfad309..039c6628 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -99,6 +99,28 @@ pub const Options = struct { causal: bool = true, }; +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + const InputSpecs = @FieldType(zml.ops.ManualPartitioning(Attention.forward), "in_specs"); + + in_specs: struct { + q: Logical, + k: Logical, + v: Logical, + token_index: Logical = .replicated, + /// Omit to replicate backend metadata tensors. + metadata: ?@FieldType(InputSpecs, "metadata") = null, + }, + out_specs: Logical, + + /// Shards the `.h` dimension of Q, K, V and the output over `mesh_axis`. + /// Other dimensions, token indices and backend metadata are replicated. + pub fn head(comptime mesh_axis: @EnumLiteral()) Partitioning { + const heads: Logical = .parse(.{ .h = mesh_axis }); + return .{ .in_specs = .{ .q = heads, .k = heads, .v = heads }, .out_specs = heads }; + } +}; + /// Causal attention as used in transformers. /// /// **Shapes**: @@ -110,27 +132,53 @@ pub const Options = struct { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - return attentionOpts(q, k, v, token_index, metadata, parameters, .{}); +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + return attentionOpts(q, k, v, token_index, metadata, parameters, .{}, mesh, partitioning); } /// `attention`, causal or not depending on `opts`. -pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { - return switch (parameters) { - .stablehlo => b: { - // Generate the attention mask. - const seq_len = k.dim(.k); - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, - // then slice into it, but XLA is able to optimize this correctly. - const attn_mask: ?zml.Tensor = if (opts.causal) - zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null) - .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, q.dtype()), token_index.appendAxes(.{.coord}), .{}) - else - null; - - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); - break :b attn_output; +pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; + return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, .{ + .in_specs = .{ + .q = partitioning.in_specs.q, + .k = partitioning.in_specs.k, + .v = partitioning.in_specs.v, + .token_index = partitioning.in_specs.token_index, + .metadata = partitioning.in_specs.metadata orelse switch (metadata) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |metadata_| .{ .flash_attention = switch (metadata_) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = .{ .num_tokens = .replicated } }, + } }, + }, }, - .flash_attention => flash_attention.attention(q, k, v, token_index, metadata.flash_attention, parameters.flash_attention, opts), - }; + .out_specs = partitioning.out_specs, + }); } + +const Attention = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + token_index: zml.Tensor, + metadata: Metadata, + parameters: Parameters, + opts: Options, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .stablehlo => b: { + const seq_len = args.k.dim(.k); + const mask: ?zml.Tensor = if (args.opts.causal) + zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, args.q.dtype(), null) + .gatherSlices(zml.Shape.init(.{ .q = args.q.dim(.q) }, args.q.dtype()), args.token_index.appendAxes(.{.coord}), .{}) + else + null; + break :b zml.nn.sdpa(args.q, args.k, args.v, .{ .attn_mask = mask }); + }, + .flash_attention => flash_attention.attention(args.q, args.k, args.v, args.token_index, args.metadata.flash_attention, args.parameters.flash_attention, args.opts), + }; + } +}; diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig index c1ffa58f..36e0953b 100644 --- a/zml/kernels/flash_attention/cute/cute.zig +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -34,68 +34,51 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T stdx.debug.assert(token_index.rank() == 0, "Unbatched CuTe attention requires a scalar token position", .{}); } - const mesh = zml.Compiler.current().resolveMesh(.{.model}); - const q_sharded = q.withPartitioning(mesh, .{ .h = .model }); - const k_sharded = k.withPartitioning(mesh, .{ .h = .model }); - const v_sharded = v.withPartitioning(mesh, .{ .h = .model }); const positions = if (batched) token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) else token_index.convert(.i32); - return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions, .causal = causal }, q.shape().withPartitioning(mesh, .{ .h = .model })); -} + const q_ = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }); + const k_ = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }); + const v_ = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }); -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - positions: zml.Tensor, - causal: bool, + const splits = kernel.decodeSplits(q_.shape(), k_.shape(), decodeResources(zml.Compiler.current().platform)); - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const batched = self.q.shape().hasTag(.b) != null; - const q = if (batched) self.q.transpose(.{ .b, .q, .h, .hd }) else self.q.transpose(.{ .q, .h, .hd }); - const k = if (batched) self.k.transpose(.{ .b, .k, .h, .hd }) else self.k.transpose(.{ .k, .h, .hd }); - const v = if (batched) self.v.transpose(.{ .b, .k, .h, .hd }) else self.v.transpose(.{ .k, .h, .hd }); + const cfg: kernel.Config = .{ + .batch = if (batched) q_.dim(.b) else 1, + .query_length = q_.dim(.q), + .key_length = k_.dim(.k), + .query_heads = q_.dim(.h), + .kv_heads = k_.dim(.h), + .head_dim = q_.dim(.hd), + .dtype = zml.kernel.cute.from(q_.dtype()), + .splits = splits, + .causal = causal, + }; - const splits = kernel.decodeSplits(q.shape(), k.shape(), decodeResources(zml.Compiler.current().platform)); - - const cfg: kernel.Config = .{ - .batch = if (batched) q.dim(.b) else 1, - .query_length = q.dim(.q), - .key_length = k.dim(.k), - .query_heads = q.dim(.h), - .kv_heads = k.dim(.h), - .head_dim = q.dim(.hd), - .dtype = zml.kernel.cute.from(q.dtype()), - .splits = splits, - .causal = self.causal, - }; - - const out = if (kernel.prefillSupported(cfg)) - kernel.Prefill.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = q.shape() }, .{ .cfg = cfg }).out - else if (cfg.splits > 1) b: { - const partials = kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ - .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), - }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, - .block = kernel.block, - }).out; - break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, - .block = kernel.block, - }).out; - } else kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = q.shape() }, .{ + const out = if (kernel.prefillSupported(cfg)) + kernel.Prefill.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ .cfg = cfg }).out + else if (cfg.splits > 1) b: { + const partials = kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ + .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), + }, .{ .cfg = cfg, - .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, .block = kernel.block, }).out; + break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, + .block = kernel.block, + }).out; + } else kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .block = kernel.block, + }).out; - return out.transpose(output_shape); - } -}; + return out.transpose(q.shape()); +} /// Returns the minimum number of multiprocessors and shared memory per block across all devices in the platform. /// If there are no devices, returns 0 for both values. diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index b0a6106b..78602bd0 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -127,6 +127,7 @@ pub const Metadata = union(Backend) { /// - .k is the number of keys /// - .hd is the head dimension pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: zml.attention.Options) zml.Tensor { + zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); return switch (parameters) { .cute => cute.attention(q, k, v, token_index, opts.causal), // The NKI and Metal kernels are causal only. @@ -315,9 +316,19 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: defer token_index.deinit(); const meshes = platform.meshes.values(); + const mesh = zml.testing.tpMesh(); + const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ + .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, + .out_specs = .replicated, + }; + const AttentionCall = struct { + fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { + return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); + } + }; const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, zml.attention.attentionOpts, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts }, .{ + const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ .program_name = "attention_stablehlo", .meshes = meshes, }); @@ -326,6 +337,7 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); defer reference_d.deinit(); try reference_d.await(io); + try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); defer reference_h.free(allocator); @@ -337,8 +349,8 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: const exe = try platform.compileFn( allocator, io, - attention, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters, opts }, + AttentionCall.forward, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, .{ .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), .meshes = meshes, diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig index 905f3aa0..f98a545d 100644 --- a/zml/kernels/flash_attention/nki/nki.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -43,71 +43,17 @@ pub const Parameters = struct { /// Attention for Neuron. Both prefill and decode lower to NKI custom calls. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - // TODO: https://github.com/zml/zml/issues/914 - const tp = zml.Compiler.current().resolveMesh(.{.model}); - const q_sharded = q.withPartitioning(tp, .{ .h = .model }); - const k_sharded = k.withPartitioning(tp, .{ .h = .model }); - const v_sharded = v.withPartitioning(tp, .{ .h = .model }); - - if (q.dim(.q) == 1) { - const token_index_2d = token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) - .withPartitioning(tp, .{ .row = .replicated, .col = .replicated }); - - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - token_index: zml.Tensor, - parameters: Parameters, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - const kernel = self.parameters.decode_kernel; - return zml.ops.neuronNki( - .{ self.q, self.k, self.v, self.token_index }, - .{output}, - .{ - .name = kernel.name, - .entrypoint = kernel.entrypoint, - .source_path = kernel.source_path, - .compiler_target = self.parameters.compiler_target, - }, - )[0]; - } - }).body, - .{ - .q = q_sharded, - .k = k_sharded, - .v = v_sharded, - .token_index = token_index_2d, - .parameters = parameters, - }, - q.shape().withPartitioning(tp, .{ .h = .model }), - ).convert(q.dtype()); - } - - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - const kernel = self.parameters.prefill_kernel; - return zml.ops.neuronNki( - .{ self.q, self.k, self.v }, - .{output}, - .{ - .name = kernel.name, - .entrypoint = kernel.entrypoint, - .source_path = kernel.source_path, - .compiler_target = self.parameters.compiler_target, - }, - )[0]; - } - }).body, - .{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .parameters = parameters }, - q.shape().withPartitioning(tp, .{ .h = .model }), - ).convert(q.dtype()); + const decode = q.dim(.q) == 1; + const kernel = if (decode) parameters.decode_kernel else parameters.prefill_kernel; + const options: zml.ops.NeuronNkiOps = .{ + .name = kernel.name, + .entrypoint = kernel.entrypoint, + .source_path = kernel.source_path, + .compiler_target = parameters.compiler_target, + }; + const output = if (decode) + zml.ops.neuronNki(.{ q, k, v, token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) }, .{q.shape()}, options)[0] + else + zml.ops.neuronNki(.{ q, k, v }, .{q.shape()}, options)[0]; + return output.convert(q.dtype()); } diff --git a/zml/nn.zig b/zml/nn.zig index 56fd8b51..4569552f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2354,8 +2354,8 @@ pub fn softplus( // Using log(1 + exp(x)) directly loses precision for x < ~-16.6, causing strongly negative logits to produce zero scores. const z = x.exp(); const u = z.addConstant(1); - const log1p = u.cmp(.EQ, .scalar(1, .f32)).select(z, z.mul(u.log()).div(u.subConstant(1))); - return x.cmp(.GT, .scalar(opts.threshold, .f32)).select(x, log1p); + const log1p = u.cmp(.EQ, .scalar(1, x.dtype())).select(z, z.mul(u.log()).div(u.subConstant(1))); + return x.cmp(.GT, .scalar(opts.threshold, x.dtype())).select(x, log1p); } test "softplus" { @@ -2365,14 +2365,11 @@ test "softplus" { const platform = zml.testing.env(); const input: Tensor = .init(.{ .x = inputs.len }, .f32); const Local = struct { - fn call(x: Tensor) Tensor { - return softplus(x, .{ .threshold = 2.0 }); + fn call(x: Tensor, dtype: DataType) Tensor { + return softplus(x.convert(dtype), .{ .threshold = 2.0 }).convert(x.dtype()); } }; - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{input}, .{}); - defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes( std.testing.io, platform, @@ -2381,13 +2378,18 @@ test "softplus" { ); defer input_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); - defer result.deinit(); + for ([_]DataType{ .f32, .bf16 }) |dtype| { + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{ input, dtype }, .{}); + defer exe.deinit(); - try zml.testing.expectClose( - std.testing.io, - Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), - result, - .{}, - ); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer result.deinit(); + + try zml.testing.expectClose( + std.testing.io, + Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), + result, + .{}, + ); + } } From 2438b9c364799ee12a52ff4b1a9e14e2aa874e54 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 5 Nov 2026 13:19:58 +0100 Subject: [PATCH 16/47] zml/kernels/paged_attention: make mesh and partitioning explicit (#1024) Paged attention currently relies on input sharding metadata or partitioning choices made inside backend kernels. Ordinary graph operations can leave that metadata unspecified, making the resulting sharding depend on the backend and preceding operations. Move the manual computation boundary into the shared `pagedAttention` API using `manualComputation2`. Callers provide a logical mesh and typed input/output partitioning specs. --- .../paged_attention/cute/benchmark.zig | 27 ++- zml/kernels/paged_attention/cute/cute.zig | 93 ++++----- zml/kernels/paged_attention/cute/fixtures.zig | 2 +- .../paged_attention/cute/sm10x/sm10x.zig | 91 ++++----- .../paged_attention/cute/sm120/sm120.zig | 109 +++++------ .../paged_attention/mosaic_tpu/mosaic_tpu.zig | 74 ++----- .../paged_attention/paged_attention.zig | 80 +++++--- zml/kernels/paged_attention/tcl/tcl.zig | 2 +- zml/kernels/paged_attention/triton/triton.zig | 184 ++++++------------ 9 files changed, 278 insertions(+), 384 deletions(-) diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 9fd6a355..3e05fed8 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -54,7 +54,7 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, fixtures.forward, .{ inputs, global_case }, .{ + const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ .program_name = "paged_attention_harness", .meshes = &.{mesh}, }); @@ -115,6 +115,31 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, }); } +fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { + const Call = struct { + inputs: Inputs, + case: cases.Case, + + fn local(self: @This(), _: zml.Shape) zml.Tensor { + return fixtures.forward(self.inputs, self.case); + } + }; + const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + return zml.ops.manualComputation2(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + .in_specs = .{ .inputs = .{ + .q = heads, + .k = heads, + .v = heads, + .table = .replicated, + .lengths = .replicated, + .starts = .replicated, + .sinks = .parse(.{ ._0 = .model }), + .indices = .replicated, + } }, + .out_specs = heads, + }); +} + fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; } diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index e572cce1..ea4f36ab 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -72,57 +72,44 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z return sm10x.pagedAttention(parameters, q, k, v, opts); if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) return sm120.pagedAttention(parameters, q, k, v, opts); - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const indices = switch (opts.mask) { + .indices => |indices| indices.transpose(.{ .b, .topk }), + else => zml.Tensor.zeroes(.init(.{ q_.dim(.b), 1 }, .i32)), + }; + const cfg: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages = k_.dim(.page), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .dtype = zml.kernel.cute.from(q_.dtype()), + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd))))), + .sink = opts.sink != null, + .indexed = opts.mask == .indices, + .topk = indices.dim(-1), + }; + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); + return kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + .indices = indices, + }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, + .block = .{ 32, 1, 1 }, + }).out.transpose(q.shape()); } - -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const indices = switch (self.opts.mask) { - .indices => |indices| indices.transpose(.{ .b, .topk }), - else => zml.Tensor.zeroes(.init(.{ q.dim(.b), 1 }, .i32)), - }; - const cfg: kernel.Config = .{ - .head_dim = q.dim(.hd), - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages = k.dim(.page), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .dtype = zml.kernel.cute.from(q.dtype()), - .causal = self.opts.mask.isCausal(), - .window = self.opts.mask.slidingWindow() orelse 0, - .scale = self.opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd))))), - .sink = self.opts.sink != null, - .indexed = self.opts.mask == .indices, - .topk = indices.dim(-1), - }; - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); - return kernel.Kernel.call(.{ - .q = q, - .k = k, - .v = v, - .table = p.block_table, - .lengths = p.seq_lens, - .starts = p.query_start_len, - .sink = sink, - .indices = indices, - }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, - .block = .{ 32, 1, 1 }, - }).out.transpose(output_shape); - } -}; diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index 8c289b04..aafb8bf1 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -122,7 +122,7 @@ pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { }, }, }; - return zml.paged_attention.pagedAttention(parameters, i.q, i.k, i.v, .{ .split = .{ .k = i.k, .v = i.v } }, i.options(c)); + return zml.paged_attention.cute.pagedAttention(parameters.cute, i.q, i.k, i.v, i.options(c)); } pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index 664720ee..af068bad 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -20,60 +20,47 @@ pub fn isAvailable(platform: *const zml.Platform) bool { } pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; + var c: kernel.Config = .{ + .architecture = if (cc.minor >= 3) .sm103 else .sm100, + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .decode = decode, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); + // An unused scalar stays a constant buffer; broadcasting a dummy for + // every head would add a helper kernel to each attention invocation. + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); + const out = kernel.Attention.call(.{ .q = q_, .k = k_, .v = v_, .table = parameters.block_table, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ + .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape(), + }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); } -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const decode = !p.options_.is_prefill; - const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; - var c: kernel.Config = .{ - .architecture = if (cc.minor >= 3) .sm103 else .sm100, - .head_dim = q.dim(.hd), - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q.dtype()), - .decode = decode, - .causal = self.opts.mask.isCausal(), - .window = self.opts.mask.slidingWindow() orelse 0, - .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), - .sink = self.opts.sink != null, - }; - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); - // An unused scalar stays a constant buffer; broadcasting a dummy for - // every head would add a helper kernel to each attention invocation. - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); - const out = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = p.block_table, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ - .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape(), - }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(output_shape); - } -}; - const ArchitectureCheck = struct { fn forward(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, c: kernel.Config) zml.Tensor { const table_values: [9]i32 = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 6dfeb7e2..292e4a9f 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -19,65 +19,52 @@ pub fn isAvailable(platform: *const zml.Platform) bool { } pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + var c: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .warps_per_query = if (decode) 4 else 2, + .kv_tile = if (q_.dim(.hd) == 512) 32 else 64, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + // Mixed batches also contain single-token rows. Partition their KV + // range so one long decode cannot hold up all the prefill CTAs. + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); + const out = kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, + .block = .{ 128, 1, 1 }, + }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); } - -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const decode = !p.options_.is_prefill; - var c: kernel.Config = .{ - .head_dim = q.dim(.hd), - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q.dtype()), - .warps_per_query = if (decode) 4 else 2, - .kv_tile = if (q.dim(.hd) == 512) 32 else 64, - .causal = self.opts.mask.isCausal(), - .window = self.opts.mask.slidingWindow() orelse 0, - .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), - .sink = self.opts.sink != null, - }; - // Mixed batches also contain single-token rows. Partition their KV - // range so one long decode cannot hold up all the prefill CTAs. - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); - const out = kernel.Kernel.call(.{ - .q = q, - .k = k, - .v = v, - .table = p.block_table, - .lengths = p.seq_lens, - .starts = p.query_start_len, - .sink = sink, - }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, - .block = .{ 128, 1, 1 }, - }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(output_shape); - } -}; diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index ef9e852c..42e1861f 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -153,32 +153,28 @@ pub const mosaic_tpu = struct { "mosaic_tpu ragged paged attention q head_dim {} exceeds kernel/cache head_dim {}", .{ q.dim(.hd), target_head_dim }, ); - var q_padded = q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); - q_padded._shape._sharding = q._shape._sharding; - return q_padded; + return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const mesh = zml.Compiler.current().resolveMesh(.{.model}); - const kv_pages_partitioned = kv_pages.withPartitioning(mesh, .{ .hkv = .model }); - const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(mesh, .{ .h = .model }); + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }); const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); - const cache_head_dim: usize = @intCast(kv_pages_partitioned.dim(.hd)); + const cache_head_dim: usize = @intCast(kv_pages.dim(.hd)); stdx.debug.assert( cache_head_dim == alignedKernelHeadDim(logical_head_dim), "mosaic_tpu ragged paged attention expects KV cache head_dim {} to match aligned logical head_dim {}", .{ cache_head_dim, logical_head_dim }, ); - const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); - const query_start_len = parameters.query_start_len.withPartitioning(mesh, .{ .b = .replicated }); + const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages.dim(.hd)); + const query_start_len = parameters.query_start_len; return .{ .q = q_ragged, - .kv_pages = kv_pages_partitioned, - .seq_lens = parameters.seq_lens.withPartitioning(mesh, .{ ._0 = .replicated }), - .block_table = parameters.block_table.withPartitioning(mesh, .{ ._0 = .replicated, ._1 = .replicated }), + .kv_pages = kv_pages, + .seq_lens = parameters.seq_lens, + .block_table = parameters.block_table, .query_start_len = query_start_len, - .num_seqs = activeSequenceCount(query_start_len).withPartitioning(mesh, .{ ._0 = .replicated }), + .num_seqs = activeSequenceCount(query_start_len), }; } @@ -224,57 +220,9 @@ pub const mosaic_tpu = struct { pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); - const prepared = prepareInputs(parameters, q, kv_cache); - - const q_out = zml.ops.manualComputation( - (struct { - q: zml.Tensor, - kv_pages: zml.Tensor, - seq_lens: zml.Tensor, - block_table: zml.Tensor, - query_start_len: zml.Tensor, - num_seqs: zml.Tensor, - opts: AttentionOptions, - parameters: Parameters, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - const prepared_inputs: PreparedInputs = .{ - .q = self.q, - .kv_pages = self.kv_pages, - .seq_lens = self.seq_lens, - .block_table = self.block_table, - .query_start_len = self.query_start_len, - .num_seqs = self.num_seqs, - }; - - const cfg = buildCfg(prepared_inputs, self.parameters, self.opts); - const q_out = raggedPagedKernelCall( - prepared_inputs.q, - prepared_inputs.kv_pages, - prepared_inputs.seq_lens, - prepared_inputs.block_table, - prepared_inputs.query_start_len, - prepared_inputs.num_seqs, - cfg, - ); - stdx.debug.assert(q_out.shape().eql(output), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ q_out.shape(), output }); - return q_out; - } - }).body, - .{ - .q = prepared.q, - .kv_pages = prepared.kv_pages, - .seq_lens = prepared.seq_lens, - .block_table = prepared.block_table, - .query_start_len = prepared.query_start_len, - .num_seqs = prepared.num_seqs, - .opts = opts, - .parameters = parameters, - }, - prepared.q.shape(), - ); - + const cfg = buildCfg(prepared, parameters, opts); + const q_out = raggedPagedKernelCall(prepared.q, prepared.kv_pages, prepared.seq_lens, prepared.block_table, prepared.query_start_len, prepared.num_seqs, cfg); const restored = restoreQueryHeads(q, q_out); stdx.debug.assert(restored.shape().eql(q.shape()), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ restored.shape(), q.shape() }); return restored; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 65cf6cea..380b4cc2 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,7 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); -const cute = @import("cute/cute.zig"); +pub const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); const stablehlo = @import("stablehlo/stablehlo.zig"); @@ -247,34 +247,48 @@ pub const KvCache = union(enum) { } }; -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { +pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { _ = k; _ = v; - return switch (parameters) { - .cute => |p| switch (kv_cache) { - .split => |split| cute.pagedAttention(p, q, split.k, split.v, opts), - .dense => @panic("CuTe paged attention requires split KV pages"), - }, - .triton => |triton_parameters| switch (kv_cache) { - .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .mosaic_tpu => |mosaic_tpu_parameters| switch (kv_cache) { - .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), - .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), - }, - .metal => |metal_parameters| switch (kv_cache) { - .split => |split| metal.pagedAttention(metal_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .stablehlo => |params| stablehlo.pagedAttention(params, q, kv_cache, opts), - .tcl => |params| switch (kv_cache) { - .split => |split| tcl.paged.pagedAttention(params, q, split.k, split.v, opts), - .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), - }, - }; + const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; + return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, partitioning); } +const Attention = struct { + parameters: Parameters, + q: zml.Tensor, + kv_cache: KvCache, + opts: AttentionOptions, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .cute => |p| switch (args.kv_cache) { + .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts), + .dense => @panic("CuTe paged attention requires split KV pages"), + }, + .triton => |triton_parameters| switch (args.kv_cache) { + .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .mosaic_tpu => |mosaic_tpu_parameters| switch (args.kv_cache) { + .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts), + }, + .metal => |metal_parameters| switch (args.kv_cache) { + .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .stablehlo => |params| stablehlo.pagedAttention(params, args.q, args.kv_cache, args.opts), + .tcl => |params| switch (args.kv_cache) { + .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), + }, + }; + } +}; + test "Backend.auto selects mosaic_tpu on TPU" { const platform: zml.Platform = .{ .arena = undefined, @@ -463,11 +477,25 @@ test pagedAttention { backend_options_args.backend = backend; const parameters: Parameters = .init(.fromBackend(backend_options_args)); + const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + const ParameterSpecs = zml.meta.MapRestrict(zml.Tensor, zml.Sharding.Partitioning.Logical).map(Parameters); + const partitioning: Partitioning = .{ + .in_specs = .{ + .parameters = switch (parameters) { + inline else => |_, tag| @unionInit(ParameterSpecs, @tagName(tag), .{ .block_table = .replicated, .seq_lens = .replicated, .query_start_len = .replicated }), + }, + .q = head_spec, + .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, + .opts = .{ .mask = .{ .standard = {} }, .sink = null }, + }, + .out_specs = head_spec, + }; + const exe = try platform.compileFn( allocator, io, pagedAttention, - .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, + .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options, tp, partitioning }, .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, ); defer exe.deinit(); diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index a7c2a5f4..828811f3 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -1655,7 +1655,7 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); const Mod = struct { pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged_attention.AttentionOptions) Tensor { - return paged_attention.pagedAttention(.{ .tcl = p }, q, k, v, .{ .split = .{ .k = k, .v = v } }, opts); + return paged.pagedAttention(p, q, k, v, opts); } }; const opts: paged_attention.AttentionOptions = .{ diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index 1ccbeba8..67416285 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -274,92 +274,57 @@ pub const paged = struct { return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); } - const output = zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k_cache: zml.Tensor, - v_cache: zml.Tensor, - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - opts: AttentionOptions, - options: Options, + const cu_count = getCuCount(); + const num_heads: usize = @intCast(q.dim(.hkv) * q.dim(.hg)); + const num_kv_heads: usize = @intCast(k_cache.dim(.hkv)); + const num_queries_per_kv: usize = num_heads / num_kv_heads; + // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the + // single decode query token doesn't carry masked-out fp32 acc lanes. + // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. + const block_m: usize = if (!parameters.options_.is_prefill and isOneapiTarget()) + std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) + else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); + const block_q: usize = block_m / num_queries_per_kv; + const num_tokens: usize = @intCast(q.dim(.b)); + const num_seqs: usize = @intCast(parameters.block_table.dim(.b)); + const total_q_blocks: usize = num_tokens / block_q + num_seqs; + const target_num_prgms: usize = cu_count * 4; + const num_2d_prgms: usize = total_q_blocks * num_kv_heads; - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const parameters_: Parameters = .{ - .block_table = self.block_table, - .seq_lens = self.seq_lens, - .query_start_len = self.query_start_len, - .options_ = self.options, - }; + const sliding_window: usize = opts.mask.slidingWindow() orelse 0; - const cu_count = getCuCount(); - const num_heads: usize = @intCast(self.q.dim(.hkv) * self.q.dim(.hg)); - const num_kv_heads: usize = @intCast(self.k_cache.dim(.hkv)); - const num_queries_per_kv: usize = num_heads / num_kv_heads; - // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the - // single decode query token doesn't carry masked-out fp32 acc lanes. - // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. - const block_m: usize = if (!self.options.is_prefill and isOneapiTarget()) - std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) - else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); - const block_q: usize = block_m / num_queries_per_kv; - const num_tokens: usize = @intCast(self.q.dim(.b)); - const num_seqs: usize = @intCast(parameters_.block_table.dim(.b)); - const total_q_blocks: usize = num_tokens / block_q + num_seqs; - const target_num_prgms: usize = cu_count * 4; - const num_2d_prgms: usize = total_q_blocks * num_kv_heads; + const paged_attention_opts: PagedAttentionOptions = .{ + .cu_count = getCuCount(), + .all_decode = !parameters.options_.is_prefill, + .num_tokens = num_tokens, + .num_heads = num_heads, + .num_kv_heads = num_kv_heads, + .head_dim = @intCast(q.dim(.hd)), + .batch_size = @intCast(parameters.block_table.dim(.b)), + .block_size = @intCast(k_cache.dim(.k_chunk)), + .num_blocks = @intCast(k_cache.dim(.page)), + .max_num_block_per_seq = @intCast(parameters.block_table.dim(.p)), + .sliding_window = sliding_window, + .block_m = block_m, + .block_q = block_q, + .total_q_blocks = total_q_blocks, + .target_num_prgms = target_num_prgms, + .num_2d_prgms = num_2d_prgms, + .max_seqlen_q = parameters.options_.max_seqlen_q, + .scale = opts.scale, + }; - const sliding_window: usize = self.opts.mask.slidingWindow() orelse 0; - - const paged_attention_opts: PagedAttentionOptions = .{ - .cu_count = getCuCount(), - .all_decode = !self.options.is_prefill, - .num_tokens = num_tokens, - .num_heads = num_heads, - .num_kv_heads = num_kv_heads, - .head_dim = @intCast(self.q.dim(.hd)), - .batch_size = @intCast(parameters_.block_table.dim(.b)), - .block_size = @intCast(self.k_cache.dim(.k_chunk)), - .num_blocks = @intCast(self.k_cache.dim(.page)), - .max_num_block_per_seq = @intCast(parameters_.block_table.dim(.p)), - .sliding_window = sliding_window, - .block_m = block_m, - .block_q = block_q, - .total_q_blocks = total_q_blocks, - .target_num_prgms = target_num_prgms, - .num_2d_prgms = num_2d_prgms, - .max_seqlen_q = self.options.max_seqlen_q, - .scale = self.opts.scale, - }; - - const use_2d_kernel = use2dKernel( - paged_attention_opts.all_decode, - paged_attention_opts.batch_size, - paged_attention_opts.num_kv_heads, - ); - const output = if (use_2d_kernel) - pagedAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) - else if (isOneapiTarget()) - pagedAttention3dOneapi(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) - else - pagedAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts); - - return output; - } - }).body, - .{ - .q = q, - .k_cache = k_cache, - .v_cache = v_cache, - .block_table = parameters.block_table, - .seq_lens = parameters.seq_lens, - .query_start_len = parameters.query_start_len, - .opts = opts, - .options = parameters.options_, - }, - q.shape(), + const use_2d_kernel = use2dKernel( + paged_attention_opts.all_decode, + paged_attention_opts.batch_size, + paged_attention_opts.num_kv_heads, ); + const output = if (use_2d_kernel) + pagedAttention2d(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else if (isOneapiTarget()) + pagedAttention3dOneapi(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else + pagedAttention3d(parameters, q, k_cache, v_cache, opts, paged_attention_opts); return output; } @@ -372,51 +337,18 @@ pub const paged = struct { stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices expect to have .b, .topk tags", .{}); stdx.debug.assert(q.dtype() == k_cache.dtype() and q.dtype() == v_cache.dtype(), "Sparse attention requires matching query and KV dtypes", .{}); - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k_cache: zml.Tensor, - v_cache: zml.Tensor, - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - opts: AttentionOptions, - options: Options, - - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const parameters_: Parameters = .{ - .block_table = self.block_table, - .seq_lens = self.seq_lens, - .query_start_len = self.query_start_len, - .options_ = self.options, - }; - - const config = selectSparseConfig( - @intCast(self.opts.mask.indices.dim(.topk)), - @intCast(self.block_table.dim(.b)), - @intCast(self.q.dim(.hkv)), - getCuCount(), - !self.options.is_prefill, - ); - - return if (config.num_segments == 1) - pagedSparseAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config) - else - pagedSparseAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config); - } - }).body, - .{ - .q = q, - .k_cache = k_cache, - .v_cache = v_cache, - .block_table = parameters.block_table, - .seq_lens = parameters.seq_lens, - .query_start_len = parameters.query_start_len, - .opts = opts, - .options = parameters.options_, - }, - q.shape(), + const config = selectSparseConfig( + @intCast(opts.mask.indices.dim(.topk)), + @intCast(parameters.block_table.dim(.b)), + @intCast(q.dim(.hkv)), + getCuCount(), + !parameters.options_.is_prefill, ); + + return if (config.num_segments == 1) + pagedSparseAttention2d(parameters, q, k_cache, v_cache, opts, config) + else + pagedSparseAttention3d(parameters, q, k_cache, v_cache, opts, config); } pub fn pagedSparseAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { From 5decef1fcabb00ed653bc547c4c8dcd6a80a4b51 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Nov 2026 14:17:44 +0100 Subject: [PATCH 17/47] zml/kernels/paged_attention: enable cute by default when available (#1017) --- zml/kernels/paged_attention/paged_attention.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 380b4cc2..81f7ec3e 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -30,7 +30,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { - .cuda => .triton, + .cuda => if (cute.isAvailable(platform)) .cute else .triton, .rocm => .triton, .oneapi => .triton, .tpu => .mosaic_tpu, From 3285b7e0f42013d6079c70a641ad53da84399f91 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 5 Nov 2026 17:38:31 +0100 Subject: [PATCH 18/47] zml: FP8 block activations and paged attention query padding on Furiosa and bump plugins (#1033) - Quantize block FP8 activations to E4M3FN on Furiosa, so scaled dots run natively. Other block sizes keep BF16 activations and XLA's fallback. - Pad paged attention query groups to a power of two with zero queries, as TCC fails other group sizes. Marked as a workaround. - Bump to [202611051630.351.1.48d539621544](https://github.com/zml/xla-private/tree/202611051630.351.1.48d539621544) --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 2 +- platforms/rocm/rocm.bzl | 2 +- zml/kernels/paged_attention/tcl/tcl.zig | 18 ++++++++++++++---- zml/nn.zig | 2 +- zml/quantization.zig | 2 ++ 8 files changed, 28 insertions(+), 16 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index b33ef4df..a72c4b15 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index a0c73948..c72ef3be 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "b68b407efcd361bbe4b297a16feb080eabbae9132a7ee1a5b105e24a91d99de0", - "url": "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cuda-linux-amd64.tar.zst", + "sha256": "1ae8a7c8fc677e0f10c0032ce9ae35e5b6c3f5eadc3f54e1bbda63c219b0170d", + "url": "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "9ac41221901eba84641aef35d1f53ff69ce471c5af3844550cd086abc6916f5c", - "url": "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cuda-linux-arm64.tar.zst", + "sha256": "14a3567c629a1ea97f1dcfa2cb47b54ea08d8c4b9e35eae47a7eff33cfb53ce0", + "url": "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index fc2e5352..b623000b 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-furiosa-linux-amd64.tar.zst", - sha256 = "85edea12494278a015c1ad29cf57401217e4809bc3a0c6890d0279659bb6209d", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-furiosa-linux-amd64.tar.zst", + sha256 = "b4c0202ec27f87b82a92b014c6b58a6bd684a7acf65eebcd8ec357f8580b2451", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 2d43a991..999cedfa 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -5,7 +5,7 @@ def _metal_impl(mctx): name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", sha256 = "bdd2de3ec8293faa25b358cdf35ee3e75d69be73a4acea87b6e007a18b8f10e8", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-metal-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 0c1f7a92..5b18a9c9 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,7 +11,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-rocm-linux-amd64.tar.zst" _PLUGIN_ROCM_SHA256 = "02b86fa3a6599f6aa2b89806b8f6c0b16f11765ec04c94517ec1463a59d92175" def _rocm_package_name(name): diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index 828811f3..c9993af9 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -152,21 +152,31 @@ pub const paged = struct { cfg.chunk = if (plan.chunk > 0) plan.chunk else keys; cfg.a = (std.math.divCeil(i64, keys, cfg.chunk) catch unreachable) * cfg.chunk; const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); + // WORKAROUND (TCC 2026.5.0b0): attention over query groups of a size + // other than a power of two fails ("Inconsistent tensor group + // offsets"): pad the groups with zero queries, whose outputs are + // dropped. Repro: + // repro_flash_prefill_group6_inconsistent_offsets.py (zml/reproducer, + // platform/furiosa). Remove once any group size compiles. + const groups: i64 = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.g)) catch unreachable); + const extra = groups - cfg.g; + cfg.g = groups; + const queries = q.convert(.bf16).pad(0, .{ .hg = Tensor.Pad{ .high = extra } }); const inputs: OneShot.Inputs = .{ - .q = q.convert(.bf16), + .q = queries, .pages = table, .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), - .sink = sink, + .sink = sink.pad(0, .{ .hg = Tensor.Pad{ .high = extra } }), }; - const outputs: OneShot.Outputs = .{ .out = q.shape().withDtype(.bf16) }; + const outputs: OneShot.Outputs = .{ .out = queries.shape() }; const out = if (cfg.a == cfg.chunk) OneShot.call(inputs, outputs, .{ .cfg = cfg }).out else Flash.call(inputs, outputs, .{ .cfg = cfg }).out; - return out.convert(q.dtype()); + return out.slice(.hg, .{ .end = groups - extra }).convert(q.dtype()); } }; diff --git a/zml/nn.zig b/zml/nn.zig index 4569552f..a2b582ee 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -226,7 +226,7 @@ test "block128 scaled dot layouts" { const io = std.testing.io; const platform = zml.testing.env(); // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) - if (platform.target != .cuda) return error.SkipZigTest; + if (platform.target != .cuda and platform.target != .furiosa) return error.SkipZigTest; const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; const Local = struct { const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; diff --git a/zml/quantization.zig b/zml/quantization.zig index 89e1b210..db20619a 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -129,6 +129,8 @@ pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, .fp8_block32, .fp8_block128 => blk: { const dtype: DataType = switch (platform.target) { .cuda => .f8e4m3fn, + // RNGD contracts matching FP8 pairs in 128-wide K blocks. + .furiosa => if (quantization.scheme == .fp8_block128) .f8e4m3fn else return null, .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { .cdna4, .rdna4 => .f8e4m3fn, .cdna3 => .f8e4m3fnuz, From 630d96726bf0c39547f6106f7b917397be4e312c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Nov 2026 19:34:17 +0100 Subject: [PATCH 19/47] zml/io: add `consolidated` version of safetensors to defaults (#1035) Some models entrypoints are exposed as `consolidated.safetensors.index.json` or `consolidated.safetensors`. --- zml/safetensors.zig | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 63791ea9..f854f02f 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -595,17 +595,16 @@ pub fn resolveModelRepo( } pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, []const u8 } { - { - const index_name = "model.safetensors.index.json"; - if (repo.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { - return .{ index_file, "model.safetensors.index.json" }; - } else |_| {} - } + const default_entrypoints = .{ + "model.safetensors.index.json", + "consolidated.safetensors.index.json", + "model.safetensors", + "consolidated.safetensors", + }; - { - const model_name = "model.safetensors"; - if (repo.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { - return .{ model_file, "model.safetensors" }; + inline for (default_entrypoints) |entrypoint| { + if (repo.openFile(io, entrypoint, .{ .mode = .read_only })) |index_file| { + return .{ index_file, entrypoint }; } else |_| {} } From 837e45022a2b95a27af846eb106970ec71a5e159 Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:57:09 +0100 Subject: [PATCH 20/47] zml/kernels/paged_attention: fix stablehlo backend slot_id increment (#1027) If sequences were not 16-aligned we would have sequence leaks. Now we increase slot_id by query_start_len and pad when needed --- .../paged_attention/stablehlo/stablehlo.zig | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index ab0ffee2..7ee4df03 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -15,10 +15,11 @@ pub fn pagedAttention( } const page_size = kv_cache.split.k.dim(.k_chunk); + const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); const final_state = zml.ops.@"while"( AttentionLoop, - .{ .q = q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, + .{ .q = padded_q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, .{ // i32 to match triton conventions .slot_id = .scalar(0, .i32), @@ -28,11 +29,11 @@ pub fn pagedAttention( // Note: we do all partial softmax in f32 .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), - .out = .zeroes(q.shape()), + .out = .zeroes(padded_q.shape()), }, ); - return final_state.out; + return final_state.out.slice(.b, .{ .end = q.dim(.b) }); } const AttentionLoop = struct { @@ -93,7 +94,7 @@ const AttentionLoop = struct { const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); } /// Decode version: num_slots == 1, we multiply one query with a page of keys. @@ -112,7 +113,7 @@ const AttentionLoop = struct { const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); } }, .{ @@ -162,7 +163,11 @@ const AttentionLoop = struct { return og_softmax.merge(partial); } - pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax) State { + fn nextQueryStart(self: While, state: State) zml.Tensor { + return self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))).asScalar(); + } + + pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax, next_query_start: zml.Tensor) State { return zml.ops.if2( query_has_more_keys, // If the query has more pages: update partial softmax with attention on current page, delay updating output. @@ -181,7 +186,7 @@ const AttentionLoop = struct { .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), - .slot_id = state.slot_id.addConstant(num_slots), + .slot_id = .select(sequence_has_more_queries, state.slot_id.addConstant(num_slots), next_query_start), // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. From 488c21c89945d669f5d1f46be3cbbc2b51fe040a Mon Sep 17 00:00:00 2001 From: PierreLgol Date: Fri, 6 Nov 2026 10:29:31 +0100 Subject: [PATCH 21/47] workspace: add Helix editor configuration for Zig (#1029) adding basic support for the helix editor, to automatically work --- .helix/languages.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .helix/languages.toml diff --git a/.helix/languages.toml b/.helix/languages.toml new file mode 100644 index 00000000..529cdcf3 --- /dev/null +++ b/.helix/languages.toml @@ -0,0 +1,19 @@ +[[language]] +auto-format = true +file-types = [ "zig" ] +language-servers = [ "zls" ] +name = "zig" +roots = [ "build.zig", "build.zig.zon", "zls.build.json" ] +scope = "source.zig" + +[language.indent] +tab-width = 4 +unit = " " + +[language-server.zls] +command = "sh" +args = [ + "-c", + "exec \"$(git rev-parse --show-toplevel)/tools/zls.sh\" \"$@\"", + "zls" +] From 5b079127bb3b8f9bc1e5e8ab773b99c274277d87 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 6 Nov 2026 10:56:05 +0100 Subject: [PATCH 22/47] workspace: bump ZML plugins (#1034) Bump plugins to [202611060925.355.1.2ca211fa5fbd](https://github.com/zml/xla-private/tree/202611060925.355.1.2ca211fa5fbd) --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 4 ++-- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 2 +- platforms/rocm/rocm.bzl | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index a72c4b15..fd070b6e 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index c72ef3be..24718ee6 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -225,11 +225,11 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { "sha256": "1ae8a7c8fc677e0f10c0032ce9ae35e5b6c3f5eadc3f54e1bbda63c219b0170d", - "url": "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cuda-linux-amd64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-amd64.tar.zst", }, "arm64": { "sha256": "14a3567c629a1ea97f1dcfa2cb47b54ea08d8c4b9e35eae47a7eff33cfb53ce0", - "url": "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cuda-linux-arm64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index b623000b..ba53bfb4 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-furiosa-linux-amd64.tar.zst", - sha256 = "b4c0202ec27f87b82a92b014c6b58a6bd684a7acf65eebcd8ec357f8580b2451", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-furiosa-linux-amd64.tar.zst", + sha256 = "514b1c96a2237380a9b9e011f5b994c6617eb103a6a9dac8ff0a7f1832d1a3bd", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 999cedfa..767ef2f6 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -5,7 +5,7 @@ def _metal_impl(mctx): name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", sha256 = "bdd2de3ec8293faa25b358cdf35ee3e75d69be73a4acea87b6e007a18b8f10e8", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-metal-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 5b18a9c9..454bf626 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,7 +11,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-rocm-linux-amd64.tar.zst" _PLUGIN_ROCM_SHA256 = "02b86fa3a6599f6aa2b89806b8f6c0b16f11765ec04c94517ec1463a59d92175" def _rocm_package_name(name): From 59a888f7e2f5049e9addb9c2647cb31fc374fd17 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 6 Nov 2026 15:14:38 +0100 Subject: [PATCH 23/47] third_party/zls: allow all 0.17 zig versions (#1036) --- third_party/zls/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index d53e20be..69cc6128 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -30,7 +30,7 @@ zls_toolchain( toolchain( name = "toolchain", - target_settings = ["@zig_toolchains//:0.17.0"], + target_settings = ["@zig_toolchains//:any_0.17.release"], toolchain = ":zls", toolchain_type = "@rules_zig//zig/zls:toolchain_type", ) From 1042cba3ff15c1a852e3c2cf8f57fce877e8df52 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 9 Nov 2026 15:26:03 +0100 Subject: [PATCH 24/47] zml/kernels/moe: make mesh and partitioning explicit (#1028) MOE API currently relies on input sharding metadata or partitioning choices made inside backend kernels. Ordinary graph operations can leave that metadata unspecified, making the resulting sharding depend on the backend and preceding operations. Move the manual computation boundary into the shared `zml.kernels.moe.forwardMoe` API using `manualComputation2`. Callers provide a logical mesh and typed input/output partitioning specs. --- examples/llm/models/qwen3_5_moe/model.zig | 2 + zml/kernels/moe/cute/cute.zig | 124 ++--- .../cutlass_flashinfer/cutlass_flashinfer.zig | 101 +--- zml/kernels/moe/fly/fly.zig | 134 ++--- zml/kernels/moe/metal/metal.zig | 94 +--- zml/kernels/moe/moe.zig | 180 ++++++- zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig | 138 +---- zml/kernels/moe/stablehlo/stablehlo.zig | 15 +- zml/kernels/moe/tcl/tcl.zig | 8 +- zml/kernels/moe/tests.zig | 38 +- zml/kernels/moe/triton/triton.zig | 502 +++++++----------- zml/ops.zig | 136 ++++- 12 files changed, 641 insertions(+), 831 deletions(-) diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 1375a1bd..2a3b0ba8 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -787,6 +787,8 @@ pub const Moe = struct { self.gate_up_proj, self.down_proj, moe_backend, + zml.Compiler.current().mesh(.experts), + .expert(.experts), .{ .activation = .{ .swiglu = .{} }, .quantize_input = false, diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index c0156720..708ea757 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -38,8 +38,7 @@ pub fn validateOptions(opts: zml.moe.Options) void { /// expect block scales in the 128x4 tensor-core layout. pub fn fusedExperts( input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: zml.moe.Options, @@ -83,24 +82,43 @@ pub fn fusedExperts( else => stdx.debug.panic("cute backend only supports mxfp4 and nvfp4 quantization, got {}", .{gq.scheme}), }; - const context: Context = .{ - .input = input, - .ids = ids, - .weights = weights, - .w1 = gate_up.weight, - .s1 = gq.scales, - .w2 = down.weight, - .s2 = dq.scales, - .topk = ids.dim(.topk), - .swiglu_limit = options.activation.swiglu.limit, - .routing_weight_placement = options.routing_weight_placement, - .format = format, - .expert_parallel = gate_up.weight.shape().partition(.expert).isSharded(), - }; - return if (context.expert_parallel) - zml.ops.manualComputation(Context.body, context, input.shape()) - else - context.body(input.shape()); + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); + const routing_weights = routing.weights.convert(.f32).reshape(.{ tokens, topk }); + + return switch (format) { + .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, topk, options.activation.swiglu.limit, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + .ids = ids, + }, null), + .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, topk, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .ids = ids, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + // `alpha` is per expert; `act` is one scale shared by every + // expert, which the kernels read per expert. + .alpha1 = f.alpha1.reshape(.{experts}), + .alpha2 = f.alpha2.reshape(.{experts}), + .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), + .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), + }, .{ + .swiglu_limit = options.activation.swiglu.limit, + .routing_weight_placement = options.routing_weight_placement, + }), + }.reshape(input.shape().dims()).withTags(input.shape()); } /// The format-specific tensors: NVFP4's per-expert global scales. They stay @@ -115,69 +133,3 @@ const Format = union(enum) { act2: zml.Tensor, }, }; - -const Context = struct { - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, - topk: i64, - swiglu_limit: ?f32, - routing_weight_placement: zml.moe.RoutingWeightPlacement, - format: Format, - expert_parallel: bool, - - fn body(self: Context, _: zml.Shape) zml.Tensor { - const experts = self.w1.dim(.expert); - const hidden = self.w2.dim(1); - const intermediate = self.w2.dim(2) * 2; - const tokens = self.input.dim(.t); - var ids = self.ids.convert(.i32); - const routing_weights = self.weights.convert(.f32); - if (self.expert_parallel) { - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(experts); - const expert_end = expert_start.addConstant(experts); - // Routes of other ranks get expert -1 and are not computed here. - const local = ids.cmp(.GE, expert_start).logical(.AND, ids.cmp(.LT, expert_end)); - ids = local.select(ids.sub(expert_start), .scalar(-1, .i32)); - } - const result = switch (self.format) { - .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, self.topk, self.swiglu_limit, .{ - .x = self.input, - .routing_weights = routing_weights, - .w1 = self.w1, - .s1 = self.s1, - .w2 = self.w2, - .s2 = self.s2, - .ids = ids, - }, null), - .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, self.topk, .{ - .x = self.input, - .routing_weights = routing_weights, - .ids = ids, - .w1 = self.w1, - .s1 = self.s1, - .w2 = self.w2, - .s2 = self.s2, - // `alpha` is per expert; `act` is one scale shared by every - // expert, which the kernels read per expert. - .alpha1 = f.alpha1.reshape(.{experts}), - .alpha2 = f.alpha2.reshape(.{experts}), - .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), - .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), - }, .{ - .swiglu_limit = self.swiglu_limit, - .routing_weight_placement = self.routing_weight_placement, - }), - }.withTags(self.input.shape()); - - return if (self.expert_parallel) - zml.ops.allReduce(result, zml.Tensor.add) - else - result; - } -}; diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig index 2c7ad342..e2c31c44 100644 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig @@ -20,7 +20,7 @@ pub const Options = struct { /// obtain the first valid GEMM2 index. gemm2_tactic: i32, - pub fn fromGlobalOptions(opts: zml.moe.Options) Options { + pub fn fromOptions(opts: zml.moe.Options) Options { return .{ .workspace_query_device = 0, .activation = opts.activation, @@ -502,8 +502,7 @@ pub fn fusedExpertsNvfp4( pub fn fusedExperts( input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options, @@ -516,8 +515,11 @@ pub fn fusedExperts( @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); } - const runner_options: Options = .fromGlobalOptions(opts); - const expert_partition = gate_up.weight.shape().partition(.expert); + const topk_ids = routing.localIds(); + // The runner ignores routes whose local expert id is -1. + const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); + + const runner_options: Options = .fromOptions(opts); const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); @@ -527,95 +529,6 @@ pub fn fusedExperts( // TODO(Corentin): Do error checking on nvfp4 // Also, maybe pass `zml.nn.Linear` directly - if (expert_partition.isSharded()) { - return zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up_weight_unpacked: zml.Tensor, - down_weight_unpacked: zml.Tensor, - gate_up_input_scale: zml.Tensor, - gate_up_scales: zml.Tensor, - gate_up_global_scale: zml.Tensor, - down_input_scale: zml.Tensor, - down_scales: zml.Tensor, - down_global_scale: zml.Tensor, - activation: zml.moe.Activation, - enable_pdl: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, - workspace_query_device: i32, - - fn body( - self: @This(), - _: zml.Shape, - ) zml.Tensor { - const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const expert_end = expert_start.addConstant(local_num_experts); - - const local_route_mask = self.topk_ids - .cmp(.GE, expert_start) - .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); - const local_topk_ids = local_route_mask.select( - self.topk_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_topk_weights = local_route_mask.select( - self.topk_weights, - zml.Tensor.scalar(-1, self.topk_weights.dtype()), - ); - - const local_output = fusedExpertsNvfp4( - self.input, - self.gate_up_weight_unpacked, - self.down_weight_unpacked, - local_topk_weights, - local_topk_ids, - self.gate_up_input_scale, - self.gate_up_scales, - self.gate_up_global_scale, - self.down_input_scale, - self.down_scales, - self.down_global_scale, - .{ - .workspace_query_device = self.workspace_query_device, - .activation = self.activation, - .enable_pdl = self.enable_pdl, - .gemm1_tactic = self.gemm1_tactic, - .gemm2_tactic = self.gemm2_tactic, - }, - ); - const local_reshaped = local_output - .reshape(self.input.shape().dims()) - .withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .gate_up_weight_unpacked = gate_up_weight_unpacked, - .down_weight_unpacked = down_weight_unpacked, - .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), - .gate_up_scales = gate_up.quantization.?.scales, - .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), - .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), - .down_scales = down.quantization.?.scales, - .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), - .activation = runner_options.activation, - .enable_pdl = runner_options.enable_pdl, - .gemm1_tactic = runner_options.gemm1_tactic, - .gemm2_tactic = runner_options.gemm2_tactic, - .workspace_query_device = runner_options.workspace_query_device, - }, - input.shape(), - ); - } - return fusedExpertsNvfp4( input, gate_up_weight_unpacked, diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index e5c5b6de..59d71dc9 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -8,7 +8,6 @@ const Shape = zml.Shape; const DataType = zml.DataType; const fly = @import("moe.zig"); const ProjectionLayout = zml.moe.ProjectionLayout; -const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -17,90 +16,19 @@ test { } pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, -) zml.Tensor { - const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) - .{ .gate_up = .interleaved, .packing = .plain } - else - .{ .gate_up = .concatenated, .packing = .plain }; - - const args: FusedExpertsArgs = .{ - .hidden_states = input, - .gate_up = gate_up, - .down = down, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .activation = opts.activation, - .quantize_input = opts.quantize_input, - .gate_up_layout = layout.gate_up, - .routing_weight_placement = opts.routing_weight_placement, - }; - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (!expert_partition.isSharded()) { - return fusedExpertsImpl(args); - } - - return zml.ops.manualComputation( - (struct { - args: FusedExpertsArgs, - global_num_experts: i64, - - fn call(self: @This(), _: zml.Shape) zml.Tensor { - const local_args = self.args; - const local_num_experts = local_args.gate_up.weight.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - // Map global expert ids to local ids, or -1 for experts outside this partition. - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - var mapped_args = local_args; - mapped_args.expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - - const local_output = fusedExpertsImpl(mapped_args); - const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).call, - .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, - input.shape(), - ); -} - -pub const FusedExpertsArgs = struct { hidden_states: Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - topk_weights: Tensor, - topk_ids: Tensor, - activation: zml.moe.Activation, - expert_map: ?Tensor = null, - /// Use FP8 activations for FP8 weights; false keeps BF16 activations. - quantize_input: bool, - gate_up_layout: ProjectionLayout, - routing_weight_placement: RoutingWeightPlacement, -}; - -pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { - const hidden_states = opts.hidden_states; - const topk_weights = opts.topk_weights; - const topk_ids = opts.topk_ids; + routing_: zml.moe.LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: zml.moe.Options, +) Tensor { + const expert_map_ = routing_.expertMap(); + const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .interleaved else .concatenated; const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = opts.gate_up.quantizationScheme(); - const down_scheme = opts.down.quantizationScheme(); + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("fly backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -117,10 +45,10 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); - const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "fly backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -148,13 +76,13 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "fly backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "fly backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "fly backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (opts.expert_map) |expert_map| + if (expert_map_) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "fly backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (opts.expert_map) |expert_map| + const expert_ids = if (expert_map_) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -164,9 +92,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = opts.gate_up.bias, + .bias = gate_up_.bias, .input_scale = input_scale, - .weight_scale = opts.gate_up.quantizationScales(), + .weight_scale = gate_up_.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -175,7 +103,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -185,9 +113,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = opts.down.bias, + .bias = down_.bias, .input_scale = down_input_scale, - .weight_scale = opts.down.quantizationScales(), + .weight_scale = down_.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -203,18 +131,18 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { } // TODO(Corentin): unused for now -pub fn supports(opts: FusedExpertsArgs) bool { - if (opts.hidden_states.dtype() != .bf16 or opts.gate_up.quantizationScheme() != .mxfp4 or - opts.down.quantizationScheme() != .mxfp4 or opts.gate_up.bias != null or opts.down.bias != null or +pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: zml.moe.Options) bool { + if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or + down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; - const gate_up = opts.gate_up.weight; - const down = opts.down.weight; + const gate_up = gate_up_.weight; + const down = down_.weight; if (gate_up.rank() != 3 or down.rank() != 3) return false; - const gate_up_scale = opts.gate_up.quantizationScales() orelse return false; - const down_scale = opts.down.quantizationScales() orelse return false; - const experts = if (opts.expert_map) |map| map.dim(.expert) else gate_up.dim(0); + const gate_up_scale = gate_up_.quantizationScales() orelse return false; + const down_scale = down_.quantizationScales() orelse return false; + const experts = routing.global_num_experts; if (experts < 1 or experts > 512) return false; - var max_width: i64 = opts.hidden_states.dim(.d); + var max_width: i64 = hidden_states.dim(.d); inline for (.{ .{ gate_up, gate_up_scale }, .{ down, down_scale } }) |projection| { const weight, const scale = projection; const packing_factor: i64 = switch (weight.dtype()) { @@ -233,7 +161,7 @@ pub fn supports(opts: FusedExpertsArgs) bool { else => return false, } } - const routes = opts.hidden_states.dim(.t) * opts.topk_ids.dim(.topk); + const routes = hidden_states.dim(.t) * routing.ids.dim(.topk); return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); } diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index e129c76c..9d397a46 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -6,15 +6,6 @@ const Tensor = zml.Tensor; const log = std.log.scoped(.@"zml/moe/metal"); -pub const Options = struct { - activation: zml.moe.Activation, - global_num_experts: i64 = -1, - w1_scale: ?Tensor = null, - w2_scale: ?Tensor = null, - w1_global_scale: ?Tensor = null, - w2_global_scale: ?Tensor = null, -}; - fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); @@ -72,57 +63,30 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso } pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + hidden_states: zml.Tensor, + routing: zml.moe.LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, opts: zml.moe.Options, ) zml.Tensor { validateOptions(opts); - stdx.debug.assert(gate_up.bias == null, "metal backend doesn't support gate_up bias", .{}); - stdx.debug.assert(down.bias == null, "metal backend doesn't support down bias", .{}); + stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); + stdx.debug.assert(down_.bias == null, "metal backend doesn't support down bias", .{}); - const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; - const gate_up_global_scale: ?zml.Tensor = if (gate_up.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const gate_up_scales: ?zml.Tensor = if (gate_up_.quantization) |q| q.scales else null; + const gate_up_global_scale: ?zml.Tensor = if (gate_up_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; - const down_global_scale: ?zml.Tensor = if (down.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; + const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); - const down_weight_unpacked = zml.moe.unpackedWeight(down); - - return fusedExpertsImpl( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - .{ - .activation = opts.activation, - .global_num_experts = gate_up_weight_unpacked.dim(.expert), - .w1_scale = gate_up_scales, - .w2_scale = down_scales, - .w1_global_scale = gate_up_global_scale, - .w2_global_scale = down_global_scale, - }, - ); -} - -pub fn fusedExpertsImpl( - hidden_states: Tensor, - gate_up: Tensor, - down: Tensor, - topk_weights: Tensor, - topk_ids: Tensor, - opts: Options, -) Tensor { + const gate_up = zml.moe.unpackedWeight(gate_up_); + const down = zml.moe.unpackedWeight(down_); stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); const mode = quantMode(gate_up.dtype()); switch (mode) { .none => { - stdx.debug.assert(opts.w1_scale == null and opts.w2_scale == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); - stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(gate_up_scales == null and down_scales == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); const is_hidden_states_valid_dtype = switch (hidden_states.dtype()) { .bf16, .f16, .f32 => true, else => false, @@ -130,28 +94,28 @@ pub fn fusedExpertsImpl( stdx.debug.assert(is_hidden_states_valid_dtype, "metal backend expected hidden_states data type to be bf16, f16 or f32, got {}", .{hidden_states.dtype()}); }, .fp8 => { - stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(opts.w1_scale.?.dtype() == .bf16 and opts.w2_scale.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); - stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .bf16 and down_scales.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, .nvfp4 => { - stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(opts.w1_scale.?.dtype() == .f8e4m3fn and opts.w2_scale.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .f8e4m3fn and down_scales.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, } - const is_topk_weights_valid_dtype = switch (topk_weights.dtype()) { + const is_topk_weights_valid_dtype = switch (routing.weights.dtype()) { .bf16, .f16, .f32 => true, else => false, }; - stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{topk_weights.dtype()}); - stdx.debug.assert(topk_ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{topk_ids.dtype()}); + stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{routing.weights.dtype()}); + stdx.debug.assert(routing.ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{routing.ids.dtype()}); const d = hidden_states.dim(.d); const num_tokens = hidden_states.dim(.t); - const topk = topk_ids.dim(.topk); + const topk = routing.ids.dim(.topk); const num_routes = num_tokens * topk; const act_dtype: zml.DataType = switch (mode) { @@ -163,29 +127,29 @@ pub fn fusedExpertsImpl( const x_rows = hidden.insertAxes(.d, .{.topk}) .broad(zml.Shape.init(.{ .token = num_tokens, .topk = topk, .d = d }, act_dtype)) .merge(.{ .r = .{ .token, .topk } }); - const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const expert_ids = routing.ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }).convert(.i32); const gate_up_out = applyGateUpGlobalScale(moeGemm( x_rows, gate_up, - opts.w1_scale, + gate_up_scales, expert_ids, zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, - ), opts.w1_global_scale, expert_ids); + ), gate_up_global_scale, expert_ids); const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated).convert(act_dtype); const down_out = applyDownGlobalScale(moeGemm( activated, down, - opts.w2_scale, + down_scales, expert_ids, zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, act_dtype), mode, - ), opts.w2_global_scale, expert_ids); + ), down_global_scale, expert_ids); - const weights = topk_weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const weights = routing.weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }); const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 25ce26d6..7ad85c20 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,14 +4,14 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; -pub const tcl = @import("tcl/tcl.zig"); -pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const cute = @import("cute/cute.zig"); pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); +pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); -pub const fly = @import("fly/fly.zig"); const log = std.log.scoped(.@"zml/moe"); @@ -178,6 +178,38 @@ pub const Backend = enum { } }; +/// Routing for a local token batch and a contiguous range of experts. +/// IDs retain the model's global expert numbering, while the projection weights +/// contain only the local experts. Kernels compute those experts contributions and +/// `forwardMoe` sums them along the expert axis. +pub const LocalRouting = struct { + /// Global expert IDs for local tokens, with shape {.t, .topk} + ids: zml.Tensor, + weights: zml.Tensor, + global_num_experts: u32, + local_num_experts: u32, + /// Scalar i32 containing the first local expert's global ID. + /// Null when all experts are local and global IDs already index the weights. + expert_start: ?zml.Tensor = null, + + /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. + /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. + pub fn localIds(self: LocalRouting) zml.Tensor { + const start = self.expert_start orelse return self.ids; + const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(self.ids.sub(start), .scalar(-1, .i32)); + } + + /// Returns a lookup table from every global expert ID to its local weight + /// index or -1 if nonlocal. Returns null when every expert is local. + pub fn expertMap(self: LocalRouting) ?zml.Tensor { + const start = self.expert_start orelse return null; + const ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(ids.sub(start), .scalar(-1, .i32)); + } +}; + pub const Options = struct { activation: Activation, /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. @@ -196,6 +228,35 @@ pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Te weights.shape().hasTag(.t) == 0 and weights.shape().hasTag(.topk) == 1, "MoE expects ordered tags {{.t, .d}} for input and {{.t, .topk}} for routing", .{}); } +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + + in_specs: struct { + input: Logical, + ids: Logical, + weights: Logical, + gate_up: Logical, + down: Logical, + }, + out_specs: Logical, + + /// Shards the `.expert` dimension of both projections over `mesh_axis`. + /// Other dimensions, input tokens, routing IDs/weights and output are replicated. + pub fn expert(comptime mesh_axis: @EnumLiteral()) Partitioning { + const experts: Logical = .parse(.{ .expert = mesh_axis }); + return .{ + .in_specs = .{ + .input = .replicated, + .ids = .replicated, + .weights = .replicated, + .gate_up = experts, + .down = experts, + }, + .out_specs = .replicated, + }; + } +}; + /// Input has shape {.t, .d}; routing IDs and weights have shape {.t, .topk}. /// Returns {.t, .d}. Flatten batched input with merge(.{ .t = .{ .b, .s } }). pub fn forwardMoe( @@ -205,19 +266,114 @@ pub fn forwardMoe( gate_up: zml.nn.Linear, down: zml.nn.Linear, backend: Backend, + mesh: *const zml.Mesh, + partitioning: Partitioning, opts: Options, ) zml.Tensor { validateRoutingInputs(input, topk_ids, topk_weights); - return switch (backend) { - .stablehlo => stablehlo.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .cute => cute.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .triton => triton.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .fly => fly.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .mosaic_tpu => mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .metal => metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .tcl => tcl.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + + const shapes = .{ + .input = input.shape().withPartitioning(mesh, partitioning.in_specs.input), + .ids = topk_ids.shape().withPartitioning(mesh, partitioning.in_specs.ids), + .weights = topk_weights.shape().withPartitioning(mesh, partitioning.in_specs.weights), + .gate_up = gate_up.weight.shape().withPartitioning(mesh, partitioning.in_specs.gate_up), + .down = down.weight.shape().withPartitioning(mesh, partitioning.in_specs.down), }; + const output_shape = input.shape().withPartitioning(mesh, partitioning.out_specs); + + const tokens = shapes.input.partition(.t); + const experts = shapes.gate_up.partition(.expert); + // Local kernels require complete feature vectors and top-k routing lists. + stdx.debug.assert(shapes.input.partition(.d) == .replicated and shapes.ids.partition(.topk) == .replicated and shapes.weights.partition(.topk) == .replicated, "MoE feature and top-k dimensions must be replicated", .{}); + // The expert-axis reduction must add contributions for the same token rows. + // Sharding tokens along that axis would mix contributions for different tokens. + stdx.debug.assert(shapes.ids.partition(.t) == tokens and shapes.weights.partition(.t) == tokens and output_shape.partition(.t) == tokens and output_shape.partition(.d) == .replicated, "MoE input, routing and output must have matching token partitioning", .{}); + stdx.debug.assert(shapes.down.partition(.expert) == experts and (!experts.isSharded() or experts != tokens), "MoE projections must agree on expert partitioning; tokens must be replicated along the expert axis", .{}); + for ([_]zml.Shape{ shapes.gate_up, shapes.down }) |shape| { + stdx.debug.assert(!shape.containsPartitionSpec(.open), "MoE kernel weight partitioning must be closed", .{}); + for (0..shape.rank()) |axis| { + stdx.debug.assert(axis == shape.axis(.expert) or shape.partition(axis) == .replicated, "MoE kernel weights can only be partitioned along experts", .{}); + } + } + const expert_axis = experts.meshAxis(); + const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; + if (backend == .metal or backend == .tcl) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); + + const Call = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + backend: Backend, + opts: Options, + mesh: *const zml.Mesh, + expert_axis: ?u3, + global_experts: u32, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + const routing: LocalRouting = .{ + .ids = args.ids, + .weights = args.weights, + .global_num_experts = args.global_experts, + .local_num_experts = @intCast(args.gate_up.weight.dim(.expert)), + .expert_start = if (args.expert_axis) |axis| zml.ops.axisIndex(args.mesh, args.mesh.logical.axes.get(axis)).convert(.i32).scale(args.gate_up.weight.dim(.expert)) else null, + }; + const output = switch (args.backend) { + .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .metal => metal.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .tcl => tcl.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + }; + return if (args.expert_axis) |axis| zml.ops.allReduceOnAxis(output, zml.Tensor.add, args.mesh, args.mesh.logical.axes.get(axis)) else output; + } + }; + const args: Call = .{ + .input = input, + .ids = topk_ids, + .weights = topk_weights, + .gate_up = gate_up, + .down = down, + .backend = backend, + .opts = opts, + .mesh = mesh, + .expert_axis = if (parallel) expert_axis else null, + .global_experts = @intCast(gate_up.weight.dim(.expert)), + }; + const Logical = zml.Sharding.Partitioning.Logical; + var projection_specs: [2]zml.meta.MapRestrict(zml.Tensor, Logical).map(zml.nn.Linear) = undefined; + for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { + zml.meta.mapAlloc(struct { + fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { + const axis = ctx.axis orelse return .replicated; + if (tensor.count() == 1) return .replicated; + // Packed scales may be untagged; experts are their leading dimension. + const dim = tensor.shape().hasTag(.expert) orelse 0; + stdx.debug.assert(tensor.dim(dim) == ctx.experts, "MoE tensor must contain {} experts, got {f}", .{ ctx.experts, tensor.shape() }); + return .{ .mapping = .{ + .src = .init(&.{if (tensor.shape().hasTag(.expert) != null) zml.Shape.toTag(.expert) else zml.Shape.toTag(._0)}), + .tgt = .init(&.{axis}), + } }; + } + }.spec, zml.Compiler.current().arena.allocator(), .{ + .experts = linear.weight.dim(.expert), + .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, + }, linear, specs) catch @panic("OOM"); + } + return zml.ops.manualComputation2(Call.forward, args, output_shape, mesh, .{ + .in_specs = .{ + .input = partitioning.in_specs.input, + .ids = partitioning.in_specs.ids, + .weights = partitioning.in_specs.weights, + .gate_up = projection_specs[0], + .down = projection_specs[1], + }, + .out_specs = partitioning.out_specs, + }); } pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index 09dc360a..c583b3e4 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -12,22 +12,9 @@ const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); var gate_up_transpose: std.atomic.Value(bool) = .init(false); var down_transpose: std.atomic.Value(bool) = .init(false); -pub const Options = struct { - activation: zml.moe.Activation, - global_num_experts: i64 = -1, - expert_map: ?Tensor = null, - w1_scale: ?Tensor = null, - w2_scale: ?Tensor = null, - w1_bias: ?Tensor = null, - w2_bias: ?Tensor = null, -}; - fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - // TODO(Corentin): find where to put that - //if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; - //if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; } fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { @@ -224,120 +211,25 @@ pub fn callGmmEp( } pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + hidden_states: zml.Tensor, + routing: zml.moe.LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, opts: zml.moe.Options, ) zml.Tensor { validateOptions(opts); + const expert_map_ = routing.expertMap(); - const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; - const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; - - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (expert_partition.isSharded()) { - const global_num_experts = down.weight.dim(.expert); - const partial_output = zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - weights_gate_up: zml.Tensor, - weights_down: zml.Tensor, - activation: zml.moe.Activation, - global_num_experts: i64, - gate_up_scales: ?zml.Tensor, - bias_gate_up: ?zml.Tensor, - down_scales: ?zml.Tensor, - bias_down: ?zml.Tensor, - - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const local_num_experts = self.weights_gate_up.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - const expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_output = fusedExpertsImpl( - self.input, - self.weights_gate_up, - self.weights_down, - self.topk_weights, - self.topk_ids, - .{ - .activation = self.activation, - .global_num_experts = self.global_num_experts, - .expert_map = expert_map, - .w1_scale = self.gate_up_scales, - .w2_scale = self.down_scales, - .w1_bias = self.bias_gate_up, - .w2_bias = self.bias_down, - }, - ); - return local_output.reshape(self.input.shape()).withTags(.{ .t, .d }); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .weights_gate_up = gate_up.weight, - .weights_down = down.weight, - .activation = opts.activation, - .global_num_experts = global_num_experts, - .gate_up_scales = gate_up_scales, - .bias_gate_up = gate_up.bias, - .down_scales = down_scales, - .bias_down = down.bias, - }, - input.shape(), - ); - return zml.ops.allReduce(partial_output, zml.Tensor.add); - } - - return fusedExpertsImpl( - input, - gate_up.weight, - down.weight, - topk_weights, - topk_ids, - .{ - .activation = opts.activation, - .global_num_experts = gate_up.weight.dim(.expert), - .w1_scale = gate_up_scales, - .w2_scale = down_scales, - .w1_bias = gate_up.bias, - .w2_bias = down.bias, - }, - ); -} - -pub fn fusedExpertsImpl( - hidden_states: Tensor, - w1: Tensor, - w2: Tensor, - topk_weights: Tensor, - topk_ids: Tensor, - opts: Options, -) Tensor { const tokens = hidden_states.dim(.t); const hidden = hidden_states.reshape(.{ .token = tokens, .in = hidden_states.dim(.d) }); // TODO(Corentin): Better error message - const gate_up = canonicalizeGateUp(w1, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExpertsImpl: {}", .{e}); - const down = canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExpertsImpl: {}", .{e}); - const weights = topk_weights.reshape(.{ .token = tokens, .topk = topk_weights.dim(.topk) }); - const ids = topk_ids.reshape(.{ .token = tokens, .topk = topk_ids.dim(.topk) }); + const gate_up = canonicalizeGateUp(gate_up_.weight, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExperts: {}", .{e}); + const down = canonicalizeDown(down_.weight, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExperts: {}", .{e}); + const weights = routing.weights.reshape(.{ .token = tokens, .topk = routing.weights.dim(.topk) }); + const ids = routing.ids.reshape(.{ .token = tokens, .topk = routing.ids.dim(.topk) }); - const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); - if (opts.expert_map) |expert_map| { + const num_experts = routing.global_num_experts; + if (expert_map_) |expert_map| { if (expert_map.dtype() != .i32) stdx.debug.panic("expert_map must be of dtype i32, got {}", .{expert_map.dtype()}); if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) stdx.debug.panic("expert_map must be a 1D tensor of length num_experts, got shape {}", .{expert_map.shape()}); } @@ -345,7 +237,7 @@ pub fn fusedExpertsImpl( const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); const flat_weights = weights.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); - const flat_ids_local, const local_route_mask = if (opts.expert_map) |expert_map| blk: { + const flat_ids_local, const local_route_mask = if (expert_map_) |expert_map| blk: { const mapped = expert_map.gather(.{ .expert = flat_ids_global }, .{}).withTags(.{.route}); const route_mask = mapped.cmp(.GE, Tensor.scalar(0, .i32).broad(mapped.shape())); const safe_ids = route_mask.select(mapped, Tensor.zeroes(mapped.shape())); @@ -356,7 +248,7 @@ pub fn fusedExpertsImpl( }; const sorted_route = flat_ids_local.argsort(.route, .{ .descending = false }).rename(.{ .route = .perm }); - const hidden_sorted = if (opts.expert_map != null) + const hidden_sorted = if (expert_map_ != null) permuteSmallBatch(hidden, sorted_route, ids.dim(.topk)) else blk: { // If not sharded, we can save the sort + gather by just repeating the hidden states for each expert and gathering with the sorted route @@ -379,7 +271,7 @@ pub fn fusedExpertsImpl( .gather(.{ .token = aligned_hidden.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (opts.w1_bias) |bias| { + if (gate_up_.bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); gate_up_out = gate_up_out.add(bias_per_token); } @@ -391,7 +283,7 @@ pub fn fusedExpertsImpl( .gather(.{ .token = aligned_activated.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (opts.w2_bias) |bias| { + if (down_.bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); down_out = down_out.add(bias_per_token); } diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index e33d2a14..25c69c5c 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -157,14 +157,19 @@ pub const Options = struct { routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, }; +/// Negative expert IDs denote nonlocal routes and contribute zero, including bias. pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { + stdx.debug.assert(topk_ids.dim(.topk) > 0, "reference top-k must be positive", .{}); stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); const tokens = input.dim(.t); const k = topk_ids.dim(.topk); - stdx.debug.assert(k > 0 and k <= gate_up.weight.dim(.expert), "reference top-k must fit the expert count", .{}); - const ids = topk_ids.reshape(.{ .route = tokens * k }); + // Nonlocal routes use expert zero for safe gathers; discard their full + // contribution after the down projection, including any projection bias. + const local_routes = topk_ids.cmp(.GE, Tensor.scalar(0, .i32)); + const safe_ids = topk_ids.maximum(Tensor.scalar(0, .i32)); + const ids = safe_ids.reshape(.{ .route = tokens * k }); const weights = topk_weights.reshape(.{ .route = tokens * k }).convert(.f32); const hidden = input.reshape(.{ .token = tokens, .in = input.dim(.d) }) .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = tokens, .topk = k, .in = input.dim(.d) }, input.dtype())) @@ -176,13 +181,15 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); + output = local_routes.reshape(.{ .route = tokens * k }).broad(output.shape()).select(output, Tensor.scalar(0, output.dtype())); return output.convert(input.dtype()).convert(.f32) .reshape(.{ .t = tokens, .topk = k, .d = output.dim(.out) }) .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, ids: Tensor, weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { - return reference(input, ids, weights, gate_up, down, .{ +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { + stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); + return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, .quantize_input = opts.quantize_input, .routing_weight_placement = opts.routing_weight_placement, diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 4b7bcfb3..991280fd 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -19,10 +19,12 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` +/// `input` is `{.t, .d}`, routing IDs and weights `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). /// SwiGLU, routing weights applied after the down projection. -pub fn fusedExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { + const topk_ids = routing.ids; + const topk_weights = routing.weights; stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); stdx.debug.assert(gate_up_linear.quantization == null and down_linear.quantization == null, "TCL MoE takes bf16 experts", .{}); @@ -277,7 +279,7 @@ test "tcl fused experts run on furiosa" { const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { - return fusedExperts(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .{ + return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ .activation = .{ .swiglu = .{} }, .quantize_input = false, .routing_weight_placement = .after_down, diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index 2bdebdfe..aa0be86f 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,8 +5,8 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const reference = @import("stablehlo/stablehlo.zig"); const Placement = @import("moe.zig").RoutingWeightPlacement; +const reference = @import("stablehlo/stablehlo.zig"); const Case = struct { backend: zml.moe.Backend, @@ -22,15 +22,22 @@ const Case = struct { width: i64 = 128, intermediate: i64 = 128, experts: i64 = 8, + expert_mesh: *const zml.Mesh, + + fn inputShape(c: Case) Shape { + const shape = Shape.init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + const spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + return shape.withPartitioning(c.expert_mesh, spec); + } pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { std.debug.assert(c.backend.isAvailable(platform)); - const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); - return platform.compileFn(allocator, io, forward, .{ x, c }, .{}); + const x: Tensor = .fromShape(c.inputShape()); + return platform.compileFn(allocator, io, forward, .{ x, c }, .{ .meshes = &.{c.expert_mesh} }); } pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { - const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + const x: Tensor = .fromShape(c.inputShape()); var host = try zml.Slice.alloc(allocator, x.shape()); defer host.free(allocator); @@ -52,6 +59,7 @@ const Case = struct { var output: zml.Bufferized(Outputs) = undefined; runner.run(io, .{input}, .{&output}, .{ .wait = true }); defer zml.Buffer.deinitAll(Outputs, &output); + try std.testing.expectEqualDeep(c.inputShape()._sharding, output.actual.shape()._sharding); // BF16 intermediate rounding and different GEMM reduction orders. Every // element must pass; near-zero values use the absolute bound. @@ -269,8 +277,20 @@ fn forward(input: Tensor, c: Case) Outputs { .routing_weight_placement = c.placement, }; - const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, options); - const expected = reference.reference(input, ids, weights, canonical.gate_up, canonical.down, .{ + const data_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + const expert_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("experts") != null) .parse(.{ .expert = .experts }) else .replicated; + const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, c.expert_mesh, .{ + .in_specs = .{ + .input = data_spec, + .ids = data_spec, + .weights = data_spec, + .gate_up = expert_spec, + .down = expert_spec, + }, + .out_specs = data_spec, + }, options); + // Keep the oracle input replicated so token sharding does not change its GEMM rounding. + const expected = reference.reference(input.replicate(), ids, weights, canonical.gate_up, canonical.down, .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, @@ -346,7 +366,7 @@ const Matrix = struct { } /// Shapes vary fastest, matching the nested loops in the backend tests. - pub fn caseAt(self: Matrix, backend: zml.moe.Backend, index: usize) Case { + pub fn caseAt(self: Matrix, backend: zml.moe.Backend, expert_mesh: *const zml.Mesh, index: usize) Case { std.debug.assert(index < self.count()); var remaining = index; const shape = self.shapes[remaining % self.shapes.len]; @@ -374,6 +394,7 @@ const Matrix = struct { .width = shape.width, .intermediate = shape.intermediate, .experts = shape.experts, + .expert_mesh = expert_mesh, }; } }; @@ -414,7 +435,7 @@ const Pipeline = struct { var index: usize = 0; for (0..matrix.count()) |local_index| { - const c = matrix.caseAt(backend, local_index); + const c = matrix.caseAt(backend, self.platform.replicated_mesh, local_index); if (available) { case_queue.putOne(io, .{ .index = index, .case = c }) catch |err| return switch (err) { error.Closed => {}, @@ -591,6 +612,7 @@ test "Triton MoE prefill launch buckets fit shared memory" { for ([_]i64{ 8, 32, 96, 128, 256, 512, 1024, 2048, 3072, 4096 }) |tokens| { const case: Case = .{ .backend = .triton, + .expert_mesh = platform.replicated_mesh, .tokens = tokens, .width = 256, .intermediate = 128, diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 9d3a5529..348f5a90 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -35,8 +35,7 @@ test { pub fn fusedExperts( input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: zml.moe.Options, @@ -45,99 +44,29 @@ pub fn fusedExperts( if (gate_up.quantization) |q| switch (q.scheme) { .mxfp4 => if (zml.Compiler.current().platform.target == .cuda) { - return fusedExpertsMxfp4(input, ids, weights, gate_up, down, options); + return fusedExpertsMxfp4(input, routing, gate_up, down, options); }, else => {}, }; - return fusedExpertsBase(input, ids, weights, gate_up, down, options); + const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .interleaved else .concatenated; + return fusedExpertsBase(input, routing, gate_up, down, layout, options); } pub fn fusedExpertsBase( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, -) zml.Tensor { - const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) - .{ .gate_up = .interleaved, .packing = .plain } - else - .{ .gate_up = .concatenated, .packing = .plain }; - - const args: FusedExpertsArgs = .{ - .hidden_states = input, - .gate_up = gate_up, - .down = down, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .activation = opts.activation, - .quantize_input = opts.quantize_input, - .gate_up_layout = layout.gate_up, - .routing_weight_placement = opts.routing_weight_placement, - }; - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (!expert_partition.isSharded()) { - return fusedExpertsImpl(args); - } - - return zml.ops.manualComputation( - (struct { - args: FusedExpertsArgs, - global_num_experts: i64, - - fn call(self: @This(), _: zml.Shape) zml.Tensor { - const local_args = self.args; - const local_num_experts = local_args.gate_up.weight.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - // Map global expert ids to local ids, or -1 for experts outside this partition. - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - var mapped_args = local_args; - mapped_args.expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - - const local_output = fusedExpertsImpl(mapped_args); - const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).call, - .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, - input.shape(), - ); -} - -pub const FusedExpertsArgs = struct { hidden_states: Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - topk_weights: Tensor, - topk_ids: Tensor, - activation: zml.moe.Activation, - expert_map: ?Tensor = null, - /// Use FP8 activations for FP8 weights; false keeps BF16 activations. - quantize_input: bool, - gate_up_layout: ProjectionLayout, - routing_weight_placement: RoutingWeightPlacement, -}; - -pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { - const hidden_states = opts.hidden_states; - const topk_weights = opts.topk_weights; - const topk_ids = opts.topk_ids; + routing_: zml.moe.LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + layout: ProjectionLayout, + opts: zml.moe.Options, +) Tensor { + const expert_map_ = routing_.expertMap(); const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = opts.gate_up.quantizationScheme(); - const down_scheme = opts.down.quantizationScheme(); + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("triton backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -154,10 +83,10 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); - const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "triton backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -185,13 +114,13 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "triton backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "triton backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "triton backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (opts.expert_map) |expert_map| + if (expert_map_) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "triton backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (opts.expert_map) |expert_map| + const expert_ids = if (expert_map_) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -201,9 +130,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = opts.gate_up.bias, + .bias = gate_up_.bias, .input_scale = input_scale, - .weight_scale = opts.gate_up.quantizationScales(), + .weight_scale = gate_up_.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -212,7 +141,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -222,9 +151,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = opts.down.bias, + .bias = down_.bias, .input_scale = down_input_scale, - .weight_scale = opts.down.quantizationScales(), + .weight_scale = down_.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -765,13 +694,12 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { } const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .t = 1, .topk = 2 }) .broad(Shape.init(.{ .t = x.dim(.t), .topk = 2 }, .i32)); - return fusedExpertsImpl(.{ - .hidden_states = x, - .gate_up = gate_up, - .down = down, - .topk_ids = route.addConstant(2), - .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), - .gate_up_layout = layout, + return fusedExpertsBase(x, .{ + .ids = route.addConstant(2), + .weights = route.convert(.f32).addConstant(1).scale(0.25), + .global_num_experts = 8, + .local_num_experts = 8, + }, gate_up, down, layout, .{ .routing_weight_placement = placement, .quantize_input = false, .activation = .{ .swiglu = .{} }, @@ -873,54 +801,6 @@ pub fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(opts.activation.swiglu.scale == null, "triton backend requires swiglu scale to be null", .{}); } -/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. -/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. -pub fn fusedExpertsMxfp4( - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - options: zml.moe.Options, -) zml.Tensor { - validateOptions(options); - stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); - stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); - - const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); - const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); - - // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - - const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); - - // Note: we directly forward input sharding to manualComputation - // TODO: improve sharding in attention and moe APIs - // https://github.com/zml/zml/issues/914 - const context: Context = .{ - .input = input, - .ids = ids, - .weights = weights, - .gate_up = gate_up.weight, - .gate_up_scales = gq.scales, - .down = down.weight, - .down_scales = dq.scales, - .global_experts = gate_up.weight.dim(.expert), - .topk = ids.dim(.topk), - .limit = options.activation.swiglu.limit.?, - .routing_weight_placement = options.routing_weight_placement, - .expert_parallel = expert_parallelism, - }; - - return if (expert_parallelism) - zml.ops.manualComputation(Context.body, context, input.shape()) - else - context.body(input.shape()); -} - pub const Config = struct { tokens: i64, hidden: i64, @@ -1012,178 +892,176 @@ fn validateInputs( ); } -const Context = struct { +/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. +/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. +pub fn fusedExpertsMxfp4( input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - gate_up: zml.Tensor, - gate_up_scales: zml.Tensor, - down: zml.Tensor, - down_scales: zml.Tensor, - global_experts: i64, - topk: i64, - limit: f32, - routing_weight_placement: RoutingWeightPlacement, - expert_parallel: bool, + routing: zml.moe.LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: zml.moe.Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); - fn body(self: Context, _: zml.Shape) zml.Tensor { - const experts = self.gate_up.dim(.expert); - const hidden = self.down.dim(1); - const intermediate = self.down.dim(2) * 2; - var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); - const tokens = ids.dim(.token); - if (self.expert_parallel) { - const partition_id = zml.ops.partitionId().convert(.i32); - ids = ids.sub(partition_id.scale(experts)); - } - const x = self.input.reshape(.{ tokens, hidden }); - const scales = self.weights.convert(.f32).reshape(.{ tokens, self.topk }); + const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); - const cfg: Config = .{ - .tokens = tokens, - .hidden = hidden, - .intermediate = intermediate, - .experts = experts, - .global_experts = self.global_experts, - .topk = self.topk, - .swiglu_limit = self.limit, - .routing_weight_placement = self.routing_weight_placement, - }; + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); - validateInputs(cfg, x, ids, scales, self.gate_up, self.gate_up_scales, self.down, self.down_scales); + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); - const bn = 16; - const bk = 512; + const x = input.reshape(.{ tokens, hidden }); + const scales = routing.weights.convert(.f32).reshape(.{ tokens, topk }); - const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); + const cfg: Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = routing.global_num_experts, + .topk = topk, + .swiglu_limit = options.activation.swiglu.limit.?, + .routing_weight_placement = options.routing_weight_placement, + }; - // Quantize activations - const q = kernels.Quant.Kernel.call( - .{ .x = x }, + cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); + validateInputs(cfg, x, ids, scales, gate_up.weight, gq.scales, down.weight, dq.scales); + + const bn = 16; + const bk = 512; + + const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); + + // Quantize activations + const q = kernels.Quant.Kernel.call( + .{ .x = x }, + .{ + .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), + }, + .{ + .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, + .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + const dummy = zml.Tensor.scalar(0, .i32); + var map = dummy; + var perm = dummy; + var sched = dummy; + + if (cfg.tokens != 1) { + const r = kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, + .{ .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + + const s = kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, .{ - .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), - .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), - }, - .{ - .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, - .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, + .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .capacity = tile_count, + .group = bn, + }, + .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, .num_warps = 4, .num_stages = 1, }, ); - const dummy = zml.Tensor.scalar(0, .i32); - var map = dummy; - var perm = dummy; - var sched = dummy; - - if (cfg.tokens != 1) { - const r = kernels.Route.Kernel.call( - .{ .ids = ids }, - .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, - .{ .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, - ); - - const s = kernels.Schedule.Kernel.call( - .{ .ids = ids, .counts = r.counts, .pos = r.pos }, - .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .capacity = tile_count, - .group = bn, - }, - .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - map = s.map; - perm = s.perm; - sched = s.sched; - } - - // Each narrow tile produces 64 activations: two complete scale groups. - const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; - const m = kernels.GateUp.Kernel.call( - .{ .q = q.q, .s = q.s, .w = self.gate_up.bitCast(.u8), .ws = self.gate_up_scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, - .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.gate_up_scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), - }, - ); - - const d = kernels.Down.Kernel.call( - .{ .mid = m.mid, .ms = m.ms, .w = self.down.bitCast(.u8), .ws = self.down_scales.bitCast(.u8), .ids = ids, .sched = sched }, - .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.down_scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), - }, - ); - - const result = kernels.Combine.Kernel.call( - .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, - .{ .y = x.shape() }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - }, - .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ).y.reshape(self.input.shape().dims()).withTags(self.input.shape()); - - return if (self.expert_parallel) - zml.ops.allReduce(result, zml.Tensor.add) - else - result; + map = s.map; + perm = s.perm; + sched = s.sched; } -}; + + // Each narrow tile produces 64 activations: two complete scale groups. + const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; + const m = kernels.GateUp.Kernel.call( + .{ .q = q.q, .s = q.s, .w = gate_up.weight.bitCast(.u8), .ws = gq.scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = gq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), + }, + ); + + const d = kernels.Down.Kernel.call( + .{ .mid = m.mid, .ms = m.ms, .w = down.weight.bitCast(.u8), .ws = dq.scales.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = dq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), + }, + ); + + return kernels.Combine.Kernel.call( + .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, + .{ .y = x.shape() }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + }, + .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ).y.reshape(input.shape().dims()).withTags(input.shape()); +} diff --git a/zml/ops.zig b/zml/ops.zig index 19d15ec3..4a189035 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -8,18 +8,6 @@ const platforms = @import("platforms"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const manual_computation = @import("ops/manual_computation.zig"); -pub const manualComputation = manual_computation.manualComputation; - -const manual_computation2 = @import("ops/manual_computation2.zig"); -pub const manualComputation2 = manual_computation2.manualComputation2; -pub const ManualPartitioning = manual_computation2.ManualPartitioning; - -test { - _ = manual_computation; - _ = manual_computation2; -} - const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("mem.zig").Bufferized; const Compiler = @import("Compiler.zig"); @@ -28,15 +16,113 @@ const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); +const manual_computation = @import("ops/manual_computation.zig"); +pub const manualComputation = manual_computation.manualComputation; +const manual_computation2 = @import("ops/manual_computation2.zig"); +pub const manualComputation2 = manual_computation2.manualComputation2; +pub const ManualPartitioning = manual_computation2.ManualPartitioning; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; + +test { + _ = manual_computation; + _ = manual_computation2; +} + const log = std.log.scoped(.@"zml/Compiler"); +/// Reduces local tensor values across all processes in the computation. +/// For each tensor index, `func_` combines the values at that index from every +/// process. The result is identical on all processes. +/// +/// Inside a manual computation, `inputs` describes the local tensors at each +/// partition. For example, summing `[1, 2]` and `[3, 4]` from two partitions gives +/// `[4, 6]` at each partition. The local tensor dimensions stay unchanged. +/// +/// * `inputs` is a `Tensor`, a nonempty tuple of tensors, or an `[N]Tensor`. +/// Each input must have matching shapes and dtypes across processes. +/// * `func_` describes how to combine two scalar contributions: +/// - For one input, it must be `fn (left: Tensor, right: Tensor) Tensor`, such +/// as `Tensor.add`. This also applies to a one-element tuple or array. +/// - For multiple inputs, it takes one `ReduceArgs` per input and returns a +/// tuple or array with one scalar tensor per input, in the same order. +/// +/// Returns the same container type as `inputs`, preserving tensor attributes. +/// With one process, returns `inputs` unchanged. +/// +/// The reduction uses one group containing all processes, including mesh replicas. +/// To reduce along one logical mesh axis, use `allReduceOnAxis`. pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const count = Compiler.current().meshes[0].numDevices(); + var devices: [Platform.MAX_NUM_DEVICES]i64 = undefined; + for (devices[0..count], 0..) |*device, i| device.* = @intCast(i); + return allReduceGroups(inputs, func_, devices[0..count], count); +} + +/// Reduces local tensor values within process groups defined by `axis` in `mesh`. +/// A group contains processes with the same coordinates on every other logical +/// mesh axis. Separate replicas of the mesh belong to separate groups. Within +/// each group, `func_` combines corresponding elements and the result is identical +/// on all member processes. +/// +/// Use this inside a manual computation to combine partial results along `axis` +/// while preserving the independent computations along the other mesh axes. +/// +/// * `inputs` and `func_` follow `allReduce`'s tensor and reducer contracts. +/// * `mesh` is the logical mesh used by the manual computation and must be +/// included in the compilation's meshes. +/// * `axis` names an axis of that mesh, for example `.experts`. It can be an enum +/// literal or a mesh axis tag. The axis must exist in `mesh`. +/// +/// Returns the same container type with tensor attributes as +/// `inputs`. An axis of size one returns `inputs` unchanged. +/// +/// For a mesh with `.data = 2` and `.experts = 4`, the following sums four local +/// expert contributions independently for each of the two data shards: +/// +/// ```zig +/// const output = allReduceOnAxis(local_output, Tensor.add, mesh, .experts); +/// ``` +/// +/// Each expert shard now has the complete output for its data shard. To expose +/// this result through `manualComputation2`, an output spec such as +/// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the +/// result replicated along `.experts`. +pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical reduction axis"); + const size = mesh.grid.axis_sizes[logical_axis]; + var groups: [Platform.MAX_NUM_DEVICES]i64 = undefined; + var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + const replicas = mesh.numReplicas(); + for (0..mesh.numDevices()) |device| { + const coords = mesh.grid.logical_coords[device]; + var tile: usize = 0; + var group: usize = 0; + for (0..mesh.logical.axes.len) |dim| { + tile = tile * mesh.grid.axis_sizes[dim] + coords[dim]; + if (dim != logical_axis) group = group * mesh.grid.axis_sizes[dim] + coords[dim]; + } + group = group * replicas + replica_indices[tile]; + replica_indices[tile] += 1; + groups[group * size + coords[logical_axis]] = @intCast(device); + } + return allReduceGroups(inputs, func_, groups[0..mesh.numDevices()], size); +} + +/// Reduces corresponding tensor elements independently within each process group. +/// `inputs`, `func_` and the return value follow `allReduce`'s contract. +/// +/// `groups` identifies processes by their executable partition IDs, as returned +/// by `partitionId()`. Each consecutive `group_size` IDs defines one group. +/// +/// For example, `groups = &.{ 0, 2, 1, 3 }` and `group_size = 2` combine partitions +/// 0 and 2 together, and partitions 1 and 3 together. Both members of each group +/// receive that group's result. Groups of size one return `inputs` unchanged. +fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64, group_size: usize) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); const mlir_ctx = ctx.mlir_ctx; @@ -80,8 +166,7 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), }; - const num_devices = ctx.meshes[0].numDevices(); - if (num_devices <= 1) return inputs; + if (group_size <= 1) return inputs; const reducer_block = b: { const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); @@ -130,14 +215,9 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ break :b block; }; - var replica_group_storage: [constants.MAX_RANK]i64 = undefined; - for (0..@intCast(num_devices)) |i| { - replica_group_storage[i] = @intCast(i); - } - const replica_group_slice = replica_group_storage[0..@intCast(num_devices)]; const replica_groups_attr = mlir.Attribute.denseElements( - .rankedTensor(&.{ 1, num_devices }, .int(mlir_ctx, .i64)), - replica_group_slice, + .rankedTensor(&.{ @intCast(groups.len / group_size), @intCast(group_size) }, .int(mlir_ctx, .i64)), + groups, ); const handle = ctx.nextChannelId(); @@ -207,6 +287,20 @@ fn AllReduceReturnType(comptime InputsT: type) type { }; } +/// Returns the current process's zero based coordinate on `axis` in `mesh` +/// as a scalar u32. +/// +/// Example: for a mesh {.data = 2, .experts = 2}, each data group has expert coordinates +/// 0 and 1. Use this coordinate inside a manual computation to identify a shard +/// along one axis while `partitionId()` identifies a partition across the whole mesh. +pub fn axisIndex(mesh: *const Sharding.Mesh, axis: anytype) Tensor { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical mesh axis"); + var coords: [Platform.MAX_NUM_DEVICES]u32 = undefined; + for (coords[0..mesh.numDevices()], 0..) |*coord, i| coord.* = mesh.grid.logical_coords[i][logical_axis]; + return Tensor.constantTensor(.init(.{ .device = mesh.numDevices() }, .u32), std.mem.sliceAsBytes(coords[0..mesh.numDevices()])) + .gather(.{ .device = partitionId() }, .{}); +} + pub fn partitionId() Tensor { const ctx = Compiler.current(); const op = stablehlo.partition_id(ctx.mlir_ctx, ctx.location).appendTo(ctx.currentScope().block); From 7fbd15eee48609f095a11005493d5beef059c443 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Nov 2026 19:24:37 +0100 Subject: [PATCH 25/47] [Modularization] zml: break cycle in `zml/attention` (#1038) Break cycle between `zml/attention/attention.zig` and `zml/kernels/flash_attention/flash_attention.zig` --- zml/BUILD.bazel | 2 + zml/attention/Options.zig | 3 + zml/attention/attention.zig | 6 +- zml/kernels/flash_attention/README.md | 2 +- .../flash_attention/flash_attention.zig | 330 +----------------- zml/kernels/flash_attention/tests.zig | 330 ++++++++++++++++++ 6 files changed, 340 insertions(+), 333 deletions(-) create mode 100644 zml/attention/Options.zig create mode 100644 zml/kernels/flash_attention/tests.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index dbb2ad33..4d58fc77 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -78,6 +78,7 @@ zig_library( srcs = [ "Compiler.zig", "Sharding.zig", + "attention/Options.zig", "attention/attention.zig", "buffer.zig", "buffer/HostAccessible.zig", @@ -88,6 +89,7 @@ zig_library( "io.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", + "kernels/flash_attention/tests.zig", "kernels/moe/moe.zig", "kernels/moe/tests.zig", "kernels/paged_attention/paged_attention.zig", diff --git a/zml/attention/Options.zig b/zml/attention/Options.zig new file mode 100644 index 00000000..539f753f --- /dev/null +++ b/zml/attention/Options.zig @@ -0,0 +1,3 @@ +/// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every +/// key, as in an encoder or a diffusion transformer, and token_index is ignored. +causal: bool = true, diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 039c6628..0ac2a95c 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -93,11 +93,7 @@ pub const Metadata = union(BackendType) { } }; -pub const Options = struct { - /// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every - /// key, as in an encoder or a diffusion transformer, and token_index is ignored. - causal: bool = true, -}; +pub const Options = @import("Options.zig"); pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md index b5fe7682..81a35cf4 100644 --- a/zml/kernels/flash_attention/README.md +++ b/zml/kernels/flash_attention/README.md @@ -265,7 +265,7 @@ are not uniformly supported by the current kernels; an existing prefill path requires equal query/key lengths and assumes positions begin at zero. Treat these as current conformance gaps, not alternative definitions of causality. -The tests in [flash_attention.zig](flash_attention.zig) compare results +The tests in [tests.zig](tests.zig) compare results against `sdpa` with the gathered causal mask. The original shared cases cover `D = 64`, MHA and GQA, single-token decode at full and partial context lengths, batched decode with diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index 78602bd0..3fb898db 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -6,9 +6,11 @@ pub const cute = @import("cute/cute.zig"); pub const metal = @import("metal/metal.zig"); pub const nki = @import("nki/nki.zig"); const zml = @import("../../zml.zig"); +const Options = @import("../../attention/Options.zig"); test { std.testing.refAllDecls(@This()); + _ = @import("tests.zig"); } pub fn isAvailable(platform: *const zml.Platform) bool { @@ -126,7 +128,7 @@ pub const Metadata = union(Backend) { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: zml.attention.Options) zml.Tensor { +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); return switch (parameters) { .cute => cute.attention(q, k, v, token_index, opts.causal), @@ -135,329 +137,3 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), }; } - -test "attention: q=1,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); - - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{36}, - ); -} - -test "attention: b=4,q=1,qh=64,kh=8" { - // Full attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 63, 63, 63, 63 }, - ); - - // Partial attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 61, 57, 23, 63 }, - ); -} - -test "attention: q=1,qh=8,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{62}, - ); - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); -} - -test "attention: q=8,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{56}, - ); -} - -test "attention: prefill, q=k=300,qh=8,kh=4" { - // Neither length is a multiple of the query (128) or key (64) tiles. - try testAttention( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries start at 800: every query tile sees full key tiles before the diagonal. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{800}, - ); -} - -test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { - // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), - .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { - try testAttention( - .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), - .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { - try testAttention( - .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), - .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), - &.{ 150, 100 }, - ); -} - -test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. - try testAttentionOpts( - .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{ 0, 0 }, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), - .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { - try testAttentionOpts( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), - &.{5}, - .{ .causal = false }, - ); -} - -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); -} - -fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { - return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(Backend)); -} - -fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const Backend) !void { - const io = std.testing.io; - const allocator = std.testing.allocator; - - const platform = zml.testing.env(); - - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { - std.debug.assert(token_index_h.len == q_shape.dim(.b)); - break :b .init(.{ .b = q_shape.dim(.b) }, .u32); - } else .init(.{}, .u32); - - const max_k_idx: i64 = k_shape.dim(.k) - 1; - const max_q_idx: i64 = q_shape.dim(.q) - 1; - if (opts.causal) { - // Check for out of bound reads, token_index is ignored by non-causal attention. - for (token_index_h) |index| { - std.debug.assert(index + max_q_idx <= max_k_idx); - } - } - - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ - .q = .fromShape(q_shape), - .k = .fromShape(k_shape), - .v = .fromShape(k_shape), - .token_index = .fromShape(token_index_shape), - }; - - var q = try attentionTestInput(q_shape, 17); - defer q.deinit(); - var k = try attentionTestInput(k_shape, 29); - defer k.deinit(); - var v = try attentionTestInput(k_shape, 43); - defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); - defer token_index.deinit(); - - const meshes = platform.meshes.values(); - const mesh = zml.testing.tpMesh(); - const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ - .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, - .out_specs = .replicated, - }; - const AttentionCall = struct { - fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { - return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); - } - }; - const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; - const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ - .program_name = "attention_stablehlo", - .meshes = meshes, - }); - defer reference_exe.deinit(); - - var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); - defer reference_d.deinit(); - try reference_d.await(io); - try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); - const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); - defer reference_h.free(allocator); - - for (backends) |backend| { - if (!backend.isAvailable(platform)) continue; - - const metadata: Metadata = .init(.fromBackend(backend)); - const parameters: Parameters = .init(.fromBackend(backend)); - const exe = try platform.compileFn( - allocator, - io, - AttentionCall.forward, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, - .{ - .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .meshes = meshes, - }, - ); - defer exe.deinit(); - - var metadata_d = try metadata.initBuffer(io, platform); - defer Metadata.deinitBuffer(&metadata_d); - - var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); - defer output_d.deinit(); - try output_d.await(io); - const output_h = try output_d.toSliceAlloc(allocator, io); - defer output_h.free(allocator); - - errdefer std.log.err( - \\ Attention test failed, {0t} output doesn't match reference. - \\ - reference: {1d} - \\ - {0t}: {2d} - , .{ backend, reference_h, output_h }); - try zml.testing.expectClose(io, reference_h, output_h, .{ - .absolute_tolerance = 5e-3, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-3, - .minimum_close_fraction = 0.99, - }); - } -} - -fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { - const host = try zml.Slice.alloc(std.testing.allocator, shape); - defer host.free(std.testing.allocator); - var prng: std.Random.DefaultPrng = .init(seed); - const random = prng.random(); - switch (shape.dtype()) { - inline .bf16, .f16, .f32 => |dtype| { - for (host.items(dtype.toZigType())) |*value| { - const x = random.float(f32) * 4 - 2; - value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); - } - }, - else => unreachable, - } - return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); -} - -/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its -/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. -fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { - const Layout = struct { - b: usize, - s: usize, - h: usize, - d: usize, - - fn stride(shape: zml.Shape, tag: anytype) usize { - const axis = shape.hasTag(tag) orelse return 0; - var result: usize = 1; - for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); - return result; - } - - fn init(shape: zml.Shape) @This() { - return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; - } - - fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { - return b * self.b + s * self.s + h * self.h + d * self.d; - } - }; - const ql = Layout.init(q.shape); - const kl = Layout.init(k.shape); - const vl = Layout.init(v.shape); - const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; - const queries: usize = @intCast(q.shape.dim(.q)); - const heads: usize = @intCast(q.shape.dim(.h)); - const dim: usize = @intCast(q.shape.dim(.hd)); - const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); - const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); - defer std.testing.allocator.free(scores); - const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); - for (0..batch) |b| { - const start: usize = positions[if (positions.len == 1) 0 else b]; - for (0..queries) |i| { - for (0..heads) |h| { - var maximum: f64 = -std.math.inf(f64); - const active = scores[0 .. start + i + 1]; - for (active, 0..) |*score, j| { - var dot: f64 = 0; - for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); - score.* = dot * scale; - maximum = @max(maximum, score.*); - } - var sum: f64 = 0; - for (active) |*score| { - score.* = @exp(score.* - maximum); - sum += score.*; - } - for (0..dim) |d| { - var expected: f64 = 0; - for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); - expected /= sum; - const actual = output.items(f32)[ql.offset(b, i, h, d)]; - try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); - } - } - } - } -} diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig new file mode 100644 index 00000000..d93bd373 --- /dev/null +++ b/zml/kernels/flash_attention/tests.zig @@ -0,0 +1,330 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const flash_attention = @import("flash_attention.zig"); + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +test "attention: prefill, q=k=300,qh=8,kh=4" { + // Neither length is a multiple of the query (128) or key (64) tiles. + try testAttention( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries start at 800: every query tile sees full key tiles before the diagonal. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{800}, + ); +} + +test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { + // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), + .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { + try testAttention( + .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), + .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { + try testAttention( + .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), + .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), + &.{ 150, 100 }, + ); +} + +test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. + try testAttentionOpts( + .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{ 0, 0 }, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), + .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { + try testAttentionOpts( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), + &.{5}, + .{ .causal = false }, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); +} + +fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(flash_attention.Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const flash_attention.Backend) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + if (opts.causal) { + // Check for out of bound reads, token_index is ignored by non-causal attention. + for (token_index_h) |index| { + std.debug.assert(index + max_q_idx <= max_k_idx); + } + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + var q = try attentionTestInput(q_shape, 17); + defer q.deinit(); + var k = try attentionTestInput(k_shape, 29); + defer k.deinit(); + var v = try attentionTestInput(k_shape, 43); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); + defer token_index.deinit(); + + const meshes = platform.meshes.values(); + const mesh = zml.testing.tpMesh(); + const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ + .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, + .out_specs = .replicated, + }; + const AttentionCall = struct { + fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { + return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); + } + }; + const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; + const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; + const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ + .program_name = "attention_stablehlo", + .meshes = meshes, + }); + defer reference_exe.deinit(); + + var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); + defer reference_d.deinit(); + try reference_d.await(io); + try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); + const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); + defer reference_h.free(allocator); + + for (backends) |backend| { + if (!backend.isAvailable(platform)) continue; + + const metadata: flash_attention.Metadata = .init(.fromBackend(backend)); + const parameters: flash_attention.Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + AttentionCall.forward, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .meshes = meshes, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform); + defer flash_attention.Metadata.deinitBuffer(&metadata_d); + + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, reference_h, output_h }); + try zml.testing.expectClose(io, reference_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} + +fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { + const host = try zml.Slice.alloc(std.testing.allocator, shape); + defer host.free(std.testing.allocator); + var prng: std.Random.DefaultPrng = .init(seed); + const random = prng.random(); + switch (shape.dtype()) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType())) |*value| { + const x = random.float(f32) * 4 - 2; + value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); + } + }, + else => unreachable, + } + return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); +} + +/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its +/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. +fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { + const Layout = struct { + b: usize, + s: usize, + h: usize, + d: usize, + + fn stride(shape: zml.Shape, tag: anytype) usize { + const axis = shape.hasTag(tag) orelse return 0; + var result: usize = 1; + for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); + return result; + } + + fn init(shape: zml.Shape) @This() { + return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; + } + + fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { + return b * self.b + s * self.s + h * self.h + d * self.d; + } + }; + const ql = Layout.init(q.shape); + const kl = Layout.init(k.shape); + const vl = Layout.init(v.shape); + const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; + const queries: usize = @intCast(q.shape.dim(.q)); + const heads: usize = @intCast(q.shape.dim(.h)); + const dim: usize = @intCast(q.shape.dim(.hd)); + const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); + const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); + defer std.testing.allocator.free(scores); + const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); + for (0..batch) |b| { + const start: usize = positions[if (positions.len == 1) 0 else b]; + for (0..queries) |i| { + for (0..heads) |h| { + var maximum: f64 = -std.math.inf(f64); + const active = scores[0 .. start + i + 1]; + for (active, 0..) |*score, j| { + var dot: f64 = 0; + for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); + score.* = dot * scale; + maximum = @max(maximum, score.*); + } + var sum: f64 = 0; + for (active) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..dim) |d| { + var expected: f64 = 0; + for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); + expected /= sum; + const actual = output.items(f32)[ql.offset(b, i, h, d)]; + try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); + } + } + } + } +} From b73de969ebe10521f8643a930566e218329f581c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Nov 2026 19:24:37 +0100 Subject: [PATCH 26/47] [Modularization] zml: break cycle in `zml/kernels/paged_attention` (#1039) --- zml/BUILD.bazel | 1 + zml/kernels/paged_attention/cute/cute.zig | 6 +-- zml/kernels/paged_attention/cute/fixtures.zig | 2 + .../paged_attention/cute/sm10x/sm10x.zig | 3 +- .../paged_attention/cute/sm120/sm120.zig | 3 +- zml/kernels/paged_attention/metal/metal.zig | 3 +- .../paged_attention/mosaic_tpu/mosaic_tpu.zig | 2 +- zml/kernels/paged_attention/options.zig | 36 +++++++++++++ .../paged_attention/paged_attention.zig | 54 ++----------------- .../paged_attention/stablehlo/stablehlo.zig | 20 ++++--- zml/kernels/paged_attention/tcl/tcl.zig | 18 +++---- zml/kernels/paged_attention/triton/triton.zig | 4 +- 12 files changed, 74 insertions(+), 78 deletions(-) create mode 100644 zml/kernels/paged_attention/options.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 4d58fc77..e4ce12eb 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -92,6 +92,7 @@ zig_library( "kernels/flash_attention/tests.zig", "kernels/moe/moe.zig", "kernels/moe/tests.zig", + "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", "mem.zig", diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index ea4f36ab..ef78cc49 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -1,10 +1,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const paged = @import("../paged_attention.zig"); +pub const AttentionOptions = @import("../options.zig").AttentionOptions; +const kernel = @import("attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); -const kernel = @import("attention.zig"); pub const Options = struct { batch_size: usize, @@ -64,7 +64,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index aafb8bf1..c0815d1d 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -1,5 +1,7 @@ const std = @import("std"); + const zml = @import("zml"); + const cases = @import("cases.zig"); pub const Fixture = struct { diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index af068bad..afc4298e 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -2,7 +2,6 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -19,7 +18,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 292e4a9f..6645abf1 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -1,7 +1,6 @@ const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -18,7 +17,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 12; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index e2d68398..e8144e3b 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -3,8 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); - -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const paged_attn = @import("paged_attn.zig"); const log = std.log.scoped(.@"zml/attention/metal"); diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index 42e1861f..84716379 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -5,7 +5,7 @@ const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const ragged_attention = @import("ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig new file mode 100644 index 00000000..35ad4049 --- /dev/null +++ b/zml/kernels/paged_attention/options.zig @@ -0,0 +1,36 @@ +const Tensor = @import("../../tensor.zig").Tensor; + +pub const AttentionMask = union(enum) { + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, + /// Attend to keys at specific positions. + indices: Tensor, + + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?Tensor = null, +}; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 81f7ec3e..92e77888 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); +pub const AttentionOptions = @import("options.zig").AttentionOptions; pub const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); @@ -151,41 +152,6 @@ pub const Parameters = union(Backend) { } }; -pub const AttentionMask = union(enum) { - standard: struct { - /// Restrict attention to keys at or before the query position. - causal: bool = true, - /// Optional left window size, including the current token. - /// Non-causal attention also includes every later valid key. - sliding_window: ?u32 = null, - }, - /// Attend to keys at specific positions. - indices: zml.Tensor, - - /// Attend to every valid key in the sequence. - pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, -}; - pub const KvCache = union(enum) { split: struct { k: zml.Tensor, @@ -232,19 +198,6 @@ pub const KvCache = union(enum) { const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); return .{ page_index, offset }; } - - pub fn getPage( - self: KvCache, - page_index: zml.Tensor, - ) [2]zml.Tensor { - return switch (self) { - .split => |split| .{ - split.k.slice(split.k.axis(.page), .dynSingle(page_index)), - split.v.slice(split.k.axis(.page), .dynSingle(page_index)), - }, - .dense => @panic("TODO"), - }; - } }; pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); @@ -280,7 +233,10 @@ const Attention = struct { .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, - .stablehlo => |params| stablehlo.pagedAttention(params, args.q, args.kv_cache, args.opts), + .stablehlo => |params| switch (args.kv_cache) { + .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, .tcl => |params| switch (args.kv_cache) { .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index 7ee4df03..55ee01d5 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -1,12 +1,12 @@ const zml = @import("../../../zml.zig"); -const KvCache = @import("../paged_attention.zig").KvCache; -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); pub fn pagedAttention( parameters: triton.paged.Parameters, q: zml.Tensor, - kv_cache: KvCache, + k: zml.Tensor, + v: zml.Tensor, opts: AttentionOptions, ) zml.Tensor { switch (opts.mask) { @@ -14,12 +14,12 @@ pub fn pagedAttention( .indices => @panic("Sparse attention mask is not supported on StableHLO"), } - const page_size = kv_cache.split.k.dim(.k_chunk); + const page_size = k.dim(.k_chunk); const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); const final_state = zml.ops.@"while"( AttentionLoop, - .{ .q = padded_q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, + .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, .{ // i32 to match triton conventions .slot_id = .scalar(0, .i32), @@ -38,7 +38,8 @@ pub fn pagedAttention( const AttentionLoop = struct { q: zml.Tensor, - kv_cache: KvCache, + k: zml.Tensor, + v: zml.Tensor, parameters: triton.paged.Parameters, opts: AttentionOptions, @@ -66,7 +67,7 @@ const AttentionLoop = struct { const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); - const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); + const page_size_: u32 = @intCast(self.k.dim(.k_chunk)); return zml.ops.@"if"( struct { @@ -134,7 +135,10 @@ const AttentionLoop = struct { .{ .b, .p }, &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, ); - const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); + + const page_axis = self.k.axis(.page); + const active_k = self.k.slice(page_axis, .dynSingle(active_k_page_id)); + const active_v = self.v.slice(page_axis, .dynSingle(active_k_page_id)); const dtype = self.q.dtype(); const attn_mask: zml.Tensor = attn_mask: { diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index c9993af9..881593aa 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -3,13 +3,13 @@ //! `Repro:` files are in the reproducer repository, platform/furiosa. const std = @import("std"); -const stdx = @import("stdx"); const builder = @import("dsl/tcl/builder"); +const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const paged_attention = @import("../paged_attention.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); /// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to @@ -49,7 +49,7 @@ pub const paged = struct { /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. /// Sinks are one logit per query head. - pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { + pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); @@ -76,14 +76,14 @@ pub const paged = struct { q: Tensor, k_cache: Tensor, v_cache: Tensor, - opts: paged_attention.AttentionOptions, + opts: AttentionOptions, fn body(self: Shard, _: zml.Shape) Tensor { return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); } }; - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to @@ -1664,11 +1664,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); const Mod = struct { - pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged_attention.AttentionOptions) Tensor { + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: AttentionOptions) Tensor { return paged.pagedAttention(p, q, k, v, opts); } }; - const opts: paged_attention.AttentionOptions = .{ + const opts: AttentionOptions = .{ .mask = .{ .standard = .{ .causal = case.opts.is_causal, .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, @@ -1742,11 +1742,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), }; defer zml.Buffer.deinitAll(paged.Parameters, &bp); - var opts_d: zml.Bufferized(paged_attention.AttentionOptions) = .{ + var opts_d: zml.Bufferized(AttentionOptions) = .{ .mask = .{ .standard = {} }, .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, }; - defer zml.Buffer.deinitAll(paged_attention.AttentionOptions, &opts_d); + defer zml.Buffer.deinitAll(AttentionOptions, &opts_d); var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); defer result.deinit(); const host = try result.toSliceAlloc(allocator, io); diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index 67416285..2a12e012 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -4,10 +4,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const triton = zml.kernel.triton; -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const kernels = @import("unified_attention.zig"); -const kernels_sparse = @import("sparse_unified_attention.zig"); const kernels_oneapi = @import("unified_attention_oneapi.zig"); +const kernels_sparse = @import("sparse_unified_attention.zig"); const log = std.log.scoped(.@"zml/attention/triton"); From 2c5795920211d7143ba6af7a1571d93910a4f62a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Nov 2026 19:24:38 +0100 Subject: [PATCH 27/47] [Modularization] zml/kernels/moe: break cycle (#1041) `zml/kernels/moe.zig` is not imported by backend implementation anymore. I introduced `options.zig` to contain the shared options. I also introduced `weights.zig` to expose the unpacking helpers. --- zml/BUILD.bazel | 2 + zml/kernels/moe/cute/cute.zig | 6 +- zml/kernels/moe/cute/sm10x/moe.zig | 3 +- .../cutlass_flashinfer/cutlass_flashinfer.zig | 27 ++-- zml/kernels/moe/fly/fly.zig | 12 +- zml/kernels/moe/metal/metal.zig | 14 ++- zml/kernels/moe/moe.zig | 119 +----------------- zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig | 8 +- zml/kernels/moe/options.zig | 117 +++++++++++++++++ zml/kernels/moe/stablehlo/stablehlo.zig | 12 +- zml/kernels/moe/tcl/tcl.zig | 4 +- zml/kernels/moe/tests.zig | 19 +-- zml/kernels/moe/triton/moe.zig | 5 +- zml/kernels/moe/triton/triton.zig | 25 ++-- zml/kernels/moe/weights.zig | 10 ++ 15 files changed, 212 insertions(+), 171 deletions(-) create mode 100644 zml/kernels/moe/options.zig create mode 100644 zml/kernels/moe/weights.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e4ce12eb..3c0ace0b 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -91,7 +91,9 @@ zig_library( "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", "kernels/moe/moe.zig", + "kernels/moe/options.zig", "kernels/moe/tests.zig", + "kernels/moe/weights.zig", "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index 708ea757..14f1b090 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,6 +3,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); +const Options = @import("../options.zig").Options; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -24,7 +26,7 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { return cc.major == 10 and cc.minor == 3; } -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: Options) void { stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); @@ -41,7 +43,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index 479e6516..a2eb1792 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -12,6 +12,7 @@ pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); +const RoutingWeightPlacement = @import("../../options.zig").RoutingWeightPlacement; const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { @@ -252,7 +253,7 @@ pub const Nvfp4Inputs = struct { pub const Nvfp4Options = struct { swiglu_limit: ?f32 = null, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, + routing_weight_placement: RoutingWeightPlacement = .after_down, /// GEMM tile N; null selects it from the batch size. tile_n: ?i64 = null, }; diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig index e2c31c44..118593a8 100644 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig @@ -1,9 +1,14 @@ const std = @import("std"); -const stdx = @import("stdx"); const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); const platforms = @import("platforms"); +const stdx = @import("stdx"); + const zml = @import("../../../zml.zig"); +const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; +const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../options.zig").Activation; +const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); @@ -13,14 +18,14 @@ pub const Options = struct { /// Device used at graph-construction time to select the architecture /// library and size the XLA-owned scratch buffer. workspace_query_device: i32, - activation: zml.moe.Activation, + activation: Activation, enable_pdl: bool, gemm1_tactic: i32, /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to /// obtain the first valid GEMM2 index. gemm2_tactic: i32, - pub fn fromOptions(opts: zml.moe.Options) Options { + pub fn fromGlobalOptions(opts: GlobalOptions) Options { return .{ .workspace_query_device = 0, .activation = opts.activation, @@ -31,7 +36,7 @@ pub const Options = struct { } }; -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); @@ -79,8 +84,6 @@ const DeviceRunner = struct { runner: *fi_cutlass_moe.Runner, }; -const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; - pub const Runners = struct { runners: [max_num_devices]?DeviceRunner = @splat(null), @@ -151,7 +154,7 @@ fn currentRunners() !*Runners { return runners orelse error.RunnersNotLoaded; } -fn cutlassActivation(activation: zml.moe.Activation) i32 { +fn cutlassActivation(activation: Activation) i32 { return switch (activation) { .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, @@ -309,7 +312,7 @@ pub fn fc1BlockScaleShape( num_experts: i64, hidden_size: i64, intermediate_size: i64, - activation: zml.moe.Activation, + activation: Activation, ) zml.Shape { const rows = switch (activation) { .silu, .gelu, .relu => intermediate_size, @@ -505,7 +508,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - opts: zml.moe.Options, + opts: GlobalOptions, ) zml.Tensor { if (comptime !platforms.isEnabled(.cuda)) { @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); @@ -519,13 +522,13 @@ pub fn fusedExperts( // The runner ignores routes whose local expert id is -1. const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); - const runner_options: Options = .fromOptions(opts); + const runner_options: Options = .fromGlobalOptions(opts); const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); - const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); - const down_weight_unpacked = zml.moe.unpackedWeight(down); + const gate_up_weight_unpacked = unpackedWeight(gate_up); + const down_weight_unpacked = unpackedWeight(down); // TODO(Corentin): Do error checking on nvfp4 // Also, maybe pass `zml.nn.Linear` directly diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index 59d71dc9..e30292cf 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,8 +6,12 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; +const ExpertsLayout = @import("../options.zig").ExpertsLayout; +const ProjectionLayout = @import("../options.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Activation = @import("../options.zig").Activation; +const Options = @import("../options.zig").Options; const fly = @import("moe.zig"); -const ProjectionLayout = zml.moe.ProjectionLayout; const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -20,7 +24,7 @@ pub fn fusedExperts( routing_: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, - opts: zml.moe.Options, + opts: Options, ) Tensor { const expert_map_ = routing_.expertMap(); const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .interleaved else .concatenated; @@ -103,7 +107,7 @@ pub fn fusedExperts( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); + var activated = opts.activation.applyActivation(gate_up_out, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -131,7 +135,7 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: zml.moe.Options) bool { +pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index 9d397a46..988a4f51 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -1,12 +1,16 @@ const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; +const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../options.zig").Options; +const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); -fn validateOptions(opts: zml.moe.Options) void { +fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); } @@ -67,7 +71,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, - opts: zml.moe.Options, + opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); @@ -79,8 +83,8 @@ pub fn fusedExperts( const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const gate_up = zml.moe.unpackedWeight(gate_up_); - const down = zml.moe.unpackedWeight(down_); + const gate_up = unpackedWeight(gate_up_); + const down = unpackedWeight(down_); stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); const mode = quantMode(gate_up.dtype()); switch (mode) { @@ -138,7 +142,7 @@ pub fn fusedExperts( zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, ), gate_up_global_scale, expert_ids); - const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated).convert(act_dtype); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated).convert(act_dtype); const down_out = applyDownGlobalScale(moeGemm( activated, diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 7ad85c20..b3ca9814 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -9,6 +9,8 @@ pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zi pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +const ExpertsLayout = @import("options.zig").ExpertsLayout; +const Options = @import("options.zig").Options; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); @@ -20,60 +22,6 @@ test { _ = @import("tests.zig"); } -pub const ProjectionLayout = enum { concatenated, interleaved }; -pub const RoutingWeightPlacement = enum { before_down, after_down }; - -/// How a backend expects the expert weights to be stored -/// The packer writes this layout and forwardMoe reads it -pub const ExpertsLayout = struct { - /// Whether the two projections occupy contiguous halves or alternate. - gate_up: ProjectionLayout, - /// Backend specific transformation of the weights and scales. - packing: Packing, - - pub const Packing = enum { - /// Experts stacked as stored in the checkpoint. - plain, - /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). - swizzled_scales, - /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. - flashinfer_nvfp4, - }; -}; - -pub const ActivationKind = enum { - /// Gelu activation function - gelu, - /// ReLU activation function - relu, - /// SiLU activation function - silu, - /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) - swiglu, - /// SwiGLU activation function with step function - swiglu_step, - /// GeGlu activation function - geglu, - /// GeGlu activation function with tanh - geglu_tanh, -}; - -pub const Activation = union(ActivationKind) { - gelu: void, - relu: void, - silu: void, - swiglu: struct { - limit: ?f32 = null, - scale: ?f32 = null, - bias: ?f32 = null, - }, - swiglu_step: struct { - limit: ?f32 = null, - }, - geglu: void, - geglu_tanh: void, -}; - pub const Backend = enum { stablehlo, cute, @@ -210,14 +158,6 @@ pub const LocalRouting = struct { } }; -pub const Options = struct { - activation: Activation, - /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. - quantize_input: bool, - /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. - routing_weight_placement: RoutingWeightPlacement, -}; - /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); @@ -375,58 +315,3 @@ pub fn forwardMoe( .out_specs = partitioning.out_specs, }); } - -pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { - const quantization = linear.quantization orelse return linear.weight; - return if (zml.nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) - zml.nn.unpackFp4(linear.weight, linear.tag, linear.tag) - else - linear.weight; -} - -pub fn applyActivation(input: zml.Tensor, activation: zml.moe.Activation, layout: ProjectionLayout) zml.Tensor { - const x = input.convert(.f32); - return switch (activation) { - .gelu => x.gelu(), - .relu => x.relu(), - .silu => x.silu(), - .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { - const mid = @divFloor(x.dim(.out), 2); - var gate, var up = switch (layout) { - .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, - }; - - break :b switch (activation) { - .swiglu => |parameters| { - stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); - - const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - - // Apply limit on gate and clamp up - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - - // Apply bias - up = if (parameters.bias) |bias| up.addConstant(bias) else up; - - break :b gate.silu().mul(up); - }, - .swiglu_step => |parameters| { - gate = gate.silu(); - - const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - break :b gate.mul(up); - }, - .geglu_tanh => gate.gelu().mul(up), - .geglu => { - log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); - break :b gate.gelu().mul(up); - }, - else => unreachable, // already treated by the top-level switch - }; - }, - }; -} diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index c583b3e4..4b33b585 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,6 +5,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; +const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../options.zig").Options; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -12,7 +14,7 @@ const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); var gate_up_transpose: std.atomic.Value(bool) = .init(false); var down_transpose: std.atomic.Value(bool) = .init(false); -fn validateOptions(opts: zml.moe.Options) void { +fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); } @@ -215,7 +217,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, - opts: zml.moe.Options, + opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); const expert_map_ = routing.expertMap(); @@ -276,7 +278,7 @@ pub fn fusedExperts( gate_up_out = gate_up_out.add(bias_per_token); } - const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated); const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) diff --git a/zml/kernels/moe/options.zig b/zml/kernels/moe/options.zig new file mode 100644 index 00000000..aa68b06a --- /dev/null +++ b/zml/kernels/moe/options.zig @@ -0,0 +1,117 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const Tensor = @import("../../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/moe"); + +pub const Options = struct { + activation: Activation, + /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. + quantize_input: bool, + /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. + routing_weight_placement: RoutingWeightPlacement, +}; + +pub const ActivationKind = enum { + /// Gelu activation function + gelu, + /// ReLU activation function + relu, + /// SiLU activation function + silu, + /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) + swiglu, + /// SwiGLU activation function with step function + swiglu_step, + /// GeGlu activation function + geglu, + /// GeGlu activation function with tanh + geglu_tanh, +}; + +pub const Activation = union(ActivationKind) { + gelu: void, + relu: void, + silu: void, + swiglu: struct { + limit: ?f32 = null, + scale: ?f32 = null, + bias: ?f32 = null, + }, + swiglu_step: struct { + limit: ?f32 = null, + }, + geglu: void, + geglu_tanh: void, + + pub fn applyActivation(self: Activation, input: Tensor, layout: ProjectionLayout) Tensor { + const x = input.convert(.f32); + return switch (self) { + .gelu => x.gelu(), + .relu => x.relu(), + .silu => x.silu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + }; + + break :b switch (self) { + .swiglu => |parameters| { + stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + + // Apply limit on gate and clamp up + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + + // Apply bias + up = if (parameters.bias) |bias| up.addConstant(bias) else up; + + break :b gate.silu().mul(up); + }, + .swiglu_step => |parameters| { + gate = gate.silu(); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + break :b gate.mul(up); + }, + .geglu_tanh => gate.gelu().mul(up), + .geglu => { + log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); + break :b gate.gelu().mul(up); + }, + else => unreachable, // already treated by the top-level switch + }; + }, + }; + } +}; + +pub const RoutingWeightPlacement = enum { before_down, after_down }; + +pub const ProjectionLayout = enum { concatenated, interleaved }; + +/// How a backend expects the expert weights to be stored +/// The packer writes this layout and forwardMoe reads it +pub const ExpertsLayout = struct { + /// Whether the two projections occupy contiguous halves or alternate. + gate_up: ProjectionLayout, + /// Backend specific transformation of the weights and scales. + packing: Packing, + + pub const Packing = enum { + /// Experts stacked as stored in the checkpoint. + plain, + /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + swizzled_scales, + /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. + flashinfer_nvfp4, + }; +}; diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index 25c69c5c..7f84835b 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -1,13 +1,17 @@ //! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. //! Weights use [expert, output, input] order with concatenated gate/up columns. const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; +const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../options.zig").Activation; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; -pub fn activate(x: Tensor, activation: zml.moe.Activation) Tensor { +pub fn activate(x: Tensor, activation: Activation) Tensor { return switch (activation) { .silu => x.silu(), .gelu => x.gelu(), @@ -148,13 +152,13 @@ fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: } pub const Options = struct { - activation: zml.moe.Activation, + activation: Activation, quantize_input: bool = false, /// Model fixed backend contracts independently of optional input quantization: /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, + routing_weight_placement: RoutingWeightPlacement = .after_down, }; /// Negative expert IDs denote nonlocal routes and contribute zero, including bias. @@ -187,7 +191,7 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 991280fd..698c4d30 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -12,17 +12,17 @@ const std = @import("std"); const builder = @import("dsl/tcl/builder"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; +const Options = @import("../options.zig").Options; /// `input` is `{.t, .d}`, routing IDs and weights `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). /// SwiGLU, routing weights applied after the down projection. -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { const topk_ids = routing.ids; const topk_weights = routing.weights; stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index aa0be86f..0abb3462 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,12 +5,15 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const Placement = @import("moe.zig").RoutingWeightPlacement; +const ExpertsLayout = @import("options.zig").ExpertsLayout; +const GlobalOptions = @import("options.zig").Options; +const Activation = @import("options.zig").Activation; +const Placement = @import("options.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); const Case = struct { backend: zml.moe.Backend, - activation: zml.moe.Activation = .{ .swiglu = .{} }, + activation: Activation = .{ .swiglu = .{} }, scheme: ?zml.Quantization.Scheme = null, activation_dtype: zml.DataType = .bf16, quantize_input: bool = false, @@ -197,7 +200,7 @@ fn packFp4Bytes(weight: Tensor) Tensor { return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); } -fn prepareGateUp(tensor: Tensor, layout: zml.moe.ExpertsLayout) Tensor { +fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { return switch (layout.gate_up) { .concatenated => tensor, .interleaved => interleave(tensor), @@ -271,7 +274,7 @@ fn forward(input: Tensor, c: Case) Outputs { const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); - const options: zml.moe.Options = .{ + const options: GlobalOptions = .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, @@ -305,7 +308,7 @@ fn forward(input: Tensor, c: Case) Outputs { return .{ .actual = actual, .expected = expected }; } -const gated_activations = [_]zml.moe.Activation{ +const gated_activations = [_]Activation{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .bias = 1 } }, @@ -315,8 +318,8 @@ const gated_activations = [_]zml.moe.Activation{ .geglu, .geglu_tanh, }; -const all_activations = gated_activations ++ [_]zml.moe.Activation{ .relu, .silu, .gelu }; -const clipped_swiglu = [_]zml.moe.Activation{ +const all_activations = gated_activations ++ [_]Activation{ .relu, .silu, .gelu }; +const clipped_swiglu = [_]Activation{ .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .limit = 7 } }, }; @@ -354,7 +357,7 @@ const Format = struct { const Matrix = struct { formats: []const Format = &.{.{}}, - activations: []const zml.moe.Activation, + activations: []const Activation, quantize_inputs: []const bool = &.{false}, placements: []const Placement = &.{.after_down}, biases: []const bool = &.{false}, diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 3eb457da..0776a590 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,6 +12,7 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); @@ -1026,7 +1027,7 @@ pub const Combine = struct { topk: i64, expert_offset: i64, experts: i64, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + routing_weight_placement: RoutingWeightPlacement = .before_down, bn: i64, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); @@ -1092,7 +1093,7 @@ const GemmConfig = struct { // Swiglu clamping. Set to 0 to disable. swiglu_limit: f32, scale_stride: i64, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + routing_weight_placement: RoutingWeightPlacement = .before_down, bn: i64, bk: i64, }; diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 348f5a90..bae76744 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -6,10 +6,13 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const kernels = @import("moe.zig"); const toDType = zml.kernel.triton.from; -const ProjectionLayout = zml.moe.ProjectionLayout; -const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; +const ExpertsLayout = @import("../options.zig").ExpertsLayout; +const ProjectionLayout = @import("../options.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Activation = @import("../options.zig").Activation; +const Options = @import("../options.zig").Options; +const kernels = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/triton"); @@ -38,7 +41,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { // HACK(Corentin): only rely on the scheme of gate_up for now. @@ -59,7 +62,7 @@ pub fn fusedExpertsBase( gate_up_: zml.nn.Linear, down_: zml.nn.Linear, layout: ProjectionLayout, - opts: zml.moe.Options, + opts: Options, ) Tensor { const expert_map_ = routing_.expertMap(); const num_tokens = hidden_states.dim(.t); @@ -141,7 +144,7 @@ pub fn fusedExpertsBase( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); + var activated = opts.activation.applyActivation(gate_up_out, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -490,8 +493,8 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const io = std.testing.io; const platform = zml.testing.env(); const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, activation: zml.moe.Activation) Tensor { - return zml.moe.applyActivation(x, activation, layout); + fn forward(x: Tensor, layout: ProjectionLayout, activation: Activation) Tensor { + return activation.applyActivation(x, layout); } }; const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); @@ -506,7 +509,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { - const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; + const activation: Activation = .{ .swiglu = .{ .limit = threshold } }; var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); @@ -794,7 +797,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: Options) void { stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); @@ -899,7 +902,7 @@ pub fn fusedExpertsMxfp4( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig new file mode 100644 index 00000000..0e288856 --- /dev/null +++ b/zml/kernels/moe/weights.zig @@ -0,0 +1,10 @@ +const nn = @import("../../nn.zig"); +const Tensor = @import("../../tensor.zig").Tensor; + +pub fn unpackedWeight(linear: nn.Linear) Tensor { + const quantization = linear.quantization orelse return linear.weight; + return if (nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) + nn.unpackFp4(linear.weight, linear.tag, linear.tag) + else + linear.weight; +} From b23dbe334ae477bd4887a46ef5eba4b0c9729d69 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Nov 2026 19:24:38 +0100 Subject: [PATCH 28/47] [Modularization] zml/kernels/moe: break cycle between flashinfer_cutlass and platform (#1042) * Stopped using `zml.ops.CustomCall` because it creates cycles impossible to break. * renamed some `cutlass_flashinfer` to `flashinfer_cutlass` for consistency I didn't replace `CustomCall` because flashinfer_cutlass is the last user and will be removed soon. --- zml/BUILD.bazel | 2 +- .../moe/cutlass_flashinfer/BUILD.bazel | 8 - .../moe/flashinfer_cutlass/BUILD.bazel | 10 + .../moe/flashinfer_cutlass/contract.zig | 40 +++ .../flashinfer_cutlass.zig} | 244 ++---------------- .../moe/flashinfer_cutlass/runtime.zig | 223 ++++++++++++++++ zml/kernels/moe/moe.zig | 12 +- zml/platform.zig | 44 +++- 8 files changed, 336 insertions(+), 247 deletions(-) delete mode 100644 zml/kernels/moe/cutlass_flashinfer/BUILD.bazel create mode 100644 zml/kernels/moe/flashinfer_cutlass/BUILD.bazel create mode 100644 zml/kernels/moe/flashinfer_cutlass/contract.zig rename zml/kernels/moe/{cutlass_flashinfer/cutlass_flashinfer.zig => flashinfer_cutlass/flashinfer_cutlass.zig} (59%) create mode 100644 zml/kernels/moe/flashinfer_cutlass/runtime.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 3c0ace0b..fff3b8a8 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -119,7 +119,7 @@ zig_library( "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", "//zml/kernels/moe/cute", - "//zml/kernels/moe/cutlass_flashinfer", + "//zml/kernels/moe/flashinfer_cutlass", "//zml/kernels/moe/fly", "//zml/kernels/moe/metal", "//zml/kernels/moe/mosaic_tpu", diff --git a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel deleted file mode 100644 index 54b8604c..00000000 --- a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "cutlass_flashinfer", - srcs = [ - "cutlass_flashinfer.zig", - ], -) diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel new file mode 100644 index 00000000..c9c5d9a6 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "flashinfer_cutlass", + srcs = [ + "contract.zig", + "flashinfer_cutlass.zig", + "runtime.zig", + ], +) diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig new file mode 100644 index 00000000..b4b77eaa --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/contract.zig @@ -0,0 +1,40 @@ +pub fn Inputs(comptime T: type) type { + return struct { + hidden_states: T, + fc1_weights: T, + fc2_weights: T, + topk_weights: T, + topk_ids: T, + fc1_act_global: T, + fc1_weight_block: T, + fc1_global: T, + fc2_act_global: T, + fc2_weight_block: T, + fc2_global: T, + }; +} + +pub fn Outputs(comptime T: type) type { + return struct { + output: T, + workspace: T, + }; +} + +pub const Attributes = struct { + runners: u64 = 0, + num_tokens: i64, + hidden_size: i64, + intermediate_size: i64, + num_experts: i32, + top_k: i32, + activation: i32, + enable_pdl: bool, + fc1_act_per_expert: bool, + fc2_act_per_expert: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, +}; + +pub const compute_on_host: bool = false; +pub const name = "flashinfer_cutlass_nvfp4_routed_moe"; diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig similarity index 59% rename from zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig rename to zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig index 118593a8..d1a0e1b5 100644 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -5,12 +5,13 @@ const platforms = @import("platforms"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; const GlobalOptions = @import("../options.zig").Options; const Activation = @import("../options.zig").Activation; const unpackedWeight = @import("../weights.zig").unpackedWeight; +const contract = @import("contract.zig"); +const runtime = @import("runtime.zig"); -const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); +const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); pub const auto_tactic: i32 = -1; @@ -45,112 +46,15 @@ pub fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); } -const Input = struct { - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, -}; +const Inputs = contract.Inputs(zml.Tensor); -const Output = struct { - output: zml.Shape, - workspace: zml.Shape, -}; +const Outputs = contract.Outputs(zml.Shape); -const Attributes = struct { - runners: u64 = 0, - num_tokens: i64, - hidden_size: i64, - intermediate_size: i64, - num_experts: i32, - top_k: i32, - activation: i32, - enable_pdl: bool, - fc1_act_per_expert: bool, - fc2_act_per_expert: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, -}; +const Attributes = contract.Attributes; -const DeviceRunner = struct { - api: *fi_cutlass_moe.Api, - runner: *fi_cutlass_moe.Runner, -}; - -pub const Runners = struct { - runners: [max_num_devices]?DeviceRunner = @splat(null), - - pub fn deinit(self: *Runners) void { - for (self.runners) |device_runner| { - if (device_runner) |runner| { - _ = runner.api.runnerDestroy(runner.runner); - } - } - } - - fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { - if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; - - const index: usize = @intCast(device); - if (self.runners[index]) |runner| return runner; - - const api = try fi_cutlass_moe.apiForDevice(device); - const options = runnerOptions(device); - var runner: ?*fi_cutlass_moe.Runner = null; - try checkStatus(api, api.runnerCreate(&options, &runner)); - const result: DeviceRunner = .{ - .api = api, - .runner = runner orelse return error.RunnerInitializationFailed, - }; - self.runners[index] = result; - return result; - } -}; - -fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { - if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; - if (api.lastError()) |message| { - log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ - status, - std.mem.span(message), - }); - } else { - log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); - } - return switch (status) { - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, - else => error.FlashinferCutlassMoe, - }; -} - -fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { - var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); - options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); - options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; - options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.device = device; - options.use_fused_finalize = 1; - return options; -} - -fn runnersFromAttributes(attributes: Attributes) !*Runners { - if (attributes.runners == 0) return error.RunnersNotLoaded; - return @ptrFromInt(attributes.runners); -} - -fn currentRunners() !*Runners { +fn currentRunners() !*runtime.Runners { const platform = zml.Compiler.current().platform; - const runners = platform.state.cuda.fi_cutlass_moe_runners; + const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; return runners orelse error.RunnersNotLoaded; } @@ -166,80 +70,22 @@ fn cutlassActivation(activation: Activation) i32 { }; } -fn makeContext(attributes: Attributes) fi_cutlass_moe.Context { - var context = std.mem.zeroes(fi_cutlass_moe.Context); - context.struct_size = @sizeOf(fi_cutlass_moe.Context); - context.num_tokens = attributes.num_tokens; - context.hidden_size = attributes.hidden_size; - context.intermediate_size = attributes.intermediate_size; - context.num_experts = attributes.num_experts; - context.num_experts_on_rank = attributes.num_experts; - context.top_k = attributes.top_k; - context.tp_size = 1; - context.ep_size = 1; - context.activation = @intCast(attributes.activation); - context.enable_pdl = @intFromBool(attributes.enable_pdl); - context.swizzled_input_sf = 1; - return context; -} +pub fn call(input_tensors: Inputs, output_shapes: Outputs, attributes: Attributes) zml.ops.ShapeToTensor(Outputs) { + const opts: zml.ops.CustomCallOptions = .{ + .has_side_effect = false, + .output_operand_aliases = comptime zml.ops.customCallOutputOperandAliases(Inputs, Outputs, null), + .compute_on_host = contract.compute_on_host, + }; -fn ffiCallNvfp4( - call_frame: *zml.pjrt.ffi.CallFrame, - input: zml.pjrtx.TensorToCustomCallBuffer(Input), - output: zml.pjrtx.ShapeToCustomCallBuffer(Output), - attributes: Attributes, -) !?*zml.pjrt.ffi.Error { - const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); - const runners = try runnersFromAttributes(attributes); - const deviceRunner = try runners.ensureRunner(device); - const context = makeContext(attributes); - - var io = std.mem.zeroes(fi_cutlass_moe.Io); - io.struct_size = @sizeOf(fi_cutlass_moe.Io); - io.input = input.hidden_states.ptr; - io.token_selected_experts = @ptrCast(@alignCast(input.topk_ids.ptr)); - io.token_final_scales = @ptrCast(@alignCast(input.topk_weights.ptr)); - io.fc1_expert_weights = input.fc1_weights.ptr; - io.fc2_expert_weights = input.fc2_weights.ptr; - io.quant_scales[0] = input.fc1_act_global.ptr; - io.quant_scales[1] = input.fc1_weight_block.ptr; - io.quant_scales[2] = input.fc1_global.ptr; - io.quant_scales[3] = input.fc2_act_global.ptr; - io.quant_scales[4] = input.fc2_weight_block.ptr; - io.quant_scales[5] = input.fc2_global.ptr; - io.quant_scale_count = 6; - io.quant_scale_per_expert_mask = - (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | - (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); - io.output = output.output.ptr; - - var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); - workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); - workspace.data = output.workspace.ptr; - workspace.data_bytes = output.workspace.shape.byteSize(); - - try checkStatus( - deviceRunner.api, - deviceRunner.api.run( - deviceRunner.runner, - &context, - &io, - &workspace, - @ptrCast(call_frame.api.stream(call_frame.ctx)), - attributes.gemm1_tactic, - attributes.gemm2_tactic, - ), + return zml.ops.typedCustomCall( + contract.name, + opts, + input_tensors, + output_shapes, + attributes, ); - return null; } -const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ - .name = "flashinfer_cutlass_nvfp4_routed_moe", - // Expert mesh is owned by forwardMoe outer manual computation. - .sharding_aware = false, - .has_side_effect = false, -}); - fn computeCapability(platform: *const zml.Platform) !u16 { const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; return switch (cc.sm()) { @@ -248,40 +94,15 @@ fn computeCapability(platform: *const zml.Platform) !u16 { }; } -pub fn load( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, -) !void { - if (comptime platforms.isEnabled(.cuda)) { - var cuda_state = &platform.state.cuda; - if (cuda_state.fi_cutlass_moe_runners != null) return; - try fi_cutlass_moe.load(allocator, io, try computeCapability(platform)); - const runners = try allocator.create(Runners); - runners.* = .{}; - cuda_state.fi_cutlass_moe_runners = runners; - return; - } - return error.UnsupportedPlatform; -} - -pub fn register(platform: *const zml.Platform) !void { - if (comptime platforms.isEnabled(.cuda)) { - try routedNvfp4Call.register(platform); - return; - } - return error.UnsupportedPlatform; -} - pub fn isAvailable(platform: *const zml.Platform) bool { if (platform.target != .cuda) return false; - if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; _ = computeCapability(platform) catch return false; return true; } pub fn isNvfp4Supported(platform: *const zml.Platform) bool { - if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; const compute_capability = computeCapability(platform) catch return false; return switch (compute_capability) { 100, 103, 120 => true, @@ -289,21 +110,6 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { }; } -pub fn tacticCounts( - platform: *const zml.Platform, - device: i32, -) !struct { gemm1: i32, gemm2: i32 } { - const runners = platform.state.cuda.fi_cutlass_moe_runners orelse return error.RunnersNotLoaded; - const deviceRunner = try runners.ensureRunner(device); - var gemm1: i32 = 0; - var gemm2: i32 = 0; - try checkStatus( - deviceRunner.api, - deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), - ); - return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; -} - fn roundUp(value: i64, alignment: i64) i64 { return @divTrunc(value + alignment - 1, alignment) * alignment; } @@ -468,10 +274,10 @@ pub fn fusedExpertsNvfp4( attributes.runners = @intFromPtr(runners); const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); - const context = makeContext(attributes); + const context = runtime.makeContext(attributes); var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - checkStatus( + runtime.checkStatus( deviceRunner.api, deviceRunner.api.getWorkspaceRequirements( deviceRunner.runner, @@ -480,7 +286,7 @@ pub fn fusedExpertsNvfp4( ), ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); - const result = routedNvfp4Call.call( + const result = call( .{ .hidden_states = hidden_states, .fc1_weights = fc1_weights, diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig new file mode 100644 index 00000000..028ef892 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/runtime.zig @@ -0,0 +1,223 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const pjrt = @import("pjrt"); + +const DataType = @import("../../../dtype.zig").DataType; +const contract = @import("contract.zig"); + +const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); + +// TODO(Corentin): This was coming from Platform but we are now downstream. +// Find a way to solve that. +// Maybe the max_num_devices could be its own module because it only depends on Target. +const max_num_devices = 64; + +// TODO(Corentin): replace that with Shape when its dependencies are untangled and +// it doesn't depend on the platform anymore. +fn count(buffer: *const pjrt.ffi.Buffer) usize { + if (buffer.rank == 0) return @intCast(buffer.dims()[0]); + + var c: usize = 1; + for (buffer.dims()) |dim| c = c * @as(usize, @intCast(dim)); + + return c; +} + +fn byteSize(buffer: *const pjrt.ffi.Buffer) usize { + const dt: DataType = switch (buffer.dtype) { + .invalid => @panic("Found an invalid pjrt buffer"), + .token => @panic("Found an token pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; + return std.math.divCeil(usize, count(buffer) * dt.bitSizeOf(), 8) catch unreachable; +} + +pub fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + // TODO(Corentin): Find a place to abstract that + var input_buffers: contract.Inputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Inputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.args.buffers()[i]; + @field(input_buffers, field_name) = buf; + } + + var output_buffers: contract.Outputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Outputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.results.buffers()[i]; + @field(output_buffers, field_name) = buf; + } + + const attributes: contract.Attributes = switch (@typeInfo(contract.Attributes)) { + .void => {}, + .@"struct" => blk: { + var attrs: contract.Attributes = undefined; + inline for (@typeInfo(contract.Attributes).@"struct".field_names, @typeInfo(contract.Attributes).@"struct".field_types) |field_name, Field| { + // TODO: Use getByIndex instead? + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); + } + break :blk attrs; + }, + else => @compileError("Unsupported attributes type: " ++ @typeName(contract.Attributes)), + }; + + return handlerInner(call_frame, input_buffers, output_buffers, attributes) catch |err| + pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); +} + +fn handlerInner( + call_frame: *pjrt.ffi.CallFrame, + inputs: contract.Inputs(*const pjrt.ffi.Buffer), + outputs: contract.Outputs(*const pjrt.ffi.Buffer), + attributes: contract.Attributes, +) !?*pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = inputs.hidden_states.data; + io.token_selected_experts = @ptrCast(@alignCast(inputs.topk_ids.data)); + io.token_final_scales = @ptrCast(@alignCast(inputs.topk_weights.data)); + io.fc1_expert_weights = inputs.fc1_weights.data; + io.fc2_expert_weights = inputs.fc2_weights.data; + io.quant_scales[0] = inputs.fc1_act_global.data; + io.quant_scales[1] = inputs.fc1_weight_block.data; + io.quant_scales[2] = inputs.fc1_global.data; + io.quant_scales[3] = inputs.fc2_act_global.data; + io.quant_scales[4] = inputs.fc2_weight_block.data; + io.quant_scales[5] = inputs.fc2_global.data; + io.quant_scale_count = 6; + io.quant_scale_per_expert_mask = + (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | + (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); + io.output = outputs.output.data; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = outputs.workspace.data; + workspace.data_bytes = byteSize(outputs.workspace); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +pub fn makeContext(attributes: contract.Attributes) fi_cutlass_moe.Context { + var context = std.mem.zeroes(fi_cutlass_moe.Context); + context.struct_size = @sizeOf(fi_cutlass_moe.Context); + context.num_tokens = attributes.num_tokens; + context.hidden_size = attributes.hidden_size; + context.intermediate_size = attributes.intermediate_size; + context.num_experts = attributes.num_experts; + context.num_experts_on_rank = attributes.num_experts; + context.top_k = attributes.top_k; + context.tp_size = 1; + context.ep_size = 1; + context.activation = @intCast(attributes.activation); + context.enable_pdl = @intFromBool(attributes.enable_pdl); + context.swizzled_input_sf = 1; + return context; +} + +pub fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { + if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; + if (api.lastError()) |message| { + log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ + status, + std.mem.span(message), + }); + } else { + log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); + } + return switch (status) { + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, + else => error.FlashinferCutlassMoe, + }; +} + +fn runnersFromAttributes(attributes: contract.Attributes) !*Runners { + if (attributes.runners == 0) return error.RunnersNotLoaded; + return @ptrFromInt(attributes.runners); +} + +pub const Runners = struct { + runners: [max_num_devices]?DeviceRunner = @splat(null), + + pub fn deinit(self: *Runners) void { + for (self.runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); + } + } + } + + pub fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { + if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; + + const index: usize = @intCast(device); + if (self.runners[index]) |runner| return runner; + + const api = try fi_cutlass_moe.apiForDevice(device); + const options = runnerOptions(device); + var runner: ?*fi_cutlass_moe.Runner = null; + try checkStatus(api, api.runnerCreate(&options, &runner)); + const result: DeviceRunner = .{ + .api = api, + .runner = runner orelse return error.RunnerInitializationFailed, + }; + self.runners[index] = result; + return result; + } +}; + +const DeviceRunner = struct { + api: *fi_cutlass_moe.Api, + runner: *fi_cutlass_moe.Runner, +}; + +fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { + var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); + options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); + options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; + options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.device = device; + options.use_fused_finalize = 1; + return options; +} + +pub fn tacticCounts( + runners: *const Runners, + device: i32, +) !struct { gemm1: i32, gemm2: i32 } { + const deviceRunner = try runners.ensureRunner(device); + var gemm1: i32 = 0; + var gemm2: i32 = 0; + try checkStatus( + deviceRunner.api, + deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), + ); + return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { + try fi_cutlass_moe.load(allocator, io, sm); +} diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index b3ca9814..32bebac0 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -5,11 +5,11 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; pub const cute = @import("cute/cute.zig"); -pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); +const ExpertsLayout = @import("options.zig").ExpertsLayout; +pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const ExpertsLayout = @import("options.zig").ExpertsLayout; const Options = @import("options.zig").Options; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); @@ -45,7 +45,7 @@ pub const Backend = enum { .triton, .nvfp4 => if (cute.isNvfp4Supported(platform)) .cute - else if (cutlass_flashinfer.isNvfp4Supported(platform)) + else if (flashinfer_cutlass.isNvfp4Supported(platform)) .flashinfer_cutlass else error.UnsupportedQuantization, @@ -84,7 +84,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => true, .cute => cute.isAvailable(platform), - .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), + .flashinfer_cutlass => flashinfer_cutlass.isAvailable(platform), .fly => switch (platform.target) { .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, else => false, @@ -117,7 +117,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => {}, .cute => {}, - .flashinfer_cutlass => cutlass_flashinfer.register(platform), + .flashinfer_cutlass => flashinfer_cutlass.register(platform), .triton, .fly => {}, .mosaic_tpu => {}, .metal => {}, @@ -262,7 +262,7 @@ pub fn forwardMoe( const output = switch (args.backend) { .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .flashinfer_cutlass => flashinfer_cutlass.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), diff --git a/zml/platform.zig b/zml/platform.zig index 3b52c08e..f3510a19 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -8,6 +8,8 @@ pub const Target = @import("platforms").Platform; const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; +const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); +const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); @@ -257,12 +259,12 @@ pub const State = union(Target) { furiosa: void, pub const CudaState = struct { - fi_cutlass_moe_runners: ?*zml.moe.cutlass_flashinfer.Runners = null, + flashinfer_cutlass_moe_runners: ?*flashinfer_cutlass_runtime.Runners = null, fn deinit(self: *CudaState) void { - if (self.fi_cutlass_moe_runners) |runners| { + if (self.flashinfer_cutlass_moe_runners) |runners| { runners.deinit(); - self.fi_cutlass_moe_runners = null; + self.flashinfer_cutlass_moe_runners = null; } } }; @@ -381,16 +383,9 @@ pub const Platform = struct { switch (target) { .cuda => { - if (zml.moe.cutlass_flashinfer.load(arena, io, platform)) { - zml.moe.cutlass_flashinfer.register(platform) catch |err| { - log.warn( - "Failed to register FlashInfer CUTLASS MoE custom calls: {}", - .{err}, - ); - }; - } else |err| { - log.warn("Failed to load FlashInfer CUTLASS MoE: {}", .{err}); - } + cuda.initFlashinferCutlass(platform, io) catch |err| { + log.warn("Failed to initialize Flashinfer CUTLASS MoE: {}", .{err}); + }; }, else => {}, } @@ -877,6 +872,29 @@ pub const cuda = struct { } } else null; } + + fn initFlashinferCutlass(platform: *Platform, io: std.Io) !void { + if (comptime !Target.cuda.isEnabled()) return error.UnsupportedPlatform; + const cc = computeCapability(platform) orelse return error.UnsupportedArchitecture; + + const allocator = platform.arena.allocator(); + const runners = try allocator.create(flashinfer_cutlass_runtime.Runners); + errdefer allocator.destroy(runners); + + try flashinfer_cutlass_runtime.load(allocator, io, cc.sm()); + + runners.* = .{}; + errdefer runners.deinit(); + + try platform.registerFfi(.{ + .name = flashinfer_cutlass_contract.name, + .handler = flashinfer_cutlass_runtime.handler, + .traits = .{ .command_buffer_compatible = true }, + .platform_name = if (flashinfer_cutlass_contract.compute_on_host) "Host" else null, + }); + + platform.state.cuda.flashinfer_cutlass_moe_runners = runners; + } }; pub const rocm = struct { From 7b40b1b0f880ff6230c38abac06daf104e3c1f93 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 9 Nov 2026 20:48:45 +0100 Subject: [PATCH 29/47] zml/io: honor slices and splat in DirectMemoryWriter.drain (#1050) `DirectMemoryWriter.drain` only wrote the first slice of data and ignored the splat count. The `std.Io.Writer` contract is to write every slice in order, then repeat the last one splat times. - A one-byte splat advanced one byte per drain call. - After a partial write, `splatBytesAll` passes an empty first slice followed by the pattern. The old drain returned 0 for that forever, so the fill never finished. `drain` now writes every slice and repeats the last one splat times. --- zml/io.zig | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index 1babdcfc..ab8d8e6e 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1287,20 +1287,52 @@ pub const DirectMemoryWriter = struct { self.shard_progress = states; } - fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - const chunk = data[0]; try self.commitWindow(); + if (self.interface.end == self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - const writable = self.interface.buffer.len - self.interface.end; - if (writable == 0) return std.Io.Writer.Error.WriteFailed; + const consumed = consumed: { + var n: usize = 0; + for (data[0 .. data.len - 1]) |bytes| { + const copied = self.writeWindow(bytes); + n += copied; - const n = @min(writable, chunk.len); - @memcpy(self.interface.buffer[self.interface.end..][0..n], chunk[0..n]); + if (copied < bytes.len) break :consumed n; + } + + const pattern = data[data.len - 1]; + switch (pattern.len) { + 0 => {}, + 1 => { + const dest = self.interface.buffer[self.interface.end..]; + const count = @min(dest.len, splat); + @memset(dest[0..count], pattern[0]); + + self.interface.end += count; + n += count; + }, + else => for (0..splat) |_| { + const copied = self.writeWindow(pattern); + n += copied; + + if (copied < pattern.len) break; + }, + } + + break :consumed n; + }; + + try self.commitWindow(); + return consumed; + } + + fn writeWindow(self: *DirectMemoryWriter, bytes: []const u8) usize { + const dest = self.interface.buffer[self.interface.end..]; + const n = @min(dest.len, bytes.len); + @memcpy(dest[0..n], bytes[0..n]); self.interface.end += n; - try self.commitWindow(); - return n; } From b61eede864e2c757367e78341aa138411822150c Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Tue, 10 Nov 2026 14:23:27 +0100 Subject: [PATCH 30/47] zml/kernels: Enable MegaMoe support in llmd (#1032) This PR enable the MegaMoe kernel support in llmd by : - Making the symmetric workspace an arg of CuteOps - Adding cuda.grid_constant arg to the DSL builder, enabling GPU kernels to receive TMA tensormap - Adapting the existing mxfp4 kernel (non mega) to the new shared swizzling `interleaved_8` --- dsl/cute/builder.zig | 806 +++++++++++++++++- zml/kernel.zig | 2 + zml/kernels/moe/cute/cute.zig | 3 +- zml/kernels/moe/cute/sm10x/moe.zig | 11 +- .../moe/cute/sm10x/persistent_mxfp4.zig | 28 +- zml/kernels/moe/fly/fly.zig | 2 +- zml/kernels/moe/moe.zig | 16 +- zml/kernels/moe/options.zig | 21 +- zml/kernels/moe/tests.zig | 39 +- zml/kernels/moe/triton/triton.zig | 9 +- .../paged_attention/paged_attention.zig | 1 + zml/ops.zig | 17 +- zml/platform.zig | 6 + 13 files changed, 888 insertions(+), 73 deletions(-) diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig index 926b8419..8f791f80 100644 --- a/dsl/cute/builder.zig +++ b/dsl/cute/builder.zig @@ -16,6 +16,49 @@ const vector = @import("mlir/dialects/vector"); const mlir = @import("mlir"); const stdx = @import("stdx"); +/// `CUtensorMapDataType`. +pub const TensorMapDataType = enum(u32) { + uint8 = 0, + uint16 = 1, + uint32 = 2, + int32 = 3, + uint64 = 4, + int64 = 5, + float16 = 6, + float32 = 7, + float64 = 8, + bfloat16 = 9, + float32_ftz = 10, + tfloat32 = 11, + tfloat32_ftz = 12, + @"16u4_align8b" = 13, + @"16u4_align16b" = 14, + @"16u6_align16b" = 15, +}; + +/// `CUtensorMapSwizzle`. +pub const TensorMapSwizzle = enum(u32) { none = 0, @"32b" = 1, @"64b" = 2, @"128b" = 3, @"128b_atom_32b" = 4 }; +/// `CUtensorMapL2promotion`. +pub const TensorMapL2Promotion = enum(u32) { none = 0, @"64b" = 1, @"128b" = 2, @"256b" = 3 }; +/// `CUtensorMapInterleave`. +pub const TensorMapInterleave = enum(u32) { none = 0, @"16b" = 1, @"32b" = 2 }; +/// `CUtensorMapFloatOOBfill`. +pub const TensorMapOobFill = enum(u32) { none = 0, nan_request_zero_fma = 1 }; + +pub const TensorMapSpec = struct { + data_type: TensorMapDataType, + rank: u32, + global_dims: [5]u64 = @splat(1), + /// Byte strides of dimensions 1..rank-1. + global_strides: [4]u64 = @splat(0), + box_dims: [5]u32 = @splat(1), + element_strides: [5]u32 = @splat(1), + interleave: TensorMapInterleave = .none, + swizzle: TensorMapSwizzle, + l2_promotion: TensorMapL2Promotion = .none, + oob_fill: TensorMapOobFill = .none, +}; + const dtypes = @import("dtype.zig"); pub const DType = dtypes.DType; const isFloatDtype = dtypes.isFloatDtype; @@ -715,6 +758,7 @@ pub const ArgSpec = struct { pub const MlirTypeSpec = struct { type_: *const mlir.Type, grid_constant: bool = false, + cuda_grid_constant: bool = false, }; pub const TensorSpec = struct { @@ -823,6 +867,7 @@ pub const Builder = struct { .mlir_type => .{ .mlir_type = .{ .type_ = inner.type_, .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, + .cuda_grid_constant = if (@hasField(@TypeOf(inner), "cuda_grid_constant")) inner.cuda_grid_constant else false, } }, } }; } @@ -849,7 +894,7 @@ pub const Builder = struct { return @Struct(.auto, null, &names, &types, &@splat(.{})); } - fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { + pub fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { std.debug.assert(self.entry_block == null); const ctx = self.ctx; const scratch = self.arena.allocator(); @@ -871,7 +916,7 @@ pub const Builder = struct { var any_arg_attr = false; for (arg_attrs, args) |*attr, a| { attr.* = switch (a.kind) { - .mlir_type => |typed| if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, + .mlir_type => |typed| if (typed.cuda_grid_constant) .dict(ctx, &.{.named(ctx, "cuda.grid_constant", .unit(ctx))}) else if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, else => empty_dict, }; if (attr.* != empty_dict) any_arg_attr = true; @@ -1083,6 +1128,75 @@ pub const Builder = struct { })); } + /// `!cuda.tensor_map`: the kernel argument type of an explicitly encoded map. + pub fn tensorMapType(self: *Builder) *const mlir.Type { + return (cuda.ops.TensorMapType.get(self.ctx, .{}) catch unreachable).type_(); + } + + /// A host stack array of `values` (u64 -> i64 slots, u32 -> i32 slots): a `cuuint64_t*` / + /// `cuuint32_t*` argument of `cuTensorMapEncodeTiled`. + fn tensorMapHostArray(self: *Builder, comptime T: type, values: []const T) Value { + const ctx = self.ctx; + const elem: *const mlir.Type = if (T == u64) .int(ctx, .i64) else .int(ctx, .i32); + const ptr_type = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic); + const array = self.emit(mlir.Operation.make(ctx, "llvm.alloca", .{ + .operands = .{ .flat = &.{self.constant(.i32, @as(i32, @intCast(@max(values.len, 1)))).inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "alignment", .int(ctx, .i64, 8)) }, + .location = self.loc(), + })); + for (values, 0..) |v, i| { + const slot = self.emit(mlir.Operation.make(ctx, "llvm.getelementptr", .{ + .operands = .{ .flat = &.{array.inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, &[_]i32{@intCast(i)})) }, + .location = self.loc(), + })); + const value = if (T == u64) self.constant(.i64, @as(i64, @intCast(v))) else self.constant(.i32, @as(i32, @intCast(v))); + self.emitVoid(llvm.store(ctx, value.inner, slot.inner, .{ .alignment = @sizeOf(T) }, self.loc())); + } + return array; + } + + /// Encode a tiled map in a host function and return both the map and CUDA status. + /// The caller supplies the final device address and handles the returned status. + pub fn encodeTensorMap(self: *Builder, spec: TensorMapSpec, address: Value) struct { map: Value, result: Value } { + const ctx = self.ctx; + // The `cuda` dialect's address operand is a `ptr` dialect pointer, which zml does not register. + ctx.setAllowUnregisteredDialects(true); + const addr = self.emit(cuda.cast(ctx, (mlir.Type.parse(self.ctx, "!ptr.ptr<#ptr.generic_space>") catch @panic("invalid tensor-map IR")), address.inner, self.loc())); + const results = self.emitResults(2, cuda.ops.tensor_map_encode_tiled( + ctx, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.data_type)))).inner, + self.constant(.i32, @as(i32, @intCast(spec.rank))).inner, + addr.inner, + self.tensorMapHostArray(u64, spec.global_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u64, spec.global_strides[0 .. spec.rank - 1]).inner, + self.tensorMapHostArray(u32, spec.box_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u32, spec.element_strides[0..spec.rank]).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.interleave)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.swizzle)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.l2_promotion)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.oob_fill)))).inner, + cuda.resultType(ctx), + self.tensorMapType(), + self.loc(), + )); + return .{ .map = results[1], .result = results[0] }; + } + + /// Device side: the generic address (i64) of the `!cuda.tensor_map` kernel argument `arg` + /// (`&tensor_map` for a `__grid_constant__` parameter). + pub fn tensorMapAddress(self: *Builder, descriptor: Value) Value { + const ctx = self.ctx; + const llvm_ptr = self.emit(cuda.cast(ctx, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), descriptor.inner, self.loc())); + return self.emit(mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{llvm_ptr.inner} }, + .results = .{ .flat = &.{DType.i64.toMlir(ctx)} }, + .location = self.loc(), + })); + } + pub fn cudaResultStatus(self: *Builder, result: Value) Value { return self.emit(cuda.cast(self.ctx, .int(self.ctx, .i32), result.inner, self.loc())); } @@ -1717,20 +1831,27 @@ pub const Builder = struct { /// `projection`, like Python's `proj`, selects the modes of `tile` that /// apply, such as `b.expr(.{ 1, AlgebraToken.all, 1 })`. - pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: *const mlir.Type, projection: ?AlgebraExpr) View { + pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: ?*const mlir.Type, projection: ?AlgebraExpr) View { + if (result_type == null) return self.inferView("cute.local_tile", &.{ self.asValue(input).inner, self.asValue(tile).inner, if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord) }, if (projection) |p| &.{.named(self.ctx, "proj", self.algebraAttr(cute_ir.CoordAttr, p).attribute())} else &.{}); const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); const result = self.emit(cute_ir.local_tile( self.ctx, self.asValue(input).inner, self.asValue(tile).inner, coord_value, - result_type, + result_type.?, .{ .proj = if (projection) |p| self.algebraAttr(cute_ir.CoordAttr, p).attribute() else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } + // A null result type on the algebra operations below requests dialect inference. + fn inferView(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attrs: []const mlir.NamedAttribute) View { + const value = self.emit(mlir.Operation.make(self.ctx, name, .{ .operands = .{ .flat = operands }, .attributes = attrs, .result_type_inference = true, .location = self.loc() })); + return .{ .inner = value.inner, .kernel = self }; + } + pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.local_partition( self.ctx, @@ -1743,12 +1864,13 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: *const mlir.Type) View { + pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.slice", &.{ self.asValue(input).inner, self.makeCoord(coord) }, &.{}); const result = self.emit(cute_ir.slice( self.ctx, self.asValue(input).inner, self.makeCoord(coord), - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1767,13 +1889,17 @@ pub const Builder = struct { (cute_ir.MemRefType.get(self.ctx, .{ .ptr = memref.ptr_(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.arithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ .desc = desc.desc(), .layout = layout_value.type_().isA(cute_ir.LayoutType).?.attr() }) catch @panic("invalid regrouped descriptor")).type_() else - std.debug.panic("withLayout: expected memref or coordinate tensor, got {f}", .{input_type}); + std.debug.panic("withLayout: expected memref, coordinate tensor or shared-memory descriptor view, got {f}", .{input_type}); const iterator_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| memref.ptr_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.arithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + desc.desc() else unreachable; const iterator = self.emit(cute_ir.get_iter(self.ctx, value.inner, iterator_type, self.loc())); @@ -1781,11 +1907,12 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: *const mlir.Type) View { + pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.group_modes", &.{self.asValue(input).inner}, &.{ .named(self.ctx, "begin", .int(self.ctx, .i32, begin)), .named(self.ctx, "end", .int(self.ctx, .i32, end)) }); const result = self.emit(cute_ir.group_modes( self.ctx, self.asValue(input).inner, - result_type, + result_type.?, .int(self.ctx, .i32, begin), .int(self.ctx, .i32, end), self.loc(), @@ -1804,12 +1931,13 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { + pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.flat_divide", &.{ self.asValue(input).inner, self.asValue(tiler).inner }, &.{}); const result = self.emit(cute_ir.flat_divide( self.ctx, self.asValue(input).inner, self.asValue(tiler).inner, - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1826,39 +1954,44 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { + pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.filter_zeros", if (target_profile) |target| &.{ self.asValue(input).inner, target.inner } else &.{self.asValue(input).inner}, &.{}); const result = self.emit(cute_ir.filter_zeros( self.ctx, self.asValue(input).inner, - result_type, + result_type.?, .{ .target_profile = if (target_profile) |target| target.inner else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn getLayout(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getLayout(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_layout", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); return .{ .inner = result.inner, .kernel = self }; } /// `cute.get_iter`: the iterator of `input`, of type `result_type`. - pub fn getIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getIter(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_iter", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn getShape(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getShape(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_shape", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: *const mlir.Type) View { + pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.add_offset", &.{ self.asValue(input).inner, self.asValue(offset).inner }, &.{}); const result = self.emit(cute_ir.add_offset( self.ctx, self.asValue(input).inner, self.asValue(offset).inner, - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1895,36 +2028,39 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: *const mlir.Type) View { + pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.copy.retile", &.{ tiled.inner, self.asValue(input).inner }, &.{}); const result = self.emit(cute_ir.tiled_copy_retile( self.ctx, tiled.inner, self.asValue(input).inner, - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: *const mlir.Type) View { + pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.mma.partition", &.{ tiled.inner, self.asValue(input).inner, self.makeCoord(coord) }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); const result = self.emit(cute_ir.tiled_mma_partition( self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), - result_type, + result_type.?, .int(self.ctx, .i32, operand_id), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: *const mlir.Type) View { + pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.mma.make_fragment", &.{ tiled.inner, self.asValue(input).inner }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); const result = self.emit(cute_ir.mma_make_fragment( self.ctx, tiled.inner, self.asValue(input).inner, - result_type, + result_type.?, .int(self.ctx, .i32, operand_id), self.loc(), )); @@ -2731,8 +2867,8 @@ pub const Builder = struct { /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a /// TMA producer does before issuing its copies. - pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: u32) void { - self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); + pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: anytype) void { + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.lift(bytes).to(.i32).inner, .cta, self.loc())); } pub const MBarrierArriveOptions = struct { @@ -4203,3 +4339,615 @@ test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { defer parsed.deinit(); try std.testing.expect(parsed.operation().verify()); } + +/// Low-level CUDA primitives. Prefer typed operations where they preserve the +/// instruction semantics. Assembly fallbacks retain exact memory scopes, rounding, +/// CTA-group/cache modifiers, and tied-register dependencies for asynchronous loads. +const sm100_mma_peer_bit_mask: u32 = 0xFEFFFFFF; + +/// Side-effecting inline assembly with no result. +pub fn asmVoid(b: *Builder, text: []const u8, args: []const Value, constraints: []const u8) void { + const inner = b.arena.allocator().alloc(*const mlir.Value, args.len) catch @panic("OOM"); + for (args, inner) |a, *x| x.* = a.inner; + b.emitVoid(llvm.inline_asm(b.ctx, text, inner, .{ .constraints = constraints, .has_side_effects = true }, b.loc())); +} + +/// Inline assembly with one result of `dtype`. +pub fn asmValue(b: *Builder, text: []const u8, args: []const Value, dtype: DType, constraints: []const u8, side_effects: bool) Value { + const inner = b.arena.allocator().alloc(*const mlir.Value, args.len) catch @panic("OOM"); + for (args, inner) |a, *x| x.* = a.inner; + return b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = dtype.toMlir(b.ctx), .constraints = constraints, .has_side_effects = side_effects }, b.loc())); +} + +/// Inline assembly with `n` results of `dtype` (an `!llvm.struct` result split with +/// `llvm.extractvalue`, how LLVM represents multi-output asm). +pub fn asmValues(b: *Builder, comptime n: usize, text: []const u8, args: []const Value, dtype: DType, constraints: []const u8, side_effects: bool) [n]Value { + return asmValuesSlice(b, n, text, args, dtype, constraints, side_effects)[0..n].*; +} + +/// Inline assembly with a host-computed number of results of the same dtype. +pub fn asmValuesSlice(b: *Builder, n: usize, text: []const u8, args: []const Value, dtype: DType, constraints: []const u8, side_effects: bool) []Value { + const inner = b.arena.allocator().alloc(*const mlir.Value, args.len) catch @panic("OOM"); + for (args, inner) |a, *x| x.* = a.inner; + const elem = dtype.toMlir(b.ctx); + const struct_type = (mlir.Type.parse(b.ctx, std.fmt.allocPrint(b.arena.allocator(), "!llvm.struct<({s})>", .{structFields(b, n, elem)}) catch @panic("OOM")) catch @panic("invalid inline assembly result type")); + const packed_ = b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = struct_type, .constraints = constraints, .has_side_effects = side_effects }, b.loc())); + const out = b.arena.allocator().alloc(Value, n) catch @panic("OOM"); + for (out, 0..) |*o, i| { + o.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{packed_.inner} }, + .results = .{ .flat = &.{elem} }, + .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &[_]i64{@intCast(i)}))}, + .location = b.loc(), + })); + } + return out; +} + +pub fn structFields(b: *Builder, n: usize, elem: *const mlir.Type) []const u8 { + var w: std.Io.Writer.Allocating = .init(b.arena.allocator()); + for (0..n) |i| w.writer.print("{s}{f}", .{ if (i == 0) "" else ", ", elem }) catch @panic("OOM"); + return w.written(); +} + +pub fn ldAcq(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn ldAcqSysS32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.s32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.u32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqGpuU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.u64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn atomicAddU32(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r", true); +} + +pub fn atomicAddU64(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddSys(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.sys.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddRel(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.release.gpu.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r,~{memory}", true); +} + +pub fn stRelSys(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "st.release.sys.global.u64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddS32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redAddU32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redXorRel(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "red.release.gpu.global.xor.b64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddRel(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn redAddRelSys(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.sys.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn atomicAddBlockShared(b: *Builder, smem: Value, value: anytype) Value { + return asmValue(b, "atom.shared.cta.add.u32 $0, [$1], $2;", &.{ smem, liftAs(b, value, .i32) }, .i32, "=r,r,r,~{memory}", true); +} + +pub fn fenceAcqRelCta(b: *Builder) void { + asmVoid(b, "fence.acq_rel.cta;", &.{}, "~{memory}"); +} + +pub fn liftAs(b: *Builder, value: anytype, dt: DType) Value { + if (@TypeOf(value) == Value) return value; + return b.constant(dt, value); +} + +/// `cvta`-style widening of an `i32` shared-memory address to the 64-bit operand +/// `__cvta_generic_to_shared` gives DeepGEMM's `"l"` constraints. +pub fn zext64(b: *Builder, v: Value) Value { + if (v.dtype() == .i64) return v; + return b.emit(arith.extui(b.ctx, v.inner, DType.i64.toMlir(b.ctx), b.loc())); +} + +pub fn i16Of(b: *Builder, v: Value) Value { + return if (v.dtype() == .i16) v else b.emit(arith.trunci(b.ctx, v.inner, DType.i16.toMlir(b.ctx), b.loc())); +} + +pub fn cst32(b: *Builder, v: anytype) Value { + return b.constant(.i32, @as(i32, @bitCast(@as(u32, @truncate(@as(u64, @bitCast(@as(i64, v)))))))); +} + +pub fn cst64(b: *Builder, v: u64) Value { + return b.constant(.i64, @as(i64, @bitCast(v))); +} + +pub fn getSmIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %smid;", &.{}, .i32, "=r", false); +} + +pub fn getLaneIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %laneid;", &.{}, .i32, "=r", false); +} + +/// `ptx::get_grid_idx`: the 64-bit `%gridid`, unique per launch. +pub fn getGridIdx(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %gridid;", &.{}, .i64, "=l", true); +} + +pub fn blockRankInCluster(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %cluster_ctarank;\n", &.{}, .i32, "=r", true); +} + +/// `clock64()` (`mov.u64 %rd, %clock64;`). +pub fn clock64(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %clock64;", &.{}, .i64, "=l", true); +} + +/// Keep the original election sequence until typed election matches its small-batch performance. +pub fn electOneSync(b: *Builder) Value { + const pred = asmValue(b, + \\{ + \\.reg .b32 %rx; + \\.reg .pred %px; + \\ mov.u32 $0, 0; + \\ elect.sync %rx|%px, $1; + \\@%px mov.s32 $0, 1; + \\} + , &.{cst32(b, 0xFFFFFFFF)}, .i32, "=r,r", true); + return pred.ne(0); +} + +/// `asm volatile("trap;")` (`DG_TRAP_ONLY_DEVICE_ASSERT`, and this port's timeout). +pub fn trap(b: *Builder) void { + asmVoid(b, "trap;", &.{}, "~{memory}"); +} + +/// `ptx::sync_unaligned(num_threads, barrier_idx)`: `barrier.sync id, n` (threads of a warp may +/// reach it at different program points). +pub fn syncUnaligned(b: *Builder, num_threads: anytype, barrier_idx: anytype) void { + asmVoid(b, "barrier.sync $0, $1;", &.{ liftAs(b, barrier_idx, .i32), liftAs(b, num_threads, .i32) }, "r,r,~{memory}"); +} + +/// `cute::cluster_arrive`. +pub fn clusterArrive(b: *Builder) void { + asmVoid(b, "barrier.cluster.arrive.aligned;\n", &.{}, "~{memory}"); +} + +/// `cudaGridDependencySynchronize()` (`griddepcontrol.wait;`, PDL). +pub fn gridDependencySynchronize(b: *Builder) void { + asmVoid(b, "griddepcontrol.wait;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::warpgroup_reg_alloc`. +pub fn warpgroupRegAlloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.inc.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `cutlass::arch::warpgroup_reg_dealloc`. +pub fn warpgroupRegDealloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.dec.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `ptx::ld_volatile(const uint64_t*)`. +pub fn ldVolatileU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l", true); +} + +/// `ptx::ld_volatile(const uint32_t*)`. +pub fn ldVolatileU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const uint32_t*)`. +pub fn ldSharedU32(b: *Builder, smem: Value) Value { + return asmValue(b, "ld.shared.u32 $0, [$1];", &.{zext64(b, smem)}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const float2*)`. +pub fn ldSharedF32x2(b: *Builder, smem: Value) [2]Value { + return asmValues(b, 2, "ld.shared.v2.f32 {$0, $1}, [$2];", &.{zext64(b, smem)}, .f32, "=f,=f,l", true); +} + +/// `ptx::ld_shared(const float4*)`. +pub fn ldSharedF32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .f32, "=f,=f,=f,=f,l", true); +} + +/// `ptx::ld_shared(const uint4*)`. +pub fn ldSharedU32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.u32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .i32, "=r,=r,=r,=r,l", true); +} + +/// `ptx::st_shared(const void*, x, y, z, w)`. +pub fn stSharedU32x4(b: *Builder, smem: Value, w: [4]Value) void { + asmVoid(b, "st.shared.v4.u32 [$0], {$1, $2, $3, $4};", &.{ zext64(b, smem), w[0], w[1], w[2], w[3] }, "l,r,r,r,r"); +} + +/// `ptx::st_shared(const uint32_t*, uint32_t)`. +pub fn stSharedU32(b: *Builder, smem: Value, value: Value) void { + asmVoid(b, "st.shared.u32 [$0], $1;", &.{ zext64(b, smem), value }, "l,r"); +} + +/// `ptx::st_shared(const float2*, float2)`. +pub fn stSharedF32x2(b: *Builder, smem: Value, v: [2]Value) void { + asmVoid(b, "st.shared.v2.f32 [$0], {$1, $2};", &.{ zext64(b, smem), v[0], v[1] }, "l,f,f"); +} + +/// `ptx::st_shared_bulk(smem, num_bytes)`: zero `num_bytes` (a multiple of 8) of shared memory. +pub fn stSharedBulk(b: *Builder, smem: Value, num_bytes: u32) void { + std.debug.assert(num_bytes % 8 == 0); + asmVoid(b, "st.bulk.weak.shared::cta [$0], $1, 0;", &.{ zext64(b, smem), b.constant(.i64, num_bytes) }, "l,l,~{memory}"); +} + +/// `ptx::mapa_shared(ptr, dst_cta_idx)`: this CTA's shared address in cluster CTA `cta`. +pub fn mapaShared(b: *Builder, smem: Value, dst_cta_idx: Value) Value { + return asmValue(b, "mapa.shared::cluster.u32 $0, $1, $2;", &.{ smem, dst_cta_idx }, .i32, "=r,r,r", true); +} + +/// `ptx::st_async_cluster(dst, src, dst_cta_idx, barrier)` for `words.len / 4` 16-byte +/// chunks: `st.async` into CTA `dst_cta_idx`'s shared memory, completing the bytes on its +/// mbarrier at `barrier` (both local addresses, mapped with `mapa`). +pub fn stAsyncCluster(b: *Builder, dst: Value, words: []const Value, dst_cta_idx: Value, barrier: Value) void { + std.debug.assert(words.len % 4 == 0); + const mapped_barrier = mapaShared(b, barrier, dst_cta_idx); + const mapped_dst = mapaShared(b, dst, dst_cta_idx); + var i: usize = 0; + while (i < words.len) : (i += 4) { + asmVoid( + b, + "st.async.shared::cluster.mbarrier::complete_tx::bytes.u32.v4 [$0], {$1, $2, $3, $4}, [$5];", + &.{ mapped_dst.add(@as(i32, @intCast(i * 4))), words[i], words[i + 1], words[i + 2], words[i + 3], mapped_barrier }, + "r,r,r,r,r,r,~{memory}", + ); + } +} + +/// `ptx::SM100_U8x4_STSM_T::copy(src, smem)`: `stmatrix.m16n8.x1.trans.b8`. +pub fn stmatrixU8x4T(b: *Builder, src: Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [$0], {$1};\n", &.{ zext64(b, smem), src }, "l,r"); +} + +/// `ptx::SM90_U32x4_STSM_T::copy(s0, s1, s2, s3, smem)`: `stmatrix.x4.m8n8.trans.b16`. +pub fn stmatrixU32x4T(b: *Builder, src: [4]Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.x4.m8n8.shared.b16.trans [$0], {$1, $2, $3, $4};\n", &.{ zext64(b, smem), src[0], src[1], src[2], src[3] }, "l,r,r,r,r"); +} + +/// `ClusterBarrier::arrive()` on this CTA's barrier. +pub fn mbarrierArrive(b: *Builder, bar: Value) void { + asmVoid(b, "{\n\tmbarrier.arrive.shared::cta.b64 _, [$0];\n\t}", &.{bar}, "r,~{memory}"); +} + +/// `ClusterBarrier::arrive(cta_id, pred = true)`: arrive on the barrier at the same offset of +/// cluster CTA `cta_id`. +pub fn mbarrierArriveCluster(b: *Builder, bar: Value, cta_id: Value) void { + asmVoid( + b, + "{\n\t.reg .b32 remAddr32;\n\tmapa.shared::cluster.u32 remAddr32, $0, $1;\n\tmbarrier.arrive.shared::cluster.b64 _, [remAddr32];\n\t}", + &.{ bar, cta_id }, + "r,r,~{memory}", + ); +} + +/// `ClusterTransactionBarrier::arrive_and_expect_tx(bytes, cta_id, pred)`: on cluster CTA +/// `cta_id`'s barrier, only when `pred == 1`. +pub fn mbarrierArriveAndExpectTxCluster(b: *Builder, bar: Value, transaction_bytes: anytype, cta_id: Value, pred: Value) void { + asmVoid( + b, + "{\n\t.reg .pred p;\n\t.reg .b32 remAddr32;\n\tsetp.eq.u32 p, $2, 1;\n\t@p mapa.shared::cluster.u32 remAddr32, $0, $1;\n\t@p mbarrier.arrive.expect_tx.shared::cluster.b64 _, [remAddr32], $3;\n\t}", + &.{ bar, cta_id, pred, liftAs(b, transaction_bytes, .i32) }, + "r,r,r,r,~{memory}", + ); +} + +/// `cute::prefetch_tma_descriptor(desc)` (generic descriptor address). +pub fn prefetchTmaDescriptor(b: *Builder, desc: Value) void { + asmVoid(b, "prefetch.tensormap [$0];", &.{desc}, "l,~{memory}"); +} + +/// `SM100_TMA_2SM_LOAD_2D::copy(desc, mbar, cache_hint, smem, c0, c1)`: issued by both CTAs of +/// the pair; the barrier address is masked so the bytes complete on the leader's barrier. +pub fn tma2SmLoad2d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c0: Value, c1: Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.2d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4}], [$2], $5;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c0, c1, cst64(b, cache_hint) }, + "r,l,r,r,r,l,~{memory}", + ); +} + +/// `SM100_TMA_2SM_LOAD_4D::copy(desc, mbar, cache_hint, smem, c0, c1, c2, c3)`. +pub fn tma2SmLoad4d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c: [4]Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4, $5, $6}], [$2], $7;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c[0], c[1], c[2], c[3], cst64(b, cache_hint) }, + "r,l,r,r,r,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_load_1d(dst, src, mbar, num_bytes, hint = EVICT_FIRST)`: non-tensor bulk copy of +/// `num_bytes` from a global (possibly peer) address into this CTA's shared memory. +pub fn tmaLoad1d(b: *Builder, dst: Value, src: Value, mbar: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1], $2, [$3], $4;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), mbar, cst64(b, cache_hint) }, + "r,l,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_store_1d(dst, src, num_bytes, hint = EVICT_NORMAL)`. +pub fn tmaStore1d(b: *Builder, dst: Value, src: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.global.shared::cta.bulk_group.L2::cache_hint [$0], [$1], $2, $3;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), cst64(b, cache_hint) }, + "l,r,r,l,~{memory}", + ); +} + +/// `cute::SM90_TMA_STORE_2D::copy(desc, smem, c0, c1)`. +pub fn tmaStore2d(b: *Builder, desc: Value, smem: Value, c0: Value, c1: Value) void { + asmVoid(b, "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3}], [$1];", &.{ desc, smem, c0, c1 }, "l,r,r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::allocate(num_columns, dst)`: write the base column to shared `dst`. +pub fn tmemAllocate2Sm(b: *Builder, num_columns: u32, dst: Value) void { + asmVoid(b, "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [$0], $1;", &.{ dst, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::free(tmem_ptr, num_columns)`. +pub fn tmemFree2Sm(b: *Builder, tmem_ptr: Value, num_columns: u32) void { + asmVoid(b, "{\n\ttcgen05.dealloc.cta_group::2.sync.aligned.b32 $0, $1; \n\t}", &.{ tmem_ptr, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `ptx::tcgen05_before_thread_sync`. +pub fn tcgen05BeforeThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::before_thread_sync;", &.{}, "~{memory}"); +} + +/// `ptx::tcgen05_after_thread_sync`. +pub fn tcgen05AfterThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::after_thread_sync;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::fence_view_async_tmem_load` (`tcgen05.wait::ld`). +pub fn fenceViewAsyncTmemLoad(b: *Builder) void { + asmVoid(b, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &.{}, "~{memory}"); +} + +/// `fence_view_async_tmem_load` with the values of earlier TMEM loads threaded through it, so +/// LLVM cannot move their uses above the wait (the PTX is the same single instruction). +pub fn fenceViewAsyncTmemLoadValues(b: *Builder, comptime n: usize, values: [n]Value) [n]Value { + comptime var constraints: []const u8 = ""; + inline for (0..n) |i| constraints = constraints ++ (if (i == 0) "" else ",") ++ "=r"; + inline for (0..n) |i| constraints = constraints ++ std.fmt.comptimePrint(",{d}", .{i}); + constraints = constraints ++ ",~{memory}"; + return asmValues(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &values, .i32, constraints, true); +} + +/// `fenceViewAsyncTmemLoadValues` for a run-time number of values (the epilogue's store blocks +/// hold `STORE_BLOCK_M / ATOM_M * 8` words): the same single `tcgen05.wait::ld`, every value +/// tied through it. +pub fn fenceViewAsyncTmemLoadSlice(b: *Builder, values: []const Value) []Value { + const n = values.len; + const a = b.arena.allocator(); + var w: std.Io.Writer.Allocating = .init(a); + for (0..n) |i| w.writer.print("{s}=r", .{if (i == 0) "" else ","}) catch @panic("OOM"); + for (0..n) |i| w.writer.print(",{d}", .{i}) catch @panic("OOM"); + w.writer.writeAll(",~{memory}") catch @panic("OOM"); + return asmValuesSlice(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", values, .i32, w.written(), true); +} + +/// `cute::SM100_TMEM_LOAD_16dp256b1x::copy(src, d0, d1, d2, d3)`: lane t of the warp gets +/// rows `base + t/4` (d0, d1) and `base + 8 + t/4` (d2, d3), columns `2(t%4)`, `2(t%4)+1`. +pub fn tmemLoad16dp256b1x(b: *Builder, taddr: Value) [4]Value { + return asmValues(b, 4, "tcgen05.ld.sync.aligned.16x256b.x1.b32{$0, $1, $2, $3},[$4];\n", &.{taddr}, .i32, "=r,=r,=r,=r,r", true); +} + +/// `cute::SM100_UTCCP_4x32dp128bit_2cta::copy(src_desc, dst_taddr)`. +pub fn utccp4x32dp128bit2cta(b: *Builder, src_desc: Value, dst_taddr: Value) void { + asmVoid(b, "tcgen05.cp.cta_group::2.32x128b.warpx4 [$0], $1;", &.{ dst_taddr, src_desc }, "r,l"); +} + +/// `ptx::SM100_MMA_MXF8F6F4_2x1SM_SS::fma(desc_a, desc_b, tmem_c, scale_c, desc, tmem_sfa, +/// tmem_sfb)`: `desc` is the 64-bit runtime descriptor (`instr << 32`), `scale_c` (i32) +/// nonzero accumulates. +pub fn mmaMxf8f6f42x1SmSs(b: *Builder, desc_a: Value, desc_b: Value, tmem_c: Value, scale_c: Value, desc: Value, tmem_sfa: Value, tmem_sfb: Value) void { + const idesc = b.emit(arith.trunci(b.ctx, desc.shrLogical(32).inner, DType.i32.toMlir(b.ctx), b.loc())); + asmVoid( + b, + "{\n\t.reg .pred p;\n\tsetp.ne.b32 p, $4, 0;\n\ttcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale [$0], $1, $2, $3, [$5], [$6], p; \n\t}\n", + &.{ tmem_c, desc_a, desc_b, idesc, scale_c, tmem_sfa, tmem_sfb }, + "r,l,l,r,r,r,r", + ); +} + +/// `cutlass::arch::umma_arrive_multicast_2x1SM(barrier, cta_mask)`: inside `elect_one_sync`, +/// `tcgen05.commit` arriving on `barrier` in every CTA of `cta_mask`. +pub fn ummaArriveMulticast2x1Sm(b: *Builder, bar: Value, cta_mask: u16) void { + var elected = b.openIf(electOneSync(b)); + { + asmVoid( + b, + "{\n\ttcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [$0], $1; \n\t}", + &.{ bar, b.constant(.i16, @as(i16, @bitCast(cta_mask))) }, + "r,h,~{memory}", + ); + elected.close(); + } +} + +pub fn bits32(v: Value) Value { + return if (v.dtype() == .i32) v else v.bitCast(.i32); +} + +pub fn fromBits32(v: Value, dt: DType) Value { + return if (dt == .i32) v else v.bitCast(dt); +} + +/// `__shfl_sync(0xffffffff, v, src_lane)` / `ptx::exchange` (32-bit). +pub fn shflSync(b: *Builder, v: Value, src_lane: anytype) Value { + const r = asmValue(b, "shfl.sync.idx.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, src_lane, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_xor_sync(0xffffffff, v, lane_mask)`. +pub fn shflXorSync(b: *Builder, v: Value, lane_mask: anytype) Value { + const r = asmValue(b, "shfl.sync.bfly.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, lane_mask, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_up_sync(0xffffffff, v, delta)`. +pub fn shflUpSync(b: *Builder, v: Value, delta: anytype) Value { + const r = asmValue(b, "shfl.sync.up.b32 $0, $1, $2, 0, -1;", &.{ bits32(v), liftAs(b, delta, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__reduce_add_sync(0xffffffff, v)` (u32). +pub fn reduceAddSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.add.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__reduce_min_sync(0xffffffff, v)` (u32). +pub fn reduceMinSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.min.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__all_sync(0xffffffff, pred)` (`pred` i1; returns i1). +pub fn allSync(b: *Builder, pred: Value) Value { + const r = asmValue(b, "{ .reg .pred p, q; setp.ne.u32 p, $1, 0; vote.sync.all.pred q, p, $2; selp.u32 $0, 1, 0, q; }", &.{ pred.to(.i32), cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); + return r.ne(0); +} + +pub fn popc(b: *Builder, v: Value) Value { + return b.callIntrinsic("llvm.ctpop.i32", &.{v}, DType.i32.toMlir(b.ctx)).?; +} + +/// `__ffs(v)`: 1-based index of the lowest set bit, 0 for 0 (nvcc: `brev` + `bfind.shiftamt`). +pub fn ffs(b: *Builder, v: Value) Value { + return asmValue(b, "{ .reg .b32 r; brev.b32 r, $1; bfind.shiftamt.u32 $0, r; }", &.{v}, .i32, "=r,r", false).add(1); +} + +/// `__fns(mask, base, offset)`: position of the `offset`-th set bit of `mask` from `base`. +pub fn fns(b: *Builder, mask: Value, base: anytype, offset: Value) Value { + return asmValue(b, "fns.b32 $0, $1, $2, $3;", &.{ mask, liftAs(b, base, .i32), offset }, .i32, "=r,r,r,r", false); +} + +/// `fminf(a, b)` (`min.f32`, NaN-ignoring; `kClampAfterSilu`). +pub fn fminf(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "min.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +/// `fabsf(x)` (`abs.f32`). +pub fn fabsf(b: *Builder, x: Value) Value { + return asmValue(b, "abs.f32 $0, $1;", &.{x}, .f32, "=f,f", false); +} + +/// `math::fast_rcp(x)`. +pub fn fastRcp(b: *Builder, x: Value) Value { + return asmValue(b, "rcp.approx.ftz.f32 $0, $1;", &.{x}, .f32, "=f,f", true); +} + +/// `__expf(x)` without `--use_fast_math`: `ex2.approx.f32(x * log2(e))` (non-FTZ). +pub fn expfFast(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0f3FB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__expf(-x)`: nvcc folds the negation into the constant (`mul.f32 t, x, -log2(e)`). +pub fn expfFastNeg(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0fBFB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__float22bfloat162_rn({lo, hi})`: bf16x2 word, `lo` in the low half. +pub fn float22bfloat162Rn(b: *Builder, lo: Value, hi: Value) Value { + return asmValue(b, "{ cvt.rn.bf16x2.f32 $0, $2, $1;}\n", &.{ lo, hi }, .i32, "=r,f,f", false); +} + +/// `__hmin2(a, b)` on bf16x2 words. +pub fn hmin2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ min.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__hmax2(a, b)` on bf16x2 words. +pub fn hmax2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ max.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__bfloat1622float2(word)`: `{low, high}` f32 (exact). +pub fn bfloat1622float2(b: *Builder, word: Value) [2]Value { + return asmValues(b, 2, "{ .reg .b16 l, h; mov.b32 {l, h}, $2; cvt.f32.bf16 $0, l; cvt.f32.bf16 $1, h; }", &.{word}, .f32, "=f,=f,r", false); +} + +/// `ptx::accumulate(float2& a, nv_bfloat162 b)`: `a.x += bf16(b.x)`, `a.y += bf16(b.y)` with +/// `add.rn.f32.bf16` (one rounding, the bf16 widened exactly). +pub fn accumulateBf16x2(b: *Builder, acc: [2]Value, bf16x2: Value) [2]Value { + const lo = i16Of(b, bf16x2); + const hi = i16Of(b, bf16x2.shrLogical(16)); + return .{ + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ lo, acc[0] }, .f32, "=f,h,0", false), + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ hi, acc[1] }, .f32, "=f,h,0", false), + }; +} +/// IEEE operations used when staging quantized inputs; no FTZ or contraction. +pub fn divRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "div.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} +pub fn mulRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "mul.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +test "canonical primitives preserve aligned barriers and runtime transaction counts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "compatibility_primitives"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const bar = b.constant(.i32, 0); + const phase = b.constant(.i32, 0); + const bytes = b.threadIdx().x.add(16); + b.barrierCtaSyncAligned(1, 32); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + b.mbarrierInit(bar, 1); + b.fenceMbarrierInit(); + b.mbarrierArriveExpectTx(bar, bytes); + b.mbarrierArriveExpectTx(bar, 16); + b.mbarrierWait(bar, phase, .{ .test_first = false }); + _ = phase.bitXor(1); + b.fenceProxyAsyncShared(); + b.fenceProxyAsyncShared(); + b.cpAsyncBulkCommitGroup(); + b.cpAsyncBulkWaitGroup(0, false); + _ = b.voteBallotSync(bytes.ne(0), null); + _ = b.fmax(b.constant(.f32, 1), b.constant(.f32, 2), .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(ir, "barrier.cluster.arrive.relaxed.aligned"); + try expectContains(ir, "nvvm.mbarrier.arrive.expect_tx"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} diff --git a/zml/kernel.zig b/zml/kernel.zig index 94d6046a..d85b6a56 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -676,6 +676,7 @@ pub const cute = struct { cfg: ConfigT, zeroed_outputs: []const i32 = &.{}, scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, }; @@ -707,6 +708,7 @@ pub const cute = struct { .ir = ir, .zeroed_outputs = opts.zeroed_outputs, .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, .output_operand_aliases = aliases.constSlice(), }); diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index 14f1b090..bd482093 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -37,7 +37,8 @@ pub fn validateOptions(opts: Options) void { /// is the per-expert multiplier dequantizing each GEMM output (it folds in /// the inverse of the input's global scale), and their input scale is the /// multiplier applied to the GEMM's input before NVFP4 quantization. Both -/// expect block scales in the 128x4 tensor-core layout. +/// expect block scales in the 128x4 tensor-core layout. MXFP4 gate/up rows +/// interleave by 8 and its scale atoms are MN-major packed i32 words. pub fn fusedExperts( input: zml.Tensor, routing: zml.moe.LocalRouting, diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index a2eb1792..f651813f 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -20,11 +20,14 @@ pub const Mxfp4Inputs = struct { x: zml.Tensor, /// `[tokens, topk]` FP32. routing_weights: zml.Tensor, + /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved by 8. w1: zml.Tensor, - /// `w1` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + /// `[experts, hidden / 128, 2 * intermediate]` i32 words of four E8M0 scales, + /// in MN-major 128x4 atoms, with rows interleaved like `w1`. s1: zml.Tensor, + /// `[experts, hidden, intermediate / 2]` packed FP4. w2: zml.Tensor, - /// `w2` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + /// `[experts, intermediate / 128, hidden]` i32 words in MN-major 128x4 atoms. s2: zml.Tensor, /// `[tokens, topk]` local expert ids. ids: zml.Tensor, @@ -208,14 +211,14 @@ pub fn forwardMxfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, t .weight = a.w1, .schedule = routing.schedule, .input_quant = q1.q, - .weight_scale = a.s1.bitCast(.u8), + .weight_scale = a.s1, .input_scale = q1.s, }, routing.route_inverse, a.routing_weights.reshape(.{routes})); const down = mxfp4_persistent.downRows(down_cfg, .{ .weight = a.w2, .schedule = routing.schedule, .input_quant = q2.q, - .weight_scale = a.s2.bitCast(.u8), + .weight_scale = a.s2, .input_scale = q2.s, }, routing.route_inverse); return reduce.topk(down, routing.route_map orelse routing.schedule, .{ diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig index 21a76436..30062a3a 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -550,7 +550,7 @@ fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: .weight = .{ .ptr = cute.DType.i8 }, .schedule = .{ .ptr = cute.DType.i32 }, .input_quant = .{ .ptr = cute.DType.f8e4m3fn }, - .weight_scale = .{ .ptr = cute.DType.i8 }, + .weight_scale = .{ .ptr = cute.DType.i32 }, .input_scale = .{ .ptr = cute.DType.i8 }, }; const shape = .{ @@ -1439,11 +1439,10 @@ fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, return .{ .count = count, .route = route, .weight = weight }; } -/// Python `fused_epilogue`: thread `t` owns output row `128 * tile_m + t` -/// of the interleaved projection, so even threads hold a gate and their odd -/// neighbour the matching up value. Each even thread computes one SwiGLU -/// activation per routed row; 32 consecutive activations span two warps, -/// whose partial maxima meet in shared memory. +/// Thread `t` owns output row `128 * tile_m + t` of the interleave-8 +/// projection. Gate and up lanes are paired by XOR 8. Each gate thread +/// computes one SwiGLU activation per routed row; 32 consecutive activations +/// span two warps, whose partial maxima meet in shared memory. /// /// At decode sizes most rows of a group are padding, whose outputs are never /// read. `count` is uniform across the CTA, so whole 8-row chunks of padding @@ -1470,7 +1469,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const tid = b.threadIdx().x; const lane = tid.rem(32); const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const is_gate = tid.rem(2).eq(0); + const is_gate = tid.rem(16).lt(8); const first_row = group.mul(operands.group_rows); const f32_type = cute.DType.f32.toMlir(b.ctx); // The TMA C staging buffer is unused by this epilogue. @@ -1497,7 +1496,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo if (operands.swiglu_limit) |limit| { gate = gate.minimum(limit); } - var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); + var up_value = b.shuffleXor(value, 8).to(.bf16).to(.f32); if (operands.swiglu_limit) |limit| { up_value = up_value.maximum(-limit).minimum(limit); } @@ -1506,7 +1505,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); - // Odd threads hold no activation and must not raise the maxima. + // Up lanes hold no activation and must not raise the maxima. activation.* = b.select(is_gate, result, b.constant(.f32, 0)); const maximum = b.warpMaxNonNegative(activation.*.abs()); chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); @@ -1540,7 +1539,8 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo // The next tile reuses the scratch maxima. b.barrierCtaSyncAligned(1, 128); - const activation_index = tile_m.mul(64).add(tid.div(2)); + const activation_row = tid.div(16).mul(8).add(tid.rem(8)); + const activation_index = tile_m.mul(64).add(activation_row); for (0..row_chunks) |row_chunk| { const first = row_chunk * swiglu_chunk; const last = @min(first + swiglu_chunk, live); @@ -1830,7 +1830,13 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); - const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); + // Weight scales step M atoms first; activation scales retain K-major order. + const sfa_order = b.makeIntTuple(.{ 0, 1, 2 }); + const sfa_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, 512 }, .{ .{ 0, 1 }, sf_stride }, .{ 0, sf_stride } }, + ); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sfa_order, b.layoutType(sfa_layout)); const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index e30292cf..d041a5ab 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -27,7 +27,7 @@ pub fn fusedExperts( opts: Options, ) Tensor { const expert_map_ = routing_.expertMap(); - const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .interleaved else .concatenated; + const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 32bebac0..33b6da55 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -5,12 +5,16 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; pub const cute = @import("cute/cute.zig"); -const ExpertsLayout = @import("options.zig").ExpertsLayout; +pub const ExpertsLayout = @import("options.zig").ExpertsLayout; pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const Options = @import("options.zig").Options; +pub const Options = @import("options.zig").Options; +pub const Activation = @import("options.zig").Activation; +pub const ActivationKind = @import("options.zig").ActivationKind; +pub const ProjectionLayout = @import("options.zig").ProjectionLayout; +pub const RoutingWeightPlacement = @import("options.zig").RoutingWeightPlacement; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); @@ -103,13 +107,17 @@ pub const Backend = enum { pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { return switch (backend) { .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, - .cute => if (scheme == .mxfp4 or scheme == .nvfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, + .cute => if (scheme) |s| switch (s) { + .mxfp4 => .{ .gate_up = .row_interleaved_8, .packing = .mxfp4_interleaved8 }, + .nvfp4 => .{ .gate_up = .row_interleaved_1, .packing = .swizzled_scales }, + else => error.UnsupportedQuantization, + } else error.UnsupportedQuantization, .flashinfer_cutlass => if (scheme) |s| switch (s) { .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, else => error.UnsupportedQuantization, } else error.UnsupportedQuantization, .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, - .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, + .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .row_interleaved_1, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, }; } diff --git a/zml/kernels/moe/options.zig b/zml/kernels/moe/options.zig index aa68b06a..7f8d26de 100644 --- a/zml/kernels/moe/options.zig +++ b/zml/kernels/moe/options.zig @@ -56,7 +56,14 @@ pub const Activation = union(ActivationKind) { const mid = @divFloor(x.dim(.out), 2); var gate, var up = switch (layout) { .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + .row_interleaved_1 => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + .row_interleaved_8 => blk: { + const grouped = x.splitAxis(.out, .{ .group = @divExact(mid, 8), .pair = 2, .row = 8 }); + break :blk .{ + grouped.slice(.pair, .{ .end = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + grouped.slice(.pair, .{ .start = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + }; + }, }; break :b switch (self) { @@ -96,12 +103,16 @@ pub const Activation = union(ActivationKind) { pub const RoutingWeightPlacement = enum { before_down, after_down }; -pub const ProjectionLayout = enum { concatenated, interleaved }; +pub const ProjectionLayout = enum { + concatenated, // G0 G1 … Gn U0 U1 … Un + row_interleaved_1, // G0 U0 G1 U1 … + row_interleaved_8, // G0…G7 U0…U7 G8…G15 U8…U15 … +}; /// How a backend expects the expert weights to be stored /// The packer writes this layout and forwardMoe reads it pub const ExpertsLayout = struct { - /// Whether the two projections occupy contiguous halves or alternate. + /// Gate/up ordering before applying the backend-specific packing. gate_up: ProjectionLayout, /// Backend specific transformation of the weights and scales. packing: Packing, @@ -113,5 +124,9 @@ pub const ExpertsLayout = struct { swizzled_scales, /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. flashinfer_nvfp4, + /// MXFP4 gate/up rows interleaved by 8. E8M0 scales use 512-byte + /// 128x4 atoms in MN-major order `[expert, k/128, rows/128, 512]`, + /// represented as `[expert, k/128, rows]` i32 words of four scales. + mxfp4_interleaved8, }; }; diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index 0abb3462..f78cd1ef 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -179,20 +179,16 @@ fn makeExpertFixture(c: Case) ExpertFixture { return fixture; } -/// Interleave the gate/up halves of the output axis: [E, 2, N/2, K] -> [E, N/2, 2, K]. -fn interleave(tensor: Tensor) Tensor { - const tags = tensor.shape().tags(); - return tensor.withTags(.{ .expert, .out, .in }) - .splitAxis(.out, .{ .projection = 2, .mid = @divExact(tensor.dim(1), 2) }) - .transpose(.{ .expert, .mid, .projection, .in }) - .merge(.{ .out = .{ .mid, .projection } }).withTags(tags); +fn interleave(tensor: Tensor, rows: i64) Tensor { + return tensor.reshape(.{ .expert = tensor.dim(0), .projection = 2, .row_block = @divExact(tensor.dim(1), 2 * rows), .row = rows, .column = tensor.dim(2) }) + .transpose(.{ .expert, .row_block, .projection, .row, .column }).reshape(tensor.shape()); } /// Blackwell block scales: [E, N/128, 4, 32, K/4, 4] -> /// [E, N/128, K/4, 32, 4, 4], retaining the declared [E, N, K] shape. fn swizzleScales(scales: Tensor) Tensor { - return scales.reshape(.{ scales.dim(0), @divExact(scales.dim(1), 128), 4, 32, @divExact(scales.dim(2), 4), 4 }) - .transpose(.{ 0, 1, 4, 3, 2, 5 }).reshape(scales.shape()); + return scales.reshape(.{ .expert = scales.dim(0), .row_block = @divExact(scales.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(scales.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .row_block, .scale_word, .row_lane, .row_quad, .scale_byte }).reshape(scales.shape()); } fn packFp4Bytes(weight: Tensor) Tensor { @@ -203,7 +199,8 @@ fn packFp4Bytes(weight: Tensor) Tensor { fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { return switch (layout.gate_up) { .concatenated => tensor, - .interleaved => interleave(tensor), + .row_interleaved_1 => interleave(tensor, 1), + .row_interleaved_8 => interleave(tensor, 8), }; } @@ -221,17 +218,29 @@ fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertF } for ([_]*zml.nn.Linear{ &prepared.gate_up, &prepared.down }) |projection| { - if (layout.packing == .swizzled_scales or layout.packing == .flashinfer_nvfp4) { - const q = &projection.quantization.?; - q.scales = swizzleScales(q.scales); - q.swizzled_scales = true; + switch (layout.packing) { + .swizzled_scales, .flashinfer_nvfp4 => { + const q = &projection.quantization.?; + q.scales = swizzleScales(q.scales); + q.swizzled_scales = true; + }, + .mxfp4_interleaved8 => { + const q = &projection.quantization.?; + const sf = q.scales; + q.scales = sf.bitCast(.u8) + .reshape(.{ .expert = sf.dim(0), .row_block = @divExact(sf.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(sf.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .scale_word, .row_block, .row_lane, .row_quad, .scale_byte }) + .reshape(.{ .expert = sf.dim(0), .scale_word = @divExact(sf.dim(2), 4), .row = sf.dim(1), .scale_byte = 4 }).bitCast(.i32) + .withTags(sf.shape().transpose(.{ 0, 2, 1 })); + }, + .plain => {}, } switch (backend) { .cute => { projection.weight = packFp4Bytes(projection.weight); const q = &projection.quantization.?; - q.scales = q.scales.bitCast(.u8); if (scheme == .nvfp4) { + q.scales = q.scales.bitCast(.u8); // CuTe takes the input quantization multiplier and a GEMM // output multiplier that folds in its inverse. const input_scale = q.input_scale.?.asMultiplier(); diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index bae76744..ed6c9f5f 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -52,7 +52,7 @@ pub fn fusedExperts( else => {}, }; - const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .interleaved else .concatenated; + const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; return fusedExpertsBase(input, routing, gate_up, down, layout, options); } @@ -500,7 +500,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); const gates = [_]f32{ 0.75, -1.25, 3.5 }; const ups = [_]f32{ 0.875, -2.25, 4.5 }; - for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { var values: [6]zml.floats.BFloat16 = undefined; for (gates, ups, 0..) |gate, up, i| { values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); @@ -665,7 +665,8 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); const is_gate = switch (layout) { .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), - .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .row_interleaved_1 => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .row_interleaved_8 => columns.divByConst(8).remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), }; const dtype: DataType = if (fp4) .u8 else .bf16; // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. @@ -717,7 +718,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); var input = try zml.Buffer.fromSlice(io, platform, host); defer input.deinit(); - for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); defer exe.deinit(); diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 92e77888..aaa42b20 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); +pub const AttentionMask = @import("options.zig").AttentionMask; pub const AttentionOptions = @import("options.zig").AttentionOptions; pub const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); diff --git a/zml/ops.zig b/zml/ops.zig index 4a189035..a9ae0f8d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1532,7 +1532,15 @@ pub const CuteOps = struct { block: ?[3]i32 = null, zeroed_outputs: []const i32 = &.{}, scalars: []const i64 = &.{}, + symmetric_workspace: ?SymmetricWorkspace = null, output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + + /// Named per-device workspace shared across CuTe calls and executables. + /// Every rank allocates `bytes`; XLA passes peer addresses and the rank. + pub const SymmetricWorkspace = struct { + name: []const u8, + bytes: i64, + }; }; /// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler @@ -1553,7 +1561,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); } - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; attrs.appendSliceAssumeCapacity(&.{ .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), @@ -1578,6 +1586,13 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso for (opts.scalars, scalar_attrs) |value, *attribute| attribute.* = .int(mlir_ctx, .i64, value); attrs.appendAssumeCapacity(.named(mlir_ctx, "scalars", .array(mlir_ctx, scalar_attrs))); } + if (opts.symmetric_workspace) |workspace| { + stdx.debug.assert(workspace.name.len > 0 and workspace.bytes > 0, "zml.ops.cute({s}): symmetric_workspace needs a name and a positive size", .{opts.name}); + attrs.appendAssumeCapacity(.named(mlir_ctx, "symmetric_workspace", .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, workspace.name)), + .named(mlir_ctx, "bytes", .int(mlir_ctx, .i64, workspace.bytes)), + }))); + } const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); const op = stablehlo.custom_call( diff --git a/zml/platform.zig b/zml/platform.zig index f3510a19..6c6081aa 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -860,6 +860,12 @@ pub const cuda = struct { } }; + /// SM count of an addressable CUDA device; unavailable attributes return null. + pub fn coreCount(platform: *const zml.Platform, device_index: usize) ?u32 { + const attribute = platform.devices[device_index].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return null; + return std.math.cast(u32, attribute.int64); + } + pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { if (platform.target != .cuda) return null; const devices = platform.pjrt_client.devices(platform.pjrt_api); From 6ee4f84ea77244c9deb7fd1ffec7b908f25d3425 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 10 Nov 2026 14:24:41 +0100 Subject: [PATCH 31/47] zml: Exe.memoryStats (#1052) --- zml/exe.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zml/exe.zig b/zml/exe.zig index 85baa7e7..32c0ac48 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -387,6 +387,11 @@ pub const Exe = struct { }, } } + + pub fn memoryStats(self: *const Exe) pjrt.ApiError!pjrt.CompiledMemoryStats { + const exe = try self.exe.executable(self.platform.pjrt_api); + return try exe.getCompiledMemoryStats(self.platform.pjrt_api); + } }; /// A typed executable whose input and output buffer structures are derived from From 9f8da93eafd7b924127cda7027266ce7571cccdb Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 10 Nov 2026 18:51:08 +0100 Subject: [PATCH 32/47] zml/kernels/paged_attention/metal: fit decode in 256 threads and split it over the GPU cores (#1046) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes Metal decode on M1/M2: `fa_vec_paged` launched up to 512 threads per threadgroup, over the 384 those GPUs accept. - A threadgroup holds up to 4 query heads of one KV head times position splits, at most 256 threads. - The split count follows the PJRT core count and the number of rows. - Head dims 32 and 64 now use the vector kernel instead of the tiled one. llmd decode on M5 Max vs master: no change at short context, +4–27% at 8K, +12–74% at 24K, bs16 unchanged. Head-dim-64 models: 2–7.6× at long context. Should fix https://github.com/zml/zml/issues/1037 but needs some testing on a real M1/M2 --- zml/kernels/paged_attention/metal/README.md | 12 +- zml/kernels/paged_attention/metal/metal.zig | 88 ++- .../paged_attention/metal/paged_attn.zig | 564 ++++++++---------- 3 files changed, 300 insertions(+), 364 deletions(-) diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md index 5624de0e..8ccc6fd9 100644 --- a/zml/kernels/paged_attention/metal/README.md +++ b/zml/kernels/paged_attention/metal/README.md @@ -10,11 +10,13 @@ transposes queries and caches into the kernels' layouts, launches the the output. Queries and caches are `bf16` or `f16`; other dtypes panic. Causal `bf16` -decode with head dimension 128, 256 or 512 runs `fa_vec_paged`, which is `bf16` -only, split over four threadgroups and merged by `fa_vec_paged_reduce` at head -dimension 512 past 4096 KV positions. Everything else, `f16` decode included, -runs `paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, -pages of 8, 16 or 32 positions), then `zero_padding_rows`. +decode with head dimension 32, 64, 128, 256 or 512 runs `fa_vec_paged`, which +is `bf16` only: a threadgroup holds up to four query heads of a KV head, times +splits of the KV positions, at most 256 threads. Rows split over the positions +are merged by `fa_vec_paged_reduce`; the split count follows the GPU core count +and the number of rows. Everything else, `f16` decode included, runs +`paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, pages of +8, 16 or 32 positions), then `zero_padding_rows`. Causal and non-causal standard masks, left sliding windows, and a custom scale are passed to the kernels. Explicit key indices are rejected, and diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index e8144e3b..2cec68f5 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -85,8 +85,8 @@ const DispatchOptions = struct { scale: f32, }; -/// Launches fa_vec_paged for causal bf16 decode, split over 4 threadgroups at -/// head_dim 512 past 4096 KV positions, and paged_attention_tiled otherwise. +/// Launches fa_vec_paged for causal bf16 decode and paged_attention_tiled +/// otherwise. /// `q` and the result are `[tokens, heads, head_dim]`, the caches /// `[pages, block_size, kv_heads, head_dim]`, `block_table` /// `[seqs, max_blocks]`, `seq_lens` `[seqs]` and `query_start_len` @@ -105,7 +105,7 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table const max_blocks: i32 = @intCast(block_table.dim(1)); const vec_decode = opts.is_causal and total_q_tokens == num_seqs and - (head_dim == 128 or head_dim == 256 or head_dim == 512) and + (head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512) and dtype == .bf16; if (vec_decode) { @@ -128,35 +128,30 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table }; const kv_capacity = max_blocks * block_size; - if (head_dim == 512 and kv_capacity > 4096) { - const nwg = 4; - const nrows = total_q_tokens * num_heads; + const span = if (opts.sliding_window < 0) kv_capacity else @min(kv_capacity, opts.sliding_window + 32); + const nrows = total_q_tokens * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), span, coreCount()); - const partial = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = zml.Shape.init(.{nrows * (head_dim * nwg + 2 * nwg)}, .f32) }, .{ - .cfg = .{ .head_dim = head_dim, .nsg = 8, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = nwg }, - .grid = .{ nwg, @intCast(num_heads), @intCast(total_q_tokens) }, - .threadgroup = .{ 32, 8, 1 }, - .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, 8), - .constants = .{ .args = .of(args) }, - }).dst; + const cfg: paged_attn.FaVecPaged.Cfg = .{ + .head_dim = head_dim, + .block_size = block_size, + .position_stride = num_kv_heads * head_dim, + .heads = launch.heads, + .splits = launch.splits, + }; + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = dst }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; - return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = partial }, .{ .dst = q.shape() }, .{ - .cfg = .{ .nwg = nwg, .head_dim = head_dim }, - .grid = .{ @intCast(nrows), 1, 1 }, - .threadgroup = .{ 32, nwg, 1 }, - .constants = .{ .args = .of(@as(i32, nrows)) }, - }).dst; - } - - var idx: usize = if (kv_capacity <= 1024) 0 else if (kv_capacity <= 2048) 1 else 2; - if (head_dim == 512 and idx > 1) idx = 1; - const nsg = ([_]i32{ 4, 8, 16 })[idx]; - - return paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = q.shape() }, .{ - .cfg = .{ .head_dim = head_dim, .nsg = nsg, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = 1 }, - .grid = .{ 1, @intCast(num_heads), @intCast(total_q_tokens) }, - .threadgroup = .{ 32, @intCast(nsg), 1 }, - .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, nsg), + return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = out, .seq_lens = seq_lens, .cu_seqlens_q = query_start_len }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * paged_attn.FaVecPagedReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, paged_attn.FaVecPagedReduce.simdgroups, 1 }, .constants = .{ .args = .of(args) }, }).dst; } @@ -210,3 +205,36 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, }).out; } + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 2, .splits = 64, .splits_per_threadgroup = 4 }, vecLaunch(256, 2, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 64, .splits_per_threadgroup = 2 }, vecLaunch(512, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 3, .splits = 8, .splits_per_threadgroup = 2 }, vecLaunch(48, 6, 288, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 20429565..337432e7 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -18,25 +18,18 @@ const masked_score = -flt_max / 2.0; const max_half = 65504.0; const simd_width = 32; -fn pad2(x: i32, n: i32) i32 { - return (x + n - 1) & ~(n - 1); -} - -/// fa_vec_paged: decode attention over paged bf16 caches, one threadgroup -/// of `nsg` simdgroups per (position split, query head, query row). +/// fa_vec_paged: bf16 decode over paged caches. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. pub const FaVecPaged = struct { pub const Cfg = struct { - /// The key and value head dimension, DK = DV. + /// The key and value head dimension. head_dim: i32, - /// Simdgroups splitting the KV positions. - nsg: i32, /// The page size. block_size: i32, /// The in-page position stride, n_kv * head_dim elements. position_stride: i32, - /// Position-split threadgroups. Above 1 the kernel writes f32 partials - /// for FaVecPagedReduce. - nwg: i32, + heads: i32, + splits: i32, }; pub const Args: metal.Struct = .{ .name = "fa_vec_paged_args", .fields = &.{ @@ -66,17 +59,10 @@ pub const FaVecPaged = struct { .run = run, }); - /// The threadgroup memory of a launch with `nsg` simdgroups. - pub fn threadgroupMemory(head_dim: i32, nsg: i32) u64 { - const padded = pad2(head_dim, 128); - const bytes = (padded + 4 * 32 + 2 * padded) * nsg * 2; - return @intCast(@divFloor(bytes + 15, 16) * 16); - } - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { switch (cfg.head_dim) { - 128, 256, 512 => {}, - else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (128, 256 or 512)", .{cfg.head_dim}), + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), } const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; @@ -89,158 +75,90 @@ pub const FaVecPaged = struct { .seq_lens = int_in, .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, - .shmem_f16 = .{ .threadgroup = .half }, .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, .tgpg = .{ .threadgroups_per_grid = .uint3 }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, }); impl(b, cfg, a); } - /// The attention body. fn impl(b: *Builder, cfg: Cfg, a: anytype) void { - const DK = cfg.head_dim; - const DV = cfg.head_dim; - const NE = 1; - const C = 32; - const NSG = cfg.nsg; + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; const BS = cfg.block_size; const PS = cfg.position_stride; - const NWG = cfg.nwg; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + const num_heads = a.args.field("num_heads"); + const group = num_heads.div(a.args.field("n_kv")); + const groups = group.div(HG); const r = a.tgpig.at(2).to(.int); - const iq2 = a.tgpig.at(1).to(.ushort); - const iwg = a.tgpig.at(0).to(.short); - - const DK4 = @divExact(DK, 4); - const DV4 = @divExact(DV, 4); - - const nrows_static = a.tgpg.at(2).to(.long).mul(a.args.field("num_heads").to(.uint)); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + // Padding rows: FaVecPagedReduce zeroes them when there are splits. var padding = b.openIf(r.ge(a.cu_seqlens_q.load(a.args.field("num_seqs")))); { - var first = b.openIf(a.sgitg.eq(0)); - { - const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); - if (NWG == 1) { - const dst4 = a.dst.to(.bfloat4); - var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); - { - const zero = b.constant(.float4, 0).to(.bfloat4); - dst4.store(rid.mul(DV4).add(loop.iv), zero); - loop.close(); - } - } else { - const dst4 = a.dst.to(.float4); - const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); - var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); - { - dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), b.constant(.float4, 0)); - loop.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), 0.0); - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), -flt_max / 2.0); - lane0.close(); - } + if (NWG == 1) { + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)); + var loop = b.openFor(a.tiisg.to(.short), D4, simd_width); + { + dst4.store(loop.iv, b.constant(.float4, 0).to(.bfloat4)); + loop.close(); } - first.close(); } b.return_(); padding.close(); } - const first_seq = b.local(.int, 0); - const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); - var search = b.openLoop(.{}); - { - search.body(first_seq.lt(last_seq)); - } - { - const mid = first_seq.add(last_seq).add(1).div(2); - var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + const seq = sequenceOf(b, a, r); + const kv_len = seq.kv_len; + const kv_lo = seq.kv_lo; + const ic0_lo = kv_lo.div(C); + const live = if (NWG > 1) liveSplits(b, seq, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); { - first_seq.assign(mid); - half.elseBranch(); + b.return_(); + idle.close(); } - { - last_seq.assign(mid.sub(1)); - half.close(); - } - search.close(); } - const seq_idx = first_seq.rvalue(); - const q_pos = r.sub(a.cu_seqlens_q.load(seq_idx)); - const query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(a.cu_seqlens_q.load(seq_idx)); - const kv_total = a.seq_lens.load(seq_idx); - const kv_len = kv_total.sub(query_len).add(q_pos).add(1); - const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); - const pt = a.block_tables.add(seq_idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + const pt = a.block_tables.add(seq.idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + const k_cache = a.k_cache.add(ikv.to(.ulong).mul(D).mul(2)); + const v_cache = a.v_cache.add(ikv.to(.ulong).mul(D).mul(2)); - const PK = pad2(DK, 128); - const PK4 = @divExact(PK, 4); - const PV = pad2(DV, 128); - const PV4 = @divExact(PV, 4); - const NW = simd_width; - const NL = @divExact(NW, NE); - const SH = 4 * C; - - const shmem = a.shmem_f16; - const sq4 = shmem.add(0 * PK).to(.half4); - const ss = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float); - const ss4 = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float4); - const sm = shmem.add(a.sgitg.mul(SH)).add(2 * C).add(NSG * PK); - const so4 = shmem.add(b.lift(2).mul(a.sgitg).mul(PV)).add(NSG * PK).add(NSG * SH).to(.float4); - // This lane's output, until the simdgroups merge theirs. - const lane_so4 = so4.add(a.tiisg); - - const q = a.q.add(r.to(.ulong).mul(a.args.field("num_heads").to(.uint)).add(iq2).mul(DK).mul(2)); - const ikv2 = iq2.div(a.args.field("num_heads").div(a.args.field("n_kv"))).to(.short); - const k_cache = a.k_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); - const v_cache = a.v_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); - - const q4 = q.to(.bfloat4); - var load_q = b.openFor(a.tiisg.to(.short), PK4, NW); + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); { - const i = load_q.iv; - var value = b.openIfElse(i.lt(DK4), .{.half4}); - { - value.yieldThen(.{q4.load(i).to(.half4)}); - } - { - value.yieldElse(.{b.constant(.half4, 0)}); - } - sq4.store(i, value.results[0]); - load_q.close(); - } - var zero_o = b.openFor(b.constant(.short, 0), @divExact(DV4, NL), 1); - { - lane_so4.store(zero_o.iv.mul(NL), b.constant(.float4, 0)); - zero_o.close(); - } - var zero_s = b.openFor(a.tiisg.to(.short), SH / 4, NW); - { - ss4.store(zero_s.iv, b.constant(.float4, 0)); - zero_s.close(); + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); } - b.threadgroupBarrier(.mem_threadgroup); - - // When DV4/NL is a multiple of NW every lane updates the output, so the - // `ty == 0` guard is dropped at comptime. - const all_lanes = @rem(@divExact(DV4, NL), NW) == 0; - const full: metal.LoopOpts = .{ .unroll = .full }; const S = b.local(.float, 0.0); const M = b.local(.float, -flt_max / 2.0); - const tx = a.tiisg.rem(NL).to(.short); - const ty = a.tiisg.div(NL).to(.short); - - const ic0_lo = kv_lo.div(C); - const ic0 = b.local(.int, ic0_lo.add(iwg.mul(NSG)).add(a.sgitg)); + const ic0 = b.local(.int, ic0_lo.add(iwg)); var blocks = b.openLoop(.{}); { const ic = ic0.mul(C); @@ -250,175 +168,103 @@ pub const FaVecPaged = struct { past.close(); } - const in_kv = ic.add(a.tiisg).lt(kv_len).bitAnd(ic.add(a.tiisg).ge(kv_lo)); - sm.store(a.tiisg, b.ternary(in_kv, b.constant(.half, 0), b.constant(.half, -max_half))); - - const pq4 = sq4.add(tx); - const mqk = b.zeroedArray(.float, C / NE); - var qk = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); { const cc = qk.iv; - const p = b.min(ic.add(cc), kv_len.sub(1)); - const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); - var dots = b.openForOpts(b.constant(.short, 0), @divExact(DK4, NL), 1, full); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); { const ii = dots.iv; - const dot = b.dot(pk4.load(ii.mul(NL)).to(.float4), pq4.load(ii.mul(NL)).to(.float4)); - mqk.store(cc, mqk.load(cc).add(dot)); + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); dots.close(); } - mqk.store(cc, b.simdSum(mqk.load(cc))); + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } qk.close(); } - const score = b.fma(mqk.load(tx), a.args.field("scale"), sm.load(b.lift(NE).mul(tx).add(ty)).to(.float)); - ss.store(b.lift(NE).mul(tx).add(ty), score); - - b.simdgroupBarrier(.mem_threadgroup); + const pos = ic.add(a.tiisg.to(.int)); + const in_kv = pos.lt(kv_len).bitAnd(pos.ge(kv_lo)); + const score = b.ternary(in_kv, s.mul(a.args.field("scale")), b.constant(.float, -max_half)); const m = M.rvalue(); - const s = ss.load(a.tiisg); - - M.assign(b.simdMax(b.max(M.rvalue(), s))); - + M.assign(b.simdMax(b.max(M.rvalue(), score))); const ms = b.exp(m.sub(M)); - const vs = b.exp(s.sub(M)); - + const vs = b.exp(score.sub(M)); S.assign(S.mul(ms).add(b.simdSum(vs))); - ss.store(a.tiisg, vs); - - var rescale_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); - var rescale = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); { - const at = rescale.iv.mul(NL); - lane_so4.store(at, lane_so4.load(at).mul(ms)); + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); rescale.close(); } - if (rescale_lanes) |*l| l.close(); - b.simdgroupBarrier(.mem_threadgroup); - - const lo = b.array(.float4, @intCast(@divExact(DV4, NL))); - var zero_lo = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); - { - lo.store(zero_lo.iv, b.constant(.float4, 0)); - zero_lo.close(); - } - - const sst = ss.add(ty); - - var pv = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); { const cc = pv.iv; - const p = b.min(ic.add(cc), kv_len.sub(1)); - const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); - var dims = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); { const ii = dims.iv; - const weighted = pv4.load(ii.mul(NL)).to(.float4).mul(b.vec(.float4, .{sst.load(cc.mul(NE))})); - lo.store(ii, lo.load(ii).add(weighted)); + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); dims.close(); } pv.close(); } - var add_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); - var add_o = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); - { - const ii = add_o.iv; - const partial = lo.load(ii); - lane_so4.store(ii.mul(NL), lane_so4.load(ii.mul(NL)).add(partial)); - add_o.close(); - } - if (add_lanes) |*l| l.close(); blocks.next(); - ic0.assign(ic0.add(NWG * NSG)); + ic0.assign(ic0.add(live)); blocks.close(); } - var stats = b.openIf(a.tiisg.eq(0)); - { - ss.store(0, S); - ss.store(1, M); - stats.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const rr = b.local(.short, @divFloor(NSG, 2)); - var halving = b.openLoop(.{}); - { - halving.body(rr.gt(0)); - } - { - var group = b.openIf(a.sgitg.lt(rr)); + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); { - const S0 = ss.load(0); - const S1 = ss.load(rr.mul(@divExact(SH, 2)).add(0)); - - const M0 = ss.load(1); - const M1 = ss.load(rr.mul(@divExact(SH, 2)).add(1)); - - const M_ = b.max(M0, M1); - - const ms0 = b.exp(M0.sub(M_)); - const ms1 = b.exp(M1.sub(M_)); - - const S_ = S0.mul(ms0).add(S1.mul(ms1)); - - var lane0 = b.openIf(a.tiisg.eq(0)); - { - ss.store(0, S_); - ss.store(1, M_); - lane0.close(); - } - - var merge = b.openFor(a.tiisg.to(.short), DV4, NW); - { - const i = merge.iv; - so4.store(i, so4.load(i).mul(ms0).add(so4.load(i.add(rr.mul(PV4))).mul(ms1))); - merge.close(); - } - group.close(); + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); } - b.threadgroupBarrier(.mem_threadgroup); - rr.assign(rr.shr(1)); - halving.close(); } - - var first = b.openIf(a.sgitg.eq(0)); - { - const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); - if (NWG == 1) { - const inv_sum = b.ternary(ss.load(0).eq(0.0), 0.0, b.lift(1.0).div(ss.load(0))); - const dst4 = a.dst.to(.bfloat4); - var loop = b.openFor(a.tiisg.to(.short), DV4, NW); - { - const out = so4.load(loop.iv).mul(inv_sum).to(.bfloat4); - dst4.store(rid.mul(DV4).add(loop.iv), out); - loop.close(); - } - } else { - const dst4 = a.dst.to(.float4); - const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); - var loop = b.openFor(a.tiisg.to(.short), DV4, NW); - { - const part = so4.load(loop.iv); - dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), part); - loop.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - const sum = ss.load(0); - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), sum); - const max = ss.load(1); - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), max); - lane0.close(); - } + var first = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); } - first.close(); } + if (first) |*scope| scope.close(); } /// The address of position `p`'s bf16 row in `cache`, through the page @@ -430,85 +276,144 @@ pub const FaVecPaged = struct { } }; -/// fa_vec_paged_reduce: merges the `nwg` partials of a split-K fa_vec_paged, -/// one threadgroup per row and head. +/// The sequence of query row `r` and its causal KV range [kv_lo, kv_len). +fn sequenceOf(b: *Builder, a: anytype, r: Value) struct { idx: Value, kv_len: Value, kv_lo: Value } { + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); + var search = b.openLoop(.{}); + { + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + const idx = first_seq.rvalue(); + const q_pos = r.sub(a.cu_seqlens_q.load(idx)); + const query_len = a.cu_seqlens_q.load(idx.add(1)).sub(a.cu_seqlens_q.load(idx)); + const kv_len = a.seq_lens.load(idx).sub(query_len).add(q_pos).add(1); + const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); + return .{ .idx = idx, .kv_len = kv_len, .kv_lo = kv_lo }; +} + +fn liveSplits(b: *Builder, seq: anytype, splits: i32) Value { + const chunks = seq.kv_len.add(simd_width - 1).div(simd_width).sub(seq.kv_lo.div(simd_width)); + return b.min(splits, chunks); +} + +/// fa_vec_paged_reduce: merges the fa_vec_paged split partials. pub const FaVecPagedReduce = struct { pub const Cfg = struct { - /// The partials to merge, FaVecPaged's `nwg`. - nwg: i32, + splits: i32, /// The head dimension. head_dim: i32, }; - pub const Args: metal.Struct = .{ .name = "fa_vec_paged_reduce_args", .fields = &.{ - .{ .name = "nrows", .type = .int }, - } }; + pub const simdgroups = 8; pub const Kernel = metal.Kernel(Cfg, .{ .name = "fa_vec_paged_reduce", - .inputs = &.{"htmp"}, + .inputs = &.{ "htmp", "seq_lens", "cu_seqlens_q" }, .outputs = &.{"dst"}, .constants = &.{"args"}, .run = run, }); fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; const a = try b.declareArgs(.{ - .args = .{ .constant_struct = Args }, + .args = .{ .constant_struct = FaVecPaged.Args }, .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .seq_lens = int_in, + .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, }); - const nwg = cfg.nwg; + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_paged_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); const DV = cfg.head_dim; - const rid = a.tgpig.to(.ulong); - const iwg = a.tiisg.to(.short); - const ss = a.htmp.to(.float).add(a.args.field("nrows").to(.ulong).mul(DV).mul(nwg)); + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; - var max = b.openIfElse(iwg.lt(nwg), .{.float}); - { - max.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(1))}); - } - { - max.yieldElse(.{-flt_max / 2.0}); - } - const M = max.results[0]; - var sum = b.openIfElse(iwg.lt(nwg), .{.float}); - { - sum.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(0))}); - } - { - sum.yieldElse(.{0.0}); - } - - const m = b.simdMax(M); - const ms = b.exp(M.sub(m)); - const S = b.simdSum(sum.results[0].mul(ms)); - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - - const DV4 = @divTrunc(DV, 4); - const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(nwg)); + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); - var loop = b.openFor(a.sgitg.to(.int), DV4, nwg); + + const live = b.local(.int, 0); + var real = b.openIf(r.lt(a.cu_seqlens_q.load(a.args.field("num_seqs")))); { - const i = loop.iv; - var part_ = b.openIfElse(iwg.lt(nwg), .{.float4}); + live.assign(liveSplits(b, sequenceOf(b, a, r), NWG)); + real.close(); + } + var empty = b.openIf(live.le(0)); + { + var lane0 = b.openIf(a.tiisg.eq(0)); { - part_.yieldThen(.{htmp4.load(i.mul(nwg).add(iwg)).mul(ms)}); - } - { - part_.yieldElse(.{b.constant(.float4, 0)}); - } - const part = part_.results[0]; - const v = b.simdSum(part); - var lane0 = b.openIf(iwg.eq(0)); - { - dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + dst4.store(i, b.constant(.float4, 0).to(.bfloat4)); lane0.close(); } - loop.close(); + b.return_(); + empty.close(); + } + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); } } }; @@ -984,13 +889,14 @@ pub const PagedAttentionTiled = struct { test "the paged attention kernels emit verified modules" { const allocator = std.testing.allocator; - for ([_]i32{ 128, 256, 512 }) |head_dim| { - for ([_]i32{ 4, 8, 16 }) |nsg| { - allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .nsg = nsg, .block_size = 16, .position_stride = 2 * head_dim, .nwg = 1 })); + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .block_size = 16, .position_stride = 2 * head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); } } - allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = 512, .nsg = 8, .block_size = 16, .position_stride = 1024, .nwg = 4 })); - allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .nwg = 4, .head_dim = 512 })); for ([_]Type{ .half, .bfloat }) |dtype| { allocator.free(try ZeroPaddingRows.Kernel.emit(allocator, .{ .dtype = dtype })); for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { From 2b604ebfc64eccbcbf3703674b852f2ab9b82e78 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 10 Nov 2026 18:51:08 +0100 Subject: [PATCH 33/47] zml/kernels/flash_attention/metal: port the kernels to the Metal DSL (#1054) Replaces the `zml$flash_attn` custom call with Metal DSL kernels over contiguous K/V: - `fa_vec` + `fa_vec_reduce` for bf16 decode, at most 256 threads per threadgroup. - `fa_tiled` for prefill and f16. - Batched queries work (the old path panicked). Other dtypes and head dims fall back to sdpa. Bumps the plugins to [202611101651.363.1.d46c7471c419](https://github.com/zml/xla-private/tree/202611101651.363.1.d46c7471c419) --- platforms/cpu/cpu.bzl | 6 +- platforms/cuda/cuda.bzl | 8 +- platforms/furiosa/furiosa.bzl | 4 +- platforms/metal/metal.bzl | 4 +- platforms/rocm/rocm.bzl | 4 +- zml/kernels/flash_attention/metal/BUILD.bazel | 5 +- .../flash_attention/metal/flash_attn.zig | 733 ++++++++++++++++++ zml/kernels/flash_attention/metal/metal.zig | 129 ++- zml/kernels/flash_attention/tests.zig | 16 + 9 files changed, 883 insertions(+), 26 deletions(-) create mode 100644 zml/kernels/flash_attention/metal/flash_attn.zig diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index fd070b6e..08817afe 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 24718ee6..8482e52a 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "1ae8a7c8fc677e0f10c0032ce9ae35e5b6c3f5eadc3f54e1bbda63c219b0170d", - "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-amd64.tar.zst", + "sha256": "72b1359384ac50e727b800e81c56d7134f9d6a061a982c4e36b7fef62cf7c331", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "14a3567c629a1ea97f1dcfa2cb47b54ea08d8c4b9e35eae47a7eff33cfb53ce0", - "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-arm64.tar.zst", + "sha256": "caeefd717960fa2acec6304cd8296299e262abdba669ecf4d487e3af76d73a7b", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index ba53bfb4..81518d53 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-furiosa-linux-amd64.tar.zst", - sha256 = "514b1c96a2237380a9b9e011f5b994c6617eb103a6a9dac8ff0a7f1832d1a3bd", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-furiosa-linux-amd64.tar.zst", + sha256 = "6da755570140c9f8959ba477fef3df4c48b0619274c97429d204e177b8649e12", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 767ef2f6..71e1858a 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "bdd2de3ec8293faa25b358cdf35ee3e75d69be73a4acea87b6e007a18b8f10e8", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-metal-darwin-arm64.tar.zst", + sha256 = "89881d1a1db336e3337a9373db2c4e94f71ddfb77619dee75c0c12e2903913ae", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 454bf626..41b01b60 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "02b86fa3a6599f6aa2b89806b8f6c0b16f11765ec04c94517ec1463a59d92175" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "cdc4434f2023842f0e1a34c08aaed87f082df19f92c4ae21c9d4452a9739b604" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel index 855bdcca..f82ade45 100644 --- a/zml/kernels/flash_attention/metal/BUILD.bazel +++ b/zml/kernels/flash_attention/metal/BUILD.bazel @@ -2,5 +2,8 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "metal", - srcs = ["metal.zig"], + srcs = [ + "flash_attn.zig", + "metal.zig", + ], ) diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig new file mode 100644 index 00000000..4faa1e82 --- /dev/null +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -0,0 +1,733 @@ +//! Metal flash attention over contiguous `[batch, keys, kv_heads, head_dim]` +//! K/V, causal from `token_index`: fa_vec and fa_vec_reduce for decode, +//! fa_tiled otherwise. + +const std = @import("std"); + +const kernel = @import("../../../kernel.zig"); + +const metal = kernel.metal; +const ArgSpec = metal.ArgSpec; +const Builder = metal.Builder; +const Type = metal.Type; +const Value = metal.Value; + +const flt_max = std.math.floatMax(f32); +/// A masked score: below any real one yet finite, since fast math assumes +/// no infinities. Halved so subtracting a running maximum cannot overflow. +const masked_score = -flt_max / 2.0; +const max_half = 65504.0; +const simd_width = 32; + +/// fa_vec: bf16 decode, one query per batch row. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. +pub const FaVec = struct { + pub const Cfg = struct { + head_dim: i32, + heads: i32, + splits: i32, + }; + + pub const Args: metal.Struct = .{ .name = "fa_vec_args", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "n_kv", .type = .int }, + .{ .name = "k_len", .type = .int }, + .{ .name = "scale", .type = .float }, + } }; + + pub const HostArgs = extern struct { + num_heads: i32, + n_kv: i32, + k_len: i32, + scale: f32, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec", + .inputs = &.{ "q", "k", "v", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.head_dim) { + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), + } + const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .q = char_in, + .k = char_in, + .v = char_in, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, + }); + impl(b, cfg, a); + } + + fn impl(b: *Builder, cfg: Cfg, a: anytype) void { + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + + const num_heads = a.args.field("num_heads"); + const n_kv = a.args.field("n_kv"); + const group = num_heads.div(n_kv); + const groups = group.div(HG); + const r = a.tgpig.at(2).to(.int); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + + const kv_len = a.token_index.load(r).add(1); + const live = if (NWG > 1) liveSplits(b, kv_len, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); + { + b.return_(); + idle.close(); + } + } + + const position_bytes = n_kv.to(.ulong).mul(D * 2); + const first = r.to(.ulong).mul(a.args.field("k_len").to(.ulong)).mul(position_bytes).add(ikv.to(.ulong).mul(D * 2)); + const k0 = a.k.add(first); + const v0 = a.v.add(first); + + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); + } + + const S = b.local(.float, 0.0); + const M = b.local(.float, -flt_max / 2.0); + + const ic0 = b.local(.int, iwg); + var blocks = b.openLoop(.{}); + { + const ic = ic0.mul(C); + var past = b.openIf(ic.ge(kv_len)); + { + blocks.breakLoop(); + past.close(); + } + + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = qk.iv; + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = k0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dots.iv; + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); + dots.close(); + } + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } + qk.close(); + } + + const pos = ic.add(a.tiisg.to(.int)); + const score = b.ternary(pos.lt(kv_len), s.mul(a.args.field("scale")), b.constant(.float, -max_half)); + + const m = M.rvalue(); + M.assign(b.simdMax(b.max(M.rvalue(), score))); + const ms = b.exp(m.sub(M)); + const vs = b.exp(score.sub(M)); + S.assign(S.mul(ms).add(b.simdSum(vs))); + + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); + rescale.close(); + } + + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = pv.iv; + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = v0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dims.iv; + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); + dims.close(); + } + pv.close(); + } + + blocks.next(); + ic0.assign(ic0.add(live)); + blocks.close(); + } + + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); + } + } + var first_row = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); + } + } + if (first_row) |*scope| scope.close(); + } +}; + +fn liveSplits(b: *Builder, kv_len: Value, splits: i32) Value { + return b.min(splits, kv_len.add(simd_width - 1).div(simd_width)); +} + +/// fa_vec_reduce: merges the fa_vec split partials. +pub const FaVecReduce = struct { + pub const Cfg = struct { + splits: i32, + head_dim: i32, + }; + + pub const simdgroups = 8; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_reduce", + .inputs = &.{ "htmp", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = FaVec.Args }, + .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + }); + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); + const DV = cfg.head_dim; + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; + + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + const live = liveSplits(b, a.token_index.load(r).add(1), NWG); + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + } +}; + +/// fa_tiled: causal FlashAttention-2 over simdgroup matrices. Queries past +/// `live` output zeros. +pub const FaTiled = struct { + pub const Cfg = struct { + dtype: Type, + head_size: i32, + bq: i32 = 32, + tile_kv: i32 = 32, + num_threads: i32 = 128, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_tiled", + .inputs = &.{ "q", "k", "v", "token_index", "live" }, + .outputs = &.{"out"}, + .constants = &.{ "num_kv_heads", "scale", "q_len", "k_len" }, + .run = run, + }); + + fn supported(cfg: Cfg) bool { + const tile = tileConfig(cfg.head_size) orelse return false; + return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and + cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; + } + + pub fn tileConfig(head_size: i64) ?[3]i32 { + return switch (head_size) { + 32, 64, 96, 128 => .{ 32, 32, 128 }, + 256 => .{ 16, 16, 64 }, + 512 => .{ 8, 8, 32 }, + else => null, + }; + } + + pub fn threadgroupMemory(cfg: Cfg) u64 { + const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); + return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); + } + + /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. + fn fragCoord(b: *Builder, lane_id: Value) Value { + const qid = lane_id.to(.short).div(4).to(.short); + const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); + const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); + return b.vec(.short2, .{ fn_, fm }); + } + + fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { + var x = v.rvalue(); + inline for (.{ 1, 8 }) |mask| { + const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); + x = switch (op) { + .max => b.max(x, y), + .sum => x.add(y), + }; + } + return x; + } + + fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { + p.store(i, p.load(i).insert(lane, x)); + } + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + if (!supported(cfg)) std.debug.panic("fa_tiled: {s} at head size {d} and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); + const T = cfg.dtype; + const HEAD_SIZE = cfg.head_size; + const BQ = cfg.bq; + const TILE_KV = cfg.tile_kv; + const NUM_THREADS = cfg.num_threads; + const int_arg: ArgSpec = .{ .constant = .int }; + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = T } }, + .q = .{ .buffer = .{ .type = T, .const_ = true } }, + .k = .{ .buffer = .{ .type = T, .const_ = true } }, + .v = .{ .buffer = .{ .type = T, .const_ = true } }, + .token_index = int_in, + .live = int_in, + .num_kv_heads = int_arg, + .scale = .{ .constant = .float }, + .q_len = int_arg, + .k_len = int_arg, + .shared_mem = .{ .threadgroup = .char }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + + const NUM_SIMD_LANES = 32; + const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); + const TD = @divExact(HEAD_SIZE, 8); + const TK = @divExact(TILE_KV, 8); + if (@divExact(BQ, NUM_SG) != 8) @panic("fa_tiled: ROWS_PER_SG must be 8"); + const unroll: metal.LoopOpts = .{ .unroll = .enable }; + + const thread_idx = a.tpt.x().to(.int); + const head_idx = a.tgp.x().to(.int); + const q_pos_start = a.tgp.y().to(.int).mul(BQ); + const batch = a.tgp.z().to(.int); + const num_heads = a.tgpg.x().to(.int); + const kv_head_idx = head_idx.div(num_heads.div(a.num_kv_heads)); + const q_stride = num_heads.mul(HEAD_SIZE); + const kv_token_stride = a.num_kv_heads.mul(HEAD_SIZE); + + const block_rows = b.min(BQ, a.q_len.sub(q_pos_start)); + const live = a.live.load(0); + const valid_q = b.min(block_rows, b.max(0, live.sub(q_pos_start))); + const out_base = a.out.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + + var dead = b.openIf(valid_q.le(0)); + { + var zero = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = zero.iv; + out_base.store(i.div(HEAD_SIZE).mul(q_stride).add(i.rem(HEAD_SIZE)), b.constant(T, 0)); + zero.close(); + } + b.return_(); + dead.close(); + } + + // Query i of the batch row is at position context_len + i. + const context_len = a.token_index.load(batch); + const seq_len = context_len.add(live); + const kv_base = batch.to(.long).mul(a.k_len).mul(kv_token_stride).add(kv_head_idx.mul(HEAD_SIZE)); + + const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); + const LD = HEAD_SIZE + SMEM_PAD; + const Q_ELEMS = BQ * LD; + const KV_ELEMS = TILE_KV * LD; + + const Q_smem = a.shared_mem.to(T); + const K_smem = Q_smem.add(Q_ELEMS); + const V_smem = K_smem.add(KV_ELEMS); + + const log2e: f32 = std.math.log2e; + const scale_log2 = a.scale.mul(log2e); + + const q_base = a.q.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); + { + const i = load_q.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + var value = b.openIfElse(r.lt(valid_q), .{T}); + { + value.yieldThen(.{q_base.load(r.mul(q_stride).add(d))}); + } + { + value.yieldElse(.{b.constant(T, 0)}); + } + Q_smem.store(r.mul(LD).add(d), value.results[0]); + load_q.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const T2: Type = .vec(T.scalar, 2); + const q_reg = b.array(T2, @intCast(TD)); + var load_q_reg = b.openForOpts(0, TD, 1, unroll); + { + const d = load_q_reg.iv; + const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + q_reg.store(d, tmp.threadElements()); + load_q_reg.close(); + } + + const fc = fragCoord(b, a.lane.to(.ushort)); + const fm = fc.y(); + const fn_ = fc.x(); + + const max_score = b.local(.float, masked_score); + const sum_score = b.local(.float, 0.0); + + const o_reg = b.array(.float2, @intCast(TD)); + var zero_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(zero_o.iv, b.constant(.float2, 0)); + zero_o.close(); + } + + const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); + const last_q_pos = context_len.add(q_pos_start).add(valid_q).sub(1); + + var tiles = b.openFor(0, num_kv_tiles, 1); + { + const tile_idx = tiles.iv; + const tile_start = tile_idx.mul(TILE_KV); + + var past = b.openIf(tile_start.gt(last_q_pos)); + { + tiles.breakLoop(); + past.close(); + } + + const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; + const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); + const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; + std.debug.assert(@rem(LD, VEC) == 0); + var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); + { + const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); + const kv_pos = tile_start.add(t); + var present = b.openIfElse(kv_pos.lt(seq_len), .{}); + { + const off = kv_base.add(kv_pos.to(.long).mul(kv_token_stride)); + const k_ptr = a.k.add(off); + const v_ptr = a.v.add(off); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k_ptr.add(d).to(LoadUnit).load(0)); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v_ptr.add(d).to(LoadUnit).load(0)); + loop.close(); + } + present.elseBranch(); + } + { + const zero = b.constant(LoadUnit, 0); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + loop.close(); + } + present.close(); + } + rows.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const s_reg = b.array(.float2, @intCast(TK)); + var zero_s = b.openForOpts(0, TK, 1, unroll); + { + s_reg.store(zero_s.iv, b.constant(.float2, 0)); + zero_s.close(); + } + + var qk_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = qk_dims.iv; + const q_frag = b.matrix(T.scalar, q_reg.load(d)); + var qk_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = qk_keys.iv; + const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); + const s_frag = b.matrix(.float, s_reg.load(k)); + s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); + qk_keys.close(); + } + qk_dims.close(); + } + + const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); + const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); + const tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(tile_start.add(TILE_KV).sub(1).lt(context_len.add(q_pos_start))); + + var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); + { + var scale = b.openForOpts(0, TK, 1, unroll); + { + const k = scale.iv; + setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); + setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); + scale.close(); + } + masking.elseBranch(); + } + { + var keys = b.openForOpts(0, TK, 1, unroll); + { + const k = keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj).mul(scale_log2); + const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); + const masked = row_masked.bitOr(kv_pos.ge(seq_len)).bitOr(kv_pos.gt(q_abs_pos)); + setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); + pair.close(); + } + keys.close(); + } + masking.close(); + } + + const local_max = b.local(.float, masked_score); + var max_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = max_keys.iv; + local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); + max_keys.close(); + } + const row_max = fragRowReduce(b, .max, local_max); + + const prev_max = max_score.rvalue(); + const grown_max = b.max(prev_max, row_max); + const grew = grown_max.gt(prev_max); + const was_empty = prev_max.eq(masked_score); + const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); + const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); + // A tile masking a whole row leaves the maximum masked: zero would + // pin it, and later scores far below zero would underflow. + max_score.assign(grown_max); + + const local_sum = b.local(.float, 0.0); + var p_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = p_keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj); + const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); + setLane(s_reg, k, jj, p); + local_sum.assign(local_sum.add(p)); + pair.close(); + } + p_keys.close(); + } + const row_sum = fragRowReduce(b, .sum, local_sum); + + sum_score.assign(sum_score.mul(factor).add(row_sum)); + + var rescale_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); + rescale_o.close(); + } + + var pv_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = pv_keys.iv; + const p_frag = b.matrix(.float, s_reg.load(k)); + var pv_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = pv_dims.iv; + const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + const o_frag = b.matrix(.float, o_reg.load(d)); + o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); + pv_dims.close(); + } + pv_keys.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + tiles.close(); + } + + const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); + var normalize = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); + normalize.close(); + } + + const O_smem = Q_smem.to(.float); + var store_o = b.openForOpts(0, TD, 1, unroll); + { + const d = store_o.iv; + const o_frag = b.matrix(.float, o_reg.load(d)); + b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + store_o.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + var write_out = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = write_out.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + const value = b.ternary(r.lt(valid_q), O_smem.load(r.mul(LD).add(d)).to(T), b.constant(T, 0)); + out_base.store(r.mul(q_stride).add(d), value); + write_out.close(); + } + } +}; + +test "the flash attention kernels emit verified modules" { + const allocator = std.testing.allocator; + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVec.Kernel.emit(allocator, .{ .head_dim = head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); + } + } + for ([_]Type{ .half, .bfloat }) |dtype| { + for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { + const tile = FaTiled.tileConfig(head_size).?; + allocator.free(try FaTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2] })); + } + } +} diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig index 9327c2a1..bdfb00c0 100644 --- a/zml/kernels/flash_attention/metal/metal.zig +++ b/zml/kernels/flash_attention/metal/metal.zig @@ -1,12 +1,10 @@ const std = @import("std"); -const stdx = @import("stdx"); - const zml = @import("../../../zml.zig"); - -const log = std.log.scoped(.@"zml/attention/metal"); +const flash_attn = @import("flash_attn.zig"); pub const Metadata = struct { + /// Live queries of a single-sequence prefill; 0 means all. num_tokens: zml.Tensor, pub fn init() Metadata { @@ -23,16 +21,123 @@ pub const Metadata = struct { } }; +/// Causal attention: fa_vec for bf16 decode, fa_tiled otherwise, sdpa for +/// unsupported dtypes and head sizes. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { - const qc = q.transpose(.{ .h, .q, .hd }); - const kc = k.transpose(.{ .h, .k, .hd }); - const vc = v.transpose(.{ .h, .k, .hd }); - const tok_i32 = token_index.convert(.i32); + const dtype = q.dtype(); + const head_dim = q.dim(.hd); + if (!((dtype == .bf16 or dtype == .f16) and k.dtype() == dtype and v.dtype() == dtype and flash_attn.FaTiled.tileConfig(head_dim) != null)) { + const seq_len = k.dim(.k); + const mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) + .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, dtype), token_index.appendAxes(.{.coord}), .{}); + return zml.nn.sdpa(q, k, v, .{ .attn_mask = mask }); + } - const attn = if (q.dim(.q) > 1) - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) + const batched = q.shape().hasTag(.b) != null; + const qb = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }).insertAxes(0, .{.b}); + const kb = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const vb = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const positions = token_index.convert(.i32).broad(.init(.{ .b = qb.dim(.b) }, .i32)); + const scale = 1.0 / @sqrt(@as(f32, @floatFromInt(head_dim))); + + const vec_head_dim = head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512; + const out = if (qb.dim(.q) == 1 and dtype == .bf16 and vec_head_dim) + decode(qb, kb, vb, positions, scale) else - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); + prefill(qb, kb, vb, positions, liveQueries(qb, metadata), scale); + return (if (batched) out else out.squeeze(.b)).transpose(q.shape()); +} - return attn.transpose(q.shape()); +fn liveQueries(q: zml.Tensor, metadata: Metadata) zml.Tensor { + const all: zml.Tensor = .scalar(q.dim(.q), .i32); + if (q.dim(.b) != 1 or q.dim(.q) == 1) return all; + const num_tokens = metadata.num_tokens.convert(.i32); + return num_tokens.cmp(.GE, .scalar(1, .i32)).select(num_tokens.minimum(all), all); +} + +/// `q` is `[b, 1, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn decode(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, scale: f32) zml.Tensor { + const batch: i32 = @intCast(q.dim(.b)); + const num_heads: i32 = @intCast(q.dim(.h)); + const head_dim: i32 = @intCast(q.dim(.hd)); + const num_kv_heads: i32 = @intCast(k.dim(.h)); + const keys: i32 = @intCast(k.dim(.k)); + const nrows = batch * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), keys, coreCount()); + const args: flash_attn.FaVec.HostArgs = .{ .num_heads = num_heads, .n_kv = num_kv_heads, .k_len = keys, .scale = scale }; + + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = flash_attn.FaVec.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions }, .{ .dst = dst }, .{ + .cfg = .{ .head_dim = head_dim, .heads = launch.heads, .splits = launch.splits }, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(batch) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; + + return flash_attn.FaVecReduce.Kernel.call(.{ .htmp = out, .token_index = positions }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * flash_attn.FaVecReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, flash_attn.FaVecReduce.simdgroups, 1 }, + .constants = .{ .args = .of(args) }, + }).dst; +} + +/// `q` is `[b, q, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn prefill(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, live: zml.Tensor, scale: f32) zml.Tensor { + const queries: i32 = @intCast(q.dim(.q)); + const tile = flash_attn.FaTiled.tileConfig(q.dim(.hd)).?; + const cfg: flash_attn.FaTiled.Cfg = .{ + .dtype = zml.kernel.metal.from(q.dtype()), + .head_size = @intCast(q.dim(.hd)), + .bq = tile[0], + .tile_kv = tile[1], + .num_threads = tile[2], + }; + return flash_attn.FaTiled.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions, .live = live }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(q.dim(.h)), @intCast(@divFloor(queries + tile[0] - 1, tile[0])), @intCast(q.dim(.b)) }, + .threadgroup = .{ @intCast(tile[2]), 1, 1 }, + .threadgroup_memory = flash_attn.FaTiled.threadgroupMemory(cfg), + .constants = .{ + .num_kv_heads = .of(@as(i32, @intCast(k.dim(.h)))), + .scale = .of(scale), + .q_len = .of(queries), + .k_len = .of(@as(i32, @intCast(k.dim(.k)))), + }, + }).out; +} + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} + +test { + std.testing.refAllDecls(flash_attn); } diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig index d93bd373..c3c3dca2 100644 --- a/zml/kernels/flash_attention/tests.zig +++ b/zml/kernels/flash_attention/tests.zig @@ -46,6 +46,22 @@ test "attention: q=1,qh=8,kh=8" { ); } +test "attention: q=1,k=4096,qh=32,kh=8,hd=128" { + try testAttention( + .init(.{ .q = 1, .h = 32, .hd = 128 }, .bf16), + .init(.{ .k = 4096, .h = 8, .hd = 128 }, .bf16), + &.{3000}, + ); +} + +test "attention: q=1,k=1000,qh=8,kh=2,hd=32" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 32 }, .bf16), + .init(.{ .k = 1000, .h = 2, .hd = 32 }, .bf16), + &.{900}, + ); +} + test "attention: q=8,qh=64,kh=8" { try testAttention( .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), From 8ebed5402912e5340333d877a077930ec536e3c3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 12 Nov 2026 13:12:25 +0100 Subject: [PATCH 34/47] zml: refactor safetensors (#1048) Refactor `zml/safetensors.zig` * Drop metadata parsing * refactor code to be more "object oriented", code is owned by the `TensorRegistry` * implement a new format `model.safetensors.full_index.json` that is similar to `index.json` but actually has data inside it: ``` { "metadata": {"num_tensors":96085,"total_size":510286023000}, "weight_map": { "aligner.w1.bias": {"filename":"model-00001-of-00048.safetensors","shape":[5120],"data_offsets":[27384,37624],"dtype":"BF16"}, "aligner.w1.weight": {"filename":"model-00001-of-00048.safetensors","shape":[5120,9216],"data_offsets":[37624,94409464],"dtype":"BF16"}, "aligner.w2.bias": {"filename":"model-00001-of-00048.safetensors","shape":[5120],"data_offsets":[94409464,94419704],"dtype":"BF16"}, "aligner.w2.weight": {"filename":"model-00001-of-00048.safetensors","shape":[5120,5120],"data_offsets":[94419704,146848504],"dtype":"BF16"}, "vision.blocks.0.attn.wo.bias": {"filename":"model-00001-of-00048.safetensors","shape":[1024],"data_offsets":[146848504,146850552],"dtype":"BF16"}, "vision.blocks.0.attn.wo.weight": {"filename":"model-00001-of-00048.safetensors","shape":[1024,1024],"data_offsets":[146850552,148947704],"dtype":"BF16"}, "vision.blocks.0.attn.wqkv.bias": {"filename":"model-00001-of-00048.safetensors","shape":[3072],"data_offsets":[148947704,148953848],"dtype":"BF16"}, ... ``` I followed conventions of `model.saftensors.index.json` --- bazel/bazel.zig | 2 +- examples/io/main.zig | 20 +- tools/safetensors/BUILD.bazel | 13 + tools/safetensors/main.zig | 91 +++ zml/safetensors.zig | 1067 +++++++++++++-------------------- zml/zml.zig | 6 + 6 files changed, 539 insertions(+), 660 deletions(-) create mode 100644 tools/safetensors/BUILD.bazel create mode 100644 tools/safetensors/main.zig diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 8af28b00..67513227 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { +pub export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = envp; var threaded: std.Io.Threaded = .init_single_threaded; diff --git a/examples/io/main.zig b/examples/io/main.zig index 1cec23b0..c9dc05ea 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -132,13 +132,9 @@ pub fn main(init: std.process.Init) !void { const root = try TensorNode.init(init.arena.allocator(), ""); - var registry_it = registry.iterator(); - while (registry_it.next()) |kv| { - const name = kv.key_ptr.*; - const tensor = kv.value_ptr.*; - + for (registry.tensors.values()) |tensor| { var current = root; - var parts = std.mem.tokenizeScalar(u8, name, '.'); + var parts = std.mem.tokenizeScalar(u8, tensor.name, '.'); while (parts.next()) |part| { const gop = try current.children.getOrPut(current.allocator, part); if (!gop.found_existing) { @@ -183,16 +179,16 @@ pub fn main(init: std.process.Init) !void { const tensors = try allocator.alloc(zml.Tensor, tensor_count); defer allocator.free(tensors); - var registry_it = registry.iterator(); var load_count: usize = 0; - while (registry_it.next()) |entry| : (load_count += 1) { + for (registry.tensors.keys(), registry.tensors.values()) |name, tensor| { tensors[load_count] = switch (mesh_type) { - .replicated => store.view().createReplicatedTensor(entry.key_ptr.*, null), - .sharded => if (entry.value_ptr.shape.rank() > 0) - store.view().createTensor(entry.key_ptr.*, null, .tp, tp_partitioning) + .replicated => store.view().createReplicatedTensor(name, null), + .sharded => if (tensor.shape.rank() > 0) + store.view().createTensor(name, null, .tp, tp_partitioning) else - store.view().createReplicatedTensor(entry.key_ptr.*, null), + store.view().createReplicatedTensor(name, null), }; + load_count += 1; } const model: AllTensorsModel = .{ .tensors = tensors }; diff --git a/tools/safetensors/BUILD.bazel b/tools/safetensors/BUILD.bazel new file mode 100644 index 00000000..aff4954d --- /dev/null +++ b/tools/safetensors/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +zig_binary( + name = "safetensors", + main = "main.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + "//stdx", + "//vfs", + "//zml", + ], +) diff --git a/tools/safetensors/main.zig b/tools/safetensors/main.zig new file mode 100644 index 00000000..cf902eef --- /dev/null +++ b/tools/safetensors/main.zig @@ -0,0 +1,91 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const VFS = @import("vfs"); +const zml = @import("zml"); + +const log = std.log.scoped(.safetensors_convert); + +const Args = struct { + output: ?[]const u8 = null, + positional: struct { model: []const u8 }, + + pub const help = + \\Usage: [--output=] + \\ + \\Read safetensors headers and write model.safetensors.full_index.json in the working directory + \\Example: bazel run //tools/safetensors -- --output="model.safetensors.full_index.json" hf://deepseek-ai/DeepSeek-V4.1-Flash + ; +}; + +pub fn main(init: std.process.Init) !void { + const args = stdx.flags.parse(init.minimal.args, Args); + const allocator = init.gpa; + + const start = std.Io.Clock.now(.awake, init.io); + var client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + defer client.deinit(); + try client.initDefaultProxies(allocator, init.environ_map); + + const input = args.positional.model; + + var file_vfs: VFS.File = .init(allocator, init.io, .{ .direct_io = false }); + defer file_vfs.deinit(); + var http_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .http); + defer http_vfs.deinit(); + var https_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .https); + defer https_vfs.deinit(); + var hf_vfs: ?VFS.HF = if (std.mem.startsWith(u8, input, "hf://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (hf_vfs) |*backend| backend.deinit(); + var s3_vfs: ?VFS.S3 = if (std.mem.startsWith(u8, input, "s3://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (s3_vfs) |*backend| backend.deinit(); + var gcs_vfs: ?VFS.GCS = if (std.mem.startsWith(u8, input, "gs://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (gcs_vfs) |*backend| backend.deinit(); + + var vfs: VFS = try .init(allocator, init.io); + const io = vfs.io(); + defer vfs.deinit(); + try vfs.register("file", file_vfs.io()); + try vfs.register("http", http_vfs.io()); + try vfs.register("https", https_vfs.io()); + if (hf_vfs) |*backend| try vfs.register("hf", backend.io()); + if (s3_vfs) |*backend| try vfs.register("s3", backend.io()); + if (gcs_vfs) |*backend| try vfs.register("gs", backend.io()); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const repo = try zml.safetensors.resolveModelRepo(io, input); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + const loaded = std.Io.Clock.now(.awake, init.io); + + const index_json_file = if (args.output) |output| + try std.Io.Dir.cwd().createFile(io, output, .{}) + else + try repo.createFile(io, full_index_path, .{}); + const out_buffer = try allocator.alloc(u8, 4096); + defer allocator.free(out_buffer); + var out_writer = index_json_file.writer(io, out_buffer); + try registry.saveAsFullIndexJson(io, allocator, repo, &out_writer.interface); + const done = std.Io.Clock.now(.awake, init.io); + + log.info("Wrote {s}: {d} tensors; load {f}s, write {f}s, total {f}s", .{ + args.output orelse full_index_path, + registry.tensors.count(), + start.durationTo(loaded), + loaded.durationTo(done), + start.durationTo(done), + }); +} + +const full_index_path = "model.safetensors.full_index.json"; diff --git a/zml/safetensors.zig b/zml/safetensors.zig index f854f02f..3827fe27 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,68 +2,337 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; -const Dims = Shape.DimsArray; - -const StringBuilder = std.ArrayListUnmanaged(u8); - -pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); -pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); const log = std.log.scoped(.@"zml/safetensors"); -const BYTES_HEADER = 8; +// TODO: consider promoting this to the top level struct +// TODO: Feels like we are missing a mapping file -> tensors that would be helfpful for loading weights +/// Maps tensor names to their shape and positions in different sub files. +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, -pub const FileType = enum { - index, - safetensors, - unknown, + tensors: std.StringArrayHashMapUnmanaged(Tensor), + + /// Metadata found in `__metadata__` field of the safetensors header. + /// This is rarely use and the information there is not standardized, so we let user code handle it. + /// Here we just store the plain bytes, reach out if you need this to be already parsed. + metadata: []const u8, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = "", + }; + } + + pub fn deinit(self: *TensorRegistry) void { + self.arena.deinit(); + } + + pub fn fromRepo(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !TensorRegistry { + const file_type, const file = try resolveModelEntrypoint(io, repo); + defer file.close(io); + + var registry: TensorRegistry = .init(allocator); + switch (file_type) { + .safetensors => try registry.parseSafetensors(io, allocator, file), + .index => try registry.parseIndexJson(io, allocator, repo, file), + .full_index => try registry.parseFullIndexJson(io, allocator, repo, file), + .unknown => return error.InvalidPath, + } + return registry; + } + + pub fn fromPath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !TensorRegistry { + const file_type: FileType = .resolve(path); + switch (file_type) { + .safetensors => { + // open directly + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + var registry: TensorRegistry = .init(allocator); + try registry.parseSafetensors(io, allocator, file); + return registry; + }, + .full_index, + .index, + => { + // resolve repo as parent folder then open + const repo_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; + const repo = std.Io.Dir.openDir(.cwd(), io, repo_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + + var registry: TensorRegistry = .init(allocator); + try switch (file_type) { + .index => registry.parseIndexJson(io, allocator, repo, file), + .full_index => registry.parseFullIndexJson(io, allocator, repo, file), + .safetensors, .unknown => unreachable, + }; + return registry; + }, + .unknown => { + // the path is possibly a folder, let's try that + const repo = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return try fromRepo(allocator, io, repo); + }, + } + } + + /// Register a tensor, duplicate tensor name and uri. + /// If you create a lot of tensors sharing the same uri, + /// considere calling `registry.tensors` directly and avoid duplicated uris. + pub fn registerTensor(self: *TensorRegistry, tensor: Tensor) !void { + const allocator = self.arena.allocator(); + + const owned_name = try allocator.dupe(u8, tensor.name); + errdefer allocator.free(owned_name); + const owned_uri = try allocator.dupe(u8, tensor.file_uri); + errdefer allocator.free(owned_uri); + + const owned_tensor: Tensor = .{ + .name = owned_name, + .file_uri = owned_uri, + .shape = tensor.shape, + .offset = tensor.offset, + }; + try self.tensors.put(allocator, owned_name, owned_tensor); + } + + pub fn reader( + self: *TensorRegistry, + io: std.Io, + tensor_name: []const u8, + buffer: []u8, + ) !TensorReader { + const tensor = self.tensors.get(tensor_name) orelse { + log.err("Tensor {s} not found in registry", .{tensor_name}); + return error.TensorNotFound; + }; + + return try .init(io, tensor, buffer, .{}); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } + + /// Parse a given .safetensors file and create entries for each tensor + pub fn parseSafetensors( + self: *TensorRegistry, + io: std.Io, + scratch_allocator: std.mem.Allocator, + file: std.Io.File, + ) !void { + var tmp_state = std.heap.ArenaAllocator.init(scratch_allocator); + defer tmp_state.deinit(); + const tmp_arena = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + var owned_file_name = try self_arena.alloc(u8, std.fs.max_path_bytes); + const file_name_len = try file.realPath(io, owned_file_name); + // Try to release extra memory, but we don't care if it fails + _ = self_arena.resize(owned_file_name, file_name_len); + owned_file_name = owned_file_name[0..file_name_len]; + + const header_buf = try tmp_arena.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = file.reader(io, header_buf); + + const json_header_length: u64 = try file_reader.interface.takeInt(u64, .little); + const BYTES_HEADER = 8; + const data_start_offset = BYTES_HEADER + json_header_length; + const json_data = try tmp_arena.alloc(u8, @intCast(json_header_length)); + + try file_reader.interface.readSliceAll(json_data); + const headers: std.json.Value = try std.json.parseFromSliceLeaky( + std.json.Value, + tmp_arena, + json_data, + .{}, + ); + + try self.tensors.ensureUnusedCapacity(self_arena, headers.object.count()); + for (headers.object.keys(), headers.object.values()) |key, value| { + if (std.mem.eql(u8, key, "__metadata__")) { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(value, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + continue; + } + + const owned_name = try self_arena.dupe(u8, key); + const tensor: Tensor = try .parse(value.object, owned_file_name, owned_name, data_start_offset); + self.tensors.putAssumeCapacity(owned_name, tensor); + } + } + + pub fn parseIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var safetensors_files: std.StringArrayHashMapUnmanaged(void) = .empty; + try safetensors_files.ensureUnusedCapacity(tmp, 64); + if (index.object.get("weight_map")) |wm| { + for (wm.object.values()) |file_name| { + try safetensors_files.put(tmp, file_name.string, {}); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + for (safetensors_files.keys()) |file_name| { + const file = try repo.openFile(io, file_name, .{}); + defer file.close(io); + try self.parseSafetensors(io, tmp, file); + } + } + + pub fn parseFullIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + + var repo_uri_buf = try tmp.alloc(u8, 1024); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + + // Note: this is relatively inefficient, we read the full file, copy most of it's content, + // then reduce to TensorRegistry format. + // We could update TensorRegistry as we stream the json. + // Zig std json API are a bit awkward around streaming though. + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var realpaths: std.StringArrayHashMapUnmanaged([]const u8) = .empty; + try realpaths.ensureUnusedCapacity(tmp, 64); + const wm = index.object.get("weight_map") orelse return error.InvalidJson; + + try self.tensors.ensureUnusedCapacity(self_arena, wm.object.count()); + for (wm.object.keys(), wm.object.values()) |key, value| { + const tensor_name = try self_arena.dupe(u8, key); + const object = value.object; + const filename = value.object.get("filename").?.string; + const filename_entry = try realpaths.getOrPut(tmp, filename); + const file_uri = if (filename_entry.found_existing) + filename_entry.value_ptr.* + else uri: { + const uri = try std.fs.path.join(self_arena, &.{ repo_name, filename }); + filename_entry.value_ptr.* = uri; + break :uri uri; + }; + + const tensor_info: Tensor = try .parse(object, file_uri, tensor_name, 0); + self.tensors.putAssumeCapacity(tensor_name, tensor_info); + } + + if (index.object.get("metadata")) |metadata| { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(metadata, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + } + } + + pub fn saveAsFullIndexJson( + self: *const TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + out: *std.Io.Writer, + ) !void { + var repo_uri_buf = try gpa.alloc(u8, std.fs.max_path_bytes); + defer gpa.free(repo_uri_buf); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + var total_size: u64 = 0; + for (self.tensors.values()) |value| total_size += value.byteSize(); + + try out.print( + \\{{ + \\"metadata": {{"num_tensors":{d},"total_size":{d}}}, + \\"weight_map": {{ + \\ + , .{ self.tensors.count(), total_size }); + + const n = self.tensors.count(); + for (0.., self.tensors.keys(), self.tensors.values()) |i, key, value| { + std.debug.assert(std.mem.startsWith(u8, value.file_uri, repo_name)); + const local_name = value.file_uri[repo_name.len + 1 ..]; + const end_offset = value.offset + value.byteSize(); + const safetensors_dtype: []const u8 = dt: { + for (safetensors_dtype_to_zml_dtype.keys(), safetensors_dtype_to_zml_dtype.values()) |st_dt, zml_dt| { + if (zml_dt == value.shape._dtype) break :dt st_dt; + } + // Note: we could bitcast to u8, + // but reaching this panic implies the user didn't loaded this TensorRegistry + // from a safetensors file, and manually created the dict. + // In which case, they know what they are doing, + // and can easily insert the bitcast themselves. + std.debug.panic("Safetensors doesn't support data type {}", .{value.shape._dtype}); + }; + const comma = if (i + 1 == n) "" else ","; + try out.print( + \\ "{s}": {{"filename":"{s}","shape":{f},"data_offsets":[{d},{d}],"dtype":"{s}"}}{s} + \\ + , .{ key, local_name, std.json.fmt(value.shape.dims(), .{ .whitespace = .minified }), value.offset, end_offset, safetensors_dtype, comma }); + } + + try out.writeAll( + \\} + \\} + \\ + ); + try out.flush(); + } }; -pub fn fetchRegistry( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - model: std.Io.File, - model_path: []const u8, -) !TensorRegistry { - const file_type = resolveFiletype(model_path); - - return switch (file_type) { - .index => blk: { - const file_reader_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(file_reader_buf); - - var index_reader = model.reader(io, file_reader_buf); - - var safetensors_index = try parseSafetensorsIndex( - allocator, - &index_reader.interface, - ); - defer safetensors_index.deinit(); - - var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); - - try parseSafetensorsIndexFiles( - allocator, - io, - ®istry, - safetensors_index, - repo, - ); - - break :blk registry; - }, - .safetensors => blk: { - var registry: TensorRegistry = .init(allocator); - - try parseSafetensors(allocator, io, ®istry, model); - - break :blk registry; - }, - else => return error.InvalidPath, - }; -} - pub const TensorReader = struct { tensor: Tensor, file: std.Io.File, @@ -238,6 +507,30 @@ pub const Tensor = struct { return TensorReader.init(io, self, buffer, opts); } + pub fn parse(object: std.json.ObjectMap, file_uri: []const u8, name: []const u8, base_offset: u64) !Tensor { + const shape_field = object.get("shape").?.array; + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ name, shape_field.items.len }); + return error.TooManyDimensions; + } + + const offset_field = object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const dtype = try stringToDtype(object.get("dtype").?.string); + + var dims: Shape.DimsArray = .empty; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + return .{ + .file_uri = file_uri, + .name = name, + .shape = .init(dims.slice(), dtype), + .offset = base_offset + start, + }; + } + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ self.name, @@ -249,620 +542,100 @@ pub const Tensor = struct { } }; -pub const TensorRegistry = struct { - arena: std.heap.ArenaAllocator, +pub const FileType = enum { + full_index, + index, + safetensors, + unknown, - tensors: Tensors, - metadata: Metadatas, - - mutex: std.Io.Mutex = .init, - - pub fn init(allocator: std.mem.Allocator) TensorRegistry { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = .{}, - }; - } - - pub fn initWithMetadata( - allocator: std.mem.Allocator, - metadata: Metadatas, - ) !TensorRegistry { - var self: TensorRegistry = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = .empty, - }; - - try self.mergeMetadata(metadata); - - return self; - } - - pub fn fromRepo( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - ) !TensorRegistry { - const entrypoint_file, const entrypoint_path = try resolveModelEntrypoint(io, repo); - defer entrypoint_file.close(io); - return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); - } - - pub fn fromPath( - allocator: std.mem.Allocator, - io: std.Io, - path: []const u8, - ) !TensorRegistry { - var repo = try resolveModelRepo(io, path); - defer repo.close(io); - - const entrypoint_file, const entrypoint_path = if (std.mem.endsWith(u8, path, ".safetensors.index.json") or std.mem.endsWith(u8, path, ".safetensors")) - .{ try repo.openFile(io, path, .{ .mode = .read_only }), path } - else - try resolveModelEntrypoint(io, repo); - defer entrypoint_file.close(io); - - return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); - } - - pub fn deinit(self: *TensorRegistry) void { - const allocator = self.arena.allocator(); - self.tensors.deinit(allocator); - self.metadata.deinit(allocator); - self.arena.deinit(); - } - - pub fn mergeMetadata( - self: *TensorRegistry, - other: Metadatas, - ) !void { - // self.mutex.lock(); - // defer self.mutex.unlock(); - - const allocator = self.arena.allocator(); - - var it = other.iterator(); - while (it.next()) |entry| { - const key = try allocator.dupe(u8, entry.key_ptr.*); - const value = try entry.value_ptr.*.clone(allocator); - - const gop = try self.metadata.getOrPut(allocator, key); - if (gop.found_existing) { - gop.value_ptr.*.deinit(allocator); - gop.value_ptr.* = value; - log.debug("Overwrote existing metadata key={s} with value={f}", .{ key, value }); - } else { - gop.value_ptr.* = value; - log.debug("Added new metadata key={s} with value={f}", .{ key, value }); - } + pub fn resolve(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path, ".safetensors.full_index.json")) { + return .full_index; + } else if (std.mem.endsWith(u8, path, ".safetensors")) { + return .safetensors; + } else { + return .unknown; } } - - pub fn registerTensor( - self: *TensorRegistry, - tensor: Tensor, - ) !void { - const allocator = self.arena.allocator(); - - // self.mutex.lock(); - // defer self.mutex.unlock(); - - var tensor_copy = tensor; - - tensor_copy.name = try allocator.dupe(u8, tensor.name); - tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); - - try self.tensors.put(allocator, tensor_copy.name, tensor_copy); - } - - pub fn reader( - self: *TensorRegistry, - io: std.Io, - tensor_name: []const u8, - buffer: []u8, - ) !TensorReader { - const tensor = self.tensors.get(tensor_name) orelse { - log.err("Tensor {s} not found in registry", .{tensor_name}); - return error.TensorNotFound; - }; - - return try .init(io, tensor, buffer, .{}); - } - - pub fn iterator(self: *TensorRegistry) Tensors.Iterator { - return self.tensors.iterator(); - } - - pub fn totalBytes(self: *TensorRegistry) u64 { - var total: u64 = 0; - - var it = self.tensors.iterator(); - while (it.next()) |entry| { - total += entry.value_ptr.byteSize(); - } - - return total; - } }; -pub fn parseSafetensors( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - file: std.Io.File, -) !void { - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const header_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(header_buf); - - var reader = file.reader(io, header_buf); - - var file_uri_buf: [1024]u8 = undefined; - const file_uri_len = try file.realPath(io, &file_uri_buf); - const file_uri = file_uri_buf[0..file_uri_len]; - - const json_header_length: u64 = try reader.interface.takeInt(u64, .little); - const data_start_offset = BYTES_HEADER + json_header_length; - const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); - - try reader.interface.readSliceAll(json_data); - - const headers: std.json.Value = try std.json.parseFromSliceLeaky( - std.json.Value, - arena.allocator(), - json_data, - .{}, - ); - - var it = headers.object.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; - - if (std.mem.eql(u8, key, "__metadata__")) { - // Parse metadata into a local arena to avoid concurrent allocations - // against registry.arena (which is not thread-safe). - var local_arena = std.heap.ArenaAllocator.init(allocator); - defer local_arena.deinit(); - - const metas = try parseMetadata(local_arena.allocator(), value); - try registry.mergeMetadata(metas); - - continue; - } - - const shape_field = value.object.get("shape").?.array; - - if (shape_field.items.len > Shape.MAX_RANK) { - log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); - return error.TooManyDimensions; - } - - const offset_field = value.object.get("data_offsets").?; - const start: u64 = @intCast(offset_field.array.items[0].integer); - const dtype = try stringToDtype(value.object.get("dtype").?.string); - - var dims: Dims = .empty; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); - } - - const tensor: Tensor = .{ - .file_uri = file_uri, - .name = key, - .shape = .init(dims.slice(), dtype), - .offset = data_start_offset + start, - }; - - try registry.registerTensor(tensor); - } -} - -const SafetensorsIndex = struct { - pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); - - arena: std.heap.ArenaAllocator, - - map: Map, - metadata: Metadatas, - - pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .map = .empty, - .metadata = .empty, - }; - } - - pub fn iterator(self: *SafetensorsIndex) Map.Iterator { - return self.map.iterator(); - } - - pub fn deinit(self: *SafetensorsIndex) void { - const allocator = self.arena.allocator(); - self.map.deinit(allocator); - self.metadata.deinit(allocator); - self.arena.deinit(); - } -}; - -fn parseSafetensorsIndex( - allocator: std.mem.Allocator, - reader: *std.Io.Reader, -) !SafetensorsIndex { - var safetensors_index: SafetensorsIndex = .init(allocator); - errdefer safetensors_index.deinit(); - - const arena_allocator = safetensors_index.arena.allocator(); - var json_reader: std.json.Reader = .init(arena_allocator, reader); - - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - arena_allocator, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - const weight_map = index.object.get("weight_map"); - - if (weight_map) |wm| { - var it = wm.object.iterator(); - - while (it.next()) |entry| { - const weight_name = entry.key_ptr.*; - const filename = entry.value_ptr.string; - - const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); - - if (!map_entry.found_existing) { - map_entry.value_ptr.* = .empty; - } - - try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); - } - } else { - log.warn("No weight_map attribute found in index", .{}); - } - - if (index.object.get("metadata")) |metadata_val| { - safetensors_index.metadata = try parseMetadata(arena_allocator, metadata_val); - } - - return safetensors_index; -} - -fn parseSafetensorsIndexFiles( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - safetensors_index: SafetensorsIndex, - repo: std.Io.Dir, -) !void { - const filenames = safetensors_index.map.keys(); - for (filenames) |filename| { - const file = try repo.openFile(io, filename, .{}); - defer file.close(io); - try parseSafetensors(allocator, io, registry, file); - } -} - -pub fn resolveFiletype(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { - return .safetensors; - } else { - return .unknown; - } -} - pub const ModelPathResolutionError = error{ FileNotFound, InvalidPath, } || std.mem.Allocator.Error; -pub fn resolveModelRepo( - io: std.Io, - path: []const u8, -) ModelPathResolutionError!std.Io.Dir { - if (std.mem.endsWith(u8, path, ".safetensors.index.json") or - std.mem.endsWith(u8, path, ".safetensors")) - { - const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; +pub fn resolveModelRepo(io: std.Io, path: []const u8) ModelPathResolutionError!std.Io.Dir { + switch (FileType.resolve(path)) { + .full_index, .index, .safetensors => { + // The path is a file, let's open it's folder + const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; }; - }; - return dir; + return dir; + }, + .unknown => { + // The path is probably a directory, let's try to open it as such + const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return dir; + }, } - - const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - - return dir; } -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, []const u8 } { - const default_entrypoints = .{ - "model.safetensors.index.json", - "consolidated.safetensors.index.json", - "model.safetensors", - "consolidated.safetensors", +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { FileType, std.Io.File } { + const default_entrypoints: []const struct { FileType, []const u8 } = &.{ + .{ .full_index, "model.safetensors.full_index.json" }, + .{ .full_index, "consolidated.safetensors.full_index.json" }, + .{ .index, "model.safetensors.index.json" }, + .{ .index, "consolidated.safetensors.index.json" }, + .{ .safetensors, "model.safetensors" }, + .{ .safetensors, "consolidated.safetensors" }, }; - inline for (default_entrypoints) |entrypoint| { - if (repo.openFile(io, entrypoint, .{ .mode = .read_only })) |index_file| { - return .{ index_file, entrypoint }; - } else |_| {} + for (default_entrypoints) |entrypoint| { + const file_type, const file_name = entrypoint; + const file = repo.openFile(io, file_name, .{ .mode = .read_only }) catch continue; + + return .{ file_type, file }; } - return ModelPathResolutionError.FileNotFound; + return error.FileNotFound; } -const Metadata = union(enum) { - null: void, - int: i64, - float: f64, - bool: bool, - string: []const u8, +const safetensors_dtype_to_zml_dtype = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E8M0", .f8e8m0 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "F4_E2M1", .f4e2m1 }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, +}); - array_bool: []const bool, - array_int: []const i64, - array_float: []const f64, - array_string: []const []const u8, - - pub const ItemType = enum { - int, - float, - bool, - string, - - pub fn toZigType(comptime kind: ItemType) type { - return switch (kind) { - .int => i64, - .float => f64, - .bool => bool, - .string => []const u8, - }; - } - }; - - pub fn wrap(x: anytype) Metadata { - return switch (@TypeOf(x)) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, - inline f16, f32, f64 => .{ .float = @floatCast(x) }, - bool => .{ .bool = x }, - []const u8 => .{ .string = x }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), - }; - } - - pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { - return switch (@TypeOf(any_slice[0])) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => { - const res = try allocator.alloc(i64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @intCast(val); - return .{ .array_int = res }; - }, - inline f16, f32, f64 => { - const res = try allocator.alloc(f64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @floatCast(val); - return .{ .array_float = res }; - }, - bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, - []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), - }; - } - - pub fn format( - self: Metadata, - writer: *std.Io.Writer, - ) !void { - switch (self) { - .null => _ = try writer.write("null"), - .string => |s| try writer.print("{s}", .{s}), - .bool => |b| try writer.print("{}", .{b}), - .int => |i| try writer.print("{d}", .{i}), - .float => |f| try writer.print("{d}", .{f}), - .array_bool => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{}", .{v}); - } - try writer.writeByte(']'); - }, - .array_int => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_float => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_string => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("\"{s}\"", .{v}); - } - try writer.writeByte(']'); - }, - } - } - - pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { - switch (self) { - .string => |s| allocator.free(s), - .array_bool => |s| allocator.free(s), - .array_int => |s| allocator.free(s), - .array_float => |s| allocator.free(s), - .array_string => |s| { - for (s) |str| allocator.free(str); - allocator.free(s); - }, - else => {}, - } - } - - pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { - return switch (self) { - .null => .null, - .int => |v| .{ .int = v }, - .float => |v| .{ .float = v }, - .bool => |v| .{ .bool = v }, - .string => |s| .{ .string = try allocator.dupe(u8, s) }, - .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, - .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, - .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, - .array_string => |s| blk: { - const new_slice = try allocator.alloc([]const u8, s.len); - errdefer allocator.free(new_slice); - for (s, 0..) |str, i| { - new_slice[i] = try allocator.dupe(u8, str); - } - break :blk .{ .array_string = new_slice }; - }, - }; - } -}; - -fn stringToDtype(safetensor_type: []const u8) !DataType { - const map = std.StaticStringMap(DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E8M0", .f8e8m0 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "F4_E2M1", .f4e2m1 }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, - }); - - return map.get(safetensor_type) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); +fn stringToDtype(safetensor_dtype: []const u8) !DataType { + return safetensors_dtype_to_zml_dtype.get(safetensor_dtype) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_dtype}); return error.UnsupportedDataType; }; } - -fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { - var metadatas: Metadatas = .{}; - var prefix_buf: [1024]u8 = undefined; - var prefix = StringBuilder.initBuffer(&prefix_buf); - - try populateMetadata(allocator, &prefix, val, &metadatas); - - return metadatas; -} - -fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { - const key = prefix.items; - return switch (val) { - .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), - .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), - .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), - .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), - .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), - .array => |v| { - if (v.items.len == 0) return; - if (validSlice(v)) |item_type| { - const data: Metadata = switch (item_type) { - .bool => blk: { - const values = try allocator.alloc(bool, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.bool; - break :blk .{ .array_bool = values }; - }, - .integer => blk: { - const values = try allocator.alloc(i64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.integer; - break :blk .{ .array_int = values }; - }, - .float => blk: { - const values = try allocator.alloc(f64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.float; - break :blk .{ .array_float = values }; - }, - inline .string, .number_string => |tag| blk: { - const values = try allocator.alloc([]const u8, v.items.len); - for (v.items, 0..) |item, i| { - values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); - } - break :blk .{ .array_string = values }; - }, - .null, .array, .object => unreachable, - }; - try metadatas.put(allocator, try allocator.dupe(u8, key), data); - } else { - for (v.items, 0..) |item, i| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try populateMetadata(allocator, prefix, item, metadatas); - prefix.items.len = old_len; - } - } - }, - .object => |v| { - var obj_iter = v.iterator(); - while (obj_iter.next()) |entry| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); - prefix.items.len = old_len; - } - }, - }; -} - -/// We can only create a Zig slice out of json array, if all values -/// in the array have the same type. -fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { - if (v.items.len == 0) return null; - - const item_type: std.meta.Tag(std.json.Value) = v.items[0]; - switch (item_type) { - .null, .array, .object => return null, - else => {}, - } - - for (v.items[1..]) |item| { - if (item != item_type) - return null; - } - - return item_type; -} diff --git a/zml/zml.zig b/zml/zml.zig index 23d12e75..35c2f9df 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -81,3 +81,9 @@ pub const KiB = 1024; pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; + +comptime { + // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. + // Otherwise exported functions declared in `bazel` aren't compiled. + std.testing.refAllDecls(bazel); +} From c40fa52d2db5ca40a6a4500c94d5f04d06ae3e7d Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Thu, 12 Nov 2026 13:47:48 +0100 Subject: [PATCH 35/47] zml/moe [Platform:tcl]: support block-128 FP8 SwiGLU experts (#1053) - Select TCL for plain E4M3FN block-128 experts with matching BF16 or F32 scales. Preserve rejection of unsupported quantization. - Quantize activations per block and apply checkpoint scales during native projections. Keep BF16 materialization and F32 down and routing accumulation. - Reuse local expert routing and AllReduce for expert parallelism. Choose expert-stationary for small FP8 token counts and use the existing activation-stationary and blockwise recipes for larger counts. - Share FP8 and BF16 dispatch preparation while retaining the token padding and BF16 SiLU rounding workarounds. --- platforms/furiosa/furiosa.bzl | 4 +- zml/kernels/moe/moe.zig | 12 +- zml/kernels/moe/tcl/tcl.zig | 451 ++++++++++++++++++++++++++-------- 3 files changed, 363 insertions(+), 104 deletions(-) diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 81518d53..623c4f0d 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-furiosa-linux-amd64.tar.zst", - sha256 = "6da755570140c9f8959ba477fef3df4c48b0619274c97429d204e177b8649e12", + url = "https://mirror.zml.ai/plugins/202611121233.371.1.9ff7ddf1e19f/zml-furiosa-linux-amd64.tar.zst", + sha256 = "d732b56832b1f009cbe4d808c70c6f8d1dc3857a83eb06804fc53b39bef9dd6b", ) return mctx.extension_metadata( diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 33b6da55..2a0aa8d7 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -37,8 +37,6 @@ pub const Backend = enum { tcl, pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { - // Keep the dtype as an argument because non scheme-specific backends may depend on it later - _ = dtype; return switch (platform.target) { .cuda => b: { const s = scheme orelse break :b .triton; @@ -72,7 +70,13 @@ pub const Backend = enum { }; }, .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, - .furiosa => if (scheme == null) .tcl else error.UnsupportedQuantization, + .furiosa => b: { + const s = scheme orelse break :b .tcl; + break :b switch (s) { + .fp8_block128 => if (dtype == .f8e4m3fn) .tcl else error.UnsupportedQuantization, + .mxfp4, .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block32 => error.UnsupportedQuantization, + }; + }, .metal => b: { const s = scheme orelse break :b .metal; break :b switch (s) { @@ -245,7 +249,7 @@ pub fn forwardMoe( } const expert_axis = experts.meshAxis(); const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; - if (backend == .metal or backend == .tcl) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); + if (backend == .metal) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); const Call = struct { input: zml.Tensor, diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 698c4d30..e7d7a01b 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -9,9 +9,17 @@ //! device loop over the used chunks of `BGi` blocks gathers each block's //! expert weights and runs both projections; outputs are gathered back per //! route and weighted. +//! +//! FP8 experts split across devices (expert parallel): each device runs its own +//! experts for the routes that reach them and an all-reduce sums the devices' +//! outputs. Up to `expert_stationary_tokens` tokens, expert stationary: a +//! device loop over the used experts, a few at a time, reads each once and +//! runs it for every token, scaled by the token's routing weight (zero for +//! the tokens it does not serve); more tokens take blockwise. const std = @import("std"); const builder = @import("dsl/tcl/builder"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); @@ -19,34 +27,78 @@ const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; const Options = @import("../options.zig").Options; -/// `input` is `{.t, .d}`, routing IDs and weights `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` -/// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). +/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` +/// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. /// SwiGLU, routing weights applied after the down projection. pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { - const topk_ids = routing.ids; - const topk_weights = routing.weights; - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); + const topk_ids = routing.localIds(); + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); - stdx.debug.assert(gate_up_linear.quantization == null and down_linear.quantization == null, "TCL MoE takes bf16 experts", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); const gate_up = gate_up_linear.weight; const down = down_linear.weight; - stdx.debug.assert(input.dtype() == .bf16 and gate_up.dtype() == .bf16 and down.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes bf16 activations and experts, i32 ids", .{}); - const t = input.dim(.t); - const k = topk_ids.dim(.topk); - const cfg: Cfg = .{ - .t = t, - .k = k, + const quantized = gate_up_linear.quantization != null or down_linear.quantization != null; + const scales: ?Scales = if (quantized) .{ .gate_up = fp8Scales(gate_up_linear), .down = fp8Scales(down_linear) } else null; + stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32 and (quantized or (gate_up.dtype() == .bf16 and down.dtype() == .bf16)), "TCL MoE takes BF16 activations, BF16/block-128 FP8 experts and i32 ids", .{}); + const weights = routing.weights.convert(.f32); + const local_weights = if (scales != null) topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)) else weights; + return runExperts(input, topk_ids, local_weights, gate_up, down, scales); +} + +const Scales = struct { gate_up: Tensor, down: Tensor }; + +fn fp8Scales(linear: zml.nn.Linear) Tensor { + const q = linear.quantization orelse std.debug.panic("TCL MoE requires both FP8 projections", .{}); + stdx.debug.assert(q.scheme == .fp8_block128 and linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); + stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); + const w = linear.weight; + const scale = q.scales; + stdx.debug.assert(@mod(w.dim(.d), 128) == 0 and @mod(w.dim(.dout), 128) == 0 and scale.dim(.expert) == w.dim(.expert) and scale.dim(.d) * 128 == w.dim(.d) and scale.dim(.dout) * 128 == w.dim(.dout), "TCL MoE FP8 scales must cover 128x128 weight blocks", .{}); + return scale; +} + +/// Tokens up to which a device runs its experts stationary. +pub const expert_stationary_tokens = 64; + +fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales) Tensor { + var cfg: Cfg = .{ + .t = input.dim(.t), + .k = topk_ids.dim(.topk), .e = gate_up.dim(.expert), .d = input.dim(.d), .i = down.dim(.dout), }; stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); + if (scales) |s| { + cfg.scale_dtype = scaleDtype(s); + if (cfg.t <= expert_stationary_tokens) { + // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two + // panics its adder tree ("Fail to cover valid range", 5 tokens padded + // to 8): pad with unrouted tokens. + var padded = cfg; + padded.t = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.t)) catch unreachable); + const pad = padded.t - cfg.t; + const x = input.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.d }); + const ids = topk_ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); + const weights = topk_weights.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); + const y = ExpertStationaryFp8.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }, .{ .y = x.shape() }, .{ .cfg = padded }).y; + return y.slice(0, .{ .end = cfg.t }).reshape(input.shape()); + } + } - const x = input.reshape(.{ t, cfg.d }); - const ids = topk_ids.reshape(.{ t, k }); - const weights = topk_weights.convert(.f32).reshape(.{ t, k }); + const x = input.reshape(.{ cfg.t, cfg.d }); + const ids = topk_ids.reshape(.{ cfg.t, cfg.k }); + const weights = topk_weights.reshape(.{ cfg.t, cfg.k }); const out_shape = x.shape(); + if (scales) |s| { + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); + const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = valid.select(ids, Tensor.scalar(0, .i32)), .w = valid.select(weights, Tensor.scalar(0, .f32)), .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; + const y = if (cfg.t * cfg.k <= cfg.e) + ActivationStationaryFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y + else + BlockwiseFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y; + return y.reshape(input.shape()); + } const y = if (cfg.t * cfg.k <= cfg.e) ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y else @@ -54,7 +106,12 @@ pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear return y.reshape(input.shape()); } -pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64 }; +fn scaleDtype(scales: Scales) tcl.DType { + stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); + return if (scales.gate_up.dtype() == .bf16) .bf16 else .f32; +} + +pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64, scale_dtype: tcl.DType = .bf16 }; const Axes = struct { T: tcl.Axis, @@ -80,22 +137,112 @@ const Axes = struct { } }; -fn args(b: *tcl.Builder, a: Axes) tcl.FinishError!Args { - const t = try b.declareArgs(.{ +fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime fp8: bool) tcl.FinishError!Args { + const spec = .{ .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = .bf16, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = .bf16, .axes = &.{ a.E, a.D, a.I } }, + .gate_up = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.D, a.I } }, + }; + if (!fp8) { + const t = try b.declareArgs(spec); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + } + const IG = b.axis("IG", @divExact(cfg.i, 128)); + const DG = b.axis("DG", @divExact(cfg.d, 128)); + const I2G = b.axis("I2G", 2 * @divExact(cfg.i, 128)); + const t = try b.declareArgs(.{ + .x = spec.x, + .ids = spec.ids, + .w = spec.w, + .gate_up = spec.gate_up, + .down = spec.down, + .gate_up_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, I2G, DG } }, + .down_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, DG, IG } }, }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; } -const Args = struct { x: builder.Tensor, ids: builder.Tensor, w: builder.Tensor, gate_up: builder.Tensor, down: builder.Tensor }; +const Args = struct { + x: builder.Tensor, + ids: builder.Tensor, + w: builder.Tensor, + gate_up: builder.Tensor, + down: builder.Tensor, + gate_up_scale: ?builder.Tensor = null, + down_scale: ?builder.Tensor = null, +}; -/// `silu(gate) * up` of `h = [batch..., S2, I]` (gate first), times -/// `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor) tcl.FinishError!builder.Tensor { +const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; + +fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis) tcl.FinishError!GatheredScales { + const gu = in.gate_up_scale orelse return .{}; + const IG = b.axis("IG", @divExact(a.I.size, 128)); + const DG = b.axis("DG", @divExact(a.D.size, 128)); + const gate_up = try b.reshape(gu, &.{ a.E, a.S2, IG, DG }, .{}); + var gu_axes: [8]tcl.Axis = undefined; + var dn_axes: [8]tcl.Axis = undefined; + @memcpy(gu_axes[0..batch.len], batch); + @memcpy(dn_axes[0..batch.len], batch); + @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); + @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); + return .{ + .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), + .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), + }; +} + +fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { + var n: usize = 0; + for (axes) |a| { + if (std.mem.eql(u8, a.name, axis.name)) { + storage[n] = group; + n += 1; + storage[n] = lane; + } else storage[n] = a; + n += 1; + } + return storage[0..n]; +} + +/// Block scales multiply each K-block partial before K-block reduction. +/// Only the requested experts are gathered; their bank remains FP8 in HBM. +fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType) tcl.FinishError!builder.Tensor { + if (scale == null) { + const op = b.tensorOperation(.{}); + return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); + } + const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); + const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); + const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); + const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); + var x_axes: [16]tcl.Axis = undefined; + var w_axes: [16]tcl.Axis = undefined; + var w2_axes: [16]tcl.Axis = undefined; + var out_axes: [16]tcl.Axis = undefined; + var partial_axes: [16]tcl.Axis = undefined; + const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); + const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); + const out = blockedAxes(to, output, OG, OL, &out_axes); + @memcpy(partial_axes[0..out.len], out); + partial_axes[out.len] = CG; + + // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. + var op = b.tensorOperation(.{}); + const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); + op = b.tensorOperation(.{}); + // Native F32 reciprocal accuracy is qualified separately (within one ULP). + const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); + const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); + op = b.tensorOperation(.{}); + const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); + return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); +} + +/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` +/// (gate first), times `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, comptime fp8: bool) tcl.FinishError!builder.Tensor { var sliced: [4]tcl.Axis = undefined; var flat: [3]tcl.Axis = undefined; @memcpy(sliced[0..batch.len], batch); @@ -108,9 +255,22 @@ fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, w const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); // Interleaving requires one binary instruction; SwiGLU chains several. - const op = b.tensorOperation(.{ .tactic = .elementwise }); + var op = b.tensorOperation(.{ .tactic = .elementwise }); const g = op.fetch(gate, .{}); - var v = g.sigmoid().mulf(g).mulf(op.fetch(up, .{})); + const activated = if (fp8) blk: { + // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal + // BF16 SiLU write, even through as_dram/as_logical. Make its RNE + // rounding observable as integer bits before multiplying by up. + // Remove once the native reproducer preserves the BF16 boundary. + const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const v = op.fetch(bits, .{}); + const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); + const silu = try op.commit(rounded, .{ .dtype = .bf16 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + break :blk op.fetch(silu, .{}); + } else g.sigmoid().mulf(g); + var v = activated.mulf(op.fetch(up, .{})); if (weights) |w| v = v.mulf(op.fetch(w, .{})); return op.commit(v, .{ .dtype = .bf16 }); } @@ -152,29 +312,39 @@ fn configure(b: *tcl.Builder, blockwise: bool) void { }); } -pub const ActivationStationary = tcl.Kernel(Cfg, .{ - .name = "moe_activation_stationary", - .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a); - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); +pub const ActivationStationary = activationStationaryKernel(false); +pub const ActivationStationaryFp8 = activationStationaryKernel(true); - var op = b.tensorOperation(.{}); - const h = try op.commit(op.contract(in.x, gu, &.{ a.T, a.K, a.S2, a.I }), .{}); - // Routing weights scale the activations, so the down projection - // also sums over the experts. - const act = try gated(b, a, h, &.{ a.T, a.K }, in.w); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } - }.run, -}); +fn activationStationaryKernel(comptime fp8: bool) type { + return tcl.Kernel(Cfg, .{ + .name = if (fp8) "moe_activation_stationary_fp8" else "moe_activation_stationary", + .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, false); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, fp8); + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }); + const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (fp8) .bf16 else null); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, fp8); + if (fp8) { + const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32); + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); + } else { + // The BF16 recipe weights activations and contracts experts together. + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } + } + }.run, + }); +} /// Rows per expert block: the next power of two of the routes per expert. pub fn blockRows(cfg: Cfg) i64 { @@ -184,77 +354,162 @@ pub fn blockRows(cfg: Cfg) i64 { const blocks_per_chunk = 32; -pub const Blockwise = tcl.Kernel(Cfg, .{ - .name = "moe_blockwise", - .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, +pub const Blockwise = blockwiseKernel(false); +pub const BlockwiseFp8 = blockwiseKernel(true); + +fn blockwiseKernel(comptime fp8: bool) type { + return tcl.Kernel(Cfg, .{ + .name = if (fp8) "moe_blockwise_fp8" else "moe_blockwise", + .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, true); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, fp8); + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(if (fp8) .f32 else .bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{BGi}); + const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (fp8) .bf16 else null); + const act = try gated(b, a, h, &.{ BGi, BB }, null, fp8); + const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (fp8) .f32 else .bf16); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); + } + }.run, + }); +} + +/// Experts per `ExpertStationary` loop step: more as the routes, so the +/// experts they use, grow. +pub fn stationaryChunk(cfg: Cfg) i64 { + const routes: u64 = @intCast(std.math.divCeil(i64, cfg.t * cfg.k, 16) catch unreachable); + return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); +} + +/// The experts that `ids` use (-1 for none), `stationaryChunk` at a time in a +/// device loop: each is read once and runs for every token. Routing weights +/// scale the F32 down-projection results before summing the experts and steps. +pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ + .name = "moe_expert_stationary_fp8", + .inputs = &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" }, .outputs = &.{"y"}, .run = struct { fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, true); + configure(b, false); const a: Axes = .init(b, cfg); - const in = try args(b, a); - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const in = try args(b, a, cfg, true); + const chunk = stationaryChunk(cfg); + const steps = std.math.divCeil(i64, @min(cfg.e, cfg.t * cfg.k), chunk) catch unreachable; + const C = b.axis("C", chunk); + const Go = b.axis("Go", steps); + const J = b.axis("J", steps * chunk); const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + // Each token's routing weight per expert, and the used experts. const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); op = b.tensorOperation(.{}); + const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); + op = b.tensorOperation(.{}); const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); + // A used expert's rank among them, -1 for the others. op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + // Slot j holds the used expert of rank j and its tokens' weights; + // slots past the last hold expert 0 with no weight. + const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); + op = b.tensorOperation(.{}); + const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); + // A reduction's condition is read plainly, so broadcast it first. + op = b.tensorOperation(.{}); + const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); + op = b.tensorOperation(.{}); + const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); + const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); + const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - var loop = b.openFor(used, .{try b.scratchpad(.bf16, &.{ Go, BGi, BB, D })}); + var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const ids = try b.indexRead(step_experts, loop.iv); + const weights = try b.indexRead(step_gate, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{C}); + const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16); + const act = try gated(b, a, h, &.{ C, T }, null, true); + const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32); op = b.tensorOperation(.{}); - const h = try op.commit(op.contract(chunk, gu, &.{ BGi, BB, a.S2, I }), .{}); - const act = try gated(b, a, h, &.{ BGi, BB }, null); - op = b.tensorOperation(.{}); - const y = try op.commit(op.contract(act, dn, &.{ BGi, BB, D }), .{ .dtype = .bf16 }); + const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); + b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); } }.run, }); @@ -286,7 +541,7 @@ test "tcl fused experts run on furiosa" { }); } }; - var exe = try zml.module.compile(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, platform, .{}); + var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); From dc829cfb7d052f4929524261329d92daa4f30018 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 13 Nov 2026 10:50:15 +0100 Subject: [PATCH 36/47] [Modularization] zml/kernels/moe: replace `options.zig` with `api.zig` (#1059) Also moved `LocalRouting` to `api.zig` --- zml/BUILD.bazel | 2 +- zml/kernels/moe/{options.zig => api.zig} | 32 +++++++++++++ zml/kernels/moe/cute/cute.zig | 7 +-- zml/kernels/moe/cute/sm10x/moe.zig | 5 ++- .../flashinfer_cutlass/flashinfer_cutlass.zig | 10 ++--- zml/kernels/moe/fly/fly.zig | 15 ++++--- zml/kernels/moe/metal/metal.zig | 7 +-- zml/kernels/moe/moe.zig | 45 +++---------------- zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig | 7 +-- zml/kernels/moe/stablehlo/stablehlo.zig | 9 ++-- zml/kernels/moe/tcl/tcl.zig | 6 +-- zml/kernels/moe/tests.zig | 8 ++-- zml/kernels/moe/triton/moe.zig | 2 +- zml/kernels/moe/triton/triton.zig | 17 +++---- 14 files changed, 90 insertions(+), 82 deletions(-) rename zml/kernels/moe/{options.zig => api.zig} (75%) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index fff3b8a8..b47d4f59 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -90,8 +90,8 @@ zig_library( "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", + "kernels/moe/api.zig", "kernels/moe/moe.zig", - "kernels/moe/options.zig", "kernels/moe/tests.zig", "kernels/moe/weights.zig", "kernels/paged_attention/options.zig", diff --git a/zml/kernels/moe/options.zig b/zml/kernels/moe/api.zig similarity index 75% rename from zml/kernels/moe/options.zig rename to zml/kernels/moe/api.zig index 7f8d26de..b66593c1 100644 --- a/zml/kernels/moe/options.zig +++ b/zml/kernels/moe/api.zig @@ -130,3 +130,35 @@ pub const ExpertsLayout = struct { mxfp4_interleaved8, }; }; + +/// Routing for a local token batch and a contiguous range of experts. +/// IDs retain the model's global expert numbering, while the projection weights +/// contain only the local experts. Kernels compute those experts contributions and +/// `forwardMoe` sums them along the expert axis. +pub const LocalRouting = struct { + /// Global expert IDs for local tokens, with shape {.t, .topk} + ids: Tensor, + weights: Tensor, + global_num_experts: u32, + local_num_experts: u32, + /// Scalar i32 containing the first local expert's global ID. + /// Null when all experts are local and global IDs already index the weights. + expert_start: ?Tensor = null, + + /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. + /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. + pub fn localIds(self: LocalRouting) Tensor { + const start = self.expert_start orelse return self.ids; + const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(self.ids.sub(start), .scalar(-1, .i32)); + } + + /// Returns a lookup table from every global expert ID to its local weight + /// index or -1 if nonlocal. Returns null when every expert is local. + pub fn expertMap(self: LocalRouting) ?Tensor { + const start = self.expert_start orelse return null; + const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(ids.sub(start), .scalar(-1, .i32)); + } +}; diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index bd482093..8bc03c9c 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,8 +3,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const Options = @import("../options.zig").Options; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Options = @import("../api.zig").Options; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -41,7 +42,7 @@ pub fn validateOptions(opts: Options) void { /// interleave by 8 and its scale atoms are MN-major packed i32 words. pub fn fusedExperts( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index f651813f..ff6edec2 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -4,16 +4,17 @@ //! rows in route order, and the top-k reduction. MXFP4 applies routing weights //! before the down projection, with the reference model's BF16 rounding. const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); +const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; +const triton_kernels = @import("../../triton/moe.zig"); pub const mxfp4 = @import("mxfp4.zig"); pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); -const RoutingWeightPlacement = @import("../../options.zig").RoutingWeightPlacement; -const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { /// `[tokens, hidden]` BF16. diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig index d1a0e1b5..92317c7c 100644 --- a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -5,10 +5,12 @@ const platforms = @import("platforms"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const GlobalOptions = @import("../options.zig").Options; -const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const LocalRouting = @import("../api.zig").LocalRouting; const unpackedWeight = @import("../weights.zig").unpackedWeight; const contract = @import("contract.zig"); +const Attributes = contract.Attributes; const runtime = @import("runtime.zig"); const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); @@ -50,8 +52,6 @@ const Inputs = contract.Inputs(zml.Tensor); const Outputs = contract.Outputs(zml.Shape); -const Attributes = contract.Attributes; - fn currentRunners() !*runtime.Runners { const platform = zml.Compiler.current().platform; const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; @@ -311,7 +311,7 @@ pub fn fusedExpertsNvfp4( pub fn fusedExperts( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index d041a5ab..ba3d3c00 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,11 +6,12 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const ExpertsLayout = @import("../options.zig").ExpertsLayout; -const ProjectionLayout = @import("../options.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; -const Activation = @import("../options.zig").Activation; -const Options = @import("../options.zig").Options; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const fly = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -21,7 +22,7 @@ test { pub fn fusedExperts( hidden_states: Tensor, - routing_: zml.moe.LocalRouting, + routing_: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options, @@ -135,7 +136,7 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { +pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index 988a4f51..f835c889 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -4,8 +4,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../options.zig").Activation; -const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); @@ -68,7 +69,7 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso pub fn fusedExperts( hidden_states: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 2a0aa8d7..45f41b2a 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,17 +4,18 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; +pub const Activation = @import("api.zig").Activation; +pub const ActivationKind = @import("api.zig").ActivationKind; pub const cute = @import("cute/cute.zig"); -pub const ExpertsLayout = @import("options.zig").ExpertsLayout; +pub const ExpertsLayout = @import("api.zig").ExpertsLayout; pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); pub const fly = @import("fly/fly.zig"); +pub const LocalRouting = @import("api.zig").LocalRouting; pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const Options = @import("options.zig").Options; -pub const Activation = @import("options.zig").Activation; -pub const ActivationKind = @import("options.zig").ActivationKind; -pub const ProjectionLayout = @import("options.zig").ProjectionLayout; -pub const RoutingWeightPlacement = @import("options.zig").RoutingWeightPlacement; +pub const Options = @import("api.zig").Options; +pub const ProjectionLayout = @import("api.zig").ProjectionLayout; +pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); @@ -138,38 +139,6 @@ pub const Backend = enum { } }; -/// Routing for a local token batch and a contiguous range of experts. -/// IDs retain the model's global expert numbering, while the projection weights -/// contain only the local experts. Kernels compute those experts contributions and -/// `forwardMoe` sums them along the expert axis. -pub const LocalRouting = struct { - /// Global expert IDs for local tokens, with shape {.t, .topk} - ids: zml.Tensor, - weights: zml.Tensor, - global_num_experts: u32, - local_num_experts: u32, - /// Scalar i32 containing the first local expert's global ID. - /// Null when all experts are local and global IDs already index the weights. - expert_start: ?zml.Tensor = null, - - /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. - /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. - pub fn localIds(self: LocalRouting) zml.Tensor { - const start = self.expert_start orelse return self.ids; - const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(self.ids.sub(start), .scalar(-1, .i32)); - } - - /// Returns a lookup table from every global expert ID to its local weight - /// index or -1 if nonlocal. Returns null when every expert is local. - pub fn expertMap(self: LocalRouting) ?zml.Tensor { - const start = self.expert_start orelse return null; - const ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(ids.sub(start), .scalar(-1, .i32)); - } -}; - /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index 4b33b585..bb60594b 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,8 +5,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../options.zig").Activation; -const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -214,7 +215,7 @@ pub fn callGmmEp( pub fn fusedExperts( hidden_states: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index 7f84835b..ba723234 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -7,9 +7,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const GlobalOptions = @import("../options.zig").Options; -const Activation = @import("../options.zig").Activation; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; pub fn activate(x: Tensor, activation: Activation) Tensor { return switch (activation) { @@ -191,7 +192,7 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { +pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index e7d7a01b..6238d5f6 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -19,18 +19,18 @@ const std = @import("std"); const builder = @import("dsl/tcl/builder"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const Options = @import("../options.zig").Options; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; /// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. /// SwiGLU, routing weights applied after the down projection. -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { const topk_ids = routing.localIds(); stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index f78cd1ef..c74406c3 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,10 +5,10 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const ExpertsLayout = @import("options.zig").ExpertsLayout; -const GlobalOptions = @import("options.zig").Options; -const Activation = @import("options.zig").Activation; -const Placement = @import("options.zig").RoutingWeightPlacement; +const Activation = @import("api.zig").Activation; +const ExpertsLayout = @import("api.zig").ExpertsLayout; +const GlobalOptions = @import("api.zig").Options; +const Placement = @import("api.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); const Case = struct { diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 0776a590..6e4006fa 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,7 +12,7 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index ed6c9f5f..8ba799a5 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -7,11 +7,12 @@ const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; const toDType = zml.kernel.triton.from; -const ExpertsLayout = @import("../options.zig").ExpertsLayout; -const ProjectionLayout = @import("../options.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; -const Activation = @import("../options.zig").Activation; -const Options = @import("../options.zig").Options; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const kernels = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/triton"); @@ -38,7 +39,7 @@ test { pub fn fusedExperts( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, @@ -58,7 +59,7 @@ pub fn fusedExperts( pub fn fusedExpertsBase( hidden_states: Tensor, - routing_: zml.moe.LocalRouting, + routing_: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, layout: ProjectionLayout, @@ -900,7 +901,7 @@ fn validateInputs( /// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. pub fn fusedExpertsMxfp4( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, From 2d45ef0804132eb4309b71f9b261b52de4ddd02c Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 13 Nov 2026 13:52:07 +0100 Subject: [PATCH 37/47] zml/platform: fix `zml$print` leak (#1060) --- zml/platform.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zml/platform.zig b/zml/platform.zig index 6c6081aa..54fb90e4 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1015,12 +1015,15 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; + const buffers = call_frame.args.buffers(); + if (buffers.len <= 0) return null; + const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = call_frame.args.buffers()[0]; + const buffer = buffers[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -1044,9 +1047,13 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); + defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); + + const event = pjrt_buffer.readyEvent(pjrt_api); + defer event.deinit(pjrt_api); + try event.awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); From 8c41b898019a2511d05da9d90e54188569ef962d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 13 Nov 2026 13:52:58 +0100 Subject: [PATCH 38/47] zml/ops: manualComputation v2 (#1045) Last PR for manualComputation v2 which promote as default and only one API for manual computation. Custom calls now require to be wrapped in a manual computation too. --- dsl/tcl/README.md | 3 +- examples/sharding/main.zig | 4 +- zml/BUILD.bazel | 1 - zml/attention/attention.zig | 14 +- zml/kernel.zig | 9 +- zml/kernels/moe/moe.zig | 4 +- .../paged_attention/cute/benchmark.zig | 2 +- .../paged_attention/paged_attention.zig | 7 +- zml/kernels/paged_attention/tcl/tcl.zig | 13 - zml/nn.zig | 26 +- zml/ops.zig | 94 +--- zml/ops/manual_computation.zig | 494 +++++++++++------- zml/ops/manual_computation2.zig | 465 ----------------- zml/tensor.zig | 2 +- 14 files changed, 376 insertions(+), 762 deletions(-) delete mode 100644 zml/ops/manual_computation2.zig diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md index ec1ae61f..b01b7189 100644 --- a/dsl/tcl/README.md +++ b/dsl/tcl/README.md @@ -104,8 +104,7 @@ def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16 across the partitions; other kernels stay per chip. Its arguments and results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis `C`, sized to the chip count, across chips (each partition holds its share), - the default `.broadcast` keeps a copy per chip. Call it inside - `zml.ops.manualComputation` so it sees each partition's shards. + the default `.broadcast` keeps a copy per chip. * **Convolutions** pad and slide the input in its read: a slide replaces its axis by `window_axis, frame_axis`, then `contract` with the filter. On its own (without a slide) the padding is dropped. diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 7e519bf0..34a78bfe 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -42,7 +42,7 @@ const DemoModel = struct { const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); - return zml.ops.manualComputation2( + return zml.ops.manualComputation( (struct { y: zml.Tensor, gate: zml.Tensor, @@ -70,7 +70,7 @@ fn usage() void { \\Simple sharding demo: \\ - explicit partitioning on data/model axes \\ - tensor print - \\ - manualComputation2 with multiple inputs and one output + \\ - manualComputation with multiple inputs and one output \\ \\Mesh modes: \\ - auto: physical_mesh=auto, inferred cpu device_count=2 diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index b47d4f59..2b14c79f 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -103,7 +103,6 @@ zig_library( "nn.zig", "ops.zig", "ops/manual_computation.zig", - "ops/manual_computation2.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 0ac2a95c..edd9b4ca 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -97,7 +97,6 @@ pub const Options = @import("Options.zig"); pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; - const InputSpecs = @FieldType(zml.ops.ManualPartitioning(Attention.forward), "in_specs"); in_specs: struct { q: Logical, @@ -105,7 +104,7 @@ pub const Partitioning = struct { v: Logical, token_index: Logical = .replicated, /// Omit to replicate backend metadata tensors. - metadata: ?@FieldType(InputSpecs, "metadata") = null, + metadata: ?zml.ops.PartitionSpecs(Metadata) = null, }, out_specs: Logical, @@ -135,20 +134,13 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T /// `attention`, causal or not depending on `opts`. pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; - return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, .{ + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ .in_specs = .{ .q = partitioning.in_specs.q, .k = partitioning.in_specs.k, .v = partitioning.in_specs.v, .token_index = partitioning.in_specs.token_index, - .metadata = partitioning.in_specs.metadata orelse switch (metadata) { - .stablehlo => .{ .stablehlo = {} }, - .flash_attention => |metadata_| .{ .flash_attention = switch (metadata_) { - .cute => .{ .cute = {} }, - .nki => .{ .nki = {} }, - .metal => .{ .metal = .{ .num_tokens = .replicated } }, - } }, - }, + .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), }, .out_specs = partitioning.out_specs, }); diff --git a/zml/kernel.zig b/zml/kernel.zig index d85b6a56..f111f721 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1534,13 +1534,16 @@ test "tcl multi-chip kernels run on two furiosa chips" { const tp = zml.testing.tpMesh(); const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); const Mod = struct { - pub fn forward(x: Tensor) [2]Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); + pub fn forward(x: Tensor, mesh: *const zml.Sharding.Mesh) [2]Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); return .{ r[0], r[1] }; } }; const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{tp} }); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); defer exe.deinit(); var hx: [2 * 1024]f32 = undefined; for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 45f41b2a..eb697f36 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -266,7 +266,7 @@ pub fn forwardMoe( .global_experts = @intCast(gate_up.weight.dim(.expert)), }; const Logical = zml.Sharding.Partitioning.Logical; - var projection_specs: [2]zml.meta.MapRestrict(zml.Tensor, Logical).map(zml.nn.Linear) = undefined; + var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { zml.meta.mapAlloc(struct { fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { @@ -285,7 +285,7 @@ pub fn forwardMoe( .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, }, linear, specs) catch @panic("OOM"); } - return zml.ops.manualComputation2(Call.forward, args, output_shape, mesh, .{ + return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ .in_specs = .{ .input = partitioning.in_specs.input, .ids = partitioning.in_specs.ids, diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 3e05fed8..587af69f 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -125,7 +125,7 @@ fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation2(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index aaa42b20..4a97a38e 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -207,7 +207,7 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z _ = k; _ = v; const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; - return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, partitioning); + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); } const Attention = struct { @@ -435,12 +435,9 @@ test pagedAttention { const parameters: Parameters = .init(.fromBackend(backend_options_args)); const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - const ParameterSpecs = zml.meta.MapRestrict(zml.Tensor, zml.Sharding.Partitioning.Logical).map(Parameters); const partitioning: Partitioning = .{ .in_specs = .{ - .parameters = switch (parameters) { - inline else => |_, tag| @unionInit(ParameterSpecs, @tagName(tag), .{ .block_table = .replicated, .seq_lens = .replicated, .query_start_len = .replicated }), - }, + .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), .q = head_spec, .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, .opts = .{ .mask = .{ .standard = {} }, .sink = null }, diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index 881593aa..d04b684e 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -67,22 +67,9 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); - if (compiler.meshes[0].numDevices() > 1) return zml.ops.manualComputation(Shard.body, .{ .parameters = parameters, .q = q, .k_cache = k_cache, .v_cache = v_cache, .opts = opts }, q.shape()); return attend(parameters, q, k_cache, v_cache, opts); } - const Shard = struct { - parameters: Parameters, - q: Tensor, - k_cache: Tensor, - v_cache: Tensor, - opts: AttentionOptions, - - fn body(self: Shard, _: zml.Shape) Tensor { - return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); - } - }; - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); diff --git a/zml/nn.zig b/zml/nn.zig index a2b582ee..eceb316b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,11 +2,13 @@ const std = @import("std"); const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; @@ -320,13 +322,23 @@ fn onesGrid(shape: Shape, dt: DataType) Shape { } fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { - return ops.customCall( - "zml$scaled_dot_unmatched", - .{ in[0], in[1], in[2], in[3] }, - out_shape, - {}, - .{ .has_side_effect = false }, - ); + // Keep the unmatched-composite marker in the graph for backend lowering. + const ctx = zml.Compiler.current(); + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, + &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, + .{ + .call_target_name = "zml$scaled_dot_unmatched", + .has_side_effect = false, + .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); + } + return Tensor._result(out_shape, op.result(0)); } fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { diff --git a/zml/ops.zig b/zml/ops.zig index a9ae0f8d..f8fdba7e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -18,9 +18,10 @@ const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const manual_computation = @import("ops/manual_computation.zig"); pub const manualComputation = manual_computation.manualComputation; -const manual_computation2 = @import("ops/manual_computation2.zig"); -pub const manualComputation2 = manual_computation2.manualComputation2; -pub const ManualPartitioning = manual_computation2.ManualPartitioning; +pub const ManualPartitioning = manual_computation.ManualPartitioning; +pub const PartitionSpecs = manual_computation.PartitionSpecs; +pub const partitionSpecs = manual_computation.partitionSpecs; +pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; @@ -30,7 +31,6 @@ const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; test { _ = manual_computation; - _ = manual_computation2; } const log = std.log.scoped(.@"zml/Compiler"); @@ -89,7 +89,7 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ /// ``` /// /// Each expert shard now has the complete output for its data shard. To expose -/// this result through `manualComputation2`, an output spec such as +/// this result through `manualComputation`, an output spec such as /// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the /// result replicated along `.experts`. pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { @@ -2760,11 +2760,6 @@ pub fn CustomCall( comptime params: struct { /// Name of the custom call, must be unique. name: [:0]const u8, - /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies - /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, - /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output - /// sharding, this setting will avoid the shuffling of the buffer's data. - sharding_aware: bool, /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. has_side_effect: bool, /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. @@ -2798,23 +2793,7 @@ pub fn CustomCall( .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), .compute_on_host = params.compute_on_host, }; - if (params.sharding_aware) { - return shardingAwareTypedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } else { - return typedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } + return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); } fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { @@ -2842,44 +2821,6 @@ pub fn CustomCallOutputOperandAliases(I: type, O: type) type { ); } -pub fn shardingAwareTypedCustomCall( - comptime target_name: [:0]const u8, - comptime opts: CustomCallOptions, - input: anytype, - output: anytype, - attributes: anytype, -) ShapeToTensor(@TypeOf(output)) { - const Input = @TypeOf(input); - const Output = @TypeOf(output); - const Attributes = @TypeOf(attributes); - - var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - output_shapes[i] = @field(output, field_name); - } - - const Handler = struct { - input: Input, - attributes: Attributes, - - fn body(self: @This(), sharded_output_shapes: []const Shape) []const Tensor { - return typedCustomCall(target_name, opts, self.input, sharded_output_shapes, self.attributes); - } - }; - const output_tensors = manualComputation( - Handler.body, - .{ .input = input, .attributes = attributes }, - @as([]const Shape, &output_shapes), - ); - - // Convert the slice back to a struct - var out: ShapeToTensor(Output) = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - @field(out, field_name) = output_tensors[i]; - } - return out; -} - pub fn typedCustomCall( target_name: [:0]const u8, opts: CustomCallOptions, @@ -2894,7 +2835,7 @@ pub fn typedCustomCall( const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); - stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); + stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { @@ -2904,7 +2845,6 @@ pub fn typedCustomCall( } break :b &input_tensors; }, - // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -2920,7 +2860,6 @@ pub fn typedCustomCall( } break :b &output_shapes; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -2985,7 +2924,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + if (ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -2997,7 +2936,6 @@ pub fn typedCustomCall( } return out; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -3050,7 +2988,7 @@ test customCall { const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3061,9 +2999,17 @@ test customCall { const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); - const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + const Local = struct { + fn body(x: Tensor, output_shape: Shape) Tensor { + return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + }); + } + }; + const output = manualComputation(Local.body, input, shape, tp, .{ + .in_specs = .parse(.{ ._0 = .model }), + .out_specs = .parse(.{ ._0 = .model }), }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index 45400206..a0ec0037 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); @@ -7,44 +6,126 @@ const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("../Compiler.zig"); -const DataType = @import("../dtype.zig").DataType; +const meta = @import("../meta.zig"); const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; -const zml = struct { - // Nested to disambiguate from stdx.meta - const meta = @import("../meta.zig"); - const testing = @import("../testing.zig"); -}; - const log = std.log.scoped(.@"zml/Compiler"); -/// ZML operator describes computation across several devices, -/// but `manualComputation` allows to partition the devices in small group, -/// split the `Tensor` on those partitions, -/// and describes sub-computation that work inside a partition. +/// Partitioning declarations for the tensors in `T`. +/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. +/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes +/// `struct { x: Logical, bias: ?Logical }`. /// -/// See: -/// https://github.com/openxla/shardy/blob/main/docs/sdy_dialect.md#sdymanual_computation-sdymanualcomputationop +/// Initialize individual fields to declare different shardings, or use +/// `partitionSpecs` to apply one spec to every tensor in a value. +pub fn PartitionSpecs(comptime T: type) type { + return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); +} + +/// Declares the same sharding for a group of tensors at a `manualComputation` +/// boundary. /// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R` where: -/// - `I` is user defined and contains inputs tensors and any metadata required by the computation -/// - `O` contains the partionned output shapes, types must be `void`, `Shape` or `[]const Shape` for 0, 1 or many outputs -/// - `R` must be `void`, `Tensor, or `[]const Tensor` for 0, 1 or many outputs -/// * `inputs` must be using the same mesh (using .withPartitioning), or be fully replicated -/// * `outputs` contains the global output shapes, -/// and `manualComputation` will pass to `body_fn` the shapes of each local output shard. +/// ```zig +/// .in_specs = .{ +/// .query = .parse(.{ .h = .model }), +/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), +/// .metadata = partitionSpecs(metadata, .replicated), +/// } +/// ``` /// -/// This depart from the rest of ZML where output shapes are generally inferred, -/// but in this case since `body_fn` explains how to compute parts of the result, -/// it's not clear how to put them back together, -/// hence why the user is asked to explicitly provided the expected output shape. +/// Requires an active compiler and uses its arena for allocations. Use +/// `partitionSpecsAlloc` outside graph construction. Use named struct fields +/// when mixing tensors with non-tensor metadata. +pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { + return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); +} + +/// Builds the same declarations as `partitionSpecs` using an explicit allocator. +pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { + var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; + try meta.mapAlloc(struct { + fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { + return spec_; + } + }.assign, allocator, spec, tensors, &specs); + return specs; +} + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = PartitionSpecs(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), ) stdx.meta.FnReturn(body_fn) { const Inputs = stdx.meta.FnParam(body_fn, 0); const OutputShapes = stdx.meta.FnParam(body_fn, 1); @@ -55,13 +136,42 @@ pub fn manualComputation( defer arena_state.deinit(); const arena = arena_state.allocator(); - var owned_inputs: Inputs = inputs; - const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - defer arena.free(flat_input_tensors); + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); - var owned_output_shapes: OutputShapes = output_shapes; - const flat_output_shapes: []*Shape = zml.meta.collectMutPtrs(Shape, arena, &owned_output_shapes) catch @panic("OOM"); - defer arena.free(flat_output_shapes); + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } const WrappedInputs = struct { inputs: *const Inputs, @@ -73,15 +183,17 @@ pub fn manualComputation( outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); } }; var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &owned_output_shapes }; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + _ = manualComputationInternal( compiler, arena, + mesh, @typeName(@TypeOf(body_fn)), &wrapped_inputs, flat_input_tensors, @@ -91,6 +203,7 @@ pub fn manualComputation( ) catch |err| switch (err) { error.OutOfMemory => @panic("OOM"), }; + return outputs_struct; } @@ -101,27 +214,21 @@ const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyo noinline fn manualComputationInternal( compiler: *Compiler, arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, functor_name: []const u8, input_struct: *anyopaque, input_tensors: []*Tensor, output_struct: *anyopaque, - output_shapes: []*Shape, + global_output_shapes: []const Shape, local_fn: LocalComputationFn, ) error{OutOfMemory}![]*Tensor { const global_scope = compiler.currentScope(); const mlir_ctx = compiler.mlir_ctx; - const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { - .mesh => |m| m, - .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {d}: {f} and input {d}: {f}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), - }; - const manual_axes = collectManualAxes(mesh, input_tensors); - log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); + log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); const global_input_shapes = try arena.alloc(Shape, input_tensors.len); for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - const global_output_shapes = try arena.alloc(Shape, output_shapes.len); - for (global_output_shapes, output_shapes) |*shape, shape_ptr| shape.* = shape_ptr.*; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -135,11 +242,17 @@ noinline fn manualComputationInternal( const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } input_val.* = global_scope.value(tensor_ptr.*); } - const manual_block = mlir.Block.init(input_types, input_locs); - errdefer manual_block.deinit(); + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); const local_scope = compiler.pushBlock(manual_block); defer local_scope.pop(); @@ -151,17 +264,14 @@ noinline fn manualComputationInternal( }; } - for (output_shapes, global_output_shapes) |shape_ptr, global| { - shape_ptr.* = Sharding.shardedShape(global) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - compiler.manual_computation_depth += 1; defer compiler.manual_computation_depth -= 1; const local_outputs = try local_fn(arena, input_struct, output_struct); stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, output_shapes) |i, local_tensor, local_shape| { - stdx.debug.assert(local_shape.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, local_shape.*, local_tensor._shape }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); } const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); @@ -218,95 +328,11 @@ noinline fn manualComputationInternal( // Returned tensors describe the operation's global output shapes and sharding. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape = output_shape; + output_tensor_ptr._shape._sharding = output_shape._sharding; } return local_outputs; } -const ChosenMesh = union(enum) { - mesh: *const Sharding.Mesh, - incompatible: [2]u32, -}; - -fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ChosenMesh { - var mesh: ?*const Sharding.Mesh = null; - var mesh_source_id: u32 = @intCast(inputs.len); - for (0.., inputs) |i, input| { - if (input.shape()._sharding.mesh) |shape_mesh| { - if (mesh) |m| { - if (m != shape_mesh) - return .{ .incompatible = .{ mesh_source_id, @intCast(i) } }; - } else { - mesh = shape_mesh; - mesh_source_id = @intCast(i); - } - } - } - return .{ .mesh = mesh orelse fallback }; -} - -test chooseCommonMesh { - const fallback: Sharding.Mesh = .initTest(.fallback, .{.all}); - const first: Sharding.Mesh = .initTest(.first, .{ .data, .model }); - const second: Sharding.Mesh = .initTest(.second, .{ .experts, .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); - const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); - - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); -} - -fn collectManualAxes(mesh: *const Sharding.Mesh, inputs: []const *const Tensor) stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) { - var used: [Sharding.MAX_MESH_RANK]bool = @splat(false); - for (inputs) |input| { - const sharding = input.shape()._sharding; - if (sharding.mesh == null) continue; - std.debug.assert(sharding.mesh.? == mesh); - for (0..input.rank()) |dim| { - if (sharding.partition.get(dim).meshAxis()) |axis| used[axis] = true; - } - } - - var axes: stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) = .empty; - for (mesh.logical.axes.constSlice(), 0..) |axis, i| { - if (used[i]) axes.appendAssumeCapacity(axis); - } - return axes; -} - -test collectManualAxes { - const mesh: Sharding.Mesh = .initTest(.manual, .{ .data, .model, .unused }); - var data = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .data }); - var model = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const open = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .open }); - - const axes = collectManualAxes(&mesh, &.{ &model, &replicated, &data, &model, &open }); - data = replicated; - model = replicated; - try std.testing.expectEqual(@as(usize, 2), axes.len); - try std.testing.expectEqualStrings("data", std.mem.span(axes.get(0))); - try std.testing.expectEqualStrings("model", std.mem.span(axes.get(1))); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{ &replicated, &open }).len); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{}).len); - - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const expected = shardy.ManualAxesAttr.get(ctx, &.{ - .init(ctx, "data"), - .init(ctx, "model"), - }).attribute(); - try std.testing.expect(shardy_.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); -} - const gspmd = struct { fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); @@ -349,54 +375,172 @@ const gspmd = struct { } }; -const shardy_ = struct { - pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { - var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; - for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); - return shardy.ManualAxesAttr.get(mlir_ctx, attrs.constSlice()).attribute(); - } -}; - -test "manualComputation passes local output shapes to the callback" { +test manualComputation { + const zml = @import("../zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const mesh = zml.testing.tpMesh(); - const partitions = mesh.numPartitionsForLogicalAxis(.model); - if (partitions == 1) return error.SkipZigTest; - var compiler: Compiler = .init(allocator, io, platform, .{ .meshes = &.{mesh} }); - defer compiler.deinit(); - compiler.activate(); - defer compiler.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - defer block.deinit(); - const scope = compiler.pushBlock(block); - defer scope.pop(); - - const global_shape = Shape.init(.{ .b = 4, .h = 3 * partitions, .hd = 128 }, .bf16) - .withPartitioning(mesh, .{ .h = .model }); - const local_shape = Shape.init(.{ .b = 4, .h = 3, .hd = 128 }, .bf16); - const input = Tensor.zeroes(global_shape).withPartitioning(mesh, .{ .h = .model }); - var received_shape: ?Shape = null; - const Context = struct { - input: Tensor, - received_shape: *?Shape, + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, fn body(self: @This(), output_shape: Shape) Tensor { - self.received_shape.* = output_shape; - return self.input; + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); } }; - const output = manualComputation( - Context.body, - .{ .input = input, .received_shape = &received_shape }, - global_shape, - ); - try std.testing.expect(received_shape != null); - try std.testing.expect(received_shape.?.eql(local_shape)); - try std.testing.expect(output.shape().eql(global_shape)); - try std.testing.expectEqualDeep(global_shape._sharding, output.shape()._sharding); + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} + +test "partitionSpecs preserves optional tensors and active cache variants" { + const zml = @import("../zml.zig"); + const Logical = Sharding.Partitioning.Logical; + const Inputs = struct { + layers: []const Tensor, + bias: ?Tensor, + cache: union(enum) { disabled, values: [2]Tensor }, + scale: f32, + }; + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); + var inputs: Inputs = .{ + .layers = &.{ tensor, tensor }, + .bias = null, + .cache = .{ .values = .{ tensor, tensor } }, + .scale = 2, + }; + const heads: Logical = .parse(.{ .h = .model }); + const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); + const expected: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{ heads, heads }, + .bias = null, + .cache = .{ .values = .{ heads, heads } }, + }; + try std.testing.expectEqualDeep(expected, specs); + + inputs.layers = &.{}; + inputs.bias = tensor; + inputs.cache = .disabled; + const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); + const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{}, + .bias = .replicated, + .cache = .disabled, + }; + try std.testing.expectEqualDeep(expected_replicated, replicated); } diff --git a/zml/ops/manual_computation2.zig b/zml/ops/manual_computation2.zig deleted file mode 100644 index 2d3f0aec..00000000 --- a/zml/ops/manual_computation2.zig +++ /dev/null @@ -1,465 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const shardy = @import("mlir/dialects/shardy"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../Compiler.zig"); -const meta = @import("../meta.zig"); -const mlirx = @import("../mlirx.zig"); -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const Tensor = @import("../tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/Compiler"); - -/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. -/// Returns the type of the `partitioning` argument to `manualComputation`. -/// -/// * `in_specs` specifies the sharding of each tensor in the body's first argument. -/// It follows the argument's structure, omitting non-tensor metadata. -/// * `out_specs` specifies the sharding of each shape in the body's second argument. -/// It follows the output shapes' structure and can be omitted for a `void` argument. -/// -/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as -/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across -/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` -/// replicates the entire tensor. The specs must not contain open dimensions. -/// Slice lengths, optional presence and active union tags must match the arguments. -/// -/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single -/// output `Shape`, the partitioning can be declared as: -/// -/// ```zig -/// const partitioning: ManualPartitioning(body) = .{ -/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, -/// .out_specs = .parse(.{ .hidden = .model }), -/// }; -/// ``` -pub fn ManualPartitioning(comptime body_fn: anytype) type { - const Logical = Sharding.Partitioning.Logical; - - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - - const InputSpecs = meta.MapRestrict(Tensor, Logical).map(Inputs); - const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); - - if (OutputSpecs == void) return struct { - in_specs: InputSpecs, - out_specs: void = {}, - }; - - return struct { - in_specs: InputSpecs, - out_specs: OutputSpecs, - }; -} - -/// Describes a computation on local shards of tensors distributed across `mesh`. -/// ZML operations normally describe work on global tensors. The partitioner -/// distributes the inputs according to `in_specs`, and `body_fn` describes the work -/// performed on each shard. The results are exposed to the surrounding graph as -/// global tensors with the sharding declared by `out_specs`. -/// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: -/// - `I` contains the local input tensors and any metadata needed by the body. -/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. -/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes -/// in traversal order. -/// * `inputs` contains the global tensors and metadata. The partitioner reshards -/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. -/// * `output_shapes` contains the global output shapes. The body receives their -/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. -/// * `mesh` is the logical mesh whose axis names are referenced by the specs. -/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. -/// Each partitioned dimension must be divisible by the size of its mesh axis. -/// -/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives -/// the body a dimension of size 8. A replicated dimension keeps its global size. -/// The caller supplies the global output shapes because the body only describes -/// local results: their shapes alone do not determine the global tensor shapes. -pub fn manualComputation2( - comptime body_fn: anytype, - inputs: stdx.meta.FnParam(body_fn, 0), - output_shapes: stdx.meta.FnParam(body_fn, 1), - mesh: *const Sharding.Mesh, - partitioning: ManualPartitioning(body_fn), -) stdx.meta.FnReturn(body_fn) { - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - const Outputs = stdx.meta.FnReturn(body_fn); - - const compiler = Compiler.current(); - var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // The body may return containers borrowed from its inputs or output shapes. - const body_arena = compiler.arena.allocator(); - - var owned_inputs: Inputs = undefined; - meta.mapAlloc((struct { - fn copy(_: void, tensor: Tensor) Tensor { - return tensor; - } - }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); - - const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); - stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); - - for (flat_input_tensors, in_specs) |tensor, spec| { - tensor._shape = tensor.shape().withPartitioning(mesh, spec); - stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); - } - - var local_output_shapes: OutputShapes = undefined; - meta.mapAlloc((struct { - fn copy(_: void, shape: Shape) Shape { - return shape; - } - }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); - - const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); - const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); - stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); - - const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); - for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { - global.* = local.withPartitioning(mesh, spec); - stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); - local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - - const WrappedInputs = struct { - inputs: *const Inputs, - output_shapes: *const OutputShapes, - - fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { - const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); - const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); - - outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - - return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); - } - }; - - var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; - - _ = manualComputationInternal( - compiler, - arena, - mesh, - @typeName(@TypeOf(body_fn)), - &wrapped_inputs, - flat_input_tensors, - @ptrCast(&outputs_struct), - flat_output_shapes, - WrappedInputs.wrap, - ) catch |err| switch (err) { - error.OutOfMemory => @panic("OOM"), - }; - - return outputs_struct; -} - -const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; - -/// Reified version of manualComputation2. -/// This one is compiled exactly once, user code just need to compile the `collectPtrs` -noinline fn manualComputationInternal( - compiler: *Compiler, - arena: std.mem.Allocator, - mesh: *const Sharding.Mesh, - functor_name: []const u8, - input_struct: *anyopaque, - input_tensors: []*Tensor, - output_struct: *anyopaque, - global_output_shapes: []const Shape, - local_fn: LocalComputationFn, -) error{OutOfMemory}![]*Tensor { - const global_scope = compiler.currentScope(); - const mlir_ctx = compiler.mlir_ctx; - - log.debug("manualComputation2({s}) will use mesh {s}", .{ functor_name, mesh.name }); - - const global_input_shapes = try arena.alloc(Shape, input_tensors.len); - for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - - const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); - for (input_types, input_tensors) |*input_ty, tensor_ptr| { - const global = tensor_ptr._shape; - const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); - log.debug("\t- {f} -> {f}", .{ global, local }); - input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); - } - const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); - @memset(input_locs, compiler.unknown_location); - - const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); - for (input_values, input_tensors) |*input_val, tensor_ptr| { - if (compiler.partitioner == .gspmd) { - tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); - } - input_val.* = global_scope.value(tensor_ptr.*); - } - - const manual_block = switch (compiler.partitioner) { - .shardy => mlir.Block.init(input_types, input_locs), - .gspmd => global_scope.block, - }; - errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); - - const local_scope = compiler.pushBlock(manual_block); - defer local_scope.pop(); - for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { - const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above - tensor_ptr.* = switch (compiler.partitioner) { - .shardy => Tensor._result(local, manual_block.argument(i)), - .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), - }; - } - - compiler.manual_computation_depth += 1; - defer compiler.manual_computation_depth -= 1; - - const local_outputs = try local_fn(arena, input_struct, output_struct); - stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation2 body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { - const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); - stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation2 body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); - } - - const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |output_shape, i| { - global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); - } - - const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); - for (local_output_values, local_outputs) |*output_val, tensor_ptr| { - output_val.* = local_scope.value(tensor_ptr.*); - } - - const op: *const mlir.Operation = switch (compiler.partitioner) { - .shardy => op: { - // Shardy creates a sdy.manual_computation block. - _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); - - const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); - for (out_mesh_attrs, global_output_shapes) |*attr, shape| { - attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); - } - - break :op shardy.manual_computation( - mlir_ctx, - input_values, - global_output_types, - manual_block, - try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), - .get(mlir_ctx, out_mesh_attrs), - try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), - compiler.location, - ).appendTo(global_scope.block); - }, - .gspmd => op: { - // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier - // Shortcut to avoid empty optimization barrier - if (global_output_shapes.len == 0) return &.{}; - - const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); - const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |global_shape, i| { - global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); - } - - break :op stablehlo.optimizationBarrier( - mlir_ctx, - global_values, - global_types, - compiler.location, - ).appendTo(global_scope.block); - }, - }; - - // Returned tensors describe the operation's global output shapes and sharding. - for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { - output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape._sharding = output_shape._sharding; - } - return local_outputs; -} - -const gspmd = struct { - fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { - const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); - const full_to_shard = stablehlo.custom_call( - mlir_ctx, - &.{scope.value(local_tensor)}, - &.{local_type}, - .{ - .call_target_name = "SPMDFullToShardShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), - }, - }, - location, - ).appendTo(scope.block); - return Tensor._result(local_shape, full_to_shard.result(0)); - } - - fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { - const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); - const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); - - const shard_to_full = stablehlo.custom_call( - compiler.mlir_ctx, - &.{local_value}, - &.{global_type}, - .{ - .call_target_name = "SPMDShardToFullShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), - }, - }, - compiler.location, - ).appendTo(scope.block); - return .{ shard_to_full.result(0), global_type }; - } -}; - -test manualComputation2 { - const zml = @import("../zml.zig"); - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const mesh = zml.testing.tpMesh(); - - const LocalSum = struct { - x: Tensor, - bias: Tensor, - scale: i32, - - fn body(self: @This(), output_shape: Shape) Tensor { - return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); - } - - fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { - return manualComputation2( - body, - .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, - x.shape(), - mesh_, - .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, - ); - } - }; - - // Each shard sums two elements and broadcasts the result to its local output shape. - const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); - const values = try allocator.alloc(i32, shape.count()); - defer allocator.free(values); - - for (values, 0..) |*value, i| value.* = @intCast(i + 1); - - var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - const actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - - for (actual.constItems(i32), 0..) |value, i| { - const pair = values[i / 2 * 2 ..][0..2]; - try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); - } - - try std.testing.expect(output.shape().eqlWithTags(shape)); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } -} - -test "manualComputation2 composes across logical and physical meshes" { - const zml = @import("../zml.zig"); - if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - - const Topologies = enum { flat, torus }; - - const Model = struct { - fn increment(x: Tensor, _: Shape) Tensor { - return x.addConstant(1); - } - - fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { - const partitioning: ManualPartitioning(increment) = .{ - .in_specs = .parse(.{ .x = .model }), - .out_specs = .parse(.{ .x = .model }), - }; - - const first = manualComputation2(increment, x, x.shape(), mesh, partitioning); - const converted = first.convert(.f32); - return manualComputation2(increment, converted, converted.shape(), mesh, partitioning); - } - }; - - const shape: Shape = .init(.{ .x = 16 }, .i32); - - var values: [16]i32 = undefined; - var expected: [16]f32 = undefined; - for (&values, &expected, 0..) |*value, *result, i| { - value.* = @intCast(i); - result.* = @floatFromInt(i + 2); - } - - for ([_]Topologies{ .flat, .torus }) |topology| { - const platform = try zml.Platform.init(allocator, io, .cpu, .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, - }); - defer platform.deinit(allocator, io); - - // Fold three physical axes into one logical axis, or split one into three. - const logical: Sharding.LogicalMesh = if (topology == .torus) - try .init(.{.model}, &.{.balanced}, &.{8}) - else - try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); - - const mesh = try platform.registerMesh("manual", logical); - - var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); - try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } - } -} diff --git a/zml/tensor.zig b/zml/tensor.zig index a7782844..a09a5d38 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4813,7 +4813,7 @@ pub const Tensor = struct { defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation2((struct { + ops.manualComputation((struct { input: Tensor, name: []const u8, From a0533da7b87c83ff57f24faae015d0140ffc777b Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 13 Nov 2026 14:13:10 +0100 Subject: [PATCH 39/47] zml/kernel: introduce packages per DSL (#1061) Split `kernel.zig` into one package per DSL --- dsl/fly/README.md | 2 +- mlir/dialects/fly/REGENERATING.md | 2 +- platforms/tpu/ragged_paged.zig | 2 +- zml/BUILD.bazel | 2 +- zml/kernel.zig | 1630 ----------------- zml/kernel/BUILD.bazel | 15 + zml/kernel/cuda_tile/BUILD.bazel | 9 + zml/kernel/cuda_tile/cuda_tile.zig | 163 ++ zml/kernel/cuda_tile/test.zig | 28 + zml/kernel/cute/BUILD.bazel | 9 + zml/kernel/cute/cute.zig | 264 +++ zml/kernel/cute/test.zig | 34 + zml/kernel/fly/BUILD.bazel | 9 + zml/kernel/fly/fly.zig | 248 +++ zml/kernel/fly/test.zig | 149 ++ zml/kernel/kernel.zig | 12 + zml/kernel/metal/BUILD.bazel | 9 + zml/kernel/metal/metal.zig | 237 +++ zml/kernel/metal/test.zig | 71 + zml/kernel/mosaic_tpu/BUILD.bazel | 6 + zml/kernel/mosaic_tpu/mosaic_tpu.zig | 284 +++ zml/kernel/tcl/BUILD.bazel | 9 + zml/kernel/tcl/tcl.zig | 206 +++ zml/kernel/tcl/test.zig | 123 ++ zml/kernel/triton/BUILD.bazel | 6 + zml/kernel/triton/triton.zig | 165 ++ .../flash_attention/metal/flash_attn.zig | 3 +- .../paged_attention/metal/paged_attn.zig | 3 +- zml/zml.zig | 2 +- 29 files changed, 2063 insertions(+), 1639 deletions(-) delete mode 100644 zml/kernel.zig create mode 100644 zml/kernel/BUILD.bazel create mode 100644 zml/kernel/cuda_tile/BUILD.bazel create mode 100644 zml/kernel/cuda_tile/cuda_tile.zig create mode 100644 zml/kernel/cuda_tile/test.zig create mode 100644 zml/kernel/cute/BUILD.bazel create mode 100644 zml/kernel/cute/cute.zig create mode 100644 zml/kernel/cute/test.zig create mode 100644 zml/kernel/fly/BUILD.bazel create mode 100644 zml/kernel/fly/fly.zig create mode 100644 zml/kernel/fly/test.zig create mode 100644 zml/kernel/kernel.zig create mode 100644 zml/kernel/metal/BUILD.bazel create mode 100644 zml/kernel/metal/metal.zig create mode 100644 zml/kernel/metal/test.zig create mode 100644 zml/kernel/mosaic_tpu/BUILD.bazel create mode 100644 zml/kernel/mosaic_tpu/mosaic_tpu.zig create mode 100644 zml/kernel/tcl/BUILD.bazel create mode 100644 zml/kernel/tcl/tcl.zig create mode 100644 zml/kernel/tcl/test.zig create mode 100644 zml/kernel/triton/BUILD.bazel create mode 100644 zml/kernel/triton/triton.zig diff --git a/dsl/fly/README.md b/dsl/fly/README.md index b886988b..687c99dc 100644 --- a/dsl/fly/README.md +++ b/dsl/fly/README.md @@ -81,7 +81,7 @@ const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ ``` Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` -and `zml/kernel.zig`. +and `zml/kernel/fly/test.zig`. Model kernels live beside their consumers: diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md index b66b7abe..9673870f 100644 --- a/mlir/dialects/fly/REGENERATING.md +++ b/mlir/dialects/fly/REGENERATING.md @@ -71,7 +71,7 @@ dialect today; when one appears, extend the generator and this file. `test.zig`. - Callers that no longer compile are the point of generating: an operation whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`dsl/fly/builder.zig`, `zml/kernel.zig`, `zml/kernels/*/fly/`, + the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, and downstream llmd's `llmd/kernels/*/fly/`). - Emit the IR of the fly kernels before and after (for instance llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index 4c1c50ed..6a0fdfae 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -482,7 +482,7 @@ fn flashAttention( } /// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in -/// `zml/kernel.zig` opens the builder, calls this `run`, and finalizes the +/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the /// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 2b14c79f..f52ecdd5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -87,7 +87,6 @@ zig_library( "exe.zig", "floats.zig", "io.zig", - "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", "kernels/moe/api.zig", @@ -114,6 +113,7 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", + "//zml/kernel", "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", diff --git a/zml/kernel.zig b/zml/kernel.zig deleted file mode 100644 index f111f721..00000000 --- a/zml/kernel.zig +++ /dev/null @@ -1,1630 +0,0 @@ -const std = @import("std"); - -const cuda_tile_builder = @import("dsl/cuda_tile/builder"); -const cute_builder = @import("dsl/cute/builder"); -const fly_builder = @import("dsl/fly/builder"); -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); -const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const tcl_builder = @import("dsl/tcl/builder"); -const triton_builder = @import("dsl/triton/builder"); - -const Compiler = @import("Compiler.zig"); -const DataType = @import("dtype.zig").DataType; -const mlirx = @import("mlirx.zig"); -const ops = @import("ops.zig"); -const platform_ = @import("platform.zig"); -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub const triton = struct { - pub const Builder = triton_builder.Builder; - pub const Value = triton_builder.Value; - pub const DType = triton_builder.DType; - pub const FinishError = triton_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(triton_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - debug: bool = false, - /// Bytes for Triton's implicit global scratch argument, across all CTAs. - global_scratch_memory_size: i32 = 0, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try triton_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ttir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ttir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const call_opts: ops.TritonOps = .{ - .debug = opts.debug, - .name = name, - .ir = ttir, - .grid = opts.grid, - .num_stages = opts.num_stages, - .num_warps = opts.num_warps, - .output_operand_aliases = aliases.constSlice(), - .is_tma_allowed = opts.global_scratch_memory_size > 0, - .global_scratch_memory_size = opts.global_scratch_memory_size, - }; - - const triton_results = if (opts.global_scratch_memory_size > 0) blk: { - // XLA maps the final custom-call result to Triton's implicit - // scratch pointer; it must not appear in the TTIR signature. - const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; - const res = ops.triton(inputs_arr, extended, call_opts); - break :blk res[0..spec.outputs.len].*; - } else ops.triton(inputs_arr, outputs_arr, call_opts); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; - return results; - } - }; - } -}; - -pub const mosaic_tpu = struct { - const Builder = mosaic_tpu_builder.Builder; - const DType = mosaic_tpu_builder.DType; - const FinishError = mosaic_tpu_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(mosaic_tpu_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - dynamic_grid_bounds: []const Tensor = &.{}, - }; - - pub const CallOpts = struct { - cfg: ConfigT, - extras: CallExtras = .{}, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finishOpts(&.{}, .{ .canonicalize = true }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const backend_config = buildBackendConfig(cur.allocator, ir, .{ - .vmem_limit_bytes = opts.extras.vmem_limit_bytes, - .disable_bounds_checks = opts.extras.disable_bounds_checks, - .disable_semaphore_checks = opts.extras.disable_semaphore_checks, - .has_communication = opts.extras.has_communication, - }) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); - defer cur.allocator.free(backend_config); - - var values: [spec.inputs.len]*const mlir.Value = undefined; - inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); - - var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (&res_types, spec.outputs) |*r, fname| { - r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); - } - - const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); - - const op = callTpuCustomCall(.{ - .inputs = &values, - .result_types = &res_types, - .backend_config = backend_config, - .aliases = aliases.constSlice(), - .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, - .additional_attributes = opts.extras.additional_attributes, - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - @field(results, fname) = Tensor._result(out, op.result(i)); - } - return results; - } - }; - } - - const CustomCallConfig = struct { - body: []const u8, - has_communication: ?bool = null, - serialization_format: i64 = 1, - needs_layout_passes: bool = true, - shape_invariant_numerics: bool = false, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - }; - - const ScopedMemoryConfig = struct { - memory_space: i64 = 1, - offset: i64 = 0, - size: i64, - }; - - const BackendConfig = struct { - custom_call_config: CustomCallConfig, - scoped_memory_configs: ?[]const ScopedMemoryConfig = null, - }; - - const BackendConfigOpts = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - }; - - fn buildBackendConfig( - allocator: std.mem.Allocator, - ir: [:0]const u8, - opts: BackendConfigOpts, - ) ![]u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - const module = try mlir.Module.parse(ctx, ir); - defer module.deinit(); - - mosaic_tpu_dialect.registerMosaicSerdePass(); - const prev_allow_unregistered = ctx.allowUnregisteredDialects(); - ctx.setAllowUnregisteredDialects(true); - defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); - - const pm = mlir.PassManager.init(ctx); - defer pm.deinit(); - try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); - try pm.runOnOp(module.operation()); - - const Encoder = std.base64.standard.Encoder; - const b64 = b64: { - var bytecode: std.Io.Writer.Allocating = .init(allocator); - defer bytecode.deinit(); - try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); - const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); - break :b64 Encoder.encode(buf, bytecode.written()); - }; - defer allocator.free(b64); - - const config = BackendConfig{ - .custom_call_config = .{ - .body = b64, - .has_communication = opts.has_communication, - .disable_bounds_checks = opts.disable_bounds_checks, - .disable_semaphore_checks = opts.disable_semaphore_checks, - }, - .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| - &.{.{ .size = vmem }} - else - null, - }; - - var json: std.Io.Writer.Allocating = .init(allocator); - errdefer json.deinit(); - - try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - return try json.toOwnedSlice(); - } - - const TpuCustomCallArgs = struct { - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - backend_config: []const u8, - aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, - dynamic_grid_bounds: []const Tensor, - additional_attributes: []const mlir.NamedAttribute, - }; - - fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = Compiler.current(); - - var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; - for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); - all_inputs.appendSliceAssumeCapacity(args.inputs); - - return stablehlo.custom_call( - cur.mlir_ctx, - all_inputs.constSlice(), - args.result_types, - .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = args.backend_config }, - .has_side_effect = false, - .api_version = .original, - .additional_attributes = args.additional_attributes, - .output_operand_aliases = args.aliases, - }, - cur.location, - ).appendTo(cur.currentScope().block); - } -}; - -pub const cuda_tile = struct { - pub const Builder = cuda_tile_builder.Builder; - pub const Value = cuda_tile_builder.Value; - pub const DType = cuda_tile_builder.DType; - pub const FinishError = cuda_tile_builder.FinishError; - pub const EntryHint = cuda_tile_builder.EntryHint; - pub const Arch = cuda_tile_builder.Arch; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cuda_tile_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i4, .u4 => .i4, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - .f4e2m1 => .f4e2m1fn, - else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// No `num_warps`/`num_stages`: warp and CTA tuning is - /// `optimization_hints` inside the IR (`Opts.hints`). - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Outputs XLA zeroes before the launch, by position in - /// `spec.outputs`, ascending. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(&.{}); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const cute = struct { - pub const AlgebraExpr = cute_builder.AlgebraExpr; - pub const Builder = cute_builder.Builder; - pub const ArgSpec = cute_builder.ArgSpec; - pub const Value = cute_builder.Value; - pub const View = cute_builder.View; - pub const Atom = cute_builder.Atom; - pub const DType = cute_builder.DType; - pub const AlgebraToken = cute_builder.AlgebraToken; - pub const LayoutSpec = cute_builder.LayoutSpec; - pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; - pub const FinishError = cute_builder.FinishError; - pub const TmaConfig = cute_builder.TmaConfig; - pub const TmaFormat = cute_builder.TmaFormat; - pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; - pub const CtaGroup = cute_builder.CtaGroup; - pub const ctaCount = cute_builder.ctaCount; - pub const MBarrierScope = cute_builder.MBarrierScope; - pub const OperandMajorMode = cute_builder.OperandMajorMode; - pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; - pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; - pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; - pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; - pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; - pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cute_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4e2m1fn, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared - /// memory is what the kernel declares. - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - block: [3]i32, - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - // XLA passes one pointer per input then per output; it does - // not check the count against the kernel. - const expected = spec.inputs.len + spec.outputs.len; - if (b.args.len != expected) { - std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); - } - - return b.finish(block); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| - std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .block = opts.block, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } - - fn ProgramSpec(comptime Config: type) type { - return struct { - /// Public host entry selected by XLA. The callback may give the - /// nested CUDA kernel a different (private) symbol. - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A complete CuTe program: one or more `cuda.kernel` operations inside - /// a `gpu.module`, followed by a public `func.func` which constructs the - /// launch-time CuTe objects and calls `cuda.launch_ex`. - /// - /// This is the source-level equivalent of invoking a Python `@cute.jit` - /// object whose `__call__` builds TMA descriptors and launches an - /// `@cute.kernel`. XLA executes the generated host function on its CUDA - /// stream, so the custom call deliberately carries no external grid or - /// block dimensions. - pub fn Program( - comptime ConfigT: type, - comptime spec: ProgramSpec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.openProgram(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - return b.finishProgram(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .zeroed_outputs = opts.zeroed_outputs, - .scalars = opts.scalars, - .symmetric_workspace = opts.symmetric_workspace, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const tcl = struct { - pub const Builder = tcl_builder.Builder; - pub const TensorOperation = tcl_builder.TensorOperation; - pub const Value = tcl_builder.Value; - pub const Axis = tcl_builder.Axis; - pub const Expr = tcl_builder.Expr; - pub const Dram = tcl_builder.Dram; - pub const Context = tcl_builder.Context; - pub const DType = tcl_builder.DType; - pub const VeOp = tcl_builder.VeOp; - pub const ReduceMode = tcl_builder.ReduceMode; - pub const Predicate = tcl_builder.Predicate; - pub const Tactic = tcl_builder.Tactic; - pub const FinishError = tcl_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(tcl_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .bool, - .i4 => .i4, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4_e2, - .f8e4m3fn => .f8_e4, - .f8e5m2 => .f8_e5, - else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A TCL function compiled by the Furiosa plugin as one native unit: - /// `declareArgs` declares the inputs in order, `ret` the outputs. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - return build(allocator, cfg, null, null, null); - } - - /// `updated`, when given, receives the input each output updates - /// in place (`Builder.update`), or null. - fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try Builder.open(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - - const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); - if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { - std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); - } - - if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); - if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); - if (updated) |u| for (u, results) |*dst, r| { - dst.* = b.updatedArgument(r); - }; - - return b.finish(); - } - - fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { - var same = from(s.dtype()) == t.dtype; - if (t.shards > 1) { - // A partition holds one chip's share of a chip-split DRAM tensor. - var n: i64 = 1; - for (t.axes) |a| n *= a.size; - same = same and n == @as(i64, @intCast(s.count())) * t.shards; - } else { - same = same and s.rank() == t.axes.len; - if (same) for (t.axes, 0..) |a, i| { - same = same and a.size == s.dim(i); - }; - } - - if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - var input_shapes: [spec.inputs.len]Shape = undefined; - inline for (spec.inputs, 0..) |fname, i| { - inputs_arr[i] = @field(inputs, fname); - input_shapes[i] = inputs_arr[i].shape(); - } - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var updated: [spec.outputs.len]?usize = undefined; - const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| - std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ - .output_index = @intCast(output), - .operand_index = @intCast(i), - }); - const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - @field(results, fname) = tensor_results[i]; - } - - return results; - } - }; - } -}; - -pub const metal = struct { - pub const Builder = metal_builder.Builder; - pub const Value = metal_builder.Value; - pub const Type = metal_builder.Type; - pub const Struct = metal_builder.Struct; - pub const ArgSpec = metal_builder.ArgSpec; - pub const LoopOpts = metal_builder.LoopOpts; - pub const FinishError = metal_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(metal_builder.registerDialects); - } - - pub fn from(dt: DataType) Type { - return switch (dt) { - .bool => .bool, - .i8 => .char, - .i16 => .short, - .i32 => .int, - .i64 => .long, - .u8 => .uchar, - .u16 => .ushort, - .u32 => .uint, - .u64 => .ulong, - .f16 => .half, - .bf16 => .bfloat, - .f32 => .float, - else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), - }; - } - - /// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host - /// `extern struct` laid out like a `constant S &` argument. - pub const Constant = struct { - buffer: [256]u8 = undefined, - len: usize = 0, - - pub fn of(value: anytype) Constant { - const raw = std.mem.asBytes(&value); - var c: Constant = .{ .len = raw.len }; - @memcpy(c.buffer[0..raw.len], raw); - return c; - } - - pub fn bytes(self: *const Constant) []const u8 { - return self.buffer[0..self.len]; - } - }; - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - /// The arguments bound to input tensors. - inputs: []const [:0]const u8, - /// The arguments bound to output tensors; one named like an - /// input updates it in place. - outputs: []const [:0]const u8, - /// The `constant` arguments given by value at the call. - constants: []const [:0]const u8 = &.{}, - run: *const fn (*Builder, Config) FinishError!void, - fast_math: bool = true, - }; - } - - /// A Metal kernel: `run` declares its arguments with `declareArgs`, by - /// name, and builds its body. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const Constants = StructOf(spec.constants, Constant); - - pub const CallOpts = struct { - cfg: ConfigT, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - constants: Constants = std.mem.zeroes(Constants), - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - return build(allocator, cfg, &arguments); - } - - fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); - defer b.deinit(); - try spec.run(&b, cfg); - const ir = try b.finish(); - errdefer allocator.free(ir); - try arguments.appendSlice(allocator, b.arguments.items); - return ir; - } - - fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { - for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; - return null; - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const allocator = cur.allocator; - - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - const ir = build(allocator, opts.cfg, &arguments) catch |err| - std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); - var constants: [spec.constants.len][]const u8 = undefined; - inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); - - var bound: [spec.outputs.len]bool = @splat(false); - var bound_inputs: [spec.inputs.len]bool = @splat(false); - var bound_constants: [spec.constants.len]bool = @splat(false); - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - var bindings: std.ArrayList(i32) = .empty; - defer bindings.deinit(allocator); - for (arguments.items) |arg| { - if (arg.kind == .threadgroup) { - // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. - if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); - continue; - } - std.debug.assert(arg.index == bindings.items.len); - const input = indexOf(spec.inputs, arg.name); - const output = indexOf(spec.outputs, arg.name); - if (input) |i| bound_inputs[i] = true; - if (output) |o| bound[o] = true; - const binding: i32 = if (input) |i| blk: { - if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); - break :blk @intCast(i); - } else if (output) |o| - @intCast(spec.inputs.len + o) - else if (indexOf(spec.constants, arg.name)) |k| blk: { - if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); - bound_constants[k] = true; - break :blk -1 - @as(i32, @intCast(k)); - } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); - bindings.append(allocator, binding) catch @panic("OOM"); - } - for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); - for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); - for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); - - const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .threadgroup = opts.threadgroup, - .threadgroup_memory = opts.threadgroup_memory, - .arguments = bindings.items, - .constants = &constants, - .output_operand_aliases = aliases.constSlice(), - }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const fly = struct { - /// Threads per wavefront: 64 on CDNA, 32 on RDNA. - pub fn waveSize(p: *const platform_.Platform) i32 { - const cc = platform_.rocm.computeCapability(p) orelse return 64; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => 64, - .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, - }; - } - - /// The plugin sizes a block in wavefronts, so a thread count has to divide - /// evenly: a kernel's layouts are built for an exact number of threads and - /// spare ones would partition out of range. - fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { - const wave = waveSize(p); - if (@rem(threads, wave) != 0) { - std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); - } - return @divExact(threads, wave); - } - - pub const Builder = fly_builder.Builder; - pub const Value = fly_builder.Value; - pub const DType = fly_builder.DType; - pub const FinishError = fly_builder.FinishError; - pub const TiledCopy = fly_builder.TiledCopy; - pub const TiledMma = fly_builder.TiledMma; - pub const Arch = fly_builder.Arch; - pub const MmaFlavor = fly_builder.MmaFlavor; - pub const fly_rocdl = fly_builder.fly_rocdl; - - /// The matrix atom this device provides, or null when it has none: WMMA - /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. - pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { - const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, - .rdna3, .rdna3_5 => .gfx11_wmma, - .rdna4 => .gfx120x_wmma, - .rdna2 => null, - }; - } - pub const layout = fly_builder.layout; - pub const Layout = fly_builder.Layout; - pub const Tile = fly_builder.Tile; - pub const IntTuple = fly_builder.IntTuple; - pub const L = fly_builder.L; - pub const rowMajor = fly_builder.rowMajor; - pub const colMajor = fly_builder.colMajor; - pub const ordered = fly_builder.ordered; - pub const tile = fly_builder.tile; - pub const it = fly_builder.it; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(fly_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - .f8e5m2fnuz => .f8e5m2fnuz, - else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const arg_names = spec.inputs ++ spec.outputs; - /// By name, as `!fly.memref` views. - pub const Args = StructOf(arg_names, Value); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Threads per block. Converted to the wavefronts the plugin - /// wants using the device's wave size, which is 64 on CDNA and - /// 32 on RDNA, so a kernel states the thread count its layouts - /// were built for and runs on both. - threads: i32, - /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. - waves_per_eu: i32 = 0, - /// Required when the body uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Inputs first, then outputs; see `zeroedOutput`. A no-op - /// inside command buffers. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - /// The arguments `run` was declared with. - pub fn args(b: *Builder) Args { - var out: Args = undefined; - inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); - return out; - } - - /// `zeroed_args` index of an output. - pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); - } - - /// `shapes` is inputs, then outputs. - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { - std.debug.assert(shapes.len == arg_names.len); - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try fly_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - var specs: [arg_names.len]Builder.ArgSpec = undefined; - inline for (arg_names, 0..) |fname, i| { - const shape = shapes[i]; - specs[i] = .{ - .name = fname, - .dtype = from(shape.dtype()), - .dims = if (shape.rank() > 0) shape.dims() else null, - }; - } - try b.declareArgsLow(&specs); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var shapes: [arg_names.len]Shape = undefined; - inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); - inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; - - const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| - std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .num_warps = warpsFor(cur.platform, opts.threads), - .waves_per_eu = opts.waves_per_eu, - .shared_mem_bytes = opts.shared_mem_bytes, - .zeroed_args = opts.zeroed_args, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -const FlyVectorAdd = struct { - const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), - }; - - const K = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); - } -}; - -test "fly kernel emits a module XLA can parse" { - const shapes = [_]Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; - const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); -} - -/// C = A @ B^T with one MFMA-tiled block. -const FlyTiledMma = struct { - const Cfg = struct { flavor: fly.MmaFlavor }; - const K = fly.Kernel(Cfg, .{ - .name = "tiled_mma", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); - } -}; - -test "fly kernels run on rocm" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward`, upload both inputs, run, and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); - defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); - defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); - defer bb.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // vector add: elementwise over a ragged (100, 1000), predicated at the edge. - { - const M, const N = .{ 100, 1000 }; - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, - .threads = 8 * 16, - }).c; - } - }; - - const ha = try allocator.alloc(f32, M * N); - defer allocator.free(ha); - const hb = try allocator.alloc(f32, M * N); - defer allocator.free(hb); - for (ha, hb, 0..) |*va, *vb, i| { - va.* = @floatFromInt(i % 977); - vb.* = @floatFromInt(1000 + (i % 313)); - } - - var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); - } - - // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix - // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, - // whose verifier rejects f32 operands, so the operand type and K come from - // the flavour rather than being written into the test. - if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { - inline else => |f| { - const Elem = switch (comptime f.operandDType()) { - .f32 => f32, - .f16 => f16, - else => @compileError("unhandled MMA operand type"), - }; - const M, const N = .{ 64, 64 }; - const Kd = comptime f.blockK(); - const operand_dtype: DataType = switch (comptime f.operandDType()) { - .f32 => .f32, - .f16 => .f16, - else => unreachable, - }; - - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = Shape.init(.{ M, N }, .f32) }, .{ - .cfg = .{ .flavor = f }, - .grid = .{ 1, 1, 1 }, - .threads = comptime f.blockThreads(), - }).c; - } - }; - - var ha: [M * Kd]Elem = undefined; - var hb: [N * Kd]Elem = undefined; - for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; - for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; - - var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); - defer host.free(allocator); - - // f16 operands accumulate in f32, so the error is set by the input - // rounding rather than the sum. - const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; - const out = host.items(f32); - for (0..M) |m| for (0..N) |n| { - var acc: f32 = 0; - for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); - try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); - }; - }, - }; -} - -/// `furiosa.tcl.examples.softmax` over f32. -const TclSoftmax = struct { - const Cfg = struct { b: i64, s: i64, d: i64 }; - const K = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"x"}, - .outputs = &.{"y"}, - .run = run, - }); - - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.x, .{}); - const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); - b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); - } -}; - -test "tcl kernels run on furiosa" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; - - const ir = try TclSoftmax.K.emit(allocator, cfg); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); - - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - - const Mod = struct { - pub fn forward(x: Tensor) Tensor { - return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; - } - }; - const x: Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); - defer exe.deinit(); - - var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); - defer x_buffer.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - - const out = host.items(f32); - var row: usize = 0; - while (row < cfg.b * cfg.s) : (row += 1) { - const in = input[row * cfg.d ..][0..cfg.d]; - var max: f32 = in[0]; - for (in) |v| max = @max(max, v); - var sum: f32 = 0; - for (in) |v| sum += @exp(v - max); - for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); - } -} - -test "cute kernel emits the module the CuTe compiler takes" { - const Cfg = struct { n: i64 }; - const AddOne = cute.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - }); - const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); - var guard = b.openIf(i.lt(cfg.n)); - { - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.close(); - } - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); -} - -test "cuda_tile kernel emits a module XLA can parse" { - const Cfg = struct { n: i64 }; - const AddOne = cuda_tile.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); -} - -/// exp over a tensor split across two chips in one launch, returning the -/// split result and its all_gather. -const TclSplitExp = struct { - const Cfg = struct {}; - const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); - fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - } - fn body(ctx: struct { x: Tensor }, shapes: [2]Shape) [2]Tensor { - const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); - return .{ r.y, r.all }; - } -}; - -// Needs FURIOSA_VISIBLE_DEVICES=0,1. -test "tcl multi-chip kernels run on two furiosa chips" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const tp = zml.testing.tpMesh(); - const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); - const Mod = struct { - pub fn forward(x: Tensor, mesh: *const zml.Sharding.Mesh) [2]Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ - .in_specs = .{ .x = .parse(.{ .c = .model }) }, - .out_specs = .{ .parse(.{ .c = .model }), .replicated }, - }); - return .{ r[0], r[1] }; - } - }; - const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); - defer exe.deinit(); - var hx: [2 * 1024]f32 = undefined; - for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var res: [2]zml.Buffer = undefined; - runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); - defer for (&res) |*r| r.deinit(); - for (res) |r| { - var host = try r.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); - } -} - -/// `out[i] = a[i] * scale + b[i]` over `n` floats. -const MetalAxpy = struct { - const Cfg = struct {}; - const K = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -test "metal kernels run on metal" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - - const ir = try MetalAxpy.K.emit(allocator, .{}); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); - - const platform = zml.testing.env(); - if (platform.target != .metal) return error.SkipZigTest; - - const n = 1000; - const Mod = struct { - pub fn forward(x: Tensor, y: Tensor) Tensor { - return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ (n + 255) / 256, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, - }).out; - } - }; - const x: Tensor = .init(.{n}, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, Tensor.init(.{n}, .f32) }, platform, .{}); - defer exe.deinit(); - - var hx: [n]f32 = undefined; - var hy: [n]f32 = undefined; - for (&hx, &hy, 0..) |*a, *b, i| { - a.* = @floatFromInt(i); - b.* = 1; - } - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); - defer by.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); -} diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel new file mode 100644 index 00000000..61ab44f8 --- /dev/null +++ b/zml/kernel/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "kernel", + srcs = [ + "kernel.zig", + "//zml/kernel/cuda_tile", + "//zml/kernel/cute", + "//zml/kernel/fly", + "//zml/kernel/metal", + "//zml/kernel/mosaic_tpu", + "//zml/kernel/tcl", + "//zml/kernel/triton", + ], +) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..89b7977e --- /dev/null +++ b/zml/kernel/cuda_tile/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cuda_tile", + srcs = [ + "cuda_tile.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..3f6d8734 --- /dev/null +++ b/zml/kernel/cuda_tile/cuda_tile.zig @@ -0,0 +1,163 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +pub const Builder = cuda_tile_builder.Builder; +pub const Value = cuda_tile_builder.Value; +pub const DType = cuda_tile_builder.DType; +pub const FinishError = cuda_tile_builder.FinishError; +pub const EntryHint = cuda_tile_builder.EntryHint; +pub const Arch = cuda_tile_builder.Arch; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig new file mode 100644 index 00000000..c929b0b9 --- /dev/null +++ b/zml/kernel/cuda_tile/test.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +const cuda_tile = @import("cuda_tile.zig"); + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel new file mode 100644 index 00000000..7a613072 --- /dev/null +++ b/zml/kernel/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig new file mode 100644 index 00000000..2e8c919f --- /dev/null +++ b/zml/kernel/cute/cute.zig @@ -0,0 +1,264 @@ +const std = @import("std"); + +const cute_builder = @import("dsl/cute/builder"); +pub const AlgebraExpr = cute_builder.AlgebraExpr; +pub const Builder = cute_builder.Builder; +pub const ArgSpec = cute_builder.ArgSpec; +pub const Value = cute_builder.Value; +pub const View = cute_builder.View; +pub const Atom = cute_builder.Atom; +pub const DType = cute_builder.DType; +pub const AlgebraToken = cute_builder.AlgebraToken; +pub const LayoutSpec = cute_builder.LayoutSpec; +pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; +pub const FinishError = cute_builder.FinishError; +pub const TmaConfig = cute_builder.TmaConfig; +pub const TmaFormat = cute_builder.TmaFormat; +pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; +pub const CtaGroup = cute_builder.CtaGroup; +pub const ctaCount = cute_builder.ctaCount; +pub const MBarrierScope = cute_builder.MBarrierScope; +pub const OperandMajorMode = cute_builder.OperandMajorMode; +pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; +pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; +pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; +pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; +pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; +pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} + +fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A complete CuTe program: one or more `cuda.kernel` operations inside +/// a `gpu.module`, followed by a public `func.func` which constructs the +/// launch-time CuTe objects and calls `cuda.launch_ex`. +/// +/// This is the source-level equivalent of invoking a Python `@cute.jit` +/// object whose `__call__` builds TMA descriptors and launches an +/// `@cute.kernel`. XLA executes the generated host function on its CUDA +/// stream, so the custom call deliberately carries no external grid or +/// block dimensions. +pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig new file mode 100644 index 00000000..f4d457a3 --- /dev/null +++ b/zml/kernel/cute/test.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const cute = @import("cute.zig"); + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel new file mode 100644 index 00000000..57203521 --- /dev/null +++ b/zml/kernel/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "test.zig", + ], +) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig new file mode 100644 index 00000000..b7d6c943 --- /dev/null +++ b/zml/kernel/fly/fly.zig @@ -0,0 +1,248 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); +pub const Builder = fly_builder.Builder; +pub const Value = fly_builder.Value; +pub const DType = fly_builder.DType; +pub const FinishError = fly_builder.FinishError; +pub const TiledCopy = fly_builder.TiledCopy; +pub const TiledMma = fly_builder.TiledMma; +pub const Arch = fly_builder.Arch; +pub const MmaFlavor = fly_builder.MmaFlavor; +pub const fly_rocdl = fly_builder.fly_rocdl; +pub const layout = fly_builder.layout; +pub const Layout = fly_builder.Layout; +pub const Tile = fly_builder.Tile; +pub const IntTuple = fly_builder.IntTuple; +pub const L = fly_builder.L; +pub const rowMajor = fly_builder.rowMajor; +pub const colMajor = fly_builder.colMajor; +pub const ordered = fly_builder.ordered; +pub const tile = fly_builder.tile; +pub const it = fly_builder.it; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +/// Threads per wavefront: 64 on CDNA, 32 on RDNA. +pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; +} + +/// The plugin sizes a block in wavefronts, so a thread count has to divide +/// evenly: a kernel's layouts are built for an exact number of threads and +/// spare ones would partition out of range. +fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); +} + +/// The matrix atom this device provides, or null when it has none: WMMA +/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. +pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig new file mode 100644 index 00000000..ae788d96 --- /dev/null +++ b/zml/kernel/fly/test.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); + +const fly = @import("fly.zig"); +const zml = @import("../../zml.zig"); + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +test "fly kernels run on rocm" { + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig new file mode 100644 index 00000000..204c6204 --- /dev/null +++ b/zml/kernel/kernel.zig @@ -0,0 +1,12 @@ +pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); +pub const cute = @import("cute/cute.zig"); +pub const fly = @import("fly/fly.zig"); +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const tcl = @import("tcl/tcl.zig"); +pub const triton = @import("triton/triton.zig"); + +test { + const std = @import("std"); + std.testing.refAllDecls(@This()); +} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel new file mode 100644 index 00000000..4007beb4 --- /dev/null +++ b/zml/kernel/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + "test.zig", + ], +) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig new file mode 100644 index 00000000..9626a1de --- /dev/null +++ b/zml/kernel/metal/metal.zig @@ -0,0 +1,237 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +pub const Builder = metal_builder.Builder; +pub const Value = metal_builder.Value; +pub const Type = metal_builder.Type; +pub const Struct = metal_builder.Struct; +pub const ArgSpec = metal_builder.ArgSpec; +pub const LoopOpts = metal_builder.LoopOpts; +pub const FinishError = metal_builder.FinishError; +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); +} + +pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; +} + +/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host +/// `extern struct` laid out like a `constant S &` argument. +pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } +}; + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; +} + +/// A Metal kernel: `run` declares its arguments with `declareArgs`, by +/// name, and builds its body. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig new file mode 100644 index 00000000..0a342af0 --- /dev/null +++ b/zml/kernel/metal/test.zig @@ -0,0 +1,71 @@ +const std = @import("std"); + +const metal = @import("metal.zig"); +const zml = @import("../../zml.zig"); + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: zml.Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ba3b6b66 --- /dev/null +++ b/zml/kernel/mosaic_tpu/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = ["mosaic_tpu.zig"], +) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..a09e8368 --- /dev/null +++ b/zml/kernel/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,284 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const Builder = mosaic_tpu_builder.Builder; +const DType = mosaic_tpu_builder.DType; +const FinishError = mosaic_tpu_builder.FinishError; +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; +} + +const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, +}; + +const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, +}; + +const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, +}; + +const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, +}; + +fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, +) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); +} + +const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, +}; + +fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); +} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel new file mode 100644 index 00000000..bc7a0aa2 --- /dev/null +++ b/zml/kernel/tcl/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "tcl", + srcs = [ + "tcl.zig", + "test.zig", + ], +) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig new file mode 100644 index 00000000..023e862f --- /dev/null +++ b/zml/kernel/tcl/tcl.zig @@ -0,0 +1,206 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +pub const Builder = tcl_builder.Builder; +pub const TensorOperation = tcl_builder.TensorOperation; +pub const Value = tcl_builder.Value; +pub const Axis = tcl_builder.Axis; +pub const Expr = tcl_builder.Expr; +pub const Dram = tcl_builder.Dram; +pub const Context = tcl_builder.Context; +pub const DType = tcl_builder.DType; +pub const VeOp = tcl_builder.VeOp; +pub const ReduceMode = tcl_builder.ReduceMode; +pub const Predicate = tcl_builder.Predicate; +pub const Tactic = tcl_builder.Tactic; +pub const FinishError = tcl_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A TCL function compiled by the Furiosa plugin as one native unit: +/// `declareArgs` declares the inputs in order, `ret` the outputs. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; +} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig new file mode 100644 index 00000000..b9ae1875 --- /dev/null +++ b/zml/kernel/tcl/test.zig @@ -0,0 +1,123 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const tcl = @import("tcl.zig"); + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: zml.Tensor) zml.Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const tp = zml.testing.tpMesh(); + const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); + return .{ r[0], r[1] }; + } + }; + const x: zml.Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel new file mode 100644 index 00000000..1e992352 --- /dev/null +++ b/zml/kernel/triton/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "triton", + srcs = ["triton.zig"], +) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig new file mode 100644 index 00000000..6a91a03f --- /dev/null +++ b/zml/kernel/triton/triton.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const triton_builder = @import("dsl/triton/builder"); +pub const Builder = triton_builder.Builder; +pub const Value = triton_builder.Value; +pub const DType = triton_builder.DType; +pub const FinishError = triton_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const ops = @import("../../ops.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; +} diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig index 4faa1e82..de4b68da 100644 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -4,9 +4,8 @@ const std = @import("std"); -const kernel = @import("../../../kernel.zig"); +const metal = @import("../../../kernel/kernel.zig").metal; -const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 337432e7..04f4c1ac 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,9 +3,8 @@ const std = @import("std"); -const kernel = @import("../../../kernel.zig"); +const metal = @import("../../../kernel/kernel.zig").metal; -const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/zml.zig b/zml/zml.zig index 35c2f9df..dafc3a0f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -26,7 +26,7 @@ pub const Exe = exe.Exe; pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); -pub const kernel = @import("kernel.zig"); +pub const kernel = @import("kernel/kernel.zig"); pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; From 4af281a0bed2bb9c6f4de262b36ed635f5663f0a Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:38:46 +0100 Subject: [PATCH 40/47] stdx/flags: cleanup flag_name using mem.replaceScalar (#1068) Just porting 5253ea1362d04d976734e7fdc6e417422e6e693d from #1065 --- stdx/flags.zig | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/stdx/flags.zig b/stdx/flags.zig index 52a9e6a4..d0c4352d 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -591,25 +591,19 @@ fn fields_to_comma_list(comptime E: type) []const u8 { } } +/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. pub fn flag_name(comptime field_name: []const u8) []const u8 { - // TODO(Zig): Cleanup when this is fixed after Zig 0.11. - // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. - return comptime blk: { - assert(!std.mem.eql(u8, field_name, "positional")); - - var result: []const u8 = "--"; - var index = 0; - while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { - result = result ++ field_name[index..][0..i] ++ "-"; - index = index + i + 1; - } - result = result ++ field_name[index..]; - break :blk result; - }; + comptime assert(!std.mem.eql(u8, field_name, "positional")); + comptime var name = field_name[0..field_name.len].*; + comptime std.mem.replaceScalar(u8, &name, '_', '-'); + const dashed = name; + return "--" ++ dashed; } test flag_name { - try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); + try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); + try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); + try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); } fn flag_name_positional(comptime field_name: []const u8) []const u8 { From 3ecf1ff68be97b79519a096719666c0d4fcbb54e Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 13 Nov 2026 17:34:08 +0100 Subject: [PATCH 41/47] zml/moe: mask nonlocal BF16 routes on Furiosa (#1062) Replace nonlocal expert IDs with an in-range placeholder and zero their routing weights before dispatch. This prevents the activation-stationary and blockwise kernels from using -1 in gathers and scatters. Verified BF16 outputs against the CPU reference and coherent Qwen3.5-35B-A3B generation on two and four RNGD cards. --- zml/kernels/moe/tcl/tcl.zig | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 6238d5f6..c26fff1b 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -89,10 +89,14 @@ fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Te const x = input.reshape(.{ cfg.t, cfg.d }); const ids = topk_ids.reshape(.{ cfg.t, cfg.k }); const weights = topk_weights.reshape(.{ cfg.t, cfg.k }); + // These kernels gather every route, and blockwise also scatters each route + // into an expert slot. Give nonlocal routes a valid slot and zero weight. + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); + const safe_ids = valid.select(ids, Tensor.scalar(0, .i32)); + const safe_weights = valid.select(weights, Tensor.scalar(0, .f32)); const out_shape = x.shape(); if (scales) |s| { - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); - const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = valid.select(ids, Tensor.scalar(0, .i32)), .w = valid.select(weights, Tensor.scalar(0, .f32)), .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; + const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; const y = if (cfg.t * cfg.k <= cfg.e) ActivationStationaryFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y else @@ -100,9 +104,9 @@ fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Te return y.reshape(input.shape()); } const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y + ActivationStationary.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y else - Blockwise.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; + Blockwise.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; return y.reshape(input.shape()); } From 4c8f520251d587889d88123604a505c2b5644791 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 13 Nov 2026 17:34:09 +0100 Subject: [PATCH 42/47] zml/moe: support BF16 GEGLU-tanh on Furiosa (#1063) Pass the requested activation to both BF16 TCL MoE kernels and compute tanh GELU on the gate before multiplying by the up projection. This lets Gemma 4 compile and run with its required activation. Verified both kernel paths against a CPU reference and confirmed coherent Gemma 4 generation on two and four RNGD cards. --- zml/kernels/moe/tcl/tcl.zig | 62 ++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index c26fff1b..71653f8e 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -27,22 +27,26 @@ const tcl = zml.kernel.tcl; const Options = @import("../api.zig").Options; const LocalRouting = @import("../api.zig").LocalRouting; +const Activation = enum { swiglu, geglu_tanh }; + /// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. -/// SwiGLU, routing weights applied after the down projection. +/// Supports plain SwiGLU and, for BF16 experts, GEGLU with tanh GELU. pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { const topk_ids = routing.localIds(); stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); const gate_up = gate_up_linear.weight; const down = down_linear.weight; const quantized = gate_up_linear.quantization != null or down_linear.quantization != null; + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (!quantized and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + const activation: Activation = if (swiglu) .swiglu else .geglu_tanh; const scales: ?Scales = if (quantized) .{ .gate_up = fp8Scales(gate_up_linear), .down = fp8Scales(down_linear) } else null; stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32 and (quantized or (gate_up.dtype() == .bf16 and down.dtype() == .bf16)), "TCL MoE takes BF16 activations, BF16/block-128 FP8 experts and i32 ids", .{}); const weights = routing.weights.convert(.f32); const local_weights = if (scales != null) topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)) else weights; - return runExperts(input, topk_ids, local_weights, gate_up, down, scales); + return runExperts(input, topk_ids, local_weights, gate_up, down, scales, activation); } const Scales = struct { gate_up: Tensor, down: Tensor }; @@ -60,13 +64,14 @@ fn fp8Scales(linear: zml.nn.Linear) Tensor { /// Tokens up to which a device runs its experts stationary. pub const expert_stationary_tokens = 64; -fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales) Tensor { +fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales, activation: Activation) Tensor { var cfg: Cfg = .{ .t = input.dim(.t), .k = topk_ids.dim(.topk), .e = gate_up.dim(.expert), .d = input.dim(.d), .i = down.dim(.dout), + .activation = activation, }; stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); if (scales) |s| { @@ -115,7 +120,15 @@ fn scaleDtype(scales: Scales) tcl.DType { return if (scales.gate_up.dtype() == .bf16) .bf16 else .f32; } -pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64, scale_dtype: tcl.DType = .bf16 }; +pub const Cfg = struct { + t: i64, + k: i64, + e: i64, + d: i64, + i: i64, + scale_dtype: tcl.DType = .bf16, + activation: Activation = .swiglu, +}; const Axes = struct { T: tcl.Axis, @@ -246,7 +259,7 @@ fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?b /// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` /// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, comptime fp8: bool) tcl.FinishError!builder.Tensor { +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: Activation, comptime fp8: bool) tcl.FinishError!builder.Tensor { var sliced: [4]tcl.Axis = undefined; var flat: [3]tcl.Axis = undefined; @memcpy(sliced[0..batch.len], batch); @@ -258,7 +271,7 @@ fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, w const f = flat[0 .. batch.len + 1]; const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // Interleaving requires one binary instruction; SwiGLU chains several. + // Interleaving requires one binary instruction; these activations chain several. var op = b.tensorOperation(.{ .tactic = .elementwise }); const g = op.fetch(gate, .{}); const activated = if (fp8) blk: { @@ -273,12 +286,21 @@ fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, w const silu = try op.commit(rounded, .{ .dtype = .bf16 }); op = b.tensorOperation(.{ .tactic = .elementwise }); break :blk op.fetch(silu, .{}); - } else g.sigmoid().mulf(g); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + }; var v = activated.mulf(op.fetch(up, .{})); if (weights) |w| v = v.mulf(op.fetch(w, .{})); return op.commit(v, .{ .dtype = .bf16 }); } +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + /// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): /// IO-bound tactics for activation stationary, compute-bound for blockwise. /// Keep the compiler's default lowering mode and omit the adaptive-einsum and @@ -335,7 +357,7 @@ fn activationStationaryKernel(comptime fp8: bool) type { const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }); const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, fp8); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, cfg.activation, fp8); if (fp8) { const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32); const op = b.tensorOperation(.{}); @@ -423,7 +445,7 @@ fn blockwiseKernel(comptime fp8: bool) type { const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); const scales = try gatherScales(b, a, in, ids, &.{BGi}); const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ BGi, BB }, null, fp8); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, fp8); const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (fp8) .f32 else .bf16); try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); } @@ -506,7 +528,7 @@ pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); const scales = try gatherScales(b, a, in, ids, &.{C}); const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16); - const act = try gated(b, a, h, &.{ C, T }, null, true); + const act = try gated(b, a, h, &.{ C, T }, null, .swiglu, true); const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32); op = b.tensorOperation(.{}); const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); @@ -519,6 +541,11 @@ pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ }); test "tcl fused experts run on furiosa" { + try testFusedExperts(.{ .swiglu = .{} }); + try testFusedExperts(.geglu_tanh); +} + +fn testFusedExperts(activation: zml.moe.Activation) !void { const platform = zml.testing.env(); if (platform.target != .furiosa) return error.SkipZigTest; const allocator = std.testing.allocator; @@ -537,15 +564,15 @@ test "tcl fused experts run on furiosa" { const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ - .activation = .{ .swiglu = .{} }, + .activation = act, .quantize_input = false, .routing_weight_placement = .after_down, }); } }; - var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, .{}); + var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); @@ -597,7 +624,12 @@ test "tcl fused experts run on furiosa" { g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); } - act[n] = g / (1 + @exp(-g)) * u; + const gate = switch (activation) { + .swiglu => g / (1 + @exp(-g)), + .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), + else => unreachable, + }; + act[n] = gate * u; } for (0..d) |m| { var y: f32 = 0; From d2a69b9718e5127d9247491c4b5816d279d78095 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 13 Nov 2026 17:34:09 +0100 Subject: [PATCH 43/47] zml/moe: [refacto] consolidate Furiosa TCL kernel setup (#1066) Separate dispatch from kernel generation. Use one factory to select BF16 and block-128 FP8 kernels by format and schedule, and reuse the shared ActivationKind. --- platforms/furiosa/furiosa.bzl | 4 +- zml/kernels/moe/tcl/BUILD.bazel | 5 +- zml/kernels/moe/tcl/moe.zig | 457 +++++++++++++++++++++++ zml/kernels/moe/tcl/tcl.zig | 623 ++++++-------------------------- 4 files changed, 578 insertions(+), 511 deletions(-) create mode 100644 zml/kernels/moe/tcl/moe.zig diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 623c4f0d..afb26864 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611121233.371.1.9ff7ddf1e19f/zml-furiosa-linux-amd64.tar.zst", - sha256 = "d732b56832b1f009cbe4d808c70c6f8d1dc3857a83eb06804fc53b39bef9dd6b", + url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", + sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", ) return mctx.extension_metadata( diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel index 4057eea3..44b56e08 100644 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -2,5 +2,8 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "tcl", - srcs = ["tcl.zig"], + srcs = [ + "moe.zig", + "tcl.zig", + ], ) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig new file mode 100644 index 00000000..23fb2274 --- /dev/null +++ b/zml/kernels/moe/tcl/moe.zig @@ -0,0 +1,457 @@ +//! TCL kernel schedules for Furiosa RNGD fused experts. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); + +const zml = @import("../../../zml.zig"); +const tcl = zml.kernel.tcl; +const ActivationKind = @import("../api.zig").ActivationKind; + +pub const Format = enum { bf16, fp8_block128 }; + +/// Dispatch in `tcl.zig` uses the first matching schedule. +/// +/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected +/// expert on the whole batch to reuse its weights, with zero routing weight +/// for unrouted pairs. +/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather +/// weights for each route to avoid the overhead of grouping routes. +/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to +/// reuse weights within each block, then gather results back by route. +pub const Schedule = enum { per_route, blockwise, small_batch }; + +pub const Cfg = struct { + num_tokens: i64, + top_k: i64, + local_num_experts: i64, + hidden_size: i64, + expert_intermediate_size: i64, + weight_scale_dtype: tcl.DType = .bf16, + activation: ActivationKind = .swiglu, +}; + +const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; +const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; + +pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { + if (schedule == .small_batch and format != .fp8_block128) { + @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); + } + return tcl.Kernel(Cfg, .{ + .name = "moe_" ++ @tagName(schedule) ++ switch (format) { + .bf16 => "", + .fp8_block128 => "_fp8", + }, + .inputs = switch (format) { + .bf16 => base_inputs, + .fp8_block128 => base_inputs ++ scale_inputs, + }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, schedule); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, format); + switch (schedule) { + .per_route => try perRoute(b, a, in, cfg.activation, format), + .blockwise => try blockwise(b, a, in, cfg, format), + .small_batch => try fp8SmallBatch(b, a, in, cfg), + } + } + }.run, + }); +} + +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.num_tokens), + .K = b.axis("K", cfg.top_k), + .E = b.axis("E", cfg.local_num_experts), + .D = b.axis("D", cfg.hidden_size), + .I = b.axis("I", cfg.expert_intermediate_size), + .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } +}; + +fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { + const weight_dtype: tcl.DType = switch (format) { + .bf16 => .bf16, + .fp8_block128 => .f8_e4, + }; + const spec = .{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, + }; + if (format == .bf16) { + const t = try b.declareArgs(spec); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + } + const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); + const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); + const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); + const t = try b.declareArgs(.{ + .x = spec.x, + .ids = spec.ids, + .w = spec.w, + .gate_up = spec.gate_up, + .down = spec.down, + .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, + .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; +} + +const Args = struct { + x: builder.Tensor, + ids: builder.Tensor, + w: builder.Tensor, + gate_up: builder.Tensor, + down: builder.Tensor, + gate_up_scale: ?builder.Tensor = null, + down_scale: ?builder.Tensor = null, +}; + +const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; + +fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { + if (format == .bf16) return .{}; + const IG = b.axis("IG", @divExact(a.I.size, 128)); + const DG = b.axis("DG", @divExact(a.D.size, 128)); + const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); + var gu_axes: [8]tcl.Axis = undefined; + var dn_axes: [8]tcl.Axis = undefined; + @memcpy(gu_axes[0..batch.len], batch); + @memcpy(dn_axes[0..batch.len], batch); + @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); + @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); + return .{ + .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), + .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), + }; +} + +fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { + var n: usize = 0; + for (axes) |a| { + if (std.mem.eql(u8, a.name, axis.name)) { + storage[n] = group; + n += 1; + storage[n] = lane; + } else storage[n] = a; + n += 1; + } + return storage[0..n]; +} + +/// Block scales multiply each K-block partial before K-block reduction. +/// Only the requested experts are gathered; their bank remains FP8 in HBM. +fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { + if (format == .bf16) { + const op = b.tensorOperation(.{}); + return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); + } + const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); + const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); + const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); + const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); + var x_axes: [16]tcl.Axis = undefined; + var w_axes: [16]tcl.Axis = undefined; + var w2_axes: [16]tcl.Axis = undefined; + var out_axes: [16]tcl.Axis = undefined; + var partial_axes: [16]tcl.Axis = undefined; + const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); + const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); + const out = blockedAxes(to, output, OG, OL, &out_axes); + @memcpy(partial_axes[0..out.len], out); + partial_axes[out.len] = CG; + + // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. + var op = b.tensorOperation(.{}); + const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); + op = b.tensorOperation(.{}); + // Native F32 reciprocal accuracy is qualified separately (within one ULP). + const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); + const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); + op = b.tensorOperation(.{}); + const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); + return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); +} + +/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` +/// (gate first), times `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // The builder can infer the TCL `Interleaving` tactic for two inputs with + // matching axes, such as `gate` and `up`. That tactic allows exactly one + // binary instruction, so force `.elementwise` for this activation chain. + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + const activated = if (format == .fp8_block128) blk: { + if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); + + // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal + // BF16 SiLU write, even through as_dram/as_logical. Make its RNE + // rounding observable as integer bits before multiplying by up. + // Remove once the native reproducer preserves the BF16 boundary. + const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const v = op.fetch(bits, .{}); + const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); + const silu = try op.commit(rounded, .{ .dtype = .bf16 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + break :blk op.fetch(silu, .{}); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), + }; + var v = activated.mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); +} + +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// Use IO-bound tactics for per-route and FP8 small-batch schedules and +/// compute-bound tactics for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { + const config = comptime blk: { + const common = .{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "None", + }; + const tactics = switch (schedule) { + .blockwise => .{ + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + }, + .per_route, .small_batch => .{ + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }, + }; + // The transaction-simulation setting is only emitted for IO-bound schedules. + const common_info = @typeInfo(@TypeOf(common)).@"struct"; + const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; + const Config = @Struct( + .auto, + null, + common_info.field_names ++ tactics_info.field_names, + common_info.field_types ++ tactics_info.field_types, + common_info.field_attrs ++ tactics_info.field_attrs, + ); + var merged: Config = undefined; + for (common_info.field_names) |field| @field(merged, field) = @field(common, field); + for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); + break :blk merged; + }; + b.compilerConfig(config); +} + +/// Gather the selected experts for every token. +fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); + const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); + if (format == .fp8_block128) { + const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); + } else { + // The BF16 recipe weights activations and contracts experts together. + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } +} + +/// Rows per expert block: the next power of two of the routes per expert. +fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); +} + +const blocks_per_chunk = 32; + +/// Group routes into expert-major blocks, then run both projections in a +/// device loop over chunks of blocks. +fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); + const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); + const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); +} + +/// Number of experts processed per device-loop iteration, based on the +/// batch's route count (`cfg.num_tokens * cfg.top_k`). +fn expertsPerChunk(cfg: Cfg) i64 { + const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); + return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); +} + +/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. +/// Load each selected expert once and run it for every token. Routing weights +/// scale the F32 down-projection results before summing the experts and steps. +fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { + const chunk = expertsPerChunk(cfg); + const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; + const C = b.axis("C", chunk); + const Go = b.axis("Go", steps); + const J = b.axis("J", steps * chunk); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Each token's routing weight per expert, and the used experts. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); + op = b.tensorOperation(.{}); + const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); + // A used expert's rank among them, -1 for the others. + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); + + // Slot j holds the used expert of rank j and its tokens' weights; + // slots past the last hold expert 0 with no weight. + const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); + op = b.tensorOperation(.{}); + const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); + // A reduction's condition is read plainly, so broadcast it first. + op = b.tensorOperation(.{}); + const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); + op = b.tensorOperation(.{}); + const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); + const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); + const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); + { + const ids = try b.indexRead(step_experts, loop.iv); + const weights = try b.indexRead(step_gate, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); + const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); + const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); + const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); + op = b.tensorOperation(.{}); + const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); +} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 71653f8e..f05727ce 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -1,59 +1,88 @@ //! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. -//! -//! Few tokens (at most `experts / top_k`): activation stationary, each token -//! gathers the weights of its experts and the down projection contracts over -//! the experts and the intermediate axis at once. -//! -//! More tokens: blockwise. The kernel routes on the device: routes are placed -//! in expert-major blocks of `BB` rows (one-hot, cumulative sums), then a -//! device loop over the used chunks of `BGi` blocks gathers each block's -//! expert weights and runs both projections; outputs are gathered back per -//! route and weighted. -//! -//! FP8 experts split across devices (expert parallel): each device runs its own -//! experts for the routes that reach them and an all-reduce sums the devices' -//! outputs. Up to `expert_stationary_tokens` tokens, expert stationary: a -//! device loop over the used experts, a few at a time, reads each once and -//! runs it for every token, scaled by the token's routing weight (zero for -//! the tokens it does not serve); more tokens take blockwise. const std = @import("std"); -const builder = @import("dsl/tcl/builder"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; +const kernels = @import("moe.zig"); +const api = @import("../api.zig"); -const Activation = enum { swiglu, geglu_tanh }; - -/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` -/// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. -/// Supports plain SwiGLU and, for BF16 experts, GEGLU with tanh GELU. -pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { +/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` +/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is +/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with +/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. +pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { const topk_ids = routing.localIds(); - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); + stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); + const format = expertFormat(gate_up_linear, down_linear); + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + const gate_up = gate_up_linear.weight; const down = down_linear.weight; - const quantized = gate_up_linear.quantization != null or down_linear.quantization != null; - const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); - stdx.debug.assert(swiglu or (!quantized and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); - const activation: Activation = if (swiglu) .swiglu else .geglu_tanh; - const scales: ?Scales = if (quantized) .{ .gate_up = fp8Scales(gate_up_linear), .down = fp8Scales(down_linear) } else null; - stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32 and (quantized or (gate_up.dtype() == .bf16 and down.dtype() == .bf16)), "TCL MoE takes BF16 activations, BF16/block-128 FP8 experts and i32 ids", .{}); + const cfg: kernels.Cfg = .{ + .num_tokens = input.dim(.t), + .top_k = topk_ids.dim(.topk), + .local_num_experts = gate_up.dim(.expert), + .hidden_size = input.dim(.d), + .expert_intermediate_size = down.dim(.dout), + .weight_scale_dtype = switch (format) { + .bf16 => .bf16, + .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), + }, + .activation = std.meta.activeTag(opts.activation), + }; + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); + const weights = routing.weights.convert(.f32); - const local_weights = if (scales != null) topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)) else weights; - return runExperts(input, topk_ids, local_weights, gate_up, down, scales, activation); + const local_weights = switch (format) { + .bf16 => weights, + .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), + }; + return switch (format) { + .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), + .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ + .x = input, + .ids = topk_ids, + .w = local_weights, + .gate_up = gate_up, + .down = down, + .gate_up_scale = s.gate_up, + .down_scale = s.down, + }), + }; } const Scales = struct { gate_up: Tensor, down: Tensor }; +const Format = union(kernels.Format) { + bf16: void, + fp8_block128: Scales, +}; + +fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { + const scheme = gate_up.quantizationScheme(); + stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); + return switch (scheme orelse { + stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); + return .bf16; + }) { + .fp8_block128 => blk: { + const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; + stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); + break :blk .{ .fp8_block128 = scales }; + }, + else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), + }; +} + fn fp8Scales(linear: zml.nn.Linear) Tensor { - const q = linear.quantization orelse std.debug.panic("TCL MoE requires both FP8 projections", .{}); - stdx.debug.assert(q.scheme == .fp8_block128 and linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); + const q = linear.quantization.?; + stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); const w = linear.weight; const scale = q.scales; @@ -61,485 +90,62 @@ fn fp8Scales(linear: zml.nn.Linear) Tensor { return scale; } -/// Tokens up to which a device runs its experts stationary. -pub const expert_stationary_tokens = 64; +// Maximum total input tokens per MoE call before padding for the FP8 +// small-batch schedule. It loads each used expert's weights once and reuses +// them across the batch. +// Larger batches fall back to per-route or blockwise dispatch. +const fp8_small_batch_max_tokens = 64; -fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales, activation: Activation) Tensor { - var cfg: Cfg = .{ - .t = input.dim(.t), - .k = topk_ids.dim(.topk), - .e = gate_up.dim(.expert), - .d = input.dim(.d), - .i = down.dim(.dout), - .activation = activation, - }; - stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); - if (scales) |s| { - cfg.scale_dtype = scaleDtype(s); - if (cfg.t <= expert_stationary_tokens) { - // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two - // panics its adder tree ("Fail to cover valid range", 5 tokens padded - // to 8): pad with unrouted tokens. - var padded = cfg; - padded.t = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.t)) catch unreachable); - const pad = padded.t - cfg.t; - const x = input.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.d }); - const ids = topk_ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); - const weights = topk_weights.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); - const y = ExpertStationaryFp8.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }, .{ .y = x.shape() }, .{ .cfg = padded }).y; - return y.slice(0, .{ .end = cfg.t }).reshape(input.shape()); - } +fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { + return run(format, .small_batch, cfg, inputs); } + if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { + return run(format, .per_route, cfg, inputs); + } + return run(format, .blockwise, cfg, inputs); +} - const x = input.reshape(.{ cfg.t, cfg.d }); - const ids = topk_ids.reshape(.{ cfg.t, cfg.k }); - const weights = topk_weights.reshape(.{ cfg.t, cfg.k }); +fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + const call_cfg, const prepared = if (schedule == .small_batch) + prepareFp8SmallBatch(cfg, inputs) + else + .{ cfg, prepareRoutedInputs(cfg, inputs) }; + + const shape = prepared.x.shape(); + var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; + + if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); + return y.reshape(inputs.x.shape()); +} + +fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { + var prepared = inputs; + prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); + const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); + const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); // These kernels gather every route, and blockwise also scatters each route // into an expert slot. Give nonlocal routes a valid slot and zero weight. - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); - const safe_ids = valid.select(ids, Tensor.scalar(0, .i32)); - const safe_weights = valid.select(weights, Tensor.scalar(0, .f32)); - const out_shape = x.shape(); - if (scales) |s| { - const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; - const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationaryFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y - else - BlockwiseFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y; - return y.reshape(input.shape()); - } - const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationary.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y - else - Blockwise.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; - return y.reshape(input.shape()); + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); + prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); + prepared.w = valid.select(w, Tensor.scalar(0, .f32)); + return prepared; } -fn scaleDtype(scales: Scales) tcl.DType { - stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); - return if (scales.gate_up.dtype() == .bf16) .bf16 else .f32; +fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { + // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two + // panics its adder tree ("Fail to cover valid range", 5 tokens padded + // to 8): pad with unrouted tokens. + var padded = cfg; + padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); + const pad = padded.num_tokens - cfg.num_tokens; + var prepared = inputs; + prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); + prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + return .{ padded, prepared }; } -pub const Cfg = struct { - t: i64, - k: i64, - e: i64, - d: i64, - i: i64, - scale_dtype: tcl.DType = .bf16, - activation: Activation = .swiglu, -}; - -const Axes = struct { - T: tcl.Axis, - K: tcl.Axis, - E: tcl.Axis, - D: tcl.Axis, - I: tcl.Axis, - I2: tcl.Axis, - S2: tcl.Axis, - S1: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.t), - .K = b.axis("K", cfg.k), - .E = b.axis("E", cfg.e), - .D = b.axis("D", cfg.d), - .I = b.axis("I", cfg.i), - .I2 = b.axis("I2", 2 * cfg.i), - .S2 = b.axis("S2", 2), - .S1 = b.axis("S1", 1), - }; - } -}; - -fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime fp8: bool) tcl.FinishError!Args { - const spec = .{ - .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, - .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, - .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.D, a.I } }, - }; - if (!fp8) { - const t = try b.declareArgs(spec); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; - } - const IG = b.axis("IG", @divExact(cfg.i, 128)); - const DG = b.axis("DG", @divExact(cfg.d, 128)); - const I2G = b.axis("I2G", 2 * @divExact(cfg.i, 128)); - const t = try b.declareArgs(.{ - .x = spec.x, - .ids = spec.ids, - .w = spec.w, - .gate_up = spec.gate_up, - .down = spec.down, - .gate_up_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, I2G, DG } }, - .down_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, DG, IG } }, - }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; -} - -const Args = struct { - x: builder.Tensor, - ids: builder.Tensor, - w: builder.Tensor, - gate_up: builder.Tensor, - down: builder.Tensor, - gate_up_scale: ?builder.Tensor = null, - down_scale: ?builder.Tensor = null, -}; - -const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; - -fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis) tcl.FinishError!GatheredScales { - const gu = in.gate_up_scale orelse return .{}; - const IG = b.axis("IG", @divExact(a.I.size, 128)); - const DG = b.axis("DG", @divExact(a.D.size, 128)); - const gate_up = try b.reshape(gu, &.{ a.E, a.S2, IG, DG }, .{}); - var gu_axes: [8]tcl.Axis = undefined; - var dn_axes: [8]tcl.Axis = undefined; - @memcpy(gu_axes[0..batch.len], batch); - @memcpy(dn_axes[0..batch.len], batch); - @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); - @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); - return .{ - .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), - .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), - }; -} - -fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { - var n: usize = 0; - for (axes) |a| { - if (std.mem.eql(u8, a.name, axis.name)) { - storage[n] = group; - n += 1; - storage[n] = lane; - } else storage[n] = a; - n += 1; - } - return storage[0..n]; -} - -/// Block scales multiply each K-block partial before K-block reduction. -/// Only the requested experts are gathered; their bank remains FP8 in HBM. -fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType) tcl.FinishError!builder.Tensor { - if (scale == null) { - const op = b.tensorOperation(.{}); - return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); - } - const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); - const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); - const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); - const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); - var x_axes: [16]tcl.Axis = undefined; - var w_axes: [16]tcl.Axis = undefined; - var w2_axes: [16]tcl.Axis = undefined; - var out_axes: [16]tcl.Axis = undefined; - var partial_axes: [16]tcl.Axis = undefined; - const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); - const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); - const out = blockedAxes(to, output, OG, OL, &out_axes); - @memcpy(partial_axes[0..out.len], out); - partial_axes[out.len] = CG; - - // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. - var op = b.tensorOperation(.{}); - const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); - op = b.tensorOperation(.{}); - // Native F32 reciprocal accuracy is qualified separately (within one ULP). - const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); - const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); - op = b.tensorOperation(.{}); - const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); - return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); -} - -/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` -/// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: Activation, comptime fp8: bool) tcl.FinishError!builder.Tensor { - var sliced: [4]tcl.Axis = undefined; - var flat: [3]tcl.Axis = undefined; - @memcpy(sliced[0..batch.len], batch); - @memcpy(flat[0..batch.len], batch); - sliced[batch.len] = a.S1; - sliced[batch.len + 1] = a.I; - flat[batch.len] = a.I; - const s = sliced[0 .. batch.len + 2]; - const f = flat[0 .. batch.len + 1]; - const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); - const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // Interleaving requires one binary instruction; these activations chain several. - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const g = op.fetch(gate, .{}); - const activated = if (fp8) blk: { - // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal - // BF16 SiLU write, even through as_dram/as_logical. Make its RNE - // rounding observable as integer bits before multiplying by up. - // Remove once the native reproducer preserves the BF16 boundary. - const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const v = op.fetch(bits, .{}); - const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); - const silu = try op.commit(rounded, .{ .dtype = .bf16 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - break :blk op.fetch(silu, .{}); - } else switch (activation) { - .swiglu => g.sigmoid().mulf(g), - .geglu_tanh => geluTanh(g), - }; - var v = activated.mulf(op.fetch(up, .{})); - if (weights) |w| v = v.mulf(op.fetch(w, .{})); - return op.commit(v, .{ .dtype = .bf16 }); -} - -fn geluTanh(g: builder.Value) builder.Value { - const cube = g.mulf(g).mulf(g); - const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); - return inner.tanh().addf(1.0).mulf(g).mulf(0.5); -} - -/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): -/// IO-bound tactics for activation stationary, compute-bound for blockwise. -/// Keep the compiler's default lowering mode and omit the adaptive-einsum and -/// half-VRF settings removed in TCC 2026.5. -fn configure(b: *tcl.Builder, blockwise: bool) void { - if (blockwise) b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, - .dma_preference = 1.2, - .propagate_sparse_axis_from_op = "None", - }) else b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - .propagate_sparse_axis_from_op = "None", - }); -} - -pub const ActivationStationary = activationStationaryKernel(false); -pub const ActivationStationaryFp8 = activationStationaryKernel(true); - -fn activationStationaryKernel(comptime fp8: bool) type { - return tcl.Kernel(Cfg, .{ - .name = if (fp8) "moe_activation_stationary_fp8" else "moe_activation_stationary", - .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, fp8); - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - - const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }); - const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, cfg.activation, fp8); - if (fp8) { - const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32); - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); - } else { - // The BF16 recipe weights activations and contracts experts together. - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } - } - }.run, - }); -} - -/// Rows per expert block: the next power of two of the routes per expert. -pub fn blockRows(cfg: Cfg) i64 { - const per_expert = std.math.divCeil(i64, cfg.t * cfg.k, cfg.e) catch unreachable; - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); -} - -const blocks_per_chunk = 32; - -pub const Blockwise = blockwiseKernel(false); -pub const BlockwiseFp8 = blockwiseKernel(true); - -fn blockwiseKernel(comptime fp8: bool) type { - return tcl.Kernel(Cfg, .{ - .name = if (fp8) "moe_blockwise_fp8" else "moe_blockwise", - .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, true); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, fp8); - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. - const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); - op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); - op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); - - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(used, .{try b.scratchpad(if (fp8) .f32 else .bf16, &.{ Go, BGi, BB, D })}); - { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{BGi}); - const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, fp8); - const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (fp8) .f32 else .bf16); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); - } - }.run, - }); -} - -/// Experts per `ExpertStationary` loop step: more as the routes, so the -/// experts they use, grow. -pub fn stationaryChunk(cfg: Cfg) i64 { - const routes: u64 = @intCast(std.math.divCeil(i64, cfg.t * cfg.k, 16) catch unreachable); - return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); -} - -/// The experts that `ids` use (-1 for none), `stationaryChunk` at a time in a -/// device loop: each is read once and runs for every token. Routing weights -/// scale the F32 down-projection results before summing the experts and steps. -pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ - .name = "moe_expert_stationary_fp8", - .inputs = &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, true); - const chunk = stationaryChunk(cfg); - const steps = std.math.divCeil(i64, @min(cfg.e, cfg.t * cfg.k), chunk) catch unreachable; - const C = b.axis("C", chunk); - const Go = b.axis("Go", steps); - const J = b.axis("J", steps * chunk); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Each token's routing weight per expert, and the used experts. - const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); - op = b.tensorOperation(.{}); - const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); - // A used expert's rank among them, -1 for the others. - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - - // Slot j holds the used expert of rank j and its tokens' weights; - // slots past the last hold expert 0 with no weight. - const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); - op = b.tensorOperation(.{}); - const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); - // A reduction's condition is read plainly, so broadcast it first. - op = b.tensorOperation(.{}); - const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); - op = b.tensorOperation(.{}); - const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); - const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); - const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); - { - const ids = try b.indexRead(step_experts, loop.iv); - const weights = try b.indexRead(step_gate, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{C}); - const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16); - const act = try gated(b, a, h, &.{ C, T }, null, .swiglu, true); - const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32); - op = b.tensorOperation(.{}); - const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); - } - }.run, -}); - test "tcl fused experts run on furiosa" { try testFusedExperts(.{ .swiglu = .{} }); try testFusedExperts(.geglu_tanh); @@ -552,7 +158,8 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { const io = std.testing.io; const bf16 = zml.floats.BFloat16; - // 2 * 4 <= 8 experts is activation stationary, 16 tokens is blockwise. + // With two routes per token and eight experts, four tokens use per-route + // dispatch and sixteen use blockwise dispatch. for ([_]i64{ 4, 16 }) |t| { const k = 2; const e = 8; From 9c9161bac6fd8b6dbbfd285120c6a68b69af2844 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 13 Nov 2026 17:59:02 +0100 Subject: [PATCH 44/47] platforms/oneapi: update pjrt plugin and bundle libmkl_def (#1070) Update PJRT OneAPI plugin with fixes for Triton --- platforms/oneapi/oneapi.bzl | 5 +++-- platforms/oneapi/packages.lock.json | 6 +++--- platforms/oneapi/packages.yaml | 10 ++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 6016e389..5484bec8 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611131511.374.1.f393d1632b4f/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "374ebcad90493fb311a65d8954120b7f2e8a85d52b8641329ee712ef1cb5b23a" ONEAPI_VERSION = "2026.1" @@ -248,6 +248,7 @@ filegroup( name = "mkl_core_runtime", srcs = [ "{ONEAPI_MKL_LIB}/libmkl_core.so.3", + "{ONEAPI_MKL_LIB}/libmkl_def.so.3", "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index a3e69ad4..140ce102 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -216,7 +216,7 @@ "name": "libigc2", "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -227,7 +227,7 @@ "name": "libigdfcl2", "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -249,7 +249,7 @@ "name": "libze1", "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" ], "version": "1.32.0-1~24.04~ppa1" }, diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 272bc02c..5a829ac5 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -15,9 +15,11 @@ sources: - channel: noble main url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z - # IGC 2.41.15 is newer than the graphics snapshot above. + # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics + # packages are pinned below to keep their versions from it. Use snapshots, + # not the live PPA: the PPA deletes superseded packages from its pool. - channel: noble main - url: https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z archs: - "amd64" @@ -43,9 +45,9 @@ packages: - "libnl-genl-3-200" - "libnl-route-3-200" - "libnuma1" - - "libigdgmm12" + - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" - "libigc2 (= 2.41.15-1~24.04)" - "libigdfcl2 (= 2.41.15-1~24.04)" - - "libze-intel-gpu1" + - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" - "libze1" - "zlib1g" From f65072a740fee2b415649f0c41e58935b55410e6 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 22:09:27 +0100 Subject: [PATCH 45/47] platforms/furiosa: use all RNGD chips by default (#1083) --- platforms/furiosa/furiosa.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index ebbd9e3c..ee33fd58 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -50,7 +50,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); - try setEnv("FURIOSA_VISIBLE_DEVICES", "0", 0); const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); log.info("Loading Furiosa plugin: {s}", .{library}); From 8fcff8c462070d06056f8a85ba0c8a81f0f3a636 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 22:27:25 +0100 Subject: [PATCH 46/47] workspace: bump ZML plugins (#1082) Bump plugins to [202611132003.375.1.7d765f57a407](https://github.com/zml/xla-private/tree/202611132003.375.1.7d765f57a407) oneAPI: weight-only FP8 and FP4 (NVFP4, MXFP4) scaled dots now run in a Triton kernel that reads the weights in their stored format. --- platforms/oneapi/oneapi.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 5484bec8..a3c968d3 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611131511.374.1.f393d1632b4f/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "374ebcad90493fb311a65d8954120b7f2e8a85d52b8641329ee712ef1cb5b23a" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" ONEAPI_VERSION = "2026.1" From 1a26a051a50ed8293e8c9edb539a93ef0858ca7a Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:17:19 +0100 Subject: [PATCH 47/47] zml/tensor: return a distinct tensor from toMemory on single-memory platforms (#1087) On cpu, neuron and metal, `toMemory` returned self. Donating a result to a device buffer and its `toMemory` copy to a host buffer then called `reuseBuffer` twice on the same tensor id: the second donation replaced the first, and compilation panicked because both outputs aliased the host buffer. This broke llmd async scheduling on Metal and CPU. Return the same value under a new tensor id instead. --- zml/tensor.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index a09a5d38..ee084835 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -282,7 +282,13 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { - .cpu, .neuron, .metal => return self, + .cpu, .neuron, .metal => { + // A single memory needs no copy, but the result must still be a distinct + // tensor: `reuseBuffer` donations are keyed by tensor id. + var res = _result(self._shape, self.value()); + res._shape._sharding = self._shape._sharding; + return res; + }, .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, }