diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 67513227..8af28b00 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -pub export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { +export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = envp; var threaded: std.Io.Threaded = .init_single_threaded; diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig index 8f791f80..926b8419 100644 --- a/dsl/cute/builder.zig +++ b/dsl/cute/builder.zig @@ -16,49 +16,6 @@ const vector = @import("mlir/dialects/vector"); const mlir = @import("mlir"); const stdx = @import("stdx"); -/// `CUtensorMapDataType`. -pub const TensorMapDataType = enum(u32) { - uint8 = 0, - uint16 = 1, - uint32 = 2, - int32 = 3, - uint64 = 4, - int64 = 5, - float16 = 6, - float32 = 7, - float64 = 8, - bfloat16 = 9, - float32_ftz = 10, - tfloat32 = 11, - tfloat32_ftz = 12, - @"16u4_align8b" = 13, - @"16u4_align16b" = 14, - @"16u6_align16b" = 15, -}; - -/// `CUtensorMapSwizzle`. -pub const TensorMapSwizzle = enum(u32) { none = 0, @"32b" = 1, @"64b" = 2, @"128b" = 3, @"128b_atom_32b" = 4 }; -/// `CUtensorMapL2promotion`. -pub const TensorMapL2Promotion = enum(u32) { none = 0, @"64b" = 1, @"128b" = 2, @"256b" = 3 }; -/// `CUtensorMapInterleave`. -pub const TensorMapInterleave = enum(u32) { none = 0, @"16b" = 1, @"32b" = 2 }; -/// `CUtensorMapFloatOOBfill`. -pub const TensorMapOobFill = enum(u32) { none = 0, nan_request_zero_fma = 1 }; - -pub const TensorMapSpec = struct { - data_type: TensorMapDataType, - rank: u32, - global_dims: [5]u64 = @splat(1), - /// Byte strides of dimensions 1..rank-1. - global_strides: [4]u64 = @splat(0), - box_dims: [5]u32 = @splat(1), - element_strides: [5]u32 = @splat(1), - interleave: TensorMapInterleave = .none, - swizzle: TensorMapSwizzle, - l2_promotion: TensorMapL2Promotion = .none, - oob_fill: TensorMapOobFill = .none, -}; - const dtypes = @import("dtype.zig"); pub const DType = dtypes.DType; const isFloatDtype = dtypes.isFloatDtype; @@ -758,7 +715,6 @@ pub const ArgSpec = struct { pub const MlirTypeSpec = struct { type_: *const mlir.Type, grid_constant: bool = false, - cuda_grid_constant: bool = false, }; pub const TensorSpec = struct { @@ -867,7 +823,6 @@ pub const Builder = struct { .mlir_type => .{ .mlir_type = .{ .type_ = inner.type_, .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, - .cuda_grid_constant = if (@hasField(@TypeOf(inner), "cuda_grid_constant")) inner.cuda_grid_constant else false, } }, } }; } @@ -894,7 +849,7 @@ pub const Builder = struct { return @Struct(.auto, null, &names, &types, &@splat(.{})); } - pub fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { + fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { std.debug.assert(self.entry_block == null); const ctx = self.ctx; const scratch = self.arena.allocator(); @@ -916,7 +871,7 @@ pub const Builder = struct { var any_arg_attr = false; for (arg_attrs, args) |*attr, a| { attr.* = switch (a.kind) { - .mlir_type => |typed| if (typed.cuda_grid_constant) .dict(ctx, &.{.named(ctx, "cuda.grid_constant", .unit(ctx))}) else if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, + .mlir_type => |typed| if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, else => empty_dict, }; if (attr.* != empty_dict) any_arg_attr = true; @@ -1128,75 +1083,6 @@ pub const Builder = struct { })); } - /// `!cuda.tensor_map`: the kernel argument type of an explicitly encoded map. - pub fn tensorMapType(self: *Builder) *const mlir.Type { - return (cuda.ops.TensorMapType.get(self.ctx, .{}) catch unreachable).type_(); - } - - /// A host stack array of `values` (u64 -> i64 slots, u32 -> i32 slots): a `cuuint64_t*` / - /// `cuuint32_t*` argument of `cuTensorMapEncodeTiled`. - fn tensorMapHostArray(self: *Builder, comptime T: type, values: []const T) Value { - const ctx = self.ctx; - const elem: *const mlir.Type = if (T == u64) .int(ctx, .i64) else .int(ctx, .i32); - const ptr_type = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic); - const array = self.emit(mlir.Operation.make(ctx, "llvm.alloca", .{ - .operands = .{ .flat = &.{self.constant(.i32, @as(i32, @intCast(@max(values.len, 1)))).inner} }, - .results = .{ .flat = &.{ptr_type} }, - .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "alignment", .int(ctx, .i64, 8)) }, - .location = self.loc(), - })); - for (values, 0..) |v, i| { - const slot = self.emit(mlir.Operation.make(ctx, "llvm.getelementptr", .{ - .operands = .{ .flat = &.{array.inner} }, - .results = .{ .flat = &.{ptr_type} }, - .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, &[_]i32{@intCast(i)})) }, - .location = self.loc(), - })); - const value = if (T == u64) self.constant(.i64, @as(i64, @intCast(v))) else self.constant(.i32, @as(i32, @intCast(v))); - self.emitVoid(llvm.store(ctx, value.inner, slot.inner, .{ .alignment = @sizeOf(T) }, self.loc())); - } - return array; - } - - /// Encode a tiled map in a host function and return both the map and CUDA status. - /// The caller supplies the final device address and handles the returned status. - pub fn encodeTensorMap(self: *Builder, spec: TensorMapSpec, address: Value) struct { map: Value, result: Value } { - const ctx = self.ctx; - // The `cuda` dialect's address operand is a `ptr` dialect pointer, which zml does not register. - ctx.setAllowUnregisteredDialects(true); - const addr = self.emit(cuda.cast(ctx, (mlir.Type.parse(self.ctx, "!ptr.ptr<#ptr.generic_space>") catch @panic("invalid tensor-map IR")), address.inner, self.loc())); - const results = self.emitResults(2, cuda.ops.tensor_map_encode_tiled( - ctx, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.data_type)))).inner, - self.constant(.i32, @as(i32, @intCast(spec.rank))).inner, - addr.inner, - self.tensorMapHostArray(u64, spec.global_dims[0..spec.rank]).inner, - self.tensorMapHostArray(u64, spec.global_strides[0 .. spec.rank - 1]).inner, - self.tensorMapHostArray(u32, spec.box_dims[0..spec.rank]).inner, - self.tensorMapHostArray(u32, spec.element_strides[0..spec.rank]).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.interleave)))).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.swizzle)))).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.l2_promotion)))).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.oob_fill)))).inner, - cuda.resultType(ctx), - self.tensorMapType(), - self.loc(), - )); - return .{ .map = results[1], .result = results[0] }; - } - - /// Device side: the generic address (i64) of the `!cuda.tensor_map` kernel argument `arg` - /// (`&tensor_map` for a `__grid_constant__` parameter). - pub fn tensorMapAddress(self: *Builder, descriptor: Value) Value { - const ctx = self.ctx; - const llvm_ptr = self.emit(cuda.cast(ctx, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), descriptor.inner, self.loc())); - return self.emit(mlir.Operation.make(ctx, "llvm.ptrtoint", .{ - .operands = .{ .flat = &.{llvm_ptr.inner} }, - .results = .{ .flat = &.{DType.i64.toMlir(ctx)} }, - .location = self.loc(), - })); - } - pub fn cudaResultStatus(self: *Builder, result: Value) Value { return self.emit(cuda.cast(self.ctx, .int(self.ctx, .i32), result.inner, self.loc())); } @@ -1831,27 +1717,20 @@ pub const Builder = struct { /// `projection`, like Python's `proj`, selects the modes of `tile` that /// apply, such as `b.expr(.{ 1, AlgebraToken.all, 1 })`. - pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: ?*const mlir.Type, projection: ?AlgebraExpr) View { - if (result_type == null) return self.inferView("cute.local_tile", &.{ self.asValue(input).inner, self.asValue(tile).inner, if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord) }, if (projection) |p| &.{.named(self.ctx, "proj", self.algebraAttr(cute_ir.CoordAttr, p).attribute())} else &.{}); + pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: *const mlir.Type, projection: ?AlgebraExpr) View { const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); const result = self.emit(cute_ir.local_tile( self.ctx, self.asValue(input).inner, self.asValue(tile).inner, coord_value, - result_type.?, + result_type, .{ .proj = if (projection) |p| self.algebraAttr(cute_ir.CoordAttr, p).attribute() else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - // A null result type on the algebra operations below requests dialect inference. - fn inferView(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attrs: []const mlir.NamedAttribute) View { - const value = self.emit(mlir.Operation.make(self.ctx, name, .{ .operands = .{ .flat = operands }, .attributes = attrs, .result_type_inference = true, .location = self.loc() })); - return .{ .inner = value.inner, .kernel = self }; - } - pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.local_partition( self.ctx, @@ -1864,13 +1743,12 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.slice", &.{ self.asValue(input).inner, self.makeCoord(coord) }, &.{}); + pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.slice( self.ctx, self.asValue(input).inner, self.makeCoord(coord), - result_type.?, + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1889,17 +1767,13 @@ pub const Builder = struct { (cute_ir.MemRefType.get(self.ctx, .{ .ptr = memref.ptr_(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.arithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() - else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| - (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ .desc = desc.desc(), .layout = layout_value.type_().isA(cute_ir.LayoutType).?.attr() }) catch @panic("invalid regrouped descriptor")).type_() else - std.debug.panic("withLayout: expected memref, coordinate tensor or shared-memory descriptor view, got {f}", .{input_type}); + std.debug.panic("withLayout: expected memref or coordinate tensor, got {f}", .{input_type}); const iterator_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| memref.ptr_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.arithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() - else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| - desc.desc() else unreachable; const iterator = self.emit(cute_ir.get_iter(self.ctx, value.inner, iterator_type, self.loc())); @@ -1907,12 +1781,11 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.group_modes", &.{self.asValue(input).inner}, &.{ .named(self.ctx, "begin", .int(self.ctx, .i32, begin)), .named(self.ctx, "end", .int(self.ctx, .i32, end)) }); + pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.group_modes( self.ctx, self.asValue(input).inner, - result_type.?, + result_type, .int(self.ctx, .i32, begin), .int(self.ctx, .i32, end), self.loc(), @@ -1931,13 +1804,12 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.flat_divide", &.{ self.asValue(input).inner, self.asValue(tiler).inner }, &.{}); + pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.flat_divide( self.ctx, self.asValue(input).inner, self.asValue(tiler).inner, - result_type.?, + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1954,44 +1826,39 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.filter_zeros", if (target_profile) |target| &.{ self.asValue(input).inner, target.inner } else &.{self.asValue(input).inner}, &.{}); + pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.filter_zeros( self.ctx, self.asValue(input).inner, - result_type.?, + result_type, .{ .target_profile = if (target_profile) |target| target.inner else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn getLayout(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.get_layout", &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + pub fn getLayout(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } /// `cute.get_iter`: the iterator of `input`, of type `result_type`. - pub fn getIter(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.get_iter", &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + pub fn getIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn getShape(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.get_shape", &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + pub fn getShape(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.add_offset", &.{ self.asValue(input).inner, self.asValue(offset).inner }, &.{}); + pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.add_offset( self.ctx, self.asValue(input).inner, self.asValue(offset).inner, - result_type.?, + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -2028,39 +1895,36 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.tiled.copy.retile", &.{ tiled.inner, self.asValue(input).inner }, &.{}); + pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.tiled_copy_retile( self.ctx, tiled.inner, self.asValue(input).inner, - result_type.?, + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.tiled.mma.partition", &.{ tiled.inner, self.asValue(input).inner, self.makeCoord(coord) }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); + pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.tiled_mma_partition( self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), - result_type.?, + result_type, .int(self.ctx, .i32, operand_id), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.mma.make_fragment", &.{ tiled.inner, self.asValue(input).inner }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); + pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.mma_make_fragment( self.ctx, tiled.inner, self.asValue(input).inner, - result_type.?, + result_type, .int(self.ctx, .i32, operand_id), self.loc(), )); @@ -2867,8 +2731,8 @@ pub const Builder = struct { /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a /// TMA producer does before issuing its copies. - pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: anytype) void { - self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.lift(bytes).to(.i32).inner, .cta, self.loc())); + pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: u32) void { + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); } pub const MBarrierArriveOptions = struct { @@ -4339,615 +4203,3 @@ test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { defer parsed.deinit(); try std.testing.expect(parsed.operation().verify()); } - -/// Low-level CUDA primitives. Prefer typed operations where they preserve the -/// instruction semantics. Assembly fallbacks retain exact memory scopes, rounding, -/// CTA-group/cache modifiers, and tied-register dependencies for asynchronous loads. -const sm100_mma_peer_bit_mask: u32 = 0xFEFFFFFF; - -/// Side-effecting inline assembly with no result. -pub fn asmVoid(b: *Builder, text: []const u8, args: []const Value, constraints: []const u8) void { - const inner = b.arena.allocator().alloc(*const mlir.Value, args.len) catch @panic("OOM"); - for (args, inner) |a, *x| x.* = a.inner; - b.emitVoid(llvm.inline_asm(b.ctx, text, inner, .{ .constraints = constraints, .has_side_effects = true }, b.loc())); -} - -/// Inline assembly with one result of `dtype`. -pub fn asmValue(b: *Builder, text: []const u8, args: []const Value, dtype: DType, constraints: []const u8, side_effects: bool) Value { - const inner = b.arena.allocator().alloc(*const mlir.Value, args.len) catch @panic("OOM"); - for (args, inner) |a, *x| x.* = a.inner; - return b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = dtype.toMlir(b.ctx), .constraints = constraints, .has_side_effects = side_effects }, b.loc())); -} - -/// Inline assembly with `n` results of `dtype` (an `!llvm.struct` result split with -/// `llvm.extractvalue`, how LLVM represents multi-output asm). -pub fn asmValues(b: *Builder, comptime n: usize, text: []const u8, args: []const Value, dtype: DType, constraints: []const u8, side_effects: bool) [n]Value { - return asmValuesSlice(b, n, text, args, dtype, constraints, side_effects)[0..n].*; -} - -/// Inline assembly with a host-computed number of results of the same dtype. -pub fn asmValuesSlice(b: *Builder, n: usize, text: []const u8, args: []const Value, dtype: DType, constraints: []const u8, side_effects: bool) []Value { - const inner = b.arena.allocator().alloc(*const mlir.Value, args.len) catch @panic("OOM"); - for (args, inner) |a, *x| x.* = a.inner; - const elem = dtype.toMlir(b.ctx); - const struct_type = (mlir.Type.parse(b.ctx, std.fmt.allocPrint(b.arena.allocator(), "!llvm.struct<({s})>", .{structFields(b, n, elem)}) catch @panic("OOM")) catch @panic("invalid inline assembly result type")); - const packed_ = b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = struct_type, .constraints = constraints, .has_side_effects = side_effects }, b.loc())); - const out = b.arena.allocator().alloc(Value, n) catch @panic("OOM"); - for (out, 0..) |*o, i| { - o.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ - .operands = .{ .flat = &.{packed_.inner} }, - .results = .{ .flat = &.{elem} }, - .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &[_]i64{@intCast(i)}))}, - .location = b.loc(), - })); - } - return out; -} - -pub fn structFields(b: *Builder, n: usize, elem: *const mlir.Type) []const u8 { - var w: std.Io.Writer.Allocating = .init(b.arena.allocator()); - for (0..n) |i| w.writer.print("{s}{f}", .{ if (i == 0) "" else ", ", elem }) catch @panic("OOM"); - return w.written(); -} - -pub fn ldAcq(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.gpu.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); -} - -pub fn ldAcqSysU64(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.sys.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); -} - -pub fn ldAcqSysS32(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.sys.global.s32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); -} - -pub fn ldAcqSysU32(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.sys.global.u32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); -} - -pub fn ldAcqGpuU64(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.gpu.global.u64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); -} - -pub fn atomicAddU32(b: *Builder, ptr: Value, value: anytype) Value { - return asmValue(b, "atom.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r", true); -} - -pub fn atomicAddU64(b: *Builder, ptr: Value, value: Value) Value { - return asmValue(b, "atom.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); -} - -pub fn atomicAddSys(b: *Builder, ptr: Value, value: Value) Value { - return asmValue(b, "atom.sys.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); -} - -pub fn atomicAddRel(b: *Builder, ptr: Value, value: anytype) Value { - return asmValue(b, "atom.release.gpu.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r,~{memory}", true); -} - -pub fn stRelSys(b: *Builder, ptr: Value, value: Value) void { - asmVoid(b, "st.release.sys.global.u64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); -} - -pub fn redAddS32(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.gpu.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); -} - -pub fn redAddU32(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); -} - -pub fn redXorRel(b: *Builder, ptr: Value, value: Value) void { - asmVoid(b, "red.release.gpu.global.xor.b64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); -} - -pub fn redAddRel(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.release.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); -} - -pub fn redAddRelSys(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.release.sys.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); -} - -pub fn atomicAddBlockShared(b: *Builder, smem: Value, value: anytype) Value { - return asmValue(b, "atom.shared.cta.add.u32 $0, [$1], $2;", &.{ smem, liftAs(b, value, .i32) }, .i32, "=r,r,r,~{memory}", true); -} - -pub fn fenceAcqRelCta(b: *Builder) void { - asmVoid(b, "fence.acq_rel.cta;", &.{}, "~{memory}"); -} - -pub fn liftAs(b: *Builder, value: anytype, dt: DType) Value { - if (@TypeOf(value) == Value) return value; - return b.constant(dt, value); -} - -/// `cvta`-style widening of an `i32` shared-memory address to the 64-bit operand -/// `__cvta_generic_to_shared` gives DeepGEMM's `"l"` constraints. -pub fn zext64(b: *Builder, v: Value) Value { - if (v.dtype() == .i64) return v; - return b.emit(arith.extui(b.ctx, v.inner, DType.i64.toMlir(b.ctx), b.loc())); -} - -pub fn i16Of(b: *Builder, v: Value) Value { - return if (v.dtype() == .i16) v else b.emit(arith.trunci(b.ctx, v.inner, DType.i16.toMlir(b.ctx), b.loc())); -} - -pub fn cst32(b: *Builder, v: anytype) Value { - return b.constant(.i32, @as(i32, @bitCast(@as(u32, @truncate(@as(u64, @bitCast(@as(i64, v)))))))); -} - -pub fn cst64(b: *Builder, v: u64) Value { - return b.constant(.i64, @as(i64, @bitCast(v))); -} - -pub fn getSmIdx(b: *Builder) Value { - return asmValue(b, "mov.u32 $0, %smid;", &.{}, .i32, "=r", false); -} - -pub fn getLaneIdx(b: *Builder) Value { - return asmValue(b, "mov.u32 $0, %laneid;", &.{}, .i32, "=r", false); -} - -/// `ptx::get_grid_idx`: the 64-bit `%gridid`, unique per launch. -pub fn getGridIdx(b: *Builder) Value { - return asmValue(b, "mov.u64 $0, %gridid;", &.{}, .i64, "=l", true); -} - -pub fn blockRankInCluster(b: *Builder) Value { - return asmValue(b, "mov.u32 $0, %cluster_ctarank;\n", &.{}, .i32, "=r", true); -} - -/// `clock64()` (`mov.u64 %rd, %clock64;`). -pub fn clock64(b: *Builder) Value { - return asmValue(b, "mov.u64 $0, %clock64;", &.{}, .i64, "=l", true); -} - -/// Keep the original election sequence until typed election matches its small-batch performance. -pub fn electOneSync(b: *Builder) Value { - const pred = asmValue(b, - \\{ - \\.reg .b32 %rx; - \\.reg .pred %px; - \\ mov.u32 $0, 0; - \\ elect.sync %rx|%px, $1; - \\@%px mov.s32 $0, 1; - \\} - , &.{cst32(b, 0xFFFFFFFF)}, .i32, "=r,r", true); - return pred.ne(0); -} - -/// `asm volatile("trap;")` (`DG_TRAP_ONLY_DEVICE_ASSERT`, and this port's timeout). -pub fn trap(b: *Builder) void { - asmVoid(b, "trap;", &.{}, "~{memory}"); -} - -/// `ptx::sync_unaligned(num_threads, barrier_idx)`: `barrier.sync id, n` (threads of a warp may -/// reach it at different program points). -pub fn syncUnaligned(b: *Builder, num_threads: anytype, barrier_idx: anytype) void { - asmVoid(b, "barrier.sync $0, $1;", &.{ liftAs(b, barrier_idx, .i32), liftAs(b, num_threads, .i32) }, "r,r,~{memory}"); -} - -/// `cute::cluster_arrive`. -pub fn clusterArrive(b: *Builder) void { - asmVoid(b, "barrier.cluster.arrive.aligned;\n", &.{}, "~{memory}"); -} - -/// `cudaGridDependencySynchronize()` (`griddepcontrol.wait;`, PDL). -pub fn gridDependencySynchronize(b: *Builder) void { - asmVoid(b, "griddepcontrol.wait;", &.{}, "~{memory}"); -} - -/// `cutlass::arch::warpgroup_reg_alloc`. -pub fn warpgroupRegAlloc(b: *Builder, reg_count: u32) void { - asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.inc.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); -} - -/// `cutlass::arch::warpgroup_reg_dealloc`. -pub fn warpgroupRegDealloc(b: *Builder, reg_count: u32) void { - asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.dec.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); -} - -/// `ptx::ld_volatile(const uint64_t*)`. -pub fn ldVolatileU64(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.volatile.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l", true); -} - -/// `ptx::ld_volatile(const uint32_t*)`. -pub fn ldVolatileU32(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.volatile.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l", true); -} - -/// `ptx::ld_shared(const uint32_t*)`. -pub fn ldSharedU32(b: *Builder, smem: Value) Value { - return asmValue(b, "ld.shared.u32 $0, [$1];", &.{zext64(b, smem)}, .i32, "=r,l", true); -} - -/// `ptx::ld_shared(const float2*)`. -pub fn ldSharedF32x2(b: *Builder, smem: Value) [2]Value { - return asmValues(b, 2, "ld.shared.v2.f32 {$0, $1}, [$2];", &.{zext64(b, smem)}, .f32, "=f,=f,l", true); -} - -/// `ptx::ld_shared(const float4*)`. -pub fn ldSharedF32x4(b: *Builder, smem: Value) [4]Value { - return asmValues(b, 4, "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .f32, "=f,=f,=f,=f,l", true); -} - -/// `ptx::ld_shared(const uint4*)`. -pub fn ldSharedU32x4(b: *Builder, smem: Value) [4]Value { - return asmValues(b, 4, "ld.shared.v4.u32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .i32, "=r,=r,=r,=r,l", true); -} - -/// `ptx::st_shared(const void*, x, y, z, w)`. -pub fn stSharedU32x4(b: *Builder, smem: Value, w: [4]Value) void { - asmVoid(b, "st.shared.v4.u32 [$0], {$1, $2, $3, $4};", &.{ zext64(b, smem), w[0], w[1], w[2], w[3] }, "l,r,r,r,r"); -} - -/// `ptx::st_shared(const uint32_t*, uint32_t)`. -pub fn stSharedU32(b: *Builder, smem: Value, value: Value) void { - asmVoid(b, "st.shared.u32 [$0], $1;", &.{ zext64(b, smem), value }, "l,r"); -} - -/// `ptx::st_shared(const float2*, float2)`. -pub fn stSharedF32x2(b: *Builder, smem: Value, v: [2]Value) void { - asmVoid(b, "st.shared.v2.f32 [$0], {$1, $2};", &.{ zext64(b, smem), v[0], v[1] }, "l,f,f"); -} - -/// `ptx::st_shared_bulk(smem, num_bytes)`: zero `num_bytes` (a multiple of 8) of shared memory. -pub fn stSharedBulk(b: *Builder, smem: Value, num_bytes: u32) void { - std.debug.assert(num_bytes % 8 == 0); - asmVoid(b, "st.bulk.weak.shared::cta [$0], $1, 0;", &.{ zext64(b, smem), b.constant(.i64, num_bytes) }, "l,l,~{memory}"); -} - -/// `ptx::mapa_shared(ptr, dst_cta_idx)`: this CTA's shared address in cluster CTA `cta`. -pub fn mapaShared(b: *Builder, smem: Value, dst_cta_idx: Value) Value { - return asmValue(b, "mapa.shared::cluster.u32 $0, $1, $2;", &.{ smem, dst_cta_idx }, .i32, "=r,r,r", true); -} - -/// `ptx::st_async_cluster(dst, src, dst_cta_idx, barrier)` for `words.len / 4` 16-byte -/// chunks: `st.async` into CTA `dst_cta_idx`'s shared memory, completing the bytes on its -/// mbarrier at `barrier` (both local addresses, mapped with `mapa`). -pub fn stAsyncCluster(b: *Builder, dst: Value, words: []const Value, dst_cta_idx: Value, barrier: Value) void { - std.debug.assert(words.len % 4 == 0); - const mapped_barrier = mapaShared(b, barrier, dst_cta_idx); - const mapped_dst = mapaShared(b, dst, dst_cta_idx); - var i: usize = 0; - while (i < words.len) : (i += 4) { - asmVoid( - b, - "st.async.shared::cluster.mbarrier::complete_tx::bytes.u32.v4 [$0], {$1, $2, $3, $4}, [$5];", - &.{ mapped_dst.add(@as(i32, @intCast(i * 4))), words[i], words[i + 1], words[i + 2], words[i + 3], mapped_barrier }, - "r,r,r,r,r,r,~{memory}", - ); - } -} - -/// `ptx::SM100_U8x4_STSM_T::copy(src, smem)`: `stmatrix.m16n8.x1.trans.b8`. -pub fn stmatrixU8x4T(b: *Builder, src: Value, smem: Value) void { - asmVoid(b, "stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [$0], {$1};\n", &.{ zext64(b, smem), src }, "l,r"); -} - -/// `ptx::SM90_U32x4_STSM_T::copy(s0, s1, s2, s3, smem)`: `stmatrix.x4.m8n8.trans.b16`. -pub fn stmatrixU32x4T(b: *Builder, src: [4]Value, smem: Value) void { - asmVoid(b, "stmatrix.sync.aligned.x4.m8n8.shared.b16.trans [$0], {$1, $2, $3, $4};\n", &.{ zext64(b, smem), src[0], src[1], src[2], src[3] }, "l,r,r,r,r"); -} - -/// `ClusterBarrier::arrive()` on this CTA's barrier. -pub fn mbarrierArrive(b: *Builder, bar: Value) void { - asmVoid(b, "{\n\tmbarrier.arrive.shared::cta.b64 _, [$0];\n\t}", &.{bar}, "r,~{memory}"); -} - -/// `ClusterBarrier::arrive(cta_id, pred = true)`: arrive on the barrier at the same offset of -/// cluster CTA `cta_id`. -pub fn mbarrierArriveCluster(b: *Builder, bar: Value, cta_id: Value) void { - asmVoid( - b, - "{\n\t.reg .b32 remAddr32;\n\tmapa.shared::cluster.u32 remAddr32, $0, $1;\n\tmbarrier.arrive.shared::cluster.b64 _, [remAddr32];\n\t}", - &.{ bar, cta_id }, - "r,r,~{memory}", - ); -} - -/// `ClusterTransactionBarrier::arrive_and_expect_tx(bytes, cta_id, pred)`: on cluster CTA -/// `cta_id`'s barrier, only when `pred == 1`. -pub fn mbarrierArriveAndExpectTxCluster(b: *Builder, bar: Value, transaction_bytes: anytype, cta_id: Value, pred: Value) void { - asmVoid( - b, - "{\n\t.reg .pred p;\n\t.reg .b32 remAddr32;\n\tsetp.eq.u32 p, $2, 1;\n\t@p mapa.shared::cluster.u32 remAddr32, $0, $1;\n\t@p mbarrier.arrive.expect_tx.shared::cluster.b64 _, [remAddr32], $3;\n\t}", - &.{ bar, cta_id, pred, liftAs(b, transaction_bytes, .i32) }, - "r,r,r,r,~{memory}", - ); -} - -/// `cute::prefetch_tma_descriptor(desc)` (generic descriptor address). -pub fn prefetchTmaDescriptor(b: *Builder, desc: Value) void { - asmVoid(b, "prefetch.tensormap [$0];", &.{desc}, "l,~{memory}"); -} - -/// `SM100_TMA_2SM_LOAD_2D::copy(desc, mbar, cache_hint, smem, c0, c1)`: issued by both CTAs of -/// the pair; the barrier address is masked so the bytes complete on the leader's barrier. -pub fn tma2SmLoad2d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c0: Value, c1: Value) void { - asmVoid( - b, - "cp.async.bulk.tensor.2d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4}], [$2], $5;", - &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c0, c1, cst64(b, cache_hint) }, - "r,l,r,r,r,l,~{memory}", - ); -} - -/// `SM100_TMA_2SM_LOAD_4D::copy(desc, mbar, cache_hint, smem, c0, c1, c2, c3)`. -pub fn tma2SmLoad4d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c: [4]Value) void { - asmVoid( - b, - "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4, $5, $6}], [$2], $7;", - &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c[0], c[1], c[2], c[3], cst64(b, cache_hint) }, - "r,l,r,r,r,r,r,l,~{memory}", - ); -} - -/// `ptx::tma_load_1d(dst, src, mbar, num_bytes, hint = EVICT_FIRST)`: non-tensor bulk copy of -/// `num_bytes` from a global (possibly peer) address into this CTA's shared memory. -pub fn tmaLoad1d(b: *Builder, dst: Value, src: Value, mbar: Value, num_bytes: anytype, cache_hint: u64) void { - asmVoid( - b, - "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1], $2, [$3], $4;\n", - &.{ dst, src, liftAs(b, num_bytes, .i32), mbar, cst64(b, cache_hint) }, - "r,l,r,r,l,~{memory}", - ); -} - -/// `ptx::tma_store_1d(dst, src, num_bytes, hint = EVICT_NORMAL)`. -pub fn tmaStore1d(b: *Builder, dst: Value, src: Value, num_bytes: anytype, cache_hint: u64) void { - asmVoid( - b, - "cp.async.bulk.global.shared::cta.bulk_group.L2::cache_hint [$0], [$1], $2, $3;\n", - &.{ dst, src, liftAs(b, num_bytes, .i32), cst64(b, cache_hint) }, - "l,r,r,l,~{memory}", - ); -} - -/// `cute::SM90_TMA_STORE_2D::copy(desc, smem, c0, c1)`. -pub fn tmaStore2d(b: *Builder, desc: Value, smem: Value, c0: Value, c1: Value) void { - asmVoid(b, "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3}], [$1];", &.{ desc, smem, c0, c1 }, "l,r,r,r,~{memory}"); -} - -/// `cute::TMEM::Allocator2Sm::allocate(num_columns, dst)`: write the base column to shared `dst`. -pub fn tmemAllocate2Sm(b: *Builder, num_columns: u32, dst: Value) void { - asmVoid(b, "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [$0], $1;", &.{ dst, cst32(b, num_columns) }, "r,r,~{memory}"); -} - -/// `cute::TMEM::Allocator2Sm::free(tmem_ptr, num_columns)`. -pub fn tmemFree2Sm(b: *Builder, tmem_ptr: Value, num_columns: u32) void { - asmVoid(b, "{\n\ttcgen05.dealloc.cta_group::2.sync.aligned.b32 $0, $1; \n\t}", &.{ tmem_ptr, cst32(b, num_columns) }, "r,r,~{memory}"); -} - -/// `ptx::tcgen05_before_thread_sync`. -pub fn tcgen05BeforeThreadSync(b: *Builder) void { - asmVoid(b, "tcgen05.fence::before_thread_sync;", &.{}, "~{memory}"); -} - -/// `ptx::tcgen05_after_thread_sync`. -pub fn tcgen05AfterThreadSync(b: *Builder) void { - asmVoid(b, "tcgen05.fence::after_thread_sync;", &.{}, "~{memory}"); -} - -/// `cutlass::arch::fence_view_async_tmem_load` (`tcgen05.wait::ld`). -pub fn fenceViewAsyncTmemLoad(b: *Builder) void { - asmVoid(b, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &.{}, "~{memory}"); -} - -/// `fence_view_async_tmem_load` with the values of earlier TMEM loads threaded through it, so -/// LLVM cannot move their uses above the wait (the PTX is the same single instruction). -pub fn fenceViewAsyncTmemLoadValues(b: *Builder, comptime n: usize, values: [n]Value) [n]Value { - comptime var constraints: []const u8 = ""; - inline for (0..n) |i| constraints = constraints ++ (if (i == 0) "" else ",") ++ "=r"; - inline for (0..n) |i| constraints = constraints ++ std.fmt.comptimePrint(",{d}", .{i}); - constraints = constraints ++ ",~{memory}"; - return asmValues(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &values, .i32, constraints, true); -} - -/// `fenceViewAsyncTmemLoadValues` for a run-time number of values (the epilogue's store blocks -/// hold `STORE_BLOCK_M / ATOM_M * 8` words): the same single `tcgen05.wait::ld`, every value -/// tied through it. -pub fn fenceViewAsyncTmemLoadSlice(b: *Builder, values: []const Value) []Value { - const n = values.len; - const a = b.arena.allocator(); - var w: std.Io.Writer.Allocating = .init(a); - for (0..n) |i| w.writer.print("{s}=r", .{if (i == 0) "" else ","}) catch @panic("OOM"); - for (0..n) |i| w.writer.print(",{d}", .{i}) catch @panic("OOM"); - w.writer.writeAll(",~{memory}") catch @panic("OOM"); - return asmValuesSlice(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", values, .i32, w.written(), true); -} - -/// `cute::SM100_TMEM_LOAD_16dp256b1x::copy(src, d0, d1, d2, d3)`: lane t of the warp gets -/// rows `base + t/4` (d0, d1) and `base + 8 + t/4` (d2, d3), columns `2(t%4)`, `2(t%4)+1`. -pub fn tmemLoad16dp256b1x(b: *Builder, taddr: Value) [4]Value { - return asmValues(b, 4, "tcgen05.ld.sync.aligned.16x256b.x1.b32{$0, $1, $2, $3},[$4];\n", &.{taddr}, .i32, "=r,=r,=r,=r,r", true); -} - -/// `cute::SM100_UTCCP_4x32dp128bit_2cta::copy(src_desc, dst_taddr)`. -pub fn utccp4x32dp128bit2cta(b: *Builder, src_desc: Value, dst_taddr: Value) void { - asmVoid(b, "tcgen05.cp.cta_group::2.32x128b.warpx4 [$0], $1;", &.{ dst_taddr, src_desc }, "r,l"); -} - -/// `ptx::SM100_MMA_MXF8F6F4_2x1SM_SS::fma(desc_a, desc_b, tmem_c, scale_c, desc, tmem_sfa, -/// tmem_sfb)`: `desc` is the 64-bit runtime descriptor (`instr << 32`), `scale_c` (i32) -/// nonzero accumulates. -pub fn mmaMxf8f6f42x1SmSs(b: *Builder, desc_a: Value, desc_b: Value, tmem_c: Value, scale_c: Value, desc: Value, tmem_sfa: Value, tmem_sfb: Value) void { - const idesc = b.emit(arith.trunci(b.ctx, desc.shrLogical(32).inner, DType.i32.toMlir(b.ctx), b.loc())); - asmVoid( - b, - "{\n\t.reg .pred p;\n\tsetp.ne.b32 p, $4, 0;\n\ttcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale [$0], $1, $2, $3, [$5], [$6], p; \n\t}\n", - &.{ tmem_c, desc_a, desc_b, idesc, scale_c, tmem_sfa, tmem_sfb }, - "r,l,l,r,r,r,r", - ); -} - -/// `cutlass::arch::umma_arrive_multicast_2x1SM(barrier, cta_mask)`: inside `elect_one_sync`, -/// `tcgen05.commit` arriving on `barrier` in every CTA of `cta_mask`. -pub fn ummaArriveMulticast2x1Sm(b: *Builder, bar: Value, cta_mask: u16) void { - var elected = b.openIf(electOneSync(b)); - { - asmVoid( - b, - "{\n\ttcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [$0], $1; \n\t}", - &.{ bar, b.constant(.i16, @as(i16, @bitCast(cta_mask))) }, - "r,h,~{memory}", - ); - elected.close(); - } -} - -pub fn bits32(v: Value) Value { - return if (v.dtype() == .i32) v else v.bitCast(.i32); -} - -pub fn fromBits32(v: Value, dt: DType) Value { - return if (dt == .i32) v else v.bitCast(dt); -} - -/// `__shfl_sync(0xffffffff, v, src_lane)` / `ptx::exchange` (32-bit). -pub fn shflSync(b: *Builder, v: Value, src_lane: anytype) Value { - const r = asmValue(b, "shfl.sync.idx.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, src_lane, .i32) }, .i32, "=r,r,r", true); - return fromBits32(r, v.dtype()); -} - -/// `__shfl_xor_sync(0xffffffff, v, lane_mask)`. -pub fn shflXorSync(b: *Builder, v: Value, lane_mask: anytype) Value { - const r = asmValue(b, "shfl.sync.bfly.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, lane_mask, .i32) }, .i32, "=r,r,r", true); - return fromBits32(r, v.dtype()); -} - -/// `__shfl_up_sync(0xffffffff, v, delta)`. -pub fn shflUpSync(b: *Builder, v: Value, delta: anytype) Value { - const r = asmValue(b, "shfl.sync.up.b32 $0, $1, $2, 0, -1;", &.{ bits32(v), liftAs(b, delta, .i32) }, .i32, "=r,r,r", true); - return fromBits32(r, v.dtype()); -} - -/// `__reduce_add_sync(0xffffffff, v)` (u32). -pub fn reduceAddSync(b: *Builder, v: Value) Value { - return asmValue(b, "redux.sync.add.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); -} - -/// `__reduce_min_sync(0xffffffff, v)` (u32). -pub fn reduceMinSync(b: *Builder, v: Value) Value { - return asmValue(b, "redux.sync.min.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); -} - -/// `__all_sync(0xffffffff, pred)` (`pred` i1; returns i1). -pub fn allSync(b: *Builder, pred: Value) Value { - const r = asmValue(b, "{ .reg .pred p, q; setp.ne.u32 p, $1, 0; vote.sync.all.pred q, p, $2; selp.u32 $0, 1, 0, q; }", &.{ pred.to(.i32), cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); - return r.ne(0); -} - -pub fn popc(b: *Builder, v: Value) Value { - return b.callIntrinsic("llvm.ctpop.i32", &.{v}, DType.i32.toMlir(b.ctx)).?; -} - -/// `__ffs(v)`: 1-based index of the lowest set bit, 0 for 0 (nvcc: `brev` + `bfind.shiftamt`). -pub fn ffs(b: *Builder, v: Value) Value { - return asmValue(b, "{ .reg .b32 r; brev.b32 r, $1; bfind.shiftamt.u32 $0, r; }", &.{v}, .i32, "=r,r", false).add(1); -} - -/// `__fns(mask, base, offset)`: position of the `offset`-th set bit of `mask` from `base`. -pub fn fns(b: *Builder, mask: Value, base: anytype, offset: Value) Value { - return asmValue(b, "fns.b32 $0, $1, $2, $3;", &.{ mask, liftAs(b, base, .i32), offset }, .i32, "=r,r,r,r", false); -} - -/// `fminf(a, b)` (`min.f32`, NaN-ignoring; `kClampAfterSilu`). -pub fn fminf(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "min.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); -} - -/// `fabsf(x)` (`abs.f32`). -pub fn fabsf(b: *Builder, x: Value) Value { - return asmValue(b, "abs.f32 $0, $1;", &.{x}, .f32, "=f,f", false); -} - -/// `math::fast_rcp(x)`. -pub fn fastRcp(b: *Builder, x: Value) Value { - return asmValue(b, "rcp.approx.ftz.f32 $0, $1;", &.{x}, .f32, "=f,f", true); -} - -/// `__expf(x)` without `--use_fast_math`: `ex2.approx.f32(x * log2(e))` (non-FTZ). -pub fn expfFast(b: *Builder, x: Value) Value { - return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0f3FB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); -} - -/// `__expf(-x)`: nvcc folds the negation into the constant (`mul.f32 t, x, -log2(e)`). -pub fn expfFastNeg(b: *Builder, x: Value) Value { - return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0fBFB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); -} - -/// `__float22bfloat162_rn({lo, hi})`: bf16x2 word, `lo` in the low half. -pub fn float22bfloat162Rn(b: *Builder, lo: Value, hi: Value) Value { - return asmValue(b, "{ cvt.rn.bf16x2.f32 $0, $2, $1;}\n", &.{ lo, hi }, .i32, "=r,f,f", false); -} - -/// `__hmin2(a, b)` on bf16x2 words. -pub fn hmin2Bf16(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "{ min.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); -} - -/// `__hmax2(a, b)` on bf16x2 words. -pub fn hmax2Bf16(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "{ max.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); -} - -/// `__bfloat1622float2(word)`: `{low, high}` f32 (exact). -pub fn bfloat1622float2(b: *Builder, word: Value) [2]Value { - return asmValues(b, 2, "{ .reg .b16 l, h; mov.b32 {l, h}, $2; cvt.f32.bf16 $0, l; cvt.f32.bf16 $1, h; }", &.{word}, .f32, "=f,=f,r", false); -} - -/// `ptx::accumulate(float2& a, nv_bfloat162 b)`: `a.x += bf16(b.x)`, `a.y += bf16(b.y)` with -/// `add.rn.f32.bf16` (one rounding, the bf16 widened exactly). -pub fn accumulateBf16x2(b: *Builder, acc: [2]Value, bf16x2: Value) [2]Value { - const lo = i16Of(b, bf16x2); - const hi = i16Of(b, bf16x2.shrLogical(16)); - return .{ - asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ lo, acc[0] }, .f32, "=f,h,0", false), - asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ hi, acc[1] }, .f32, "=f,h,0", false), - }; -} -/// IEEE operations used when staging quantized inputs; no FTZ or contraction. -pub fn divRn(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "div.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); -} -pub fn mulRn(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "mul.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); -} - -test "canonical primitives preserve aligned barriers and runtime transaction counts" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "compatibility_primitives"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - const bar = b.constant(.i32, 0); - const phase = b.constant(.i32, 0); - const bytes = b.threadIdx().x.add(16); - b.barrierCtaSyncAligned(1, 32); - b.barrierClusterArriveRelaxedAligned(); - b.barrierClusterWaitAligned(); - b.mbarrierInit(bar, 1); - b.fenceMbarrierInit(); - b.mbarrierArriveExpectTx(bar, bytes); - b.mbarrierArriveExpectTx(bar, 16); - b.mbarrierWait(bar, phase, .{ .test_first = false }); - _ = phase.bitXor(1); - b.fenceProxyAsyncShared(); - b.fenceProxyAsyncShared(); - b.cpAsyncBulkCommitGroup(); - b.cpAsyncBulkWaitGroup(0, false); - _ = b.voteBallotSync(bytes.ne(0), null); - _ = b.fmax(b.constant(.f32, 1), b.constant(.f32, 2), .{}); - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "llvm.nvvm.barrier.cta.sync.aligned.count"); - try expectContains(ir, "barrier.cluster.arrive.relaxed.aligned"); - try expectContains(ir, "nvvm.mbarrier.arrive.expect_tx"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} diff --git a/dsl/fly/README.md b/dsl/fly/README.md index 687c99dc..b886988b 100644 --- a/dsl/fly/README.md +++ b/dsl/fly/README.md @@ -81,7 +81,7 @@ const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ ``` Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` -and `zml/kernel/fly/test.zig`. +and `zml/kernel.zig`. Model kernels live beside their consumers: diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md index b01b7189..ec1ae61f 100644 --- a/dsl/tcl/README.md +++ b/dsl/tcl/README.md @@ -104,7 +104,8 @@ def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16 across the partitions; other kernels stay per chip. Its arguments and results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis `C`, sized to the chip count, across chips (each partition holds its share), - the default `.broadcast` keeps a copy per chip. + the default `.broadcast` keeps a copy per chip. Call it inside + `zml.ops.manualComputation` so it sees each partition's shards. * **Convolutions** pad and slide the input in its read: a slide replaces its axis by `window_axis, frame_axis`, then `contract` with the filter. On its own (without a slide) the padding is dropped. diff --git a/examples/io/main.zig b/examples/io/main.zig index c9dc05ea..1cec23b0 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -132,9 +132,13 @@ pub fn main(init: std.process.Init) !void { const root = try TensorNode.init(init.arena.allocator(), ""); - for (registry.tensors.values()) |tensor| { + var registry_it = registry.iterator(); + while (registry_it.next()) |kv| { + const name = kv.key_ptr.*; + const tensor = kv.value_ptr.*; + var current = root; - var parts = std.mem.tokenizeScalar(u8, tensor.name, '.'); + var parts = std.mem.tokenizeScalar(u8, name, '.'); while (parts.next()) |part| { const gop = try current.children.getOrPut(current.allocator, part); if (!gop.found_existing) { @@ -179,16 +183,16 @@ pub fn main(init: std.process.Init) !void { const tensors = try allocator.alloc(zml.Tensor, tensor_count); defer allocator.free(tensors); + var registry_it = registry.iterator(); var load_count: usize = 0; - for (registry.tensors.keys(), registry.tensors.values()) |name, tensor| { + while (registry_it.next()) |entry| : (load_count += 1) { tensors[load_count] = switch (mesh_type) { - .replicated => store.view().createReplicatedTensor(name, null), - .sharded => if (tensor.shape.rank() > 0) - store.view().createTensor(name, null, .tp, tp_partitioning) + .replicated => store.view().createReplicatedTensor(entry.key_ptr.*, null), + .sharded => if (entry.value_ptr.shape.rank() > 0) + store.view().createTensor(entry.key_ptr.*, null, .tp, tp_partitioning) else - store.view().createReplicatedTensor(name, null), + store.view().createReplicatedTensor(entry.key_ptr.*, null), }; - load_count += 1; } const model: AllTensorsModel = .{ .tensors = tensors }; diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index ecc5abf8..86020cce 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -173,9 +173,6 @@ pub const Session = struct { var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); defer active_length_buf.deinit(); - const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); - const params = self.compiled_model.params; var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); @@ -185,7 +182,7 @@ pub const Session = struct { if (token_id == self.config.eos_token_id) break :generation; - const token = try decoder.feedOne(token_id, out_tokens_buffer); + const token = try decoder.feedOne(token_id); if (self.think_start) |think_start| if (token_id == think_start) { try stdout.writeAll("\x1b[2m"); }; @@ -221,7 +218,7 @@ pub const Session = struct { try current_token_buffer.toSlice(self.io, self.generated_token_slice); } - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.writeAll(try decoder.finalize()); try stdout.flush(); } }; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 8a6532e6..ce3aa4c4 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -172,9 +172,6 @@ pub const Session = struct { var decoder: zml.tokenizer.Tokenizer.Decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - const decoder_out_buffer: []u8 = try self.allocator.alloc(u8, 256); - defer self.allocator.free(decoder_out_buffer); - var last_token_id: u32 = self.last_generated_token; var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); @@ -188,7 +185,7 @@ pub const Session = struct { generation: while (true) { if (isEosToken(self.config, last_token_id)) break :generation; - try stdout.writeAll(try decoder.feedOne(last_token_id, decoder_out_buffer)); + try stdout.writeAll(try decoder.feedOne(last_token_id)); try stdout.flush(); try all_tokens.append(self.allocator, last_token_id); @@ -207,7 +204,7 @@ pub const Session = struct { last_token_id = try current_token_buffer.getValue(u32, self.io); } - try stdout.writeAll(try decoder.finalize(decoder_out_buffer)); + try stdout.writeAll(try decoder.finalize()); try stdout.flush(); } }; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index 5121fd30..28df02f3 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -138,9 +138,6 @@ pub const Session = struct { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); @@ -153,7 +150,7 @@ pub const Session = struct { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.eos_token_id) break :generation; - const token = try decoder.feedOne(token_id, out_tokens_buffer); + const token = try decoder.feedOne(token_id); if (self.think_start) |think_start| if (token_id == think_start) { try stdout.writeAll("\x1b[2m"); }; @@ -179,7 +176,7 @@ pub const Session = struct { try current_token_buffer.toSlice(self.io, self.generated_token_slice); } - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.writeAll(try decoder.finalize()); try stdout.flush(); } }; diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 2a3b0ba8..1375a1bd 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -787,8 +787,6 @@ pub const Moe = struct { self.gate_up_proj, self.down_proj, moe_backend, - zml.Compiler.current().mesh(.experts), - .expert(.experts), .{ .activation = .{ .swiglu = .{} }, .quantize_input = false, diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index 6a819f26..7b6c6cb2 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -147,9 +147,6 @@ pub const Session = struct { pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - const out_tokens_buffer = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); @@ -161,7 +158,7 @@ pub const Session = struct { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.eos_token_id) break :generation; - const token = try decoder.feedOne(token_id, out_tokens_buffer); + const token = try decoder.feedOne(token_id); if (self.think_start) |think_start| if (token_id == think_start) try stdout.writeAll("\x1b[2m"); try stdout.writeAll(token); if (self.think_end) |think_end| if (token_id == think_end) try stdout.writeAll("\x1b[0m"); @@ -183,7 +180,7 @@ pub const Session = struct { try current_token_buffer.toSlice(self.io, self.generated_token_slice); } - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.writeAll(try decoder.finalize()); try stdout.flush(); } }; diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 34a78bfe..7e519bf0 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -42,7 +42,7 @@ const DemoModel = struct { const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); - return zml.ops.manualComputation( + return zml.ops.manualComputation2( (struct { y: zml.Tensor, gate: zml.Tensor, @@ -70,7 +70,7 @@ fn usage() void { \\Simple sharding demo: \\ - explicit partitioning on data/model axes \\ - tensor print - \\ - manualComputation with multiple inputs and one output + \\ - manualComputation2 with multiple inputs and one output \\ \\Mesh modes: \\ - auto: physical_mesh=auto, inferred cpu device_count=2 diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md index 9673870f..b66b7abe 100644 --- a/mlir/dialects/fly/REGENERATING.md +++ b/mlir/dialects/fly/REGENERATING.md @@ -71,7 +71,7 @@ dialect today; when one appears, extend the generator and this file. `test.zig`. - Callers that no longer compile are the point of generating: an operation whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, + the same change (`dsl/fly/builder.zig`, `zml/kernel.zig`, `zml/kernels/*/fly/`, and downstream llmd's `llmd/kernels/*/fly/`). - Emit the IR of the fly kernels before and after (for instance llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 08817afe..fd070b6e 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 8482e52a..24718ee6 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "72b1359384ac50e727b800e81c56d7134f9d6a061a982c4e36b7fef62cf7c331", - "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-amd64.tar.zst", + "sha256": "1ae8a7c8fc677e0f10c0032ce9ae35e5b6c3f5eadc3f54e1bbda63c219b0170d", + "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "caeefd717960fa2acec6304cd8296299e262abdba669ecf4d487e3af76d73a7b", - "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-arm64.tar.zst", + "sha256": "14a3567c629a1ea97f1dcfa2cb47b54ea08d8c4b9e35eae47a7eff33cfb53ce0", + "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index afb26864..ba53bfb4 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", - sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-furiosa-linux-amd64.tar.zst", + sha256 = "514b1c96a2237380a9b9e011f5b994c6617eb103a6a9dac8ff0a7f1832d1a3bd", ) return mctx.extension_metadata( diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index ee33fd58..ebbd9e3c 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -50,6 +50,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); + try setEnv("FURIOSA_VISIBLE_DEVICES", "0", 0); const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); log.info("Loading Furiosa plugin: {s}", .{library}); diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 71e1858a..767ef2f6 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "89881d1a1db336e3337a9373db2c4e94f71ddfb77619dee75c0c12e2903913ae", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-metal-darwin-arm64.tar.zst", + sha256 = "bdd2de3ec8293faa25b358cdf35ee3e75d69be73a4acea87b6e007a18b8f10e8", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index a3c968d3..6016e389 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" ONEAPI_VERSION = "2026.1" @@ -248,7 +248,6 @@ filegroup( name = "mkl_core_runtime", srcs = [ "{ONEAPI_MKL_LIB}/libmkl_core.so.3", - "{ONEAPI_MKL_LIB}/libmkl_def.so.3", "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index 140ce102..a3e69ad4 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -216,7 +216,7 @@ "name": "libigc2", "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -227,7 +227,7 @@ "name": "libigdfcl2", "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -249,7 +249,7 @@ "name": "libze1", "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" ], "version": "1.32.0-1~24.04~ppa1" }, diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 5a829ac5..272bc02c 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -15,11 +15,9 @@ sources: - channel: noble main url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z - # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics - # packages are pinned below to keep their versions from it. Use snapshots, - # not the live PPA: the PPA deletes superseded packages from its pool. + # IGC 2.41.15 is newer than the graphics snapshot above. - channel: noble main - url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z + url: https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu archs: - "amd64" @@ -45,9 +43,9 @@ packages: - "libnl-genl-3-200" - "libnl-route-3-200" - "libnuma1" - - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" + - "libigdgmm12" - "libigc2 (= 2.41.15-1~24.04)" - "libigdfcl2 (= 2.41.15-1~24.04)" - - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" + - "libze-intel-gpu1" - "libze1" - "zlib1g" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 41b01b60..454bf626 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "cdc4434f2023842f0e1a34c08aaed87f082df19f92c4ae21c9d4452a9739b604" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "02b86fa3a6599f6aa2b89806b8f6c0b16f11765ec04c94517ec1463a59d92175" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index 6a0fdfae..4c1c50ed 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -482,7 +482,7 @@ fn flashAttention( } /// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in -/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the +/// `zml/kernel.zig` opens the builder, calls this `run`, and finalizes the /// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ diff --git a/stdx/flags.zig b/stdx/flags.zig index d0c4352d..52a9e6a4 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -591,19 +591,25 @@ fn fields_to_comma_list(comptime E: type) []const u8 { } } -/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. pub fn flag_name(comptime field_name: []const u8) []const u8 { - comptime assert(!std.mem.eql(u8, field_name, "positional")); - comptime var name = field_name[0..field_name.len].*; - comptime std.mem.replaceScalar(u8, &name, '_', '-'); - const dashed = name; - return "--" ++ dashed; + // TODO(Zig): Cleanup when this is fixed after Zig 0.11. + // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. + return comptime blk: { + assert(!std.mem.eql(u8, field_name, "positional")); + + var result: []const u8 = "--"; + var index = 0; + while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { + result = result ++ field_name[index..][0..i] ++ "-"; + index = index + i + 1; + } + result = result ++ field_name[index..]; + break :blk result; + }; } test flag_name { - try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); - try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); - try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); + try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); } fn flag_name_positional(comptime field_name: []const u8) []const u8 { diff --git a/tools/safetensors/BUILD.bazel b/tools/safetensors/BUILD.bazel deleted file mode 100644 index aff4954d..00000000 --- a/tools/safetensors/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") - -zig_binary( - name = "safetensors", - main = "main.zig", - visibility = ["//visibility:public"], - zigopts = ["-lc"], - deps = [ - "//stdx", - "//vfs", - "//zml", - ], -) diff --git a/tools/safetensors/main.zig b/tools/safetensors/main.zig deleted file mode 100644 index cf902eef..00000000 --- a/tools/safetensors/main.zig +++ /dev/null @@ -1,91 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); -const VFS = @import("vfs"); -const zml = @import("zml"); - -const log = std.log.scoped(.safetensors_convert); - -const Args = struct { - output: ?[]const u8 = null, - positional: struct { model: []const u8 }, - - pub const help = - \\Usage: [--output=] - \\ - \\Read safetensors headers and write model.safetensors.full_index.json in the working directory - \\Example: bazel run //tools/safetensors -- --output="model.safetensors.full_index.json" hf://deepseek-ai/DeepSeek-V4.1-Flash - ; -}; - -pub fn main(init: std.process.Init) !void { - const args = stdx.flags.parse(init.minimal.args, Args); - const allocator = init.gpa; - - const start = std.Io.Clock.now(.awake, init.io); - var client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - defer client.deinit(); - try client.initDefaultProxies(allocator, init.environ_map); - - const input = args.positional.model; - - var file_vfs: VFS.File = .init(allocator, init.io, .{ .direct_io = false }); - defer file_vfs.deinit(); - var http_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .http); - defer http_vfs.deinit(); - var https_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .https); - defer https_vfs.deinit(); - var hf_vfs: ?VFS.HF = if (std.mem.startsWith(u8, input, "hf://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (hf_vfs) |*backend| backend.deinit(); - var s3_vfs: ?VFS.S3 = if (std.mem.startsWith(u8, input, "s3://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (s3_vfs) |*backend| backend.deinit(); - var gcs_vfs: ?VFS.GCS = if (std.mem.startsWith(u8, input, "gs://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (gcs_vfs) |*backend| backend.deinit(); - - var vfs: VFS = try .init(allocator, init.io); - const io = vfs.io(); - defer vfs.deinit(); - try vfs.register("file", file_vfs.io()); - try vfs.register("http", http_vfs.io()); - try vfs.register("https", https_vfs.io()); - if (hf_vfs) |*backend| try vfs.register("hf", backend.io()); - if (s3_vfs) |*backend| try vfs.register("s3", backend.io()); - if (gcs_vfs) |*backend| try vfs.register("gs", backend.io()); - - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - const repo = try zml.safetensors.resolveModelRepo(io, input); - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - const loaded = std.Io.Clock.now(.awake, init.io); - - const index_json_file = if (args.output) |output| - try std.Io.Dir.cwd().createFile(io, output, .{}) - else - try repo.createFile(io, full_index_path, .{}); - const out_buffer = try allocator.alloc(u8, 4096); - defer allocator.free(out_buffer); - var out_writer = index_json_file.writer(io, out_buffer); - try registry.saveAsFullIndexJson(io, allocator, repo, &out_writer.interface); - const done = std.Io.Clock.now(.awake, init.io); - - log.info("Wrote {s}: {d} tensors; load {f}s, write {f}s, total {f}s", .{ - args.output orelse full_index_path, - registry.tensors.count(), - start.durationTo(loaded), - loaded.durationTo(done), - start.durationTo(done), - }); -} - -const full_index_path = "model.safetensors.full_index.json"; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f52ecdd5..dbb2ad33 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -78,7 +78,6 @@ zig_library( srcs = [ "Compiler.zig", "Sharding.zig", - "attention/Options.zig", "attention/attention.zig", "buffer.zig", "buffer/HostAccessible.zig", @@ -87,13 +86,10 @@ zig_library( "exe.zig", "floats.zig", "io.zig", + "kernel.zig", "kernels/flash_attention/flash_attention.zig", - "kernels/flash_attention/tests.zig", - "kernels/moe/api.zig", "kernels/moe/moe.zig", "kernels/moe/tests.zig", - "kernels/moe/weights.zig", - "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", "mem.zig", @@ -102,6 +98,7 @@ zig_library( "nn.zig", "ops.zig", "ops/manual_computation.zig", + "ops/manual_computation2.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", @@ -113,12 +110,11 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", - "//zml/kernel", "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", "//zml/kernels/moe/cute", - "//zml/kernels/moe/flashinfer_cutlass", + "//zml/kernels/moe/cutlass_flashinfer", "//zml/kernels/moe/fly", "//zml/kernels/moe/metal", "//zml/kernels/moe/mosaic_tpu", diff --git a/zml/attention/Options.zig b/zml/attention/Options.zig deleted file mode 100644 index 539f753f..00000000 --- a/zml/attention/Options.zig +++ /dev/null @@ -1,3 +0,0 @@ -/// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every -/// key, as in an encoder or a diffusion transformer, and token_index is ignored. -causal: bool = true, diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index edd9b4ca..039c6628 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -93,10 +93,15 @@ pub const Metadata = union(BackendType) { } }; -pub const Options = @import("Options.zig"); +pub const Options = struct { + /// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every + /// key, as in an encoder or a diffusion transformer, and token_index is ignored. + causal: bool = true, +}; pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; + const InputSpecs = @FieldType(zml.ops.ManualPartitioning(Attention.forward), "in_specs"); in_specs: struct { q: Logical, @@ -104,7 +109,7 @@ pub const Partitioning = struct { v: Logical, token_index: Logical = .replicated, /// Omit to replicate backend metadata tensors. - metadata: ?zml.ops.PartitionSpecs(Metadata) = null, + metadata: ?@FieldType(InputSpecs, "metadata") = null, }, out_specs: Logical, @@ -134,13 +139,20 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T /// `attention`, causal or not depending on `opts`. pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; - return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ + return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, .{ .in_specs = .{ .q = partitioning.in_specs.q, .k = partitioning.in_specs.k, .v = partitioning.in_specs.v, .token_index = partitioning.in_specs.token_index, - .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), + .metadata = partitioning.in_specs.metadata orelse switch (metadata) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |metadata_| .{ .flash_attention = switch (metadata_) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = .{ .num_tokens = .replicated } }, + } }, + }, }, .out_specs = partitioning.out_specs, }); diff --git a/zml/exe.zig b/zml/exe.zig index 32c0ac48..85baa7e7 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -387,11 +387,6 @@ pub const Exe = struct { }, } } - - pub fn memoryStats(self: *const Exe) pjrt.ApiError!pjrt.CompiledMemoryStats { - const exe = try self.exe.executable(self.platform.pjrt_api); - return try exe.getCompiledMemoryStats(self.platform.pjrt_api); - } }; /// A typed executable whose input and output buffer structures are derived from diff --git a/zml/io.zig b/zml/io.zig index ab8d8e6e..1babdcfc 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1287,52 +1287,20 @@ pub const DirectMemoryWriter = struct { self.shard_progress = states; } - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + const chunk = data[0]; try self.commitWindow(); - if (self.interface.end == self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - const consumed = consumed: { - var n: usize = 0; - for (data[0 .. data.len - 1]) |bytes| { - const copied = self.writeWindow(bytes); - n += copied; + const writable = self.interface.buffer.len - self.interface.end; + if (writable == 0) return std.Io.Writer.Error.WriteFailed; - if (copied < bytes.len) break :consumed n; - } - - const pattern = data[data.len - 1]; - switch (pattern.len) { - 0 => {}, - 1 => { - const dest = self.interface.buffer[self.interface.end..]; - const count = @min(dest.len, splat); - @memset(dest[0..count], pattern[0]); - - self.interface.end += count; - n += count; - }, - else => for (0..splat) |_| { - const copied = self.writeWindow(pattern); - n += copied; - - if (copied < pattern.len) break; - }, - } - - break :consumed n; - }; - - try self.commitWindow(); - return consumed; - } - - fn writeWindow(self: *DirectMemoryWriter, bytes: []const u8) usize { - const dest = self.interface.buffer[self.interface.end..]; - const n = @min(dest.len, bytes.len); - @memcpy(dest[0..n], bytes[0..n]); + const n = @min(writable, chunk.len); + @memcpy(self.interface.buffer[self.interface.end..][0..n], chunk[0..n]); self.interface.end += n; + try self.commitWindow(); + return n; } diff --git a/zml/kernel.zig b/zml/kernel.zig new file mode 100644 index 00000000..94d6046a --- /dev/null +++ b/zml/kernel.zig @@ -0,0 +1,1625 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +const cute_builder = @import("dsl/cute/builder"); +const fly_builder = @import("dsl/fly/builder"); +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +const triton_builder = @import("dsl/triton/builder"); + +const Compiler = @import("Compiler.zig"); +const DataType = @import("dtype.zig").DataType; +const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); +const platform_ = @import("platform.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub const triton = struct { + pub const Builder = triton_builder.Builder; + pub const Value = triton_builder.Value; + pub const DType = triton_builder.DType; + pub const FinishError = triton_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; + } +}; + +pub const mosaic_tpu = struct { + const Builder = mosaic_tpu_builder.Builder; + const DType = mosaic_tpu_builder.DType; + const FinishError = mosaic_tpu_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; + } + + const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + }; + + const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, + }; + + const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, + }; + + const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + }; + + fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, + ) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); + } + + const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, + }; + + fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); + } +}; + +pub const cuda_tile = struct { + pub const Builder = cuda_tile_builder.Builder; + pub const Value = cuda_tile_builder.Value; + pub const DType = cuda_tile_builder.DType; + pub const FinishError = cuda_tile_builder.FinishError; + pub const EntryHint = cuda_tile_builder.EntryHint; + pub const Arch = cuda_tile_builder.Arch; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const cute = struct { + pub const AlgebraExpr = cute_builder.AlgebraExpr; + pub const Builder = cute_builder.Builder; + pub const ArgSpec = cute_builder.ArgSpec; + pub const Value = cute_builder.Value; + pub const View = cute_builder.View; + pub const Atom = cute_builder.Atom; + pub const DType = cute_builder.DType; + pub const AlgebraToken = cute_builder.AlgebraToken; + pub const LayoutSpec = cute_builder.LayoutSpec; + pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; + pub const FinishError = cute_builder.FinishError; + pub const TmaConfig = cute_builder.TmaConfig; + pub const TmaFormat = cute_builder.TmaFormat; + pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; + pub const CtaGroup = cute_builder.CtaGroup; + pub const ctaCount = cute_builder.ctaCount; + pub const MBarrierScope = cute_builder.MBarrierScope; + pub const OperandMajorMode = cute_builder.OperandMajorMode; + pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; + pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; + pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; + pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; + pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; + pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } + + fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + /// A complete CuTe program: one or more `cuda.kernel` operations inside + /// a `gpu.module`, followed by a public `func.func` which constructs the + /// launch-time CuTe objects and calls `cuda.launch_ex`. + /// + /// This is the source-level equivalent of invoking a Python `@cute.jit` + /// object whose `__call__` builds TMA descriptors and launches an + /// `@cute.kernel`. XLA executes the generated host function on its CUDA + /// stream, so the custom call deliberately carries no external grid or + /// block dimensions. + pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const tcl = struct { + pub const Builder = tcl_builder.Builder; + pub const TensorOperation = tcl_builder.TensorOperation; + pub const Value = tcl_builder.Value; + pub const Axis = tcl_builder.Axis; + pub const Expr = tcl_builder.Expr; + pub const Dram = tcl_builder.Dram; + pub const Context = tcl_builder.Context; + pub const DType = tcl_builder.DType; + pub const VeOp = tcl_builder.VeOp; + pub const ReduceMode = tcl_builder.ReduceMode; + pub const Predicate = tcl_builder.Predicate; + pub const Tactic = tcl_builder.Tactic; + pub const FinishError = tcl_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + /// A TCL function compiled by the Furiosa plugin as one native unit: + /// `declareArgs` declares the inputs in order, `ret` the outputs. + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; + } +}; + +pub const metal = struct { + pub const Builder = metal_builder.Builder; + pub const Value = metal_builder.Value; + pub const Type = metal_builder.Type; + pub const Struct = metal_builder.Struct; + pub const ArgSpec = metal_builder.ArgSpec; + pub const LoopOpts = metal_builder.LoopOpts; + pub const FinishError = metal_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); + } + + pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; + } + + /// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host + /// `extern struct` laid out like a `constant S &` argument. + pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } + }; + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; + } + + /// A Metal kernel: `run` declares its arguments with `declareArgs`, by + /// name, and builds its body. + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const fly = struct { + /// Threads per wavefront: 64 on CDNA, 32 on RDNA. + pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; + } + + /// The plugin sizes a block in wavefronts, so a thread count has to divide + /// evenly: a kernel's layouts are built for an exact number of threads and + /// spare ones would partition out of range. + fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); + } + + pub const Builder = fly_builder.Builder; + pub const Value = fly_builder.Value; + pub const DType = fly_builder.DType; + pub const FinishError = fly_builder.FinishError; + pub const TiledCopy = fly_builder.TiledCopy; + pub const TiledMma = fly_builder.TiledMma; + pub const Arch = fly_builder.Arch; + pub const MmaFlavor = fly_builder.MmaFlavor; + pub const fly_rocdl = fly_builder.fly_rocdl; + + /// The matrix atom this device provides, or null when it has none: WMMA + /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. + pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; + } + pub const layout = fly_builder.layout; + pub const Layout = fly_builder.Layout; + pub const Tile = fly_builder.Tile; + pub const IntTuple = fly_builder.IntTuple; + pub const L = fly_builder.L; + pub const rowMajor = fly_builder.rowMajor; + pub const colMajor = fly_builder.colMajor; + pub const ordered = fly_builder.ordered; + pub const tile = fly_builder.tile; + pub const it = fly_builder.it; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernels run on rocm" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: Tensor, b: Tensor) Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: Tensor, b: Tensor) Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: Tensor) Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: Tensor }, shapes: [2]Shape) [2]Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const tp = zml.testing.tpMesh(); + const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: Tensor) [2]Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); + return .{ r[0], r[1] }; + } + }; + const x: Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{tp} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: Tensor, y: Tensor) Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel deleted file mode 100644 index 61ab44f8..00000000 --- a/zml/kernel/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "kernel", - srcs = [ - "kernel.zig", - "//zml/kernel/cuda_tile", - "//zml/kernel/cute", - "//zml/kernel/fly", - "//zml/kernel/metal", - "//zml/kernel/mosaic_tpu", - "//zml/kernel/tcl", - "//zml/kernel/triton", - ], -) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel deleted file mode 100644 index 89b7977e..00000000 --- a/zml/kernel/cuda_tile/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "cuda_tile", - srcs = [ - "cuda_tile.zig", - "test.zig", - ], -) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig deleted file mode 100644 index 3f6d8734..00000000 --- a/zml/kernel/cuda_tile/cuda_tile.zig +++ /dev/null @@ -1,163 +0,0 @@ -const std = @import("std"); - -const cuda_tile_builder = @import("dsl/cuda_tile/builder"); -pub const Builder = cuda_tile_builder.Builder; -pub const Value = cuda_tile_builder.Value; -pub const DType = cuda_tile_builder.DType; -pub const FinishError = cuda_tile_builder.FinishError; -pub const EntryHint = cuda_tile_builder.EntryHint; -pub const Arch = cuda_tile_builder.Arch; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cuda_tile_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i4, .u4 => .i4, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - .f4e2m1 => .f4e2m1fn, - else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// No `num_warps`/`num_stages`: warp and CTA tuning is - /// `optimization_hints` inside the IR (`Opts.hints`). - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Outputs XLA zeroes before the launch, by position in - /// `spec.outputs`, ascending. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(&.{}); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig deleted file mode 100644 index c929b0b9..00000000 --- a/zml/kernel/cuda_tile/test.zig +++ /dev/null @@ -1,28 +0,0 @@ -const std = @import("std"); - -const cuda_tile = @import("cuda_tile.zig"); - -test "cuda_tile kernel emits a module XLA can parse" { - const Cfg = struct { n: i64 }; - const AddOne = cuda_tile.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); -} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel deleted file mode 100644 index 7a613072..00000000 --- a/zml/kernel/cute/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "cute", - srcs = [ - "cute.zig", - "test.zig", - ], -) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig deleted file mode 100644 index 2e8c919f..00000000 --- a/zml/kernel/cute/cute.zig +++ /dev/null @@ -1,264 +0,0 @@ -const std = @import("std"); - -const cute_builder = @import("dsl/cute/builder"); -pub const AlgebraExpr = cute_builder.AlgebraExpr; -pub const Builder = cute_builder.Builder; -pub const ArgSpec = cute_builder.ArgSpec; -pub const Value = cute_builder.Value; -pub const View = cute_builder.View; -pub const Atom = cute_builder.Atom; -pub const DType = cute_builder.DType; -pub const AlgebraToken = cute_builder.AlgebraToken; -pub const LayoutSpec = cute_builder.LayoutSpec; -pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; -pub const FinishError = cute_builder.FinishError; -pub const TmaConfig = cute_builder.TmaConfig; -pub const TmaFormat = cute_builder.TmaFormat; -pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; -pub const CtaGroup = cute_builder.CtaGroup; -pub const ctaCount = cute_builder.ctaCount; -pub const MBarrierScope = cute_builder.MBarrierScope; -pub const OperandMajorMode = cute_builder.OperandMajorMode; -pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; -pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; -pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; -pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; -pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; -pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cute_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4e2m1fn, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared - /// memory is what the kernel declares. - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - block: [3]i32, - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - // XLA passes one pointer per input then per output; it does - // not check the count against the kernel. - const expected = spec.inputs.len + spec.outputs.len; - if (b.args.len != expected) { - std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); - } - - return b.finish(block); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| - std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .block = opts.block, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} - -fn ProgramSpec(comptime Config: type) type { - return struct { - /// Public host entry selected by XLA. The callback may give the - /// nested CUDA kernel a different (private) symbol. - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -/// A complete CuTe program: one or more `cuda.kernel` operations inside -/// a `gpu.module`, followed by a public `func.func` which constructs the -/// launch-time CuTe objects and calls `cuda.launch_ex`. -/// -/// This is the source-level equivalent of invoking a Python `@cute.jit` -/// object whose `__call__` builds TMA descriptors and launches an -/// `@cute.kernel`. XLA executes the generated host function on its CUDA -/// stream, so the custom call deliberately carries no external grid or -/// block dimensions. -pub fn Program( - comptime ConfigT: type, - comptime spec: ProgramSpec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.openProgram(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - return b.finishProgram(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .zeroed_outputs = opts.zeroed_outputs, - .scalars = opts.scalars, - .symmetric_workspace = opts.symmetric_workspace, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig deleted file mode 100644 index f4d457a3..00000000 --- a/zml/kernel/cute/test.zig +++ /dev/null @@ -1,34 +0,0 @@ -const std = @import("std"); - -const cute = @import("cute.zig"); - -test "cute kernel emits the module the CuTe compiler takes" { - const Cfg = struct { n: i64 }; - const AddOne = cute.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - }); - const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); - var guard = b.openIf(i.lt(cfg.n)); - { - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.close(); - } - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); -} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel deleted file mode 100644 index 57203521..00000000 --- a/zml/kernel/fly/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "fly", - srcs = [ - "fly.zig", - "test.zig", - ], -) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig deleted file mode 100644 index b7d6c943..00000000 --- a/zml/kernel/fly/fly.zig +++ /dev/null @@ -1,248 +0,0 @@ -const std = @import("std"); - -const fly_builder = @import("dsl/fly/builder"); -pub const Builder = fly_builder.Builder; -pub const Value = fly_builder.Value; -pub const DType = fly_builder.DType; -pub const FinishError = fly_builder.FinishError; -pub const TiledCopy = fly_builder.TiledCopy; -pub const TiledMma = fly_builder.TiledMma; -pub const Arch = fly_builder.Arch; -pub const MmaFlavor = fly_builder.MmaFlavor; -pub const fly_rocdl = fly_builder.fly_rocdl; -pub const layout = fly_builder.layout; -pub const Layout = fly_builder.Layout; -pub const Tile = fly_builder.Tile; -pub const IntTuple = fly_builder.IntTuple; -pub const L = fly_builder.L; -pub const rowMajor = fly_builder.rowMajor; -pub const colMajor = fly_builder.colMajor; -pub const ordered = fly_builder.ordered; -pub const tile = fly_builder.tile; -pub const it = fly_builder.it; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -/// Threads per wavefront: 64 on CDNA, 32 on RDNA. -pub fn waveSize(p: *const platform_.Platform) i32 { - const cc = platform_.rocm.computeCapability(p) orelse return 64; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => 64, - .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, - }; -} - -/// The plugin sizes a block in wavefronts, so a thread count has to divide -/// evenly: a kernel's layouts are built for an exact number of threads and -/// spare ones would partition out of range. -fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { - const wave = waveSize(p); - if (@rem(threads, wave) != 0) { - std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); - } - return @divExact(threads, wave); -} - -/// The matrix atom this device provides, or null when it has none: WMMA -/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. -pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { - const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, - .rdna3, .rdna3_5 => .gfx11_wmma, - .rdna4 => .gfx120x_wmma, - .rdna2 => null, - }; -} -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(fly_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - .f8e5m2fnuz => .f8e5m2fnuz, - else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const arg_names = spec.inputs ++ spec.outputs; - /// By name, as `!fly.memref` views. - pub const Args = StructOf(arg_names, Value); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Threads per block. Converted to the wavefronts the plugin - /// wants using the device's wave size, which is 64 on CDNA and - /// 32 on RDNA, so a kernel states the thread count its layouts - /// were built for and runs on both. - threads: i32, - /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. - waves_per_eu: i32 = 0, - /// Required when the body uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Inputs first, then outputs; see `zeroedOutput`. A no-op - /// inside command buffers. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - /// The arguments `run` was declared with. - pub fn args(b: *Builder) Args { - var out: Args = undefined; - inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); - return out; - } - - /// `zeroed_args` index of an output. - pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); - } - - /// `shapes` is inputs, then outputs. - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { - std.debug.assert(shapes.len == arg_names.len); - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try fly_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - var specs: [arg_names.len]Builder.ArgSpec = undefined; - inline for (arg_names, 0..) |fname, i| { - const shape = shapes[i]; - specs[i] = .{ - .name = fname, - .dtype = from(shape.dtype()), - .dims = if (shape.rank() > 0) shape.dims() else null, - }; - } - try b.declareArgsLow(&specs); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var shapes: [arg_names.len]Shape = undefined; - inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); - inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; - - const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| - std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .num_warps = warpsFor(cur.platform, opts.threads), - .waves_per_eu = opts.waves_per_eu, - .shared_mem_bytes = opts.shared_mem_bytes, - .zeroed_args = opts.zeroed_args, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig deleted file mode 100644 index ae788d96..00000000 --- a/zml/kernel/fly/test.zig +++ /dev/null @@ -1,149 +0,0 @@ -const std = @import("std"); - -const fly_builder = @import("dsl/fly/builder"); - -const fly = @import("fly.zig"); -const zml = @import("../../zml.zig"); - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -const FlyVectorAdd = struct { - const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), - }; - - const K = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); - } -}; - -/// C = A @ B^T with one MFMA-tiled block. -const FlyTiledMma = struct { - const Cfg = struct { flavor: fly.MmaFlavor }; - const K = fly.Kernel(Cfg, .{ - .name = "tiled_mma", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); - } -}; - -test "fly kernel emits a module XLA can parse" { - const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; - const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); -} - -test "fly kernels run on rocm" { - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward`, upload both inputs, run, and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); - defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); - defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); - defer bb.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // vector add: elementwise over a ragged (100, 1000), predicated at the edge. - { - const M, const N = .{ 100, 1000 }; - const Mod = struct { - pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, - .threads = 8 * 16, - }).c; - } - }; - - const ha = try allocator.alloc(f32, M * N); - defer allocator.free(ha); - const hb = try allocator.alloc(f32, M * N); - defer allocator.free(hb); - for (ha, hb, 0..) |*va, *vb, i| { - va.* = @floatFromInt(i % 977); - vb.* = @floatFromInt(1000 + (i % 313)); - } - - var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); - } - - // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix - // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, - // whose verifier rejects f32 operands, so the operand type and K come from - // the flavour rather than being written into the test. - if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { - inline else => |f| { - const Elem = switch (comptime f.operandDType()) { - .f32 => f32, - .f16 => f16, - else => @compileError("unhandled MMA operand type"), - }; - const M, const N = .{ 64, 64 }; - const Kd = comptime f.blockK(); - const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { - .f32 => .f32, - .f16 => .f16, - else => unreachable, - }; - - const Mod = struct { - pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ - .cfg = .{ .flavor = f }, - .grid = .{ 1, 1, 1 }, - .threads = comptime f.blockThreads(), - }).c; - } - }; - - var ha: [M * Kd]Elem = undefined; - var hb: [N * Kd]Elem = undefined; - for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; - for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; - - var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); - defer host.free(allocator); - - // f16 operands accumulate in f32, so the error is set by the input - // rounding rather than the sum. - const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; - const out = host.items(f32); - for (0..M) |m| for (0..N) |n| { - var acc: f32 = 0; - for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); - try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); - }; - }, - }; -} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig deleted file mode 100644 index 204c6204..00000000 --- a/zml/kernel/kernel.zig +++ /dev/null @@ -1,12 +0,0 @@ -pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); -pub const cute = @import("cute/cute.zig"); -pub const fly = @import("fly/fly.zig"); -pub const metal = @import("metal/metal.zig"); -pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const tcl = @import("tcl/tcl.zig"); -pub const triton = @import("triton/triton.zig"); - -test { - const std = @import("std"); - std.testing.refAllDecls(@This()); -} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel deleted file mode 100644 index 4007beb4..00000000 --- a/zml/kernel/metal/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "metal", - srcs = [ - "metal.zig", - "test.zig", - ], -) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig deleted file mode 100644 index 9626a1de..00000000 --- a/zml/kernel/metal/metal.zig +++ /dev/null @@ -1,237 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -pub const Builder = metal_builder.Builder; -pub const Value = metal_builder.Value; -pub const Type = metal_builder.Type; -pub const Struct = metal_builder.Struct; -pub const ArgSpec = metal_builder.ArgSpec; -pub const LoopOpts = metal_builder.LoopOpts; -pub const FinishError = metal_builder.FinishError; -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(metal_builder.registerDialects); -} - -pub fn from(dt: DataType) Type { - return switch (dt) { - .bool => .bool, - .i8 => .char, - .i16 => .short, - .i32 => .int, - .i64 => .long, - .u8 => .uchar, - .u16 => .ushort, - .u32 => .uint, - .u64 => .ulong, - .f16 => .half, - .bf16 => .bfloat, - .f32 => .float, - else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), - }; -} - -/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host -/// `extern struct` laid out like a `constant S &` argument. -pub const Constant = struct { - buffer: [256]u8 = undefined, - len: usize = 0, - - pub fn of(value: anytype) Constant { - const raw = std.mem.asBytes(&value); - var c: Constant = .{ .len = raw.len }; - @memcpy(c.buffer[0..raw.len], raw); - return c; - } - - pub fn bytes(self: *const Constant) []const u8 { - return self.buffer[0..self.len]; - } -}; - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - /// The arguments bound to input tensors. - inputs: []const [:0]const u8, - /// The arguments bound to output tensors; one named like an - /// input updates it in place. - outputs: []const [:0]const u8, - /// The `constant` arguments given by value at the call. - constants: []const [:0]const u8 = &.{}, - run: *const fn (*Builder, Config) FinishError!void, - fast_math: bool = true, - }; -} - -/// A Metal kernel: `run` declares its arguments with `declareArgs`, by -/// name, and builds its body. -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const Constants = StructOf(spec.constants, Constant); - - pub const CallOpts = struct { - cfg: ConfigT, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - constants: Constants = std.mem.zeroes(Constants), - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - return build(allocator, cfg, &arguments); - } - - fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); - defer b.deinit(); - try spec.run(&b, cfg); - const ir = try b.finish(); - errdefer allocator.free(ir); - try arguments.appendSlice(allocator, b.arguments.items); - return ir; - } - - fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { - for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; - return null; - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const allocator = cur.allocator; - - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - const ir = build(allocator, opts.cfg, &arguments) catch |err| - std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); - var constants: [spec.constants.len][]const u8 = undefined; - inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); - - var bound: [spec.outputs.len]bool = @splat(false); - var bound_inputs: [spec.inputs.len]bool = @splat(false); - var bound_constants: [spec.constants.len]bool = @splat(false); - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - var bindings: std.ArrayList(i32) = .empty; - defer bindings.deinit(allocator); - for (arguments.items) |arg| { - if (arg.kind == .threadgroup) { - // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. - if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); - continue; - } - std.debug.assert(arg.index == bindings.items.len); - const input = indexOf(spec.inputs, arg.name); - const output = indexOf(spec.outputs, arg.name); - if (input) |i| bound_inputs[i] = true; - if (output) |o| bound[o] = true; - const binding: i32 = if (input) |i| blk: { - if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); - break :blk @intCast(i); - } else if (output) |o| - @intCast(spec.inputs.len + o) - else if (indexOf(spec.constants, arg.name)) |k| blk: { - if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); - bound_constants[k] = true; - break :blk -1 - @as(i32, @intCast(k)); - } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); - bindings.append(allocator, binding) catch @panic("OOM"); - } - for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); - for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); - for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); - - const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .threadgroup = opts.threadgroup, - .threadgroup_memory = opts.threadgroup_memory, - .arguments = bindings.items, - .constants = &constants, - .output_operand_aliases = aliases.constSlice(), - }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig deleted file mode 100644 index 0a342af0..00000000 --- a/zml/kernel/metal/test.zig +++ /dev/null @@ -1,71 +0,0 @@ -const std = @import("std"); - -const metal = @import("metal.zig"); -const zml = @import("../../zml.zig"); - -/// `out[i] = a[i] * scale + b[i]` over `n` floats. -const MetalAxpy = struct { - const Cfg = struct {}; - const K = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -test "metal kernels run on metal" { - const allocator = std.testing.allocator; - - const ir = try MetalAxpy.K.emit(allocator, .{}); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); - - const platform = zml.testing.env(); - if (platform.target != .metal) return error.SkipZigTest; - - const n = 1000; - const Mod = struct { - pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { - return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ (n + 255) / 256, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, - }).out; - } - }; - const x: zml.Tensor = .init(.{n}, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); - defer exe.deinit(); - - var hx: [n]f32 = undefined; - var hy: [n]f32 = undefined; - for (&hx, &hy, 0..) |*a, *b, i| { - a.* = @floatFromInt(i); - b.* = 1; - } - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); - defer by.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); -} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel deleted file mode 100644 index ba3b6b66..00000000 --- a/zml/kernel/mosaic_tpu/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "mosaic_tpu", - srcs = ["mosaic_tpu.zig"], -) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig deleted file mode 100644 index a09e8368..00000000 --- a/zml/kernel/mosaic_tpu/mosaic_tpu.zig +++ /dev/null @@ -1,284 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); -const Builder = mosaic_tpu_builder.Builder; -const DType = mosaic_tpu_builder.DType; -const FinishError = mosaic_tpu_builder.FinishError; -const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(mosaic_tpu_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - dynamic_grid_bounds: []const Tensor = &.{}, - }; - - pub const CallOpts = struct { - cfg: ConfigT, - extras: CallExtras = .{}, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finishOpts(&.{}, .{ .canonicalize = true }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const backend_config = buildBackendConfig(cur.allocator, ir, .{ - .vmem_limit_bytes = opts.extras.vmem_limit_bytes, - .disable_bounds_checks = opts.extras.disable_bounds_checks, - .disable_semaphore_checks = opts.extras.disable_semaphore_checks, - .has_communication = opts.extras.has_communication, - }) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); - defer cur.allocator.free(backend_config); - - var values: [spec.inputs.len]*const mlir.Value = undefined; - inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); - - var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (&res_types, spec.outputs) |*r, fname| { - r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); - } - - const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); - - const op = callTpuCustomCall(.{ - .inputs = &values, - .result_types = &res_types, - .backend_config = backend_config, - .aliases = aliases.constSlice(), - .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, - .additional_attributes = opts.extras.additional_attributes, - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - @field(results, fname) = Tensor._result(out, op.result(i)); - } - return results; - } - }; -} - -const CustomCallConfig = struct { - body: []const u8, - has_communication: ?bool = null, - serialization_format: i64 = 1, - needs_layout_passes: bool = true, - shape_invariant_numerics: bool = false, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, -}; - -const ScopedMemoryConfig = struct { - memory_space: i64 = 1, - offset: i64 = 0, - size: i64, -}; - -const BackendConfig = struct { - custom_call_config: CustomCallConfig, - scoped_memory_configs: ?[]const ScopedMemoryConfig = null, -}; - -const BackendConfigOpts = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, -}; - -fn buildBackendConfig( - allocator: std.mem.Allocator, - ir: [:0]const u8, - opts: BackendConfigOpts, -) ![]u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - const module = try mlir.Module.parse(ctx, ir); - defer module.deinit(); - - mosaic_tpu_dialect.registerMosaicSerdePass(); - const prev_allow_unregistered = ctx.allowUnregisteredDialects(); - ctx.setAllowUnregisteredDialects(true); - defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); - - const pm = mlir.PassManager.init(ctx); - defer pm.deinit(); - try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); - try pm.runOnOp(module.operation()); - - const Encoder = std.base64.standard.Encoder; - const b64 = b64: { - var bytecode: std.Io.Writer.Allocating = .init(allocator); - defer bytecode.deinit(); - try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); - const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); - break :b64 Encoder.encode(buf, bytecode.written()); - }; - defer allocator.free(b64); - - const config = BackendConfig{ - .custom_call_config = .{ - .body = b64, - .has_communication = opts.has_communication, - .disable_bounds_checks = opts.disable_bounds_checks, - .disable_semaphore_checks = opts.disable_semaphore_checks, - }, - .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| - &.{.{ .size = vmem }} - else - null, - }; - - var json: std.Io.Writer.Allocating = .init(allocator); - errdefer json.deinit(); - - try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - return try json.toOwnedSlice(); -} - -const TpuCustomCallArgs = struct { - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - backend_config: []const u8, - aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, - dynamic_grid_bounds: []const Tensor, - additional_attributes: []const mlir.NamedAttribute, -}; - -fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = Compiler.current(); - - var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; - for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); - all_inputs.appendSliceAssumeCapacity(args.inputs); - - return stablehlo.custom_call( - cur.mlir_ctx, - all_inputs.constSlice(), - args.result_types, - .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = args.backend_config }, - .has_side_effect = false, - .api_version = .original, - .additional_attributes = args.additional_attributes, - .output_operand_aliases = args.aliases, - }, - cur.location, - ).appendTo(cur.currentScope().block); -} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel deleted file mode 100644 index bc7a0aa2..00000000 --- a/zml/kernel/tcl/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "tcl", - srcs = [ - "tcl.zig", - "test.zig", - ], -) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig deleted file mode 100644 index 023e862f..00000000 --- a/zml/kernel/tcl/tcl.zig +++ /dev/null @@ -1,206 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const tcl_builder = @import("dsl/tcl/builder"); -pub const Builder = tcl_builder.Builder; -pub const TensorOperation = tcl_builder.TensorOperation; -pub const Value = tcl_builder.Value; -pub const Axis = tcl_builder.Axis; -pub const Expr = tcl_builder.Expr; -pub const Dram = tcl_builder.Dram; -pub const Context = tcl_builder.Context; -pub const DType = tcl_builder.DType; -pub const VeOp = tcl_builder.VeOp; -pub const ReduceMode = tcl_builder.ReduceMode; -pub const Predicate = tcl_builder.Predicate; -pub const Tactic = tcl_builder.Tactic; -pub const FinishError = tcl_builder.FinishError; - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(tcl_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .bool, - .i4 => .i4, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4_e2, - .f8e4m3fn => .f8_e4, - .f8e5m2 => .f8_e5, - else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -/// A TCL function compiled by the Furiosa plugin as one native unit: -/// `declareArgs` declares the inputs in order, `ret` the outputs. -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - return build(allocator, cfg, null, null, null); - } - - /// `updated`, when given, receives the input each output updates - /// in place (`Builder.update`), or null. - fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try Builder.open(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - - const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); - if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { - std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); - } - - if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); - if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); - if (updated) |u| for (u, results) |*dst, r| { - dst.* = b.updatedArgument(r); - }; - - return b.finish(); - } - - fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { - var same = from(s.dtype()) == t.dtype; - if (t.shards > 1) { - // A partition holds one chip's share of a chip-split DRAM tensor. - var n: i64 = 1; - for (t.axes) |a| n *= a.size; - same = same and n == @as(i64, @intCast(s.count())) * t.shards; - } else { - same = same and s.rank() == t.axes.len; - if (same) for (t.axes, 0..) |a, i| { - same = same and a.size == s.dim(i); - }; - } - - if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - var input_shapes: [spec.inputs.len]Shape = undefined; - inline for (spec.inputs, 0..) |fname, i| { - inputs_arr[i] = @field(inputs, fname); - input_shapes[i] = inputs_arr[i].shape(); - } - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var updated: [spec.outputs.len]?usize = undefined; - const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| - std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ - .output_index = @intCast(output), - .operand_index = @intCast(i), - }); - const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - @field(results, fname) = tensor_results[i]; - } - - return results; - } - }; -} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig deleted file mode 100644 index b9ae1875..00000000 --- a/zml/kernel/tcl/test.zig +++ /dev/null @@ -1,123 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const tcl = @import("tcl.zig"); - -/// `furiosa.tcl.examples.softmax` over f32. -const TclSoftmax = struct { - const Cfg = struct { b: i64, s: i64, d: i64 }; - const K = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"x"}, - .outputs = &.{"y"}, - .run = run, - }); - - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.x, .{}); - const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); - b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); - } -}; - -test "tcl kernels run on furiosa" { - const allocator = std.testing.allocator; - const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; - - const ir = try TclSoftmax.K.emit(allocator, cfg); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); - - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - - const Mod = struct { - pub fn forward(x: zml.Tensor) zml.Tensor { - return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; - } - }; - const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); - defer exe.deinit(); - - var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); - defer x_buffer.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - - const out = host.items(f32); - var row: usize = 0; - while (row < cfg.b * cfg.s) : (row += 1) { - const in = input[row * cfg.d ..][0..cfg.d]; - var max: f32 = in[0]; - for (in) |v| max = @max(max, v); - var sum: f32 = 0; - for (in) |v| sum += @exp(v - max); - for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); - } -} - -/// exp over a tensor split across two chips in one launch, returning the -/// split result and its all_gather. -const TclSplitExp = struct { - const Cfg = struct {}; - const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); - fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - } - fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { - const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); - return .{ r.y, r.all }; - } -}; - -// Needs FURIOSA_VISIBLE_DEVICES=0,1. -test "tcl multi-chip kernels run on two furiosa chips" { - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const tp = zml.testing.tpMesh(); - const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); - const Mod = struct { - pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ - .in_specs = .{ .x = .parse(.{ .c = .model }) }, - .out_specs = .{ .parse(.{ .c = .model }), .replicated }, - }); - return .{ r[0], r[1] }; - } - }; - const x: zml.Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); - defer exe.deinit(); - var hx: [2 * 1024]f32 = undefined; - for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var res: [2]zml.Buffer = undefined; - runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); - defer for (&res) |*r| r.deinit(); - for (res) |r| { - var host = try r.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); - } -} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel deleted file mode 100644 index 1e992352..00000000 --- a/zml/kernel/triton/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "triton", - srcs = ["triton.zig"], -) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig deleted file mode 100644 index 6a91a03f..00000000 --- a/zml/kernel/triton/triton.zig +++ /dev/null @@ -1,165 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const triton_builder = @import("dsl/triton/builder"); -pub const Builder = triton_builder.Builder; -pub const Value = triton_builder.Value; -pub const DType = triton_builder.DType; -pub const FinishError = triton_builder.FinishError; - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const ops = @import("../../ops.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(triton_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - debug: bool = false, - /// Bytes for Triton's implicit global scratch argument, across all CTAs. - global_scratch_memory_size: i32 = 0, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try triton_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ttir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ttir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const call_opts: ops.TritonOps = .{ - .debug = opts.debug, - .name = name, - .ir = ttir, - .grid = opts.grid, - .num_stages = opts.num_stages, - .num_warps = opts.num_warps, - .output_operand_aliases = aliases.constSlice(), - .is_tma_allowed = opts.global_scratch_memory_size > 0, - .global_scratch_memory_size = opts.global_scratch_memory_size, - }; - - const triton_results = if (opts.global_scratch_memory_size > 0) blk: { - // XLA maps the final custom-call result to Triton's implicit - // scratch pointer; it must not appear in the TTIR signature. - const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; - const res = ops.triton(inputs_arr, extended, call_opts); - break :blk res[0..spec.outputs.len].*; - } else ops.triton(inputs_arr, outputs_arr, call_opts); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; - return results; - } - }; -} diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md index 81a35cf4..b5fe7682 100644 --- a/zml/kernels/flash_attention/README.md +++ b/zml/kernels/flash_attention/README.md @@ -265,7 +265,7 @@ are not uniformly supported by the current kernels; an existing prefill path requires equal query/key lengths and assumes positions begin at zero. Treat these as current conformance gaps, not alternative definitions of causality. -The tests in [tests.zig](tests.zig) compare results +The tests in [flash_attention.zig](flash_attention.zig) compare results against `sdpa` with the gathered causal mask. The original shared cases cover `D = 64`, MHA and GQA, single-token decode at full and partial context lengths, batched decode with diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index 3fb898db..78602bd0 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -6,11 +6,9 @@ pub const cute = @import("cute/cute.zig"); pub const metal = @import("metal/metal.zig"); pub const nki = @import("nki/nki.zig"); const zml = @import("../../zml.zig"); -const Options = @import("../../attention/Options.zig"); test { std.testing.refAllDecls(@This()); - _ = @import("tests.zig"); } pub fn isAvailable(platform: *const zml.Platform) bool { @@ -128,7 +126,7 @@ pub const Metadata = union(Backend) { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: zml.attention.Options) zml.Tensor { zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); return switch (parameters) { .cute => cute.attention(q, k, v, token_index, opts.causal), @@ -137,3 +135,329 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), }; } + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +test "attention: prefill, q=k=300,qh=8,kh=4" { + // Neither length is a multiple of the query (128) or key (64) tiles. + try testAttention( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries start at 800: every query tile sees full key tiles before the diagonal. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{800}, + ); +} + +test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { + // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), + .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { + try testAttention( + .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), + .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { + try testAttention( + .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), + .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), + &.{ 150, 100 }, + ); +} + +test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. + try testAttentionOpts( + .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{ 0, 0 }, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), + .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { + try testAttentionOpts( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), + &.{5}, + .{ .causal = false }, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); +} + +fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const Backend) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + if (opts.causal) { + // Check for out of bound reads, token_index is ignored by non-causal attention. + for (token_index_h) |index| { + std.debug.assert(index + max_q_idx <= max_k_idx); + } + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + var q = try attentionTestInput(q_shape, 17); + defer q.deinit(); + var k = try attentionTestInput(k_shape, 29); + defer k.deinit(); + var v = try attentionTestInput(k_shape, 43); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); + defer token_index.deinit(); + + const meshes = platform.meshes.values(); + const mesh = zml.testing.tpMesh(); + const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ + .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, + .out_specs = .replicated, + }; + const AttentionCall = struct { + fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { + return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); + } + }; + const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; + const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; + const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ + .program_name = "attention_stablehlo", + .meshes = meshes, + }); + defer reference_exe.deinit(); + + var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); + defer reference_d.deinit(); + try reference_d.await(io); + try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); + const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); + defer reference_h.free(allocator); + + for (backends) |backend| { + if (!backend.isAvailable(platform)) continue; + + const metadata: Metadata = .init(.fromBackend(backend)); + const parameters: Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + AttentionCall.forward, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .meshes = meshes, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform); + defer Metadata.deinitBuffer(&metadata_d); + + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, reference_h, output_h }); + try zml.testing.expectClose(io, reference_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} + +fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { + const host = try zml.Slice.alloc(std.testing.allocator, shape); + defer host.free(std.testing.allocator); + var prng: std.Random.DefaultPrng = .init(seed); + const random = prng.random(); + switch (shape.dtype()) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType())) |*value| { + const x = random.float(f32) * 4 - 2; + value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); + } + }, + else => unreachable, + } + return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); +} + +/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its +/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. +fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { + const Layout = struct { + b: usize, + s: usize, + h: usize, + d: usize, + + fn stride(shape: zml.Shape, tag: anytype) usize { + const axis = shape.hasTag(tag) orelse return 0; + var result: usize = 1; + for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); + return result; + } + + fn init(shape: zml.Shape) @This() { + return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; + } + + fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { + return b * self.b + s * self.s + h * self.h + d * self.d; + } + }; + const ql = Layout.init(q.shape); + const kl = Layout.init(k.shape); + const vl = Layout.init(v.shape); + const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; + const queries: usize = @intCast(q.shape.dim(.q)); + const heads: usize = @intCast(q.shape.dim(.h)); + const dim: usize = @intCast(q.shape.dim(.hd)); + const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); + const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); + defer std.testing.allocator.free(scores); + const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); + for (0..batch) |b| { + const start: usize = positions[if (positions.len == 1) 0 else b]; + for (0..queries) |i| { + for (0..heads) |h| { + var maximum: f64 = -std.math.inf(f64); + const active = scores[0 .. start + i + 1]; + for (active, 0..) |*score, j| { + var dot: f64 = 0; + for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); + score.* = dot * scale; + maximum = @max(maximum, score.*); + } + var sum: f64 = 0; + for (active) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..dim) |d| { + var expected: f64 = 0; + for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); + expected /= sum; + const actual = output.items(f32)[ql.offset(b, i, h, d)]; + try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); + } + } + } + } +} diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel index f82ade45..855bdcca 100644 --- a/zml/kernels/flash_attention/metal/BUILD.bazel +++ b/zml/kernels/flash_attention/metal/BUILD.bazel @@ -2,8 +2,5 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "metal", - srcs = [ - "flash_attn.zig", - "metal.zig", - ], + srcs = ["metal.zig"], ) diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig deleted file mode 100644 index de4b68da..00000000 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ /dev/null @@ -1,732 +0,0 @@ -//! Metal flash attention over contiguous `[batch, keys, kv_heads, head_dim]` -//! K/V, causal from `token_index`: fa_vec and fa_vec_reduce for decode, -//! fa_tiled otherwise. - -const std = @import("std"); - -const metal = @import("../../../kernel/kernel.zig").metal; - -const ArgSpec = metal.ArgSpec; -const Builder = metal.Builder; -const Type = metal.Type; -const Value = metal.Value; - -const flt_max = std.math.floatMax(f32); -/// A masked score: below any real one yet finite, since fast math assumes -/// no infinities. Halved so subtracting a running maximum cannot overflow. -const masked_score = -flt_max / 2.0; -const max_half = 65504.0; -const simd_width = 32; - -/// fa_vec: bf16 decode, one query per batch row. Simdgroup s takes query head -/// s % heads of the threadgroup's KV head group and position split s / heads. -pub const FaVec = struct { - pub const Cfg = struct { - head_dim: i32, - heads: i32, - splits: i32, - }; - - pub const Args: metal.Struct = .{ .name = "fa_vec_args", .fields = &.{ - .{ .name = "num_heads", .type = .int }, - .{ .name = "n_kv", .type = .int }, - .{ .name = "k_len", .type = .int }, - .{ .name = "scale", .type = .float }, - } }; - - pub const HostArgs = extern struct { - num_heads: i32, - n_kv: i32, - k_len: i32, - scale: f32, - }; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_vec", - .inputs = &.{ "q", "k", "v", "token_index" }, - .outputs = &.{"dst"}, - .constants = &.{"args"}, - .run = run, - }); - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - switch (cfg.head_dim) { - 32, 64, 128, 256, 512 => {}, - else => std.debug.panic("fa_vec: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), - } - const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; - const a = try b.declareArgs(.{ - .args = .{ .constant_struct = Args }, - .q = char_in, - .k = char_in, - .v = char_in, - .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, - .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, - .tgpg = .{ .threadgroups_per_grid = .uint3 }, - .tiisg = .{ .thread_index_in_simdgroup = .ushort }, - .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - .nsg = .{ .simdgroups_per_threadgroup = .ushort }, - }); - impl(b, cfg, a); - } - - fn impl(b: *Builder, cfg: Cfg, a: anytype) void { - const D = cfg.head_dim; - const D4 = @divExact(D, 4); - // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. - const NL = @min(D4, simd_width); - const R = @divExact(simd_width, NL); - const NI = @divExact(D4, NL); - const C = simd_width; - const HG = cfg.heads; - const NWG = cfg.splits; - const full: metal.LoopOpts = .{ .unroll = .full }; - - const num_heads = a.args.field("num_heads"); - const n_kv = a.args.field("n_kv"); - const group = num_heads.div(n_kv); - const groups = group.div(HG); - const r = a.tgpig.at(2).to(.int); - const tg_head = a.tgpig.at(1).to(.int); - const ikv = tg_head.div(groups); - const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); - const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); - const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); - const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); - - const kv_len = a.token_index.load(r).add(1); - const live = if (NWG > 1) liveSplits(b, kv_len, NWG) else b.lift(@as(i32, 1)); - - if (NWG > 1) { - var idle = b.openIf(iwg.ge(live)); - { - b.return_(); - idle.close(); - } - } - - const position_bytes = n_kv.to(.ulong).mul(D * 2); - const first = r.to(.ulong).mul(a.args.field("k_len").to(.ulong)).mul(position_bytes).add(ikv.to(.ulong).mul(D * 2)); - const k0 = a.k.add(first); - const v0 = a.v.add(first); - - const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); - const sub = a.tiisg.div(NL).to(.int); - const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); - const qr = b.array(.float4, @intCast(NI)); - const o = b.array(.float4, @intCast(NI)); - var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = init.iv; - qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); - o.store(ii, b.constant(.float4, 0)); - init.close(); - } - - const S = b.local(.float, 0.0); - const M = b.local(.float, -flt_max / 2.0); - - const ic0 = b.local(.int, iwg); - var blocks = b.openLoop(.{}); - { - const ic = ic0.mul(C); - var past = b.openIf(ic.ge(kv_len)); - { - blocks.breakLoop(); - past.close(); - } - - const s = b.local(.float, 0.0); - var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); - { - const cc = qk.iv; - const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); - const pk4 = k0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); - const d = b.local(.float, 0.0); - var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = dots.iv; - d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); - dots.close(); - } - if (R == 1) { - s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); - } else { - var x = d.rvalue(); - var mask: i32 = 1; - while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); - const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); - s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); - } - qk.close(); - } - - const pos = ic.add(a.tiisg.to(.int)); - const score = b.ternary(pos.lt(kv_len), s.mul(a.args.field("scale")), b.constant(.float, -max_half)); - - const m = M.rvalue(); - M.assign(b.simdMax(b.max(M.rvalue(), score))); - const ms = b.exp(m.sub(M)); - const vs = b.exp(score.sub(M)); - S.assign(S.mul(ms).add(b.simdSum(vs))); - - var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(rescale.iv, o.load(rescale.iv).mul(ms)); - rescale.close(); - } - - var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); - { - const cc = pv.iv; - const row = cc.to(.int).mul(R).add(sub); - const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); - const pv4 = v0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); - const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); - var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = dims.iv; - o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); - dims.close(); - } - pv.close(); - } - - blocks.next(); - ic0.assign(ic0.add(live)); - blocks.close(); - } - - var mask: i32 = NL; - while (mask < simd_width) : (mask *= 2) { - var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); - sum.close(); - } - } - var first_row = if (R > 1) b.openIf(sub.eq(0)) else null; - if (NWG == 1) { - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); - out.close(); - } - } else { - // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). - const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); - dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); - out.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); - dst1.store(b.lift(2).mul(iwg), S); - dst1.store(b.lift(2).mul(iwg).add(1), M); - lane0.close(); - } - } - if (first_row) |*scope| scope.close(); - } -}; - -fn liveSplits(b: *Builder, kv_len: Value, splits: i32) Value { - return b.min(splits, kv_len.add(simd_width - 1).div(simd_width)); -} - -/// fa_vec_reduce: merges the fa_vec split partials. -pub const FaVecReduce = struct { - pub const Cfg = struct { - splits: i32, - head_dim: i32, - }; - - pub const simdgroups = 8; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_vec_reduce", - .inputs = &.{ "htmp", "token_index" }, - .outputs = &.{"dst"}, - .constants = &.{"args"}, - .run = run, - }); - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .args = .{ .constant_struct = FaVec.Args }, - .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, - .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, - .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, - .tgpg = .{ .threadgroups_per_grid = .uint2 }, - .tiisg = .{ .thread_index_in_simdgroup = .ushort }, - .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - }); - const NWG = cfg.splits; - if (NWG > 4 * simd_width) std.debug.panic("fa_vec_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); - const NP = @divFloor(NWG + simd_width - 1, simd_width); - const DV = cfg.head_dim; - const DV4 = @divExact(DV, 4); - const full: metal.LoopOpts = .{ .unroll = .full }; - - const rid = a.tgpig.at(0).to(.ulong); - const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); - const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); - const live = liveSplits(b, a.token_index.load(r).add(1), NWG); - - const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); - const Mj = b.array(.float, @intCast(NP)); - const Sj = b.array(.float, @intCast(NP)); - const mx = b.local(.float, -flt_max / 2.0); - var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = load.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - const ok = idx.lt(live); - Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); - Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); - mx.assign(b.max(mx.rvalue(), Mj.load(j))); - load.close(); - } - const m = b.simdMax(mx.rvalue()); - const sum = b.local(.float, 0.0); - var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = weigh.iv; - Mj.store(j, b.exp(Mj.load(j).sub(m))); - sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); - weigh.close(); - } - const S = b.simdSum(sum.rvalue()); - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - - const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); - const acc = b.local(.float4, b.constant(.float4, 0)); - var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = parts.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - var ok = b.openIf(idx.lt(live)); - { - acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); - ok.close(); - } - parts.close(); - } - const v = b.simdSum(acc.rvalue()); - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst4.store(i, v.mul(inv_sum).to(.bfloat4)); - lane0.close(); - } - } -}; - -/// fa_tiled: causal FlashAttention-2 over simdgroup matrices. Queries past -/// `live` output zeros. -pub const FaTiled = struct { - pub const Cfg = struct { - dtype: Type, - head_size: i32, - bq: i32 = 32, - tile_kv: i32 = 32, - num_threads: i32 = 128, - }; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_tiled", - .inputs = &.{ "q", "k", "v", "token_index", "live" }, - .outputs = &.{"out"}, - .constants = &.{ "num_kv_heads", "scale", "q_len", "k_len" }, - .run = run, - }); - - fn supported(cfg: Cfg) bool { - const tile = tileConfig(cfg.head_size) orelse return false; - return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and - cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; - } - - pub fn tileConfig(head_size: i64) ?[3]i32 { - return switch (head_size) { - 32, 64, 96, 128 => .{ 32, 32, 128 }, - 256 => .{ 16, 16, 64 }, - 512 => .{ 8, 8, 32 }, - else => null, - }; - } - - pub fn threadgroupMemory(cfg: Cfg) u64 { - const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); - return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); - } - - /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. - fn fragCoord(b: *Builder, lane_id: Value) Value { - const qid = lane_id.to(.short).div(4).to(.short); - const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); - const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); - return b.vec(.short2, .{ fn_, fm }); - } - - fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { - var x = v.rvalue(); - inline for (.{ 1, 8 }) |mask| { - const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); - x = switch (op) { - .max => b.max(x, y), - .sum => x.add(y), - }; - } - return x; - } - - fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { - p.store(i, p.load(i).insert(lane, x)); - } - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - if (!supported(cfg)) std.debug.panic("fa_tiled: {s} at head size {d} and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); - const T = cfg.dtype; - const HEAD_SIZE = cfg.head_size; - const BQ = cfg.bq; - const TILE_KV = cfg.tile_kv; - const NUM_THREADS = cfg.num_threads; - const int_arg: ArgSpec = .{ .constant = .int }; - const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; - const a = try b.declareArgs(.{ - .out = .{ .buffer = .{ .type = T } }, - .q = .{ .buffer = .{ .type = T, .const_ = true } }, - .k = .{ .buffer = .{ .type = T, .const_ = true } }, - .v = .{ .buffer = .{ .type = T, .const_ = true } }, - .token_index = int_in, - .live = int_in, - .num_kv_heads = int_arg, - .scale = .{ .constant = .float }, - .q_len = int_arg, - .k_len = int_arg, - .shared_mem = .{ .threadgroup = .char }, - .tgp = .{ .threadgroup_position_in_grid = .uint3 }, - .tgpg = .{ .threadgroups_per_grid = .uint3 }, - .tpt = .{ .thread_position_in_threadgroup = .uint3 }, - .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, - .lane = .{ .thread_index_in_simdgroup = .uint }, - }); - - const NUM_SIMD_LANES = 32; - const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); - const TD = @divExact(HEAD_SIZE, 8); - const TK = @divExact(TILE_KV, 8); - if (@divExact(BQ, NUM_SG) != 8) @panic("fa_tiled: ROWS_PER_SG must be 8"); - const unroll: metal.LoopOpts = .{ .unroll = .enable }; - - const thread_idx = a.tpt.x().to(.int); - const head_idx = a.tgp.x().to(.int); - const q_pos_start = a.tgp.y().to(.int).mul(BQ); - const batch = a.tgp.z().to(.int); - const num_heads = a.tgpg.x().to(.int); - const kv_head_idx = head_idx.div(num_heads.div(a.num_kv_heads)); - const q_stride = num_heads.mul(HEAD_SIZE); - const kv_token_stride = a.num_kv_heads.mul(HEAD_SIZE); - - const block_rows = b.min(BQ, a.q_len.sub(q_pos_start)); - const live = a.live.load(0); - const valid_q = b.min(block_rows, b.max(0, live.sub(q_pos_start))); - const out_base = a.out.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); - - var dead = b.openIf(valid_q.le(0)); - { - var zero = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); - { - const i = zero.iv; - out_base.store(i.div(HEAD_SIZE).mul(q_stride).add(i.rem(HEAD_SIZE)), b.constant(T, 0)); - zero.close(); - } - b.return_(); - dead.close(); - } - - // Query i of the batch row is at position context_len + i. - const context_len = a.token_index.load(batch); - const seq_len = context_len.add(live); - const kv_base = batch.to(.long).mul(a.k_len).mul(kv_token_stride).add(kv_head_idx.mul(HEAD_SIZE)); - - const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); - const LD = HEAD_SIZE + SMEM_PAD; - const Q_ELEMS = BQ * LD; - const KV_ELEMS = TILE_KV * LD; - - const Q_smem = a.shared_mem.to(T); - const K_smem = Q_smem.add(Q_ELEMS); - const V_smem = K_smem.add(KV_ELEMS); - - const log2e: f32 = std.math.log2e; - const scale_log2 = a.scale.mul(log2e); - - const q_base = a.q.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); - var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); - { - const i = load_q.iv; - const r = i.div(HEAD_SIZE); - const d = i.rem(HEAD_SIZE); - var value = b.openIfElse(r.lt(valid_q), .{T}); - { - value.yieldThen(.{q_base.load(r.mul(q_stride).add(d))}); - } - { - value.yieldElse(.{b.constant(T, 0)}); - } - Q_smem.store(r.mul(LD).add(d), value.results[0]); - load_q.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const T2: Type = .vec(T.scalar, 2); - const q_reg = b.array(T2, @intCast(TD)); - var load_q_reg = b.openForOpts(0, TD, 1, unroll); - { - const d = load_q_reg.iv; - const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - q_reg.store(d, tmp.threadElements()); - load_q_reg.close(); - } - - const fc = fragCoord(b, a.lane.to(.ushort)); - const fm = fc.y(); - const fn_ = fc.x(); - - const max_score = b.local(.float, masked_score); - const sum_score = b.local(.float, 0.0); - - const o_reg = b.array(.float2, @intCast(TD)); - var zero_o = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(zero_o.iv, b.constant(.float2, 0)); - zero_o.close(); - } - - const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); - const last_q_pos = context_len.add(q_pos_start).add(valid_q).sub(1); - - var tiles = b.openFor(0, num_kv_tiles, 1); - { - const tile_idx = tiles.iv; - const tile_start = tile_idx.mul(TILE_KV); - - var past = b.openIf(tile_start.gt(last_q_pos)); - { - tiles.breakLoop(); - past.close(); - } - - const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; - const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); - const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; - std.debug.assert(@rem(LD, VEC) == 0); - var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); - { - const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); - const kv_pos = tile_start.add(t); - var present = b.openIfElse(kv_pos.lt(seq_len), .{}); - { - const off = kv_base.add(kv_pos.to(.long).mul(kv_token_stride)); - const k_ptr = a.k.add(off); - const v_ptr = a.v.add(off); - var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); - { - const d = loop.iv; - K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k_ptr.add(d).to(LoadUnit).load(0)); - V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v_ptr.add(d).to(LoadUnit).load(0)); - loop.close(); - } - present.elseBranch(); - } - { - const zero = b.constant(LoadUnit, 0); - var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); - { - const d = loop.iv; - K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); - V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); - loop.close(); - } - present.close(); - } - rows.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const s_reg = b.array(.float2, @intCast(TK)); - var zero_s = b.openForOpts(0, TK, 1, unroll); - { - s_reg.store(zero_s.iv, b.constant(.float2, 0)); - zero_s.close(); - } - - var qk_dims = b.openForOpts(0, TD, 1, unroll); - { - const d = qk_dims.iv; - const q_frag = b.matrix(T.scalar, q_reg.load(d)); - var qk_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = qk_keys.iv; - const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); - const s_frag = b.matrix(.float, s_reg.load(k)); - s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); - qk_keys.close(); - } - qk_dims.close(); - } - - const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); - const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); - const tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(tile_start.add(TILE_KV).sub(1).lt(context_len.add(q_pos_start))); - - var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); - { - var scale = b.openForOpts(0, TK, 1, unroll); - { - const k = scale.iv; - setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); - setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); - scale.close(); - } - masking.elseBranch(); - } - { - var keys = b.openForOpts(0, TK, 1, unroll); - { - const k = keys.iv; - var pair = b.openForOpts(0, 2, 1, unroll); - { - const jj = pair.iv; - const s = s_reg.load(k).at(jj).mul(scale_log2); - const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); - const masked = row_masked.bitOr(kv_pos.ge(seq_len)).bitOr(kv_pos.gt(q_abs_pos)); - setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); - pair.close(); - } - keys.close(); - } - masking.close(); - } - - const local_max = b.local(.float, masked_score); - var max_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = max_keys.iv; - local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); - max_keys.close(); - } - const row_max = fragRowReduce(b, .max, local_max); - - const prev_max = max_score.rvalue(); - const grown_max = b.max(prev_max, row_max); - const grew = grown_max.gt(prev_max); - const was_empty = prev_max.eq(masked_score); - const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); - const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); - // A tile masking a whole row leaves the maximum masked: zero would - // pin it, and later scores far below zero would underflow. - max_score.assign(grown_max); - - const local_sum = b.local(.float, 0.0); - var p_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = p_keys.iv; - var pair = b.openForOpts(0, 2, 1, unroll); - { - const jj = pair.iv; - const s = s_reg.load(k).at(jj); - const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); - setLane(s_reg, k, jj, p); - local_sum.assign(local_sum.add(p)); - pair.close(); - } - p_keys.close(); - } - const row_sum = fragRowReduce(b, .sum, local_sum); - - sum_score.assign(sum_score.mul(factor).add(row_sum)); - - var rescale_o = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); - rescale_o.close(); - } - - var pv_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = pv_keys.iv; - const p_frag = b.matrix(.float, s_reg.load(k)); - var pv_dims = b.openForOpts(0, TD, 1, unroll); - { - const d = pv_dims.iv; - const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - const o_frag = b.matrix(.float, o_reg.load(d)); - o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); - pv_dims.close(); - } - pv_keys.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - tiles.close(); - } - - const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); - var normalize = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); - normalize.close(); - } - - const O_smem = Q_smem.to(.float); - var store_o = b.openForOpts(0, TD, 1, unroll); - { - const d = store_o.iv; - const o_frag = b.matrix(.float, o_reg.load(d)); - b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - store_o.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - var write_out = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); - { - const i = write_out.iv; - const r = i.div(HEAD_SIZE); - const d = i.rem(HEAD_SIZE); - const value = b.ternary(r.lt(valid_q), O_smem.load(r.mul(LD).add(d)).to(T), b.constant(T, 0)); - out_base.store(r.mul(q_stride).add(d), value); - write_out.close(); - } - } -}; - -test "the flash attention kernels emit verified modules" { - const allocator = std.testing.allocator; - for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { - for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { - allocator.free(try FaVec.Kernel.emit(allocator, .{ .head_dim = head_dim, .heads = launch[0], .splits = launch[1] })); - } - for ([_]i32{ 8, 128 }) |splits| { - allocator.free(try FaVecReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); - } - } - for ([_]Type{ .half, .bfloat }) |dtype| { - for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { - const tile = FaTiled.tileConfig(head_size).?; - allocator.free(try FaTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2] })); - } - } -} diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig index bdfb00c0..9327c2a1 100644 --- a/zml/kernels/flash_attention/metal/metal.zig +++ b/zml/kernels/flash_attention/metal/metal.zig @@ -1,10 +1,12 @@ const std = @import("std"); +const stdx = @import("stdx"); + const zml = @import("../../../zml.zig"); -const flash_attn = @import("flash_attn.zig"); + +const log = std.log.scoped(.@"zml/attention/metal"); pub const Metadata = struct { - /// Live queries of a single-sequence prefill; 0 means all. num_tokens: zml.Tensor, pub fn init() Metadata { @@ -21,123 +23,16 @@ pub const Metadata = struct { } }; -/// Causal attention: fa_vec for bf16 decode, fa_tiled otherwise, sdpa for -/// unsupported dtypes and head sizes. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { - const dtype = q.dtype(); - const head_dim = q.dim(.hd); - if (!((dtype == .bf16 or dtype == .f16) and k.dtype() == dtype and v.dtype() == dtype and flash_attn.FaTiled.tileConfig(head_dim) != null)) { - const seq_len = k.dim(.k); - const mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) - .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, dtype), token_index.appendAxes(.{.coord}), .{}); - return zml.nn.sdpa(q, k, v, .{ .attn_mask = mask }); - } + const qc = q.transpose(.{ .h, .q, .hd }); + const kc = k.transpose(.{ .h, .k, .hd }); + const vc = v.transpose(.{ .h, .k, .hd }); + const tok_i32 = token_index.convert(.i32); - const batched = q.shape().hasTag(.b) != null; - const qb = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }).insertAxes(0, .{.b}); - const kb = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); - const vb = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); - const positions = token_index.convert(.i32).broad(.init(.{ .b = qb.dim(.b) }, .i32)); - const scale = 1.0 / @sqrt(@as(f32, @floatFromInt(head_dim))); - - const vec_head_dim = head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512; - const out = if (qb.dim(.q) == 1 and dtype == .bf16 and vec_head_dim) - decode(qb, kb, vb, positions, scale) + const attn = if (q.dim(.q) > 1) + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) else - prefill(qb, kb, vb, positions, liveQueries(qb, metadata), scale); - return (if (batched) out else out.squeeze(.b)).transpose(q.shape()); -} + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); -fn liveQueries(q: zml.Tensor, metadata: Metadata) zml.Tensor { - const all: zml.Tensor = .scalar(q.dim(.q), .i32); - if (q.dim(.b) != 1 or q.dim(.q) == 1) return all; - const num_tokens = metadata.num_tokens.convert(.i32); - return num_tokens.cmp(.GE, .scalar(1, .i32)).select(num_tokens.minimum(all), all); -} - -/// `q` is `[b, 1, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. -fn decode(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, scale: f32) zml.Tensor { - const batch: i32 = @intCast(q.dim(.b)); - const num_heads: i32 = @intCast(q.dim(.h)); - const head_dim: i32 = @intCast(q.dim(.hd)); - const num_kv_heads: i32 = @intCast(k.dim(.h)); - const keys: i32 = @intCast(k.dim(.k)); - const nrows = batch * num_heads; - const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), keys, coreCount()); - const args: flash_attn.FaVec.HostArgs = .{ .num_heads = num_heads, .n_kv = num_kv_heads, .k_len = keys, .scale = scale }; - - const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); - const out = flash_attn.FaVec.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions }, .{ .dst = dst }, .{ - .cfg = .{ .head_dim = head_dim, .heads = launch.heads, .splits = launch.splits }, - .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(batch) }, - .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, - .constants = .{ .args = .of(args) }, - }).dst; - if (launch.splits == 1) return out; - - return flash_attn.FaVecReduce.Kernel.call(.{ .htmp = out, .token_index = positions }, .{ .dst = q.shape() }, .{ - .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, - .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * flash_attn.FaVecReduce.simdgroups)), 1 }, - .threadgroup = .{ 32, flash_attn.FaVecReduce.simdgroups, 1 }, - .constants = .{ .args = .of(args) }, - }).dst; -} - -/// `q` is `[b, q, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. -fn prefill(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, live: zml.Tensor, scale: f32) zml.Tensor { - const queries: i32 = @intCast(q.dim(.q)); - const tile = flash_attn.FaTiled.tileConfig(q.dim(.hd)).?; - const cfg: flash_attn.FaTiled.Cfg = .{ - .dtype = zml.kernel.metal.from(q.dtype()), - .head_size = @intCast(q.dim(.hd)), - .bq = tile[0], - .tile_kv = tile[1], - .num_threads = tile[2], - }; - return flash_attn.FaTiled.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions, .live = live }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(q.dim(.h)), @intCast(@divFloor(queries + tile[0] - 1, tile[0])), @intCast(q.dim(.b)) }, - .threadgroup = .{ @intCast(tile[2]), 1, 1 }, - .threadgroup_memory = flash_attn.FaTiled.threadgroupMemory(cfg), - .constants = .{ - .num_kv_heads = .of(@as(i32, @intCast(k.dim(.h)))), - .scale = .of(scale), - .q_len = .of(queries), - .k_len = .of(@as(i32, @intCast(k.dim(.k)))), - }, - }).out; -} - -const VecLaunch = struct { - heads: i32, - splits: i32, - splits_per_threadgroup: i32, -}; - -/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a -/// row's length exit early, so `splits` follows the core and row counts. -fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { - var heads: i32 = @min(group, 4); - while (@rem(group, heads) != 0) heads -= 1; - const want: i32 = if (rows <= 2 * cores) 128 else 64; - const chunks = @divFloor(span + 31, 32); - var splits: i32 = 1; - while (splits * 2 <= @min(want, chunks)) splits *= 2; - return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; -} - -fn coreCount() i32 { - const platform = zml.Compiler.current().platform; - const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse - std.debug.panic("Metal device has no core_count attribute", .{}); - return @intCast(attribute.int64); -} - -test vecLaunch { - try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); -} - -test { - std.testing.refAllDecls(flash_attn); + return attn.transpose(q.shape()); } diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig deleted file mode 100644 index c3c3dca2..00000000 --- a/zml/kernels/flash_attention/tests.zig +++ /dev/null @@ -1,346 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const flash_attention = @import("flash_attention.zig"); - -test "attention: q=1,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); - - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{36}, - ); -} - -test "attention: b=4,q=1,qh=64,kh=8" { - // Full attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 63, 63, 63, 63 }, - ); - - // Partial attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 61, 57, 23, 63 }, - ); -} - -test "attention: q=1,qh=8,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{62}, - ); - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); -} - -test "attention: q=1,k=4096,qh=32,kh=8,hd=128" { - try testAttention( - .init(.{ .q = 1, .h = 32, .hd = 128 }, .bf16), - .init(.{ .k = 4096, .h = 8, .hd = 128 }, .bf16), - &.{3000}, - ); -} - -test "attention: q=1,k=1000,qh=8,kh=2,hd=32" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 32 }, .bf16), - .init(.{ .k = 1000, .h = 2, .hd = 32 }, .bf16), - &.{900}, - ); -} - -test "attention: q=8,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{56}, - ); -} - -test "attention: prefill, q=k=300,qh=8,kh=4" { - // Neither length is a multiple of the query (128) or key (64) tiles. - try testAttention( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries start at 800: every query tile sees full key tiles before the diagonal. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{800}, - ); -} - -test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { - // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), - .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { - try testAttention( - .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), - .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { - try testAttention( - .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), - .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), - &.{ 150, 100 }, - ); -} - -test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. - try testAttentionOpts( - .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{ 0, 0 }, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), - .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { - try testAttentionOpts( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), - &.{5}, - .{ .causal = false }, - ); -} - -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); -} - -fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { - return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(flash_attention.Backend)); -} - -fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const flash_attention.Backend) !void { - const io = std.testing.io; - const allocator = std.testing.allocator; - - const platform = zml.testing.env(); - - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { - std.debug.assert(token_index_h.len == q_shape.dim(.b)); - break :b .init(.{ .b = q_shape.dim(.b) }, .u32); - } else .init(.{}, .u32); - - const max_k_idx: i64 = k_shape.dim(.k) - 1; - const max_q_idx: i64 = q_shape.dim(.q) - 1; - if (opts.causal) { - // Check for out of bound reads, token_index is ignored by non-causal attention. - for (token_index_h) |index| { - std.debug.assert(index + max_q_idx <= max_k_idx); - } - } - - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ - .q = .fromShape(q_shape), - .k = .fromShape(k_shape), - .v = .fromShape(k_shape), - .token_index = .fromShape(token_index_shape), - }; - - var q = try attentionTestInput(q_shape, 17); - defer q.deinit(); - var k = try attentionTestInput(k_shape, 29); - defer k.deinit(); - var v = try attentionTestInput(k_shape, 43); - defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); - defer token_index.deinit(); - - const meshes = platform.meshes.values(); - const mesh = zml.testing.tpMesh(); - const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ - .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, - .out_specs = .replicated, - }; - const AttentionCall = struct { - fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { - return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); - } - }; - const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; - const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ - .program_name = "attention_stablehlo", - .meshes = meshes, - }); - defer reference_exe.deinit(); - - var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); - defer reference_d.deinit(); - try reference_d.await(io); - try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); - const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); - defer reference_h.free(allocator); - - for (backends) |backend| { - if (!backend.isAvailable(platform)) continue; - - const metadata: flash_attention.Metadata = .init(.fromBackend(backend)); - const parameters: flash_attention.Parameters = .init(.fromBackend(backend)); - const exe = try platform.compileFn( - allocator, - io, - AttentionCall.forward, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, - .{ - .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .meshes = meshes, - }, - ); - defer exe.deinit(); - - var metadata_d = try metadata.initBuffer(io, platform); - defer flash_attention.Metadata.deinitBuffer(&metadata_d); - - var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); - defer output_d.deinit(); - try output_d.await(io); - const output_h = try output_d.toSliceAlloc(allocator, io); - defer output_h.free(allocator); - - errdefer std.log.err( - \\ Attention test failed, {0t} output doesn't match reference. - \\ - reference: {1d} - \\ - {0t}: {2d} - , .{ backend, reference_h, output_h }); - try zml.testing.expectClose(io, reference_h, output_h, .{ - .absolute_tolerance = 5e-3, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-3, - .minimum_close_fraction = 0.99, - }); - } -} - -fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { - const host = try zml.Slice.alloc(std.testing.allocator, shape); - defer host.free(std.testing.allocator); - var prng: std.Random.DefaultPrng = .init(seed); - const random = prng.random(); - switch (shape.dtype()) { - inline .bf16, .f16, .f32 => |dtype| { - for (host.items(dtype.toZigType())) |*value| { - const x = random.float(f32) * 4 - 2; - value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); - } - }, - else => unreachable, - } - return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); -} - -/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its -/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. -fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { - const Layout = struct { - b: usize, - s: usize, - h: usize, - d: usize, - - fn stride(shape: zml.Shape, tag: anytype) usize { - const axis = shape.hasTag(tag) orelse return 0; - var result: usize = 1; - for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); - return result; - } - - fn init(shape: zml.Shape) @This() { - return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; - } - - fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { - return b * self.b + s * self.s + h * self.h + d * self.d; - } - }; - const ql = Layout.init(q.shape); - const kl = Layout.init(k.shape); - const vl = Layout.init(v.shape); - const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; - const queries: usize = @intCast(q.shape.dim(.q)); - const heads: usize = @intCast(q.shape.dim(.h)); - const dim: usize = @intCast(q.shape.dim(.hd)); - const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); - const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); - defer std.testing.allocator.free(scores); - const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); - for (0..batch) |b| { - const start: usize = positions[if (positions.len == 1) 0 else b]; - for (0..queries) |i| { - for (0..heads) |h| { - var maximum: f64 = -std.math.inf(f64); - const active = scores[0 .. start + i + 1]; - for (active, 0..) |*score, j| { - var dot: f64 = 0; - for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); - score.* = dot * scale; - maximum = @max(maximum, score.*); - } - var sum: f64 = 0; - for (active) |*score| { - score.* = @exp(score.* - maximum); - sum += score.*; - } - for (0..dim) |d| { - var expected: f64 = 0; - for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); - expected /= sum; - const actual = output.items(f32)[ql.offset(b, i, h, d)]; - try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); - } - } - } - } -} diff --git a/zml/kernels/moe/api.zig b/zml/kernels/moe/api.zig deleted file mode 100644 index b66593c1..00000000 --- a/zml/kernels/moe/api.zig +++ /dev/null @@ -1,164 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const Tensor = @import("../../tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/moe"); - -pub const Options = struct { - activation: Activation, - /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. - quantize_input: bool, - /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. - routing_weight_placement: RoutingWeightPlacement, -}; - -pub const ActivationKind = enum { - /// Gelu activation function - gelu, - /// ReLU activation function - relu, - /// SiLU activation function - silu, - /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) - swiglu, - /// SwiGLU activation function with step function - swiglu_step, - /// GeGlu activation function - geglu, - /// GeGlu activation function with tanh - geglu_tanh, -}; - -pub const Activation = union(ActivationKind) { - gelu: void, - relu: void, - silu: void, - swiglu: struct { - limit: ?f32 = null, - scale: ?f32 = null, - bias: ?f32 = null, - }, - swiglu_step: struct { - limit: ?f32 = null, - }, - geglu: void, - geglu_tanh: void, - - pub fn applyActivation(self: Activation, input: Tensor, layout: ProjectionLayout) Tensor { - const x = input.convert(.f32); - return switch (self) { - .gelu => x.gelu(), - .relu => x.relu(), - .silu => x.silu(), - .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { - const mid = @divFloor(x.dim(.out), 2); - var gate, var up = switch (layout) { - .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .row_interleaved_1 => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, - .row_interleaved_8 => blk: { - const grouped = x.splitAxis(.out, .{ .group = @divExact(mid, 8), .pair = 2, .row = 8 }); - break :blk .{ - grouped.slice(.pair, .{ .end = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), - grouped.slice(.pair, .{ .start = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), - }; - }, - }; - - break :b switch (self) { - .swiglu => |parameters| { - stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); - - const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - - // Apply limit on gate and clamp up - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - - // Apply bias - up = if (parameters.bias) |bias| up.addConstant(bias) else up; - - break :b gate.silu().mul(up); - }, - .swiglu_step => |parameters| { - gate = gate.silu(); - - const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - break :b gate.mul(up); - }, - .geglu_tanh => gate.gelu().mul(up), - .geglu => { - log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); - break :b gate.gelu().mul(up); - }, - else => unreachable, // already treated by the top-level switch - }; - }, - }; - } -}; - -pub const RoutingWeightPlacement = enum { before_down, after_down }; - -pub const ProjectionLayout = enum { - concatenated, // G0 G1 … Gn U0 U1 … Un - row_interleaved_1, // G0 U0 G1 U1 … - row_interleaved_8, // G0…G7 U0…U7 G8…G15 U8…U15 … -}; - -/// How a backend expects the expert weights to be stored -/// The packer writes this layout and forwardMoe reads it -pub const ExpertsLayout = struct { - /// Gate/up ordering before applying the backend-specific packing. - gate_up: ProjectionLayout, - /// Backend specific transformation of the weights and scales. - packing: Packing, - - pub const Packing = enum { - /// Experts stacked as stored in the checkpoint. - plain, - /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). - swizzled_scales, - /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. - flashinfer_nvfp4, - /// MXFP4 gate/up rows interleaved by 8. E8M0 scales use 512-byte - /// 128x4 atoms in MN-major order `[expert, k/128, rows/128, 512]`, - /// represented as `[expert, k/128, rows]` i32 words of four scales. - mxfp4_interleaved8, - }; -}; - -/// Routing for a local token batch and a contiguous range of experts. -/// IDs retain the model's global expert numbering, while the projection weights -/// contain only the local experts. Kernels compute those experts contributions and -/// `forwardMoe` sums them along the expert axis. -pub const LocalRouting = struct { - /// Global expert IDs for local tokens, with shape {.t, .topk} - ids: Tensor, - weights: Tensor, - global_num_experts: u32, - local_num_experts: u32, - /// Scalar i32 containing the first local expert's global ID. - /// Null when all experts are local and global IDs already index the weights. - expert_start: ?Tensor = null, - - /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. - /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. - pub fn localIds(self: LocalRouting) Tensor { - const start = self.expert_start orelse return self.ids; - const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(self.ids.sub(start), .scalar(-1, .i32)); - } - - /// Returns a lookup table from every global expert ID to its local weight - /// index or -1 if nonlocal. Returns null when every expert is local. - pub fn expertMap(self: LocalRouting) ?Tensor { - const start = self.expert_start orelse return null; - const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(ids.sub(start), .scalar(-1, .i32)); - } -}; diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index 8bc03c9c..c0156720 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,9 +3,6 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const Options = @import("../api.zig").Options; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -27,7 +24,7 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { return cc.major == 10 and cc.minor == 3; } -pub fn validateOptions(opts: Options) void { +pub fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); @@ -38,14 +35,14 @@ pub fn validateOptions(opts: Options) void { /// is the per-expert multiplier dequantizing each GEMM output (it folds in /// the inverse of the input's global scale), and their input scale is the /// multiplier applied to the GEMM's input before NVFP4 quantization. Both -/// expect block scales in the 128x4 tensor-core layout. MXFP4 gate/up rows -/// interleave by 8 and its scale atoms are MN-major packed i32 words. +/// expect block scales in the 128x4 tensor-core layout. pub fn fusedExperts( input: zml.Tensor, - routing: LocalRouting, + ids: zml.Tensor, + weights: zml.Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: Options, + options: zml.moe.Options, ) zml.Tensor { validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); @@ -86,43 +83,24 @@ pub fn fusedExperts( else => stdx.debug.panic("cute backend only supports mxfp4 and nvfp4 quantization, got {}", .{gq.scheme}), }; - const hidden = down.weight.dim(1); - const intermediate = down.weight.dim(2) * 2; - const topk = routing.ids.dim(.topk); - const experts = gate_up.weight.dim(.expert); - const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); - const tokens = ids.dim(.token); - const routing_weights = routing.weights.convert(.f32).reshape(.{ tokens, topk }); - - return switch (format) { - .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, topk, options.activation.swiglu.limit, .{ - .x = input.reshape(.{ tokens, hidden }), - .routing_weights = routing_weights, - .w1 = gate_up.weight, - .s1 = gq.scales, - .w2 = down.weight, - .s2 = dq.scales, - .ids = ids, - }, null), - .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, topk, .{ - .x = input.reshape(.{ tokens, hidden }), - .routing_weights = routing_weights, - .ids = ids, - .w1 = gate_up.weight, - .s1 = gq.scales, - .w2 = down.weight, - .s2 = dq.scales, - // `alpha` is per expert; `act` is one scale shared by every - // expert, which the kernels read per expert. - .alpha1 = f.alpha1.reshape(.{experts}), - .alpha2 = f.alpha2.reshape(.{experts}), - .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), - .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), - }, .{ - .swiglu_limit = options.activation.swiglu.limit, - .routing_weight_placement = options.routing_weight_placement, - }), - }.reshape(input.shape().dims()).withTags(input.shape()); + const context: Context = .{ + .input = input, + .ids = ids, + .weights = weights, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + .topk = ids.dim(.topk), + .swiglu_limit = options.activation.swiglu.limit, + .routing_weight_placement = options.routing_weight_placement, + .format = format, + .expert_parallel = gate_up.weight.shape().partition(.expert).isSharded(), + }; + return if (context.expert_parallel) + zml.ops.manualComputation(Context.body, context, input.shape()) + else + context.body(input.shape()); } /// The format-specific tensors: NVFP4's per-expert global scales. They stay @@ -137,3 +115,69 @@ const Format = union(enum) { act2: zml.Tensor, }, }; + +const Context = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + w1: zml.Tensor, + s1: zml.Tensor, + w2: zml.Tensor, + s2: zml.Tensor, + topk: i64, + swiglu_limit: ?f32, + routing_weight_placement: zml.moe.RoutingWeightPlacement, + format: Format, + expert_parallel: bool, + + fn body(self: Context, _: zml.Shape) zml.Tensor { + const experts = self.w1.dim(.expert); + const hidden = self.w2.dim(1); + const intermediate = self.w2.dim(2) * 2; + const tokens = self.input.dim(.t); + var ids = self.ids.convert(.i32); + const routing_weights = self.weights.convert(.f32); + if (self.expert_parallel) { + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(experts); + const expert_end = expert_start.addConstant(experts); + // Routes of other ranks get expert -1 and are not computed here. + const local = ids.cmp(.GE, expert_start).logical(.AND, ids.cmp(.LT, expert_end)); + ids = local.select(ids.sub(expert_start), .scalar(-1, .i32)); + } + const result = switch (self.format) { + .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, self.topk, self.swiglu_limit, .{ + .x = self.input, + .routing_weights = routing_weights, + .w1 = self.w1, + .s1 = self.s1, + .w2 = self.w2, + .s2 = self.s2, + .ids = ids, + }, null), + .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, self.topk, .{ + .x = self.input, + .routing_weights = routing_weights, + .ids = ids, + .w1 = self.w1, + .s1 = self.s1, + .w2 = self.w2, + .s2 = self.s2, + // `alpha` is per expert; `act` is one scale shared by every + // expert, which the kernels read per expert. + .alpha1 = f.alpha1.reshape(.{experts}), + .alpha2 = f.alpha2.reshape(.{experts}), + .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), + .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), + }, .{ + .swiglu_limit = self.swiglu_limit, + .routing_weight_placement = self.routing_weight_placement, + }), + }.withTags(self.input.shape()); + + return if (self.expert_parallel) + zml.ops.allReduce(result, zml.Tensor.add) + else + result; + } +}; diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index ff6edec2..479e6516 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -4,31 +4,26 @@ //! rows in route order, and the top-k reduction. MXFP4 applies routing weights //! before the down projection, with the reference model's BF16 rounding. const std = @import("std"); - const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; -const triton_kernels = @import("../../triton/moe.zig"); pub const mxfp4 = @import("mxfp4.zig"); pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); +const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { /// `[tokens, hidden]` BF16. x: zml.Tensor, /// `[tokens, topk]` FP32. routing_weights: zml.Tensor, - /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved by 8. w1: zml.Tensor, - /// `[experts, hidden / 128, 2 * intermediate]` i32 words of four E8M0 scales, - /// in MN-major 128x4 atoms, with rows interleaved like `w1`. + /// `w1` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). s1: zml.Tensor, - /// `[experts, hidden, intermediate / 2]` packed FP4. w2: zml.Tensor, - /// `[experts, intermediate / 128, hidden]` i32 words in MN-major 128x4 atoms. + /// `w2` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). s2: zml.Tensor, /// `[tokens, topk]` local expert ids. ids: zml.Tensor, @@ -212,14 +207,14 @@ pub fn forwardMxfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, t .weight = a.w1, .schedule = routing.schedule, .input_quant = q1.q, - .weight_scale = a.s1, + .weight_scale = a.s1.bitCast(.u8), .input_scale = q1.s, }, routing.route_inverse, a.routing_weights.reshape(.{routes})); const down = mxfp4_persistent.downRows(down_cfg, .{ .weight = a.w2, .schedule = routing.schedule, .input_quant = q2.q, - .weight_scale = a.s2, + .weight_scale = a.s2.bitCast(.u8), .input_scale = q2.s, }, routing.route_inverse); return reduce.topk(down, routing.route_map orelse routing.schedule, .{ @@ -257,7 +252,7 @@ pub const Nvfp4Inputs = struct { pub const Nvfp4Options = struct { swiglu_limit: ?f32 = null, - routing_weight_placement: RoutingWeightPlacement = .after_down, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, /// GEMM tile N; null selects it from the batch size. tile_n: ?i64 = null, }; diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig index 30062a3a..21a76436 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -550,7 +550,7 @@ fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: .weight = .{ .ptr = cute.DType.i8 }, .schedule = .{ .ptr = cute.DType.i32 }, .input_quant = .{ .ptr = cute.DType.f8e4m3fn }, - .weight_scale = .{ .ptr = cute.DType.i32 }, + .weight_scale = .{ .ptr = cute.DType.i8 }, .input_scale = .{ .ptr = cute.DType.i8 }, }; const shape = .{ @@ -1439,10 +1439,11 @@ fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, return .{ .count = count, .route = route, .weight = weight }; } -/// Thread `t` owns output row `128 * tile_m + t` of the interleave-8 -/// projection. Gate and up lanes are paired by XOR 8. Each gate thread -/// computes one SwiGLU activation per routed row; 32 consecutive activations -/// span two warps, whose partial maxima meet in shared memory. +/// Python `fused_epilogue`: thread `t` owns output row `128 * tile_m + t` +/// of the interleaved projection, so even threads hold a gate and their odd +/// neighbour the matching up value. Each even thread computes one SwiGLU +/// activation per routed row; 32 consecutive activations span two warps, +/// whose partial maxima meet in shared memory. /// /// At decode sizes most rows of a group are padding, whose outputs are never /// read. `count` is uniform across the CTA, so whole 8-row chunks of padding @@ -1469,7 +1470,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const tid = b.threadIdx().x; const lane = tid.rem(32); const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const is_gate = tid.rem(16).lt(8); + const is_gate = tid.rem(2).eq(0); const first_row = group.mul(operands.group_rows); const f32_type = cute.DType.f32.toMlir(b.ctx); // The TMA C staging buffer is unused by this epilogue. @@ -1496,7 +1497,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo if (operands.swiglu_limit) |limit| { gate = gate.minimum(limit); } - var up_value = b.shuffleXor(value, 8).to(.bf16).to(.f32); + var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); if (operands.swiglu_limit) |limit| { up_value = up_value.maximum(-limit).minimum(limit); } @@ -1505,7 +1506,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); - // Up lanes hold no activation and must not raise the maxima. + // Odd threads hold no activation and must not raise the maxima. activation.* = b.select(is_gate, result, b.constant(.f32, 0)); const maximum = b.warpMaxNonNegative(activation.*.abs()); chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); @@ -1539,8 +1540,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo // The next tile reuses the scratch maxima. b.barrierCtaSyncAligned(1, 128); - const activation_row = tid.div(16).mul(8).add(tid.rem(8)); - const activation_index = tile_m.mul(64).add(activation_row); + const activation_index = tile_m.mul(64).add(tid.div(2)); for (0..row_chunks) |row_chunk| { const first = row_chunk * swiglu_chunk; const last = @min(first + swiglu_chunk, live); @@ -1830,13 +1830,7 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); - // Weight scales step M atoms first; activation scales retain K-major order. - const sfa_order = b.makeIntTuple(.{ 0, 1, 2 }); - const sfa_layout = b.layoutSpec( - .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, - .{ .{ .{ 16, 4 }, 512 }, .{ .{ 0, 1 }, sf_stride }, .{ 0, sf_stride } }, - ); - const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sfa_order, b.layoutType(sfa_layout)); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); diff --git a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel new file mode 100644 index 00000000..54b8604c --- /dev/null +++ b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cutlass_flashinfer", + srcs = [ + "cutlass_flashinfer.zig", + ], +) diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig new file mode 100644 index 00000000..2c7ad342 --- /dev/null +++ b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig @@ -0,0 +1,633 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const platforms = @import("platforms"); +const zml = @import("../../../zml.zig"); + +const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); + +pub const auto_tactic: i32 = -1; + +pub const Options = struct { + /// Device used at graph-construction time to select the architecture + /// library and size the XLA-owned scratch buffer. + workspace_query_device: i32, + activation: zml.moe.Activation, + enable_pdl: bool, + gemm1_tactic: i32, + /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to + /// obtain the first valid GEMM2 index. + gemm2_tactic: i32, + + pub fn fromGlobalOptions(opts: zml.moe.Options) Options { + return .{ + .workspace_query_device = 0, + .activation = opts.activation, + .enable_pdl = false, + .gemm1_tactic = auto_tactic, + .gemm2_tactic = auto_tactic, + }; + } +}; + +pub fn validateOptions(opts: zml.moe.Options) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); +} + +const Input = struct { + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, +}; + +const Output = struct { + output: zml.Shape, + workspace: zml.Shape, +}; + +const Attributes = struct { + runners: u64 = 0, + num_tokens: i64, + hidden_size: i64, + intermediate_size: i64, + num_experts: i32, + top_k: i32, + activation: i32, + enable_pdl: bool, + fc1_act_per_expert: bool, + fc2_act_per_expert: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, +}; + +const DeviceRunner = struct { + api: *fi_cutlass_moe.Api, + runner: *fi_cutlass_moe.Runner, +}; + +const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; + +pub const Runners = struct { + runners: [max_num_devices]?DeviceRunner = @splat(null), + + pub fn deinit(self: *Runners) void { + for (self.runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); + } + } + } + + fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { + if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; + + const index: usize = @intCast(device); + if (self.runners[index]) |runner| return runner; + + const api = try fi_cutlass_moe.apiForDevice(device); + const options = runnerOptions(device); + var runner: ?*fi_cutlass_moe.Runner = null; + try checkStatus(api, api.runnerCreate(&options, &runner)); + const result: DeviceRunner = .{ + .api = api, + .runner = runner orelse return error.RunnerInitializationFailed, + }; + self.runners[index] = result; + return result; + } +}; + +fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { + if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; + if (api.lastError()) |message| { + log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ + status, + std.mem.span(message), + }); + } else { + log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); + } + return switch (status) { + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, + else => error.FlashinferCutlassMoe, + }; +} + +fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { + var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); + options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); + options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; + options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.device = device; + options.use_fused_finalize = 1; + return options; +} + +fn runnersFromAttributes(attributes: Attributes) !*Runners { + if (attributes.runners == 0) return error.RunnersNotLoaded; + return @ptrFromInt(attributes.runners); +} + +fn currentRunners() !*Runners { + const platform = zml.Compiler.current().platform; + const runners = platform.state.cuda.fi_cutlass_moe_runners; + return runners orelse error.RunnersNotLoaded; +} + +fn cutlassActivation(activation: zml.moe.Activation) i32 { + return switch (activation) { + .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, + .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, + .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, + .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, + .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, + .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, + .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, + }; +} + +fn makeContext(attributes: Attributes) fi_cutlass_moe.Context { + var context = std.mem.zeroes(fi_cutlass_moe.Context); + context.struct_size = @sizeOf(fi_cutlass_moe.Context); + context.num_tokens = attributes.num_tokens; + context.hidden_size = attributes.hidden_size; + context.intermediate_size = attributes.intermediate_size; + context.num_experts = attributes.num_experts; + context.num_experts_on_rank = attributes.num_experts; + context.top_k = attributes.top_k; + context.tp_size = 1; + context.ep_size = 1; + context.activation = @intCast(attributes.activation); + context.enable_pdl = @intFromBool(attributes.enable_pdl); + context.swizzled_input_sf = 1; + return context; +} + +fn ffiCallNvfp4( + call_frame: *zml.pjrt.ffi.CallFrame, + input: zml.pjrtx.TensorToCustomCallBuffer(Input), + output: zml.pjrtx.ShapeToCustomCallBuffer(Output), + attributes: Attributes, +) !?*zml.pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = input.hidden_states.ptr; + io.token_selected_experts = @ptrCast(@alignCast(input.topk_ids.ptr)); + io.token_final_scales = @ptrCast(@alignCast(input.topk_weights.ptr)); + io.fc1_expert_weights = input.fc1_weights.ptr; + io.fc2_expert_weights = input.fc2_weights.ptr; + io.quant_scales[0] = input.fc1_act_global.ptr; + io.quant_scales[1] = input.fc1_weight_block.ptr; + io.quant_scales[2] = input.fc1_global.ptr; + io.quant_scales[3] = input.fc2_act_global.ptr; + io.quant_scales[4] = input.fc2_weight_block.ptr; + io.quant_scales[5] = input.fc2_global.ptr; + io.quant_scale_count = 6; + io.quant_scale_per_expert_mask = + (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | + (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); + io.output = output.output.ptr; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = output.workspace.ptr; + workspace.data_bytes = output.workspace.shape.byteSize(); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ + .name = "flashinfer_cutlass_nvfp4_routed_moe", + // Expert mesh is owned by forwardMoe outer manual computation. + .sharding_aware = false, + .has_side_effect = false, +}); + +fn computeCapability(platform: *const zml.Platform) !u16 { + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; + return switch (cc.sm()) { + 100, 103, 120 => |sm| sm, + else => error.UnsupportedArchitecture, + }; +} + +pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, +) !void { + if (comptime platforms.isEnabled(.cuda)) { + var cuda_state = &platform.state.cuda; + if (cuda_state.fi_cutlass_moe_runners != null) return; + try fi_cutlass_moe.load(allocator, io, try computeCapability(platform)); + const runners = try allocator.create(Runners); + runners.* = .{}; + cuda_state.fi_cutlass_moe_runners = runners; + return; + } + return error.UnsupportedPlatform; +} + +pub fn register(platform: *const zml.Platform) !void { + if (comptime platforms.isEnabled(.cuda)) { + try routedNvfp4Call.register(platform); + return; + } + return error.UnsupportedPlatform; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + _ = computeCapability(platform) catch return false; + return true; +} + +pub fn isNvfp4Supported(platform: *const zml.Platform) bool { + if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + const compute_capability = computeCapability(platform) catch return false; + return switch (compute_capability) { + 100, 103, 120 => true, + else => false, + }; +} + +pub fn tacticCounts( + platform: *const zml.Platform, + device: i32, +) !struct { gemm1: i32, gemm2: i32 } { + const runners = platform.state.cuda.fi_cutlass_moe_runners orelse return error.RunnersNotLoaded; + const deviceRunner = try runners.ensureRunner(device); + var gemm1: i32 = 0; + var gemm2: i32 = 0; + try checkStatus( + deviceRunner.api, + deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), + ); + return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; +} + +fn roundUp(value: i64, alignment: i64) i64 { + return @divTrunc(value + alignment - 1, alignment) * alignment; +} + +pub fn fc1BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, + activation: zml.moe.Activation, +) zml.Shape { + const rows = switch (activation) { + .silu, .gelu, .relu => intermediate_size, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, + }; + return .init( + .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, + .f8e4m3fn, + ); +} + +pub fn fc2BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, +) zml.Shape { + return .init( + .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, + .f8e4m3fn, + ); +} + +fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { + return tensor.dtype() == .f32 and + (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); +} + +fn validateInputs( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) !Attributes { + if (hidden_states.dtype() != .bf16 or + fc1_weights.dtype() != .f4e2m1 or + fc2_weights.dtype() != .f4e2m1 or + topk_weights.dtype() != .f32 or + topk_ids.dtype() != .i32 or + fc1_weight_block.dtype() != .f8e4m3fn or + fc2_weight_block.dtype() != .f8e4m3fn or + fc1_global.dtype() != .f32 or + fc2_global.dtype() != .f32) + { + return error.UnsupportedType; + } + if (hidden_states.rank() != 2 or + fc1_weights.rank() != 3 or + fc2_weights.rank() != 3 or + topk_weights.rank() != 2 or + topk_ids.rank() != 2) + { + return error.InvalidShape; + } + + const tokens = hidden_states.dim(.t); + const hidden_size = hidden_states.dim(.d); + const num_experts = fc1_weights.dim(0); + const fc1_rows = fc1_weights.dim(1); + const intermediate_size = switch (options.activation) { + .silu, .gelu, .relu => fc1_rows, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), + }; + const top_k = topk_ids.dim(.topk); + + if (tokens <= 0 or hidden_size <= 0 or + num_experts <= 0 or intermediate_size <= 0 or + @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or + top_k <= 0 or top_k > num_experts) + { + return error.InvalidShape; + } + if (fc1_weights.dim(2) != hidden_size or + fc2_weights.dim(0) != num_experts or + fc2_weights.dim(1) != hidden_size or + fc2_weights.dim(2) != intermediate_size or + topk_weights.dim(.t) != tokens or + topk_weights.dim(.topk) != top_k or + topk_ids.dim(.t) != tokens or + !fc1_weight_block.shape().eql(fc1BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + options.activation, + )) or + !fc2_weight_block.shape().eql(fc2BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + )) or + !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or + !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or + !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or + !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) + { + return error.InvalidShape; + } + + return .{ + .num_tokens = tokens, + .hidden_size = hidden_size, + .intermediate_size = intermediate_size, + .num_experts = @intCast(num_experts), + .top_k = @intCast(top_k), + .activation = cutlassActivation(options.activation), + .enable_pdl = options.enable_pdl, + .fc1_act_per_expert = fc1_act_global.rank() == 1, + .fc2_act_per_expert = fc2_act_global.rank() == 1, + .gemm1_tactic = options.gemm1_tactic, + .gemm2_tactic = options.gemm2_tactic, + }; +} + +/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. +/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout +/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. +pub fn fusedExpertsNvfp4( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) zml.Tensor { + const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); + var attributes = validateInputs( + hidden_states, + fc1_weights, + fc2_weights, + topk_weights, + topk_ids, + fc1_act_global, + fc1_weight_block, + fc1_global, + fc2_act_global, + fc2_weight_block, + fc2_global, + options, + ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); + + attributes.runners = @intFromPtr(runners); + const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); + const context = makeContext(attributes); + var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); + requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); + checkStatus( + deviceRunner.api, + deviceRunner.api.getWorkspaceRequirements( + deviceRunner.runner, + &context, + &requirements, + ), + ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); + + const result = routedNvfp4Call.call( + .{ + .hidden_states = hidden_states, + .fc1_weights = fc1_weights, + .fc2_weights = fc2_weights, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .fc1_act_global = fc1_act_global, + .fc1_weight_block = fc1_weight_block, + .fc1_global = fc1_global, + .fc2_act_global = fc2_act_global, + .fc2_weight_block = fc2_weight_block, + .fc2_global = fc2_global, + }, + .{ + .output = hidden_states.shape(), + .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), + }, + attributes, + ); + return result.output; +} + +pub fn fusedExperts( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, +) zml.Tensor { + if (comptime !platforms.isEnabled(.cuda)) { + @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); + } + + if (gate_up.bias != null or down.bias != null) { + @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); + } + + const runner_options: Options = .fromGlobalOptions(opts); + const expert_partition = gate_up.weight.shape().partition(.expert); + + const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); + + const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); + const down_weight_unpacked = zml.moe.unpackedWeight(down); + + // TODO(Corentin): Do error checking on nvfp4 + // Also, maybe pass `zml.nn.Linear` directly + if (expert_partition.isSharded()) { + return zml.ops.manualComputation( + (struct { + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up_weight_unpacked: zml.Tensor, + down_weight_unpacked: zml.Tensor, + gate_up_input_scale: zml.Tensor, + gate_up_scales: zml.Tensor, + gate_up_global_scale: zml.Tensor, + down_input_scale: zml.Tensor, + down_scales: zml.Tensor, + down_global_scale: zml.Tensor, + activation: zml.moe.Activation, + enable_pdl: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, + workspace_query_device: i32, + + fn body( + self: @This(), + _: zml.Shape, + ) zml.Tensor { + const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const expert_end = expert_start.addConstant(local_num_experts); + + const local_route_mask = self.topk_ids + .cmp(.GE, expert_start) + .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); + const local_topk_ids = local_route_mask.select( + self.topk_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + const local_topk_weights = local_route_mask.select( + self.topk_weights, + zml.Tensor.scalar(-1, self.topk_weights.dtype()), + ); + + const local_output = fusedExpertsNvfp4( + self.input, + self.gate_up_weight_unpacked, + self.down_weight_unpacked, + local_topk_weights, + local_topk_ids, + self.gate_up_input_scale, + self.gate_up_scales, + self.gate_up_global_scale, + self.down_input_scale, + self.down_scales, + self.down_global_scale, + .{ + .workspace_query_device = self.workspace_query_device, + .activation = self.activation, + .enable_pdl = self.enable_pdl, + .gemm1_tactic = self.gemm1_tactic, + .gemm2_tactic = self.gemm2_tactic, + }, + ); + const local_reshaped = local_output + .reshape(self.input.shape().dims()) + .withTags(.{ .t, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .gate_up_weight_unpacked = gate_up_weight_unpacked, + .down_weight_unpacked = down_weight_unpacked, + .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), + .gate_up_scales = gate_up.quantization.?.scales, + .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), + .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), + .down_scales = down.quantization.?.scales, + .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), + .activation = runner_options.activation, + .enable_pdl = runner_options.enable_pdl, + .gemm1_tactic = runner_options.gemm1_tactic, + .gemm2_tactic = runner_options.gemm2_tactic, + .workspace_query_device = runner_options.workspace_query_device, + }, + input.shape(), + ); + } + + return fusedExpertsNvfp4( + input, + gate_up_weight_unpacked, + down_weight_unpacked, + topk_weights, + topk_ids, + gate_up.quantization.?.input_scale.?.asMultiplier(), + gate_up.quantization.?.scales, + gate_up.quantization.?.global_scale.?.asMultiplier(), + down.quantization.?.input_scale.?.asMultiplier(), + down.quantization.?.scales, + down.quantization.?.global_scale.?.asMultiplier(), + runner_options, + ); +} diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel deleted file mode 100644 index c9c5d9a6..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "flashinfer_cutlass", - srcs = [ - "contract.zig", - "flashinfer_cutlass.zig", - "runtime.zig", - ], -) diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig deleted file mode 100644 index b4b77eaa..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/contract.zig +++ /dev/null @@ -1,40 +0,0 @@ -pub fn Inputs(comptime T: type) type { - return struct { - hidden_states: T, - fc1_weights: T, - fc2_weights: T, - topk_weights: T, - topk_ids: T, - fc1_act_global: T, - fc1_weight_block: T, - fc1_global: T, - fc2_act_global: T, - fc2_weight_block: T, - fc2_global: T, - }; -} - -pub fn Outputs(comptime T: type) type { - return struct { - output: T, - workspace: T, - }; -} - -pub const Attributes = struct { - runners: u64 = 0, - num_tokens: i64, - hidden_size: i64, - intermediate_size: i64, - num_experts: i32, - top_k: i32, - activation: i32, - enable_pdl: bool, - fc1_act_per_expert: bool, - fc2_act_per_expert: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, -}; - -pub const compute_on_host: bool = false; -pub const name = "flashinfer_cutlass_nvfp4_routed_moe"; diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig deleted file mode 100644 index 92317c7c..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig +++ /dev/null @@ -1,355 +0,0 @@ -const std = @import("std"); - -const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); -const platforms = @import("platforms"); -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const LocalRouting = @import("../api.zig").LocalRouting; -const unpackedWeight = @import("../weights.zig").unpackedWeight; -const contract = @import("contract.zig"); -const Attributes = contract.Attributes; -const runtime = @import("runtime.zig"); - -const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); - -pub const auto_tactic: i32 = -1; - -pub const Options = struct { - /// Device used at graph-construction time to select the architecture - /// library and size the XLA-owned scratch buffer. - workspace_query_device: i32, - activation: Activation, - enable_pdl: bool, - gemm1_tactic: i32, - /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to - /// obtain the first valid GEMM2 index. - gemm2_tactic: i32, - - pub fn fromGlobalOptions(opts: GlobalOptions) Options { - return .{ - .workspace_query_device = 0, - .activation = opts.activation, - .enable_pdl = false, - .gemm1_tactic = auto_tactic, - .gemm2_tactic = auto_tactic, - }; - } -}; - -pub fn validateOptions(opts: GlobalOptions) void { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); - stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); - stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); - stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); -} - -const Inputs = contract.Inputs(zml.Tensor); - -const Outputs = contract.Outputs(zml.Shape); - -fn currentRunners() !*runtime.Runners { - const platform = zml.Compiler.current().platform; - const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; - return runners orelse error.RunnersNotLoaded; -} - -fn cutlassActivation(activation: Activation) i32 { - return switch (activation) { - .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, - .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, - .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, - .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, - .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, - .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, - .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, - }; -} - -pub fn call(input_tensors: Inputs, output_shapes: Outputs, attributes: Attributes) zml.ops.ShapeToTensor(Outputs) { - const opts: zml.ops.CustomCallOptions = .{ - .has_side_effect = false, - .output_operand_aliases = comptime zml.ops.customCallOutputOperandAliases(Inputs, Outputs, null), - .compute_on_host = contract.compute_on_host, - }; - - return zml.ops.typedCustomCall( - contract.name, - opts, - input_tensors, - output_shapes, - attributes, - ); -} - -fn computeCapability(platform: *const zml.Platform) !u16 { - const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; - return switch (cc.sm()) { - 100, 103, 120 => |sm| sm, - else => error.UnsupportedArchitecture, - }; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; - _ = computeCapability(platform) catch return false; - return true; -} - -pub fn isNvfp4Supported(platform: *const zml.Platform) bool { - if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; - const compute_capability = computeCapability(platform) catch return false; - return switch (compute_capability) { - 100, 103, 120 => true, - else => false, - }; -} - -fn roundUp(value: i64, alignment: i64) i64 { - return @divTrunc(value + alignment - 1, alignment) * alignment; -} - -pub fn fc1BlockScaleShape( - num_experts: i64, - hidden_size: i64, - intermediate_size: i64, - activation: Activation, -) zml.Shape { - const rows = switch (activation) { - .silu, .gelu, .relu => intermediate_size, - .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, - }; - return .init( - .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, - .f8e4m3fn, - ); -} - -pub fn fc2BlockScaleShape( - num_experts: i64, - hidden_size: i64, - intermediate_size: i64, -) zml.Shape { - return .init( - .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, - .f8e4m3fn, - ); -} - -fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { - return tensor.dtype() == .f32 and - (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); -} - -fn validateInputs( - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, - options: Options, -) !Attributes { - if (hidden_states.dtype() != .bf16 or - fc1_weights.dtype() != .f4e2m1 or - fc2_weights.dtype() != .f4e2m1 or - topk_weights.dtype() != .f32 or - topk_ids.dtype() != .i32 or - fc1_weight_block.dtype() != .f8e4m3fn or - fc2_weight_block.dtype() != .f8e4m3fn or - fc1_global.dtype() != .f32 or - fc2_global.dtype() != .f32) - { - return error.UnsupportedType; - } - if (hidden_states.rank() != 2 or - fc1_weights.rank() != 3 or - fc2_weights.rank() != 3 or - topk_weights.rank() != 2 or - topk_ids.rank() != 2) - { - return error.InvalidShape; - } - - const tokens = hidden_states.dim(.t); - const hidden_size = hidden_states.dim(.d); - const num_experts = fc1_weights.dim(0); - const fc1_rows = fc1_weights.dim(1); - const intermediate_size = switch (options.activation) { - .silu, .gelu, .relu => fc1_rows, - .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), - }; - const top_k = topk_ids.dim(.topk); - - if (tokens <= 0 or hidden_size <= 0 or - num_experts <= 0 or intermediate_size <= 0 or - @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or - top_k <= 0 or top_k > num_experts) - { - return error.InvalidShape; - } - if (fc1_weights.dim(2) != hidden_size or - fc2_weights.dim(0) != num_experts or - fc2_weights.dim(1) != hidden_size or - fc2_weights.dim(2) != intermediate_size or - topk_weights.dim(.t) != tokens or - topk_weights.dim(.topk) != top_k or - topk_ids.dim(.t) != tokens or - !fc1_weight_block.shape().eql(fc1BlockScaleShape( - num_experts, - hidden_size, - intermediate_size, - options.activation, - )) or - !fc2_weight_block.shape().eql(fc2BlockScaleShape( - num_experts, - hidden_size, - intermediate_size, - )) or - !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or - !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or - !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or - !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) - { - return error.InvalidShape; - } - - return .{ - .num_tokens = tokens, - .hidden_size = hidden_size, - .intermediate_size = intermediate_size, - .num_experts = @intCast(num_experts), - .top_k = @intCast(top_k), - .activation = cutlassActivation(options.activation), - .enable_pdl = options.enable_pdl, - .fc1_act_per_expert = fc1_act_global.rank() == 1, - .fc2_act_per_expert = fc2_act_global.rank() == 1, - .gemm1_tactic = options.gemm1_tactic, - .gemm2_tactic = options.gemm2_tactic, - }; -} - -/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. -/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout -/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. -pub fn fusedExpertsNvfp4( - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, - options: Options, -) zml.Tensor { - const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); - var attributes = validateInputs( - hidden_states, - fc1_weights, - fc2_weights, - topk_weights, - topk_ids, - fc1_act_global, - fc1_weight_block, - fc1_global, - fc2_act_global, - fc2_weight_block, - fc2_global, - options, - ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); - - attributes.runners = @intFromPtr(runners); - const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); - const context = runtime.makeContext(attributes); - var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); - requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - runtime.checkStatus( - deviceRunner.api, - deviceRunner.api.getWorkspaceRequirements( - deviceRunner.runner, - &context, - &requirements, - ), - ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); - - const result = call( - .{ - .hidden_states = hidden_states, - .fc1_weights = fc1_weights, - .fc2_weights = fc2_weights, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .fc1_act_global = fc1_act_global, - .fc1_weight_block = fc1_weight_block, - .fc1_global = fc1_global, - .fc2_act_global = fc2_act_global, - .fc2_weight_block = fc2_weight_block, - .fc2_global = fc2_global, - }, - .{ - .output = hidden_states.shape(), - .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), - }, - attributes, - ); - return result.output; -} - -pub fn fusedExperts( - input: zml.Tensor, - routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: GlobalOptions, -) zml.Tensor { - if (comptime !platforms.isEnabled(.cuda)) { - @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); - } - - if (gate_up.bias != null or down.bias != null) { - @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); - } - - const topk_ids = routing.localIds(); - // The runner ignores routes whose local expert id is -1. - const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); - - const runner_options: Options = .fromGlobalOptions(opts); - - const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); - - const gate_up_weight_unpacked = unpackedWeight(gate_up); - const down_weight_unpacked = unpackedWeight(down); - - // TODO(Corentin): Do error checking on nvfp4 - // Also, maybe pass `zml.nn.Linear` directly - return fusedExpertsNvfp4( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - gate_up.quantization.?.input_scale.?.asMultiplier(), - gate_up.quantization.?.scales, - gate_up.quantization.?.global_scale.?.asMultiplier(), - down.quantization.?.input_scale.?.asMultiplier(), - down.quantization.?.scales, - down.quantization.?.global_scale.?.asMultiplier(), - runner_options, - ); -} diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig deleted file mode 100644 index 028ef892..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/runtime.zig +++ /dev/null @@ -1,223 +0,0 @@ -const std = @import("std"); - -const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); -const pjrt = @import("pjrt"); - -const DataType = @import("../../../dtype.zig").DataType; -const contract = @import("contract.zig"); - -const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); - -// TODO(Corentin): This was coming from Platform but we are now downstream. -// Find a way to solve that. -// Maybe the max_num_devices could be its own module because it only depends on Target. -const max_num_devices = 64; - -// TODO(Corentin): replace that with Shape when its dependencies are untangled and -// it doesn't depend on the platform anymore. -fn count(buffer: *const pjrt.ffi.Buffer) usize { - if (buffer.rank == 0) return @intCast(buffer.dims()[0]); - - var c: usize = 1; - for (buffer.dims()) |dim| c = c * @as(usize, @intCast(dim)); - - return c; -} - -fn byteSize(buffer: *const pjrt.ffi.Buffer) usize { - const dt: DataType = switch (buffer.dtype) { - .invalid => @panic("Found an invalid pjrt buffer"), - .token => @panic("Found an token pjrt buffer"), - inline else => |tag| @field(DataType, @tagName(tag)), - }; - return std.math.divCeil(usize, count(buffer) * dt.bitSizeOf(), 8) catch unreachable; -} - -pub fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - // TODO(Corentin): Find a place to abstract that - var input_buffers: contract.Inputs(*const pjrt.ffi.Buffer) = undefined; - inline for (@typeInfo(contract.Inputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { - const buf = call_frame.args.buffers()[i]; - @field(input_buffers, field_name) = buf; - } - - var output_buffers: contract.Outputs(*const pjrt.ffi.Buffer) = undefined; - inline for (@typeInfo(contract.Outputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { - const buf = call_frame.results.buffers()[i]; - @field(output_buffers, field_name) = buf; - } - - const attributes: contract.Attributes = switch (@typeInfo(contract.Attributes)) { - .void => {}, - .@"struct" => blk: { - var attrs: contract.Attributes = undefined; - inline for (@typeInfo(contract.Attributes).@"struct".field_names, @typeInfo(contract.Attributes).@"struct".field_types) |field_name, Field| { - // TODO: Use getByIndex instead? - const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse - @panic("Attribute not found: " ++ field_name); - @field(attrs, field_name) = attribute.get(Field); - } - break :blk attrs; - }, - else => @compileError("Unsupported attributes type: " ++ @typeName(contract.Attributes)), - }; - - return handlerInner(call_frame, input_buffers, output_buffers, attributes) catch |err| - pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); -} - -fn handlerInner( - call_frame: *pjrt.ffi.CallFrame, - inputs: contract.Inputs(*const pjrt.ffi.Buffer), - outputs: contract.Outputs(*const pjrt.ffi.Buffer), - attributes: contract.Attributes, -) !?*pjrt.ffi.Error { - const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); - const runners = try runnersFromAttributes(attributes); - const deviceRunner = try runners.ensureRunner(device); - const context = makeContext(attributes); - - var io = std.mem.zeroes(fi_cutlass_moe.Io); - io.struct_size = @sizeOf(fi_cutlass_moe.Io); - io.input = inputs.hidden_states.data; - io.token_selected_experts = @ptrCast(@alignCast(inputs.topk_ids.data)); - io.token_final_scales = @ptrCast(@alignCast(inputs.topk_weights.data)); - io.fc1_expert_weights = inputs.fc1_weights.data; - io.fc2_expert_weights = inputs.fc2_weights.data; - io.quant_scales[0] = inputs.fc1_act_global.data; - io.quant_scales[1] = inputs.fc1_weight_block.data; - io.quant_scales[2] = inputs.fc1_global.data; - io.quant_scales[3] = inputs.fc2_act_global.data; - io.quant_scales[4] = inputs.fc2_weight_block.data; - io.quant_scales[5] = inputs.fc2_global.data; - io.quant_scale_count = 6; - io.quant_scale_per_expert_mask = - (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | - (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); - io.output = outputs.output.data; - - var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); - workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); - workspace.data = outputs.workspace.data; - workspace.data_bytes = byteSize(outputs.workspace); - - try checkStatus( - deviceRunner.api, - deviceRunner.api.run( - deviceRunner.runner, - &context, - &io, - &workspace, - @ptrCast(call_frame.api.stream(call_frame.ctx)), - attributes.gemm1_tactic, - attributes.gemm2_tactic, - ), - ); - return null; -} - -pub fn makeContext(attributes: contract.Attributes) fi_cutlass_moe.Context { - var context = std.mem.zeroes(fi_cutlass_moe.Context); - context.struct_size = @sizeOf(fi_cutlass_moe.Context); - context.num_tokens = attributes.num_tokens; - context.hidden_size = attributes.hidden_size; - context.intermediate_size = attributes.intermediate_size; - context.num_experts = attributes.num_experts; - context.num_experts_on_rank = attributes.num_experts; - context.top_k = attributes.top_k; - context.tp_size = 1; - context.ep_size = 1; - context.activation = @intCast(attributes.activation); - context.enable_pdl = @intFromBool(attributes.enable_pdl); - context.swizzled_input_sf = 1; - return context; -} - -pub fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { - if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; - if (api.lastError()) |message| { - log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ - status, - std.mem.span(message), - }); - } else { - log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); - } - return switch (status) { - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, - else => error.FlashinferCutlassMoe, - }; -} - -fn runnersFromAttributes(attributes: contract.Attributes) !*Runners { - if (attributes.runners == 0) return error.RunnersNotLoaded; - return @ptrFromInt(attributes.runners); -} - -pub const Runners = struct { - runners: [max_num_devices]?DeviceRunner = @splat(null), - - pub fn deinit(self: *Runners) void { - for (self.runners) |device_runner| { - if (device_runner) |runner| { - _ = runner.api.runnerDestroy(runner.runner); - } - } - } - - pub fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { - if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; - - const index: usize = @intCast(device); - if (self.runners[index]) |runner| return runner; - - const api = try fi_cutlass_moe.apiForDevice(device); - const options = runnerOptions(device); - var runner: ?*fi_cutlass_moe.Runner = null; - try checkStatus(api, api.runnerCreate(&options, &runner)); - const result: DeviceRunner = .{ - .api = api, - .runner = runner orelse return error.RunnerInitializationFailed, - }; - self.runners[index] = result; - return result; - } -}; - -const DeviceRunner = struct { - api: *fi_cutlass_moe.Api, - runner: *fi_cutlass_moe.Runner, -}; - -fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { - var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); - options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); - options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; - options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.device = device; - options.use_fused_finalize = 1; - return options; -} - -pub fn tacticCounts( - runners: *const Runners, - device: i32, -) !struct { gemm1: i32, gemm2: i32 } { - const deviceRunner = try runners.ensureRunner(device); - var gemm1: i32 = 0; - var gemm2: i32 = 0; - try checkStatus( - deviceRunner.api, - deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), - ); - return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; -} - -pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { - try fi_cutlass_moe.load(allocator, io, sm); -} diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index ba3d3c00..e5c5b6de 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,13 +6,9 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; const fly = @import("moe.zig"); +const ProjectionLayout = zml.moe.ProjectionLayout; +const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -21,19 +17,90 @@ test { } pub fn fusedExperts( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, +) zml.Tensor { + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) + .{ .gate_up = .interleaved, .packing = .plain } + else + .{ .gate_up = .concatenated, .packing = .plain }; + + const args: FusedExpertsArgs = .{ + .hidden_states = input, + .gate_up = gate_up, + .down = down, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .activation = opts.activation, + .quantize_input = opts.quantize_input, + .gate_up_layout = layout.gate_up, + .routing_weight_placement = opts.routing_weight_placement, + }; + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (!expert_partition.isSharded()) { + return fusedExpertsImpl(args); + } + + return zml.ops.manualComputation( + (struct { + args: FusedExpertsArgs, + global_num_experts: i64, + + fn call(self: @This(), _: zml.Shape) zml.Tensor { + const local_args = self.args; + const local_num_experts = local_args.gate_up.weight.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + // Map global expert ids to local ids, or -1 for experts outside this partition. + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + var mapped_args = local_args; + mapped_args.expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + + const local_output = fusedExpertsImpl(mapped_args); + const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).call, + .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, + input.shape(), + ); +} + +pub const FusedExpertsArgs = struct { hidden_states: Tensor, - routing_: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - opts: Options, -) Tensor { - const expert_map_ = routing_.expertMap(); - const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + topk_weights: Tensor, + topk_ids: Tensor, + activation: zml.moe.Activation, + expert_map: ?Tensor = null, + /// Use FP8 activations for FP8 weights; false keeps BF16 activations. + quantize_input: bool, + gate_up_layout: ProjectionLayout, + routing_weight_placement: RoutingWeightPlacement, +}; + +pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { + const hidden_states = opts.hidden_states; + const topk_weights = opts.topk_weights; + const topk_ids = opts.topk_ids; const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = gate_up_.quantizationScheme(); - const down_scheme = down_.quantizationScheme(); + const gate_up_scheme = opts.gate_up.quantizationScheme(); + const down_scheme = opts.down.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("fly backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -50,10 +117,10 @@ pub fn fusedExperts( }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); - const down = down_.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); + const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "fly backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -81,13 +148,13 @@ pub fn fusedExperts( stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "fly backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "fly backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "fly backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (expert_map_) |expert_map| + if (opts.expert_map) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "fly backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (expert_map_) |expert_map| + const expert_ids = if (opts.expert_map) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -97,9 +164,9 @@ pub fn fusedExperts( const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = gate_up_.bias, + .bias = opts.gate_up.bias, .input_scale = input_scale, - .weight_scale = gate_up_.quantizationScales(), + .weight_scale = opts.gate_up.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -108,7 +175,7 @@ pub fn fusedExperts( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = opts.activation.applyActivation(gate_up_out, layout); + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -118,9 +185,9 @@ pub fn fusedExperts( const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = down_.bias, + .bias = opts.down.bias, .input_scale = down_input_scale, - .weight_scale = down_.quantizationScales(), + .weight_scale = opts.down.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -136,18 +203,18 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { - if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or - down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or +pub fn supports(opts: FusedExpertsArgs) bool { + if (opts.hidden_states.dtype() != .bf16 or opts.gate_up.quantizationScheme() != .mxfp4 or + opts.down.quantizationScheme() != .mxfp4 or opts.gate_up.bias != null or opts.down.bias != null or opts.routing_weight_placement != .before_down) return false; - const gate_up = gate_up_.weight; - const down = down_.weight; + const gate_up = opts.gate_up.weight; + const down = opts.down.weight; if (gate_up.rank() != 3 or down.rank() != 3) return false; - const gate_up_scale = gate_up_.quantizationScales() orelse return false; - const down_scale = down_.quantizationScales() orelse return false; - const experts = routing.global_num_experts; + const gate_up_scale = opts.gate_up.quantizationScales() orelse return false; + const down_scale = opts.down.quantizationScales() orelse return false; + const experts = if (opts.expert_map) |map| map.dim(.expert) else gate_up.dim(0); if (experts < 1 or experts > 512) return false; - var max_width: i64 = hidden_states.dim(.d); + var max_width: i64 = opts.hidden_states.dim(.d); inline for (.{ .{ gate_up, gate_up_scale }, .{ down, down_scale } }) |projection| { const weight, const scale = projection; const packing_factor: i64 = switch (weight.dtype()) { @@ -166,7 +233,7 @@ pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.L else => return false, } } - const routes = hidden_states.dim(.t) * routing.ids.dim(.topk); + const routes = opts.hidden_states.dim(.t) * opts.topk_ids.dim(.topk); return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); } diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index f835c889..e129c76c 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -1,17 +1,21 @@ const std = @import("std"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; -const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); -fn validateOptions(opts: GlobalOptions) void { +pub const Options = struct { + activation: zml.moe.Activation, + global_num_experts: i64 = -1, + w1_scale: ?Tensor = null, + w2_scale: ?Tensor = null, + w1_global_scale: ?Tensor = null, + w2_global_scale: ?Tensor = null, +}; + +fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); } @@ -68,30 +72,57 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso } pub fn fusedExperts( - hidden_states: zml.Tensor, - routing: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - opts: GlobalOptions, + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, ) zml.Tensor { validateOptions(opts); - stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); - stdx.debug.assert(down_.bias == null, "metal backend doesn't support down bias", .{}); + stdx.debug.assert(gate_up.bias == null, "metal backend doesn't support gate_up bias", .{}); + stdx.debug.assert(down.bias == null, "metal backend doesn't support down bias", .{}); - const gate_up_scales: ?zml.Tensor = if (gate_up_.quantization) |q| q.scales else null; - const gate_up_global_scale: ?zml.Tensor = if (gate_up_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; + const gate_up_global_scale: ?zml.Tensor = if (gate_up.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; - const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; + const down_global_scale: ?zml.Tensor = if (down.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const gate_up = unpackedWeight(gate_up_); - const down = unpackedWeight(down_); + const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); + const down_weight_unpacked = zml.moe.unpackedWeight(down); + + return fusedExpertsImpl( + input, + gate_up_weight_unpacked, + down_weight_unpacked, + topk_weights, + topk_ids, + .{ + .activation = opts.activation, + .global_num_experts = gate_up_weight_unpacked.dim(.expert), + .w1_scale = gate_up_scales, + .w2_scale = down_scales, + .w1_global_scale = gate_up_global_scale, + .w2_global_scale = down_global_scale, + }, + ); +} + +pub fn fusedExpertsImpl( + hidden_states: Tensor, + gate_up: Tensor, + down: Tensor, + topk_weights: Tensor, + topk_ids: Tensor, + opts: Options, +) Tensor { stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); const mode = quantMode(gate_up.dtype()); switch (mode) { .none => { - stdx.debug.assert(gate_up_scales == null and down_scales == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); - stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(opts.w1_scale == null and opts.w2_scale == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); const is_hidden_states_valid_dtype = switch (hidden_states.dtype()) { .bf16, .f16, .f32 => true, else => false, @@ -99,28 +130,28 @@ pub fn fusedExperts( stdx.debug.assert(is_hidden_states_valid_dtype, "metal backend expected hidden_states data type to be bf16, f16 or f32, got {}", .{hidden_states.dtype()}); }, .fp8 => { - stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(gate_up_scales.?.dtype() == .bf16 and down_scales.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); - stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); + stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(opts.w1_scale.?.dtype() == .bf16 and opts.w2_scale.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); + stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, .nvfp4 => { - stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(gate_up_scales.?.dtype() == .f8e4m3fn and down_scales.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); + stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(opts.w1_scale.?.dtype() == .f8e4m3fn and opts.w2_scale.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, } - const is_topk_weights_valid_dtype = switch (routing.weights.dtype()) { + const is_topk_weights_valid_dtype = switch (topk_weights.dtype()) { .bf16, .f16, .f32 => true, else => false, }; - stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{routing.weights.dtype()}); - stdx.debug.assert(routing.ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{routing.ids.dtype()}); + stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{topk_weights.dtype()}); + stdx.debug.assert(topk_ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{topk_ids.dtype()}); const d = hidden_states.dim(.d); const num_tokens = hidden_states.dim(.t); - const topk = routing.ids.dim(.topk); + const topk = topk_ids.dim(.topk); const num_routes = num_tokens * topk; const act_dtype: zml.DataType = switch (mode) { @@ -132,29 +163,29 @@ pub fn fusedExperts( const x_rows = hidden.insertAxes(.d, .{.topk}) .broad(zml.Shape.init(.{ .token = num_tokens, .topk = topk, .d = d }, act_dtype)) .merge(.{ .r = .{ .token, .topk } }); - const expert_ids = routing.ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }).convert(.i32); const gate_up_out = applyGateUpGlobalScale(moeGemm( x_rows, gate_up, - gate_up_scales, + opts.w1_scale, expert_ids, zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, - ), gate_up_global_scale, expert_ids); - const activated = opts.activation.applyActivation(gate_up_out, .concatenated).convert(act_dtype); + ), opts.w1_global_scale, expert_ids); + const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated).convert(act_dtype); const down_out = applyDownGlobalScale(moeGemm( activated, down, - down_scales, + opts.w2_scale, expert_ids, zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, act_dtype), mode, - ), down_global_scale, expert_ids); + ), opts.w2_global_scale, expert_ids); - const weights = routing.weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const weights = topk_weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }); const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index eb697f36..25ce26d6 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,21 +4,14 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; -pub const Activation = @import("api.zig").Activation; -pub const ActivationKind = @import("api.zig").ActivationKind; +pub const tcl = @import("tcl/tcl.zig"); +pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const cute = @import("cute/cute.zig"); -pub const ExpertsLayout = @import("api.zig").ExpertsLayout; -pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); -pub const fly = @import("fly/fly.zig"); -pub const LocalRouting = @import("api.zig").LocalRouting; +pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const Options = @import("api.zig").Options; -pub const ProjectionLayout = @import("api.zig").ProjectionLayout; -pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; -pub const stablehlo = @import("stablehlo/stablehlo.zig"); -pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); +pub const fly = @import("fly/fly.zig"); const log = std.log.scoped(.@"zml/moe"); @@ -27,6 +20,60 @@ test { _ = @import("tests.zig"); } +pub const ProjectionLayout = enum { concatenated, interleaved }; +pub const RoutingWeightPlacement = enum { before_down, after_down }; + +/// How a backend expects the expert weights to be stored +/// The packer writes this layout and forwardMoe reads it +pub const ExpertsLayout = struct { + /// Whether the two projections occupy contiguous halves or alternate. + gate_up: ProjectionLayout, + /// Backend specific transformation of the weights and scales. + packing: Packing, + + pub const Packing = enum { + /// Experts stacked as stored in the checkpoint. + plain, + /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + swizzled_scales, + /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. + flashinfer_nvfp4, + }; +}; + +pub const ActivationKind = enum { + /// Gelu activation function + gelu, + /// ReLU activation function + relu, + /// SiLU activation function + silu, + /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) + swiglu, + /// SwiGLU activation function with step function + swiglu_step, + /// GeGlu activation function + geglu, + /// GeGlu activation function with tanh + geglu_tanh, +}; + +pub const Activation = union(ActivationKind) { + gelu: void, + relu: void, + silu: void, + swiglu: struct { + limit: ?f32 = null, + scale: ?f32 = null, + bias: ?f32 = null, + }, + swiglu_step: struct { + limit: ?f32 = null, + }, + geglu: void, + geglu_tanh: void, +}; + pub const Backend = enum { stablehlo, cute, @@ -38,6 +85,8 @@ pub const Backend = enum { tcl, pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { + // Keep the dtype as an argument because non scheme-specific backends may depend on it later + _ = dtype; return switch (platform.target) { .cuda => b: { const s = scheme orelse break :b .triton; @@ -48,7 +97,7 @@ pub const Backend = enum { .triton, .nvfp4 => if (cute.isNvfp4Supported(platform)) .cute - else if (flashinfer_cutlass.isNvfp4Supported(platform)) + else if (cutlass_flashinfer.isNvfp4Supported(platform)) .flashinfer_cutlass else error.UnsupportedQuantization, @@ -71,13 +120,7 @@ pub const Backend = enum { }; }, .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, - .furiosa => b: { - const s = scheme orelse break :b .tcl; - break :b switch (s) { - .fp8_block128 => if (dtype == .f8e4m3fn) .tcl else error.UnsupportedQuantization, - .mxfp4, .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block32 => error.UnsupportedQuantization, - }; - }, + .furiosa => if (scheme == null) .tcl else error.UnsupportedQuantization, .metal => b: { const s = scheme orelse break :b .metal; break :b switch (s) { @@ -93,7 +136,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => true, .cute => cute.isAvailable(platform), - .flashinfer_cutlass => flashinfer_cutlass.isAvailable(platform), + .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), .fly => switch (platform.target) { .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, else => false, @@ -112,17 +155,13 @@ pub const Backend = enum { pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { return switch (backend) { .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, - .cute => if (scheme) |s| switch (s) { - .mxfp4 => .{ .gate_up = .row_interleaved_8, .packing = .mxfp4_interleaved8 }, - .nvfp4 => .{ .gate_up = .row_interleaved_1, .packing = .swizzled_scales }, - else => error.UnsupportedQuantization, - } else error.UnsupportedQuantization, + .cute => if (scheme == .mxfp4 or scheme == .nvfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, .flashinfer_cutlass => if (scheme) |s| switch (s) { .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, else => error.UnsupportedQuantization, } else error.UnsupportedQuantization, .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, - .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .row_interleaved_1, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, + .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, }; } @@ -130,7 +169,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => {}, .cute => {}, - .flashinfer_cutlass => flashinfer_cutlass.register(platform), + .flashinfer_cutlass => cutlass_flashinfer.register(platform), .triton, .fly => {}, .mosaic_tpu => {}, .metal => {}, @@ -139,6 +178,14 @@ pub const Backend = enum { } }; +pub const Options = struct { + activation: Activation, + /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. + quantize_input: bool, + /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. + routing_weight_placement: RoutingWeightPlacement, +}; + /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); @@ -149,35 +196,6 @@ pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Te weights.shape().hasTag(.t) == 0 and weights.shape().hasTag(.topk) == 1, "MoE expects ordered tags {{.t, .d}} for input and {{.t, .topk}} for routing", .{}); } -pub const Partitioning = struct { - const Logical = zml.Sharding.Partitioning.Logical; - - in_specs: struct { - input: Logical, - ids: Logical, - weights: Logical, - gate_up: Logical, - down: Logical, - }, - out_specs: Logical, - - /// Shards the `.expert` dimension of both projections over `mesh_axis`. - /// Other dimensions, input tokens, routing IDs/weights and output are replicated. - pub fn expert(comptime mesh_axis: @EnumLiteral()) Partitioning { - const experts: Logical = .parse(.{ .expert = mesh_axis }); - return .{ - .in_specs = .{ - .input = .replicated, - .ids = .replicated, - .weights = .replicated, - .gate_up = experts, - .down = experts, - }, - .out_specs = .replicated, - }; - } -}; - /// Input has shape {.t, .d}; routing IDs and weights have shape {.t, .topk}. /// Returns {.t, .d}. Flatten batched input with merge(.{ .t = .{ .b, .s } }). pub fn forwardMoe( @@ -187,112 +205,72 @@ pub fn forwardMoe( gate_up: zml.nn.Linear, down: zml.nn.Linear, backend: Backend, - mesh: *const zml.Mesh, - partitioning: Partitioning, opts: Options, ) zml.Tensor { validateRoutingInputs(input, topk_ids, topk_weights); + return switch (backend) { + .stablehlo => stablehlo.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .cute => cute.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .triton => triton.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .fly => fly.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .mosaic_tpu => mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .metal => metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .tcl => tcl.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + }; +} - const shapes = .{ - .input = input.shape().withPartitioning(mesh, partitioning.in_specs.input), - .ids = topk_ids.shape().withPartitioning(mesh, partitioning.in_specs.ids), - .weights = topk_weights.shape().withPartitioning(mesh, partitioning.in_specs.weights), - .gate_up = gate_up.weight.shape().withPartitioning(mesh, partitioning.in_specs.gate_up), - .down = down.weight.shape().withPartitioning(mesh, partitioning.in_specs.down), +pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { + const quantization = linear.quantization orelse return linear.weight; + return if (zml.nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) + zml.nn.unpackFp4(linear.weight, linear.tag, linear.tag) + else + linear.weight; +} + +pub fn applyActivation(input: zml.Tensor, activation: zml.moe.Activation, layout: ProjectionLayout) zml.Tensor { + const x = input.convert(.f32); + return switch (activation) { + .gelu => x.gelu(), + .relu => x.relu(), + .silu => x.silu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + }; + + break :b switch (activation) { + .swiglu => |parameters| { + stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); + + const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + + // Apply limit on gate and clamp up + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + + // Apply bias + up = if (parameters.bias) |bias| up.addConstant(bias) else up; + + break :b gate.silu().mul(up); + }, + .swiglu_step => |parameters| { + gate = gate.silu(); + + const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + break :b gate.mul(up); + }, + .geglu_tanh => gate.gelu().mul(up), + .geglu => { + log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); + break :b gate.gelu().mul(up); + }, + else => unreachable, // already treated by the top-level switch + }; + }, }; - const output_shape = input.shape().withPartitioning(mesh, partitioning.out_specs); - - const tokens = shapes.input.partition(.t); - const experts = shapes.gate_up.partition(.expert); - // Local kernels require complete feature vectors and top-k routing lists. - stdx.debug.assert(shapes.input.partition(.d) == .replicated and shapes.ids.partition(.topk) == .replicated and shapes.weights.partition(.topk) == .replicated, "MoE feature and top-k dimensions must be replicated", .{}); - // The expert-axis reduction must add contributions for the same token rows. - // Sharding tokens along that axis would mix contributions for different tokens. - stdx.debug.assert(shapes.ids.partition(.t) == tokens and shapes.weights.partition(.t) == tokens and output_shape.partition(.t) == tokens and output_shape.partition(.d) == .replicated, "MoE input, routing and output must have matching token partitioning", .{}); - stdx.debug.assert(shapes.down.partition(.expert) == experts and (!experts.isSharded() or experts != tokens), "MoE projections must agree on expert partitioning; tokens must be replicated along the expert axis", .{}); - for ([_]zml.Shape{ shapes.gate_up, shapes.down }) |shape| { - stdx.debug.assert(!shape.containsPartitionSpec(.open), "MoE kernel weight partitioning must be closed", .{}); - for (0..shape.rank()) |axis| { - stdx.debug.assert(axis == shape.axis(.expert) or shape.partition(axis) == .replicated, "MoE kernel weights can only be partitioned along experts", .{}); - } - } - const expert_axis = experts.meshAxis(); - const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; - if (backend == .metal) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); - - const Call = struct { - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - backend: Backend, - opts: Options, - mesh: *const zml.Mesh, - expert_axis: ?u3, - global_experts: u32, - - fn forward(args: @This(), _: zml.Shape) zml.Tensor { - const routing: LocalRouting = .{ - .ids = args.ids, - .weights = args.weights, - .global_num_experts = args.global_experts, - .local_num_experts = @intCast(args.gate_up.weight.dim(.expert)), - .expert_start = if (args.expert_axis) |axis| zml.ops.axisIndex(args.mesh, args.mesh.logical.axes.get(axis)).convert(.i32).scale(args.gate_up.weight.dim(.expert)) else null, - }; - const output = switch (args.backend) { - .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .flashinfer_cutlass => flashinfer_cutlass.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .metal => metal.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .tcl => tcl.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - }; - return if (args.expert_axis) |axis| zml.ops.allReduceOnAxis(output, zml.Tensor.add, args.mesh, args.mesh.logical.axes.get(axis)) else output; - } - }; - const args: Call = .{ - .input = input, - .ids = topk_ids, - .weights = topk_weights, - .gate_up = gate_up, - .down = down, - .backend = backend, - .opts = opts, - .mesh = mesh, - .expert_axis = if (parallel) expert_axis else null, - .global_experts = @intCast(gate_up.weight.dim(.expert)), - }; - const Logical = zml.Sharding.Partitioning.Logical; - var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; - for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { - zml.meta.mapAlloc(struct { - fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { - const axis = ctx.axis orelse return .replicated; - if (tensor.count() == 1) return .replicated; - // Packed scales may be untagged; experts are their leading dimension. - const dim = tensor.shape().hasTag(.expert) orelse 0; - stdx.debug.assert(tensor.dim(dim) == ctx.experts, "MoE tensor must contain {} experts, got {f}", .{ ctx.experts, tensor.shape() }); - return .{ .mapping = .{ - .src = .init(&.{if (tensor.shape().hasTag(.expert) != null) zml.Shape.toTag(.expert) else zml.Shape.toTag(._0)}), - .tgt = .init(&.{axis}), - } }; - } - }.spec, zml.Compiler.current().arena.allocator(), .{ - .experts = linear.weight.dim(.expert), - .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, - }, linear, specs) catch @panic("OOM"); - } - return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ - .in_specs = .{ - .input = partitioning.in_specs.input, - .ids = partitioning.in_specs.ids, - .weights = partitioning.in_specs.weights, - .gate_up = projection_specs[0], - .down = projection_specs[1], - }, - .out_specs = partitioning.out_specs, - }); } diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index bb60594b..09dc360a 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,9 +5,6 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -15,9 +12,22 @@ const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); var gate_up_transpose: std.atomic.Value(bool) = .init(false); var down_transpose: std.atomic.Value(bool) = .init(false); -fn validateOptions(opts: GlobalOptions) void { +pub const Options = struct { + activation: zml.moe.Activation, + global_num_experts: i64 = -1, + expert_map: ?Tensor = null, + w1_scale: ?Tensor = null, + w2_scale: ?Tensor = null, + w1_bias: ?Tensor = null, + w2_bias: ?Tensor = null, +}; + +fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + // TODO(Corentin): find where to put that + //if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; + //if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; } fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { @@ -214,25 +224,120 @@ pub fn callGmmEp( } pub fn fusedExperts( - hidden_states: zml.Tensor, - routing: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - opts: GlobalOptions, + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, ) zml.Tensor { validateOptions(opts); - const expert_map_ = routing.expertMap(); + const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; + const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; + + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (expert_partition.isSharded()) { + const global_num_experts = down.weight.dim(.expert); + const partial_output = zml.ops.manualComputation( + (struct { + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + weights_gate_up: zml.Tensor, + weights_down: zml.Tensor, + activation: zml.moe.Activation, + global_num_experts: i64, + gate_up_scales: ?zml.Tensor, + bias_gate_up: ?zml.Tensor, + down_scales: ?zml.Tensor, + bias_down: ?zml.Tensor, + + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const local_num_experts = self.weights_gate_up.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + const expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + const local_output = fusedExpertsImpl( + self.input, + self.weights_gate_up, + self.weights_down, + self.topk_weights, + self.topk_ids, + .{ + .activation = self.activation, + .global_num_experts = self.global_num_experts, + .expert_map = expert_map, + .w1_scale = self.gate_up_scales, + .w2_scale = self.down_scales, + .w1_bias = self.bias_gate_up, + .w2_bias = self.bias_down, + }, + ); + return local_output.reshape(self.input.shape()).withTags(.{ .t, .d }); + } + }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .weights_gate_up = gate_up.weight, + .weights_down = down.weight, + .activation = opts.activation, + .global_num_experts = global_num_experts, + .gate_up_scales = gate_up_scales, + .bias_gate_up = gate_up.bias, + .down_scales = down_scales, + .bias_down = down.bias, + }, + input.shape(), + ); + return zml.ops.allReduce(partial_output, zml.Tensor.add); + } + + return fusedExpertsImpl( + input, + gate_up.weight, + down.weight, + topk_weights, + topk_ids, + .{ + .activation = opts.activation, + .global_num_experts = gate_up.weight.dim(.expert), + .w1_scale = gate_up_scales, + .w2_scale = down_scales, + .w1_bias = gate_up.bias, + .w2_bias = down.bias, + }, + ); +} + +pub fn fusedExpertsImpl( + hidden_states: Tensor, + w1: Tensor, + w2: Tensor, + topk_weights: Tensor, + topk_ids: Tensor, + opts: Options, +) Tensor { const tokens = hidden_states.dim(.t); const hidden = hidden_states.reshape(.{ .token = tokens, .in = hidden_states.dim(.d) }); // TODO(Corentin): Better error message - const gate_up = canonicalizeGateUp(gate_up_.weight, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExperts: {}", .{e}); - const down = canonicalizeDown(down_.weight, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExperts: {}", .{e}); - const weights = routing.weights.reshape(.{ .token = tokens, .topk = routing.weights.dim(.topk) }); - const ids = routing.ids.reshape(.{ .token = tokens, .topk = routing.ids.dim(.topk) }); + const gate_up = canonicalizeGateUp(w1, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExpertsImpl: {}", .{e}); + const down = canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExpertsImpl: {}", .{e}); + const weights = topk_weights.reshape(.{ .token = tokens, .topk = topk_weights.dim(.topk) }); + const ids = topk_ids.reshape(.{ .token = tokens, .topk = topk_ids.dim(.topk) }); - const num_experts = routing.global_num_experts; - if (expert_map_) |expert_map| { + const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); + if (opts.expert_map) |expert_map| { if (expert_map.dtype() != .i32) stdx.debug.panic("expert_map must be of dtype i32, got {}", .{expert_map.dtype()}); if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) stdx.debug.panic("expert_map must be a 1D tensor of length num_experts, got shape {}", .{expert_map.shape()}); } @@ -240,7 +345,7 @@ pub fn fusedExperts( const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); const flat_weights = weights.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); - const flat_ids_local, const local_route_mask = if (expert_map_) |expert_map| blk: { + const flat_ids_local, const local_route_mask = if (opts.expert_map) |expert_map| blk: { const mapped = expert_map.gather(.{ .expert = flat_ids_global }, .{}).withTags(.{.route}); const route_mask = mapped.cmp(.GE, Tensor.scalar(0, .i32).broad(mapped.shape())); const safe_ids = route_mask.select(mapped, Tensor.zeroes(mapped.shape())); @@ -251,7 +356,7 @@ pub fn fusedExperts( }; const sorted_route = flat_ids_local.argsort(.route, .{ .descending = false }).rename(.{ .route = .perm }); - const hidden_sorted = if (expert_map_ != null) + const hidden_sorted = if (opts.expert_map != null) permuteSmallBatch(hidden, sorted_route, ids.dim(.topk)) else blk: { // If not sharded, we can save the sort + gather by just repeating the hidden states for each expert and gathering with the sorted route @@ -274,19 +379,19 @@ pub fn fusedExperts( .gather(.{ .token = aligned_hidden.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (gate_up_.bias) |bias| { + if (opts.w1_bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); gate_up_out = gate_up_out.add(bias_per_token); } - const activated = opts.activation.applyActivation(gate_up_out, .concatenated); + const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated); const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) .gather(.{ .token = aligned_activated.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (down_.bias) |bias| { + if (opts.w2_bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); down_out = down_out.add(bias_per_token); } diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index ba723234..e33d2a14 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -1,18 +1,13 @@ //! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. //! Weights use [expert, output, input] order with concatenated gate/up columns. const std = @import("std"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; -pub fn activate(x: Tensor, activation: Activation) Tensor { +pub fn activate(x: Tensor, activation: zml.moe.Activation) Tensor { return switch (activation) { .silu => x.silu(), .gelu => x.gelu(), @@ -153,28 +148,23 @@ fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: } pub const Options = struct { - activation: Activation, + activation: zml.moe.Activation, quantize_input: bool = false, /// Model fixed backend contracts independently of optional input quantization: /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, - routing_weight_placement: RoutingWeightPlacement = .after_down, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, }; -/// Negative expert IDs denote nonlocal routes and contribute zero, including bias. pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { - stdx.debug.assert(topk_ids.dim(.topk) > 0, "reference top-k must be positive", .{}); stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); const tokens = input.dim(.t); const k = topk_ids.dim(.topk); - // Nonlocal routes use expert zero for safe gathers; discard their full - // contribution after the down projection, including any projection bias. - const local_routes = topk_ids.cmp(.GE, Tensor.scalar(0, .i32)); - const safe_ids = topk_ids.maximum(Tensor.scalar(0, .i32)); - const ids = safe_ids.reshape(.{ .route = tokens * k }); + stdx.debug.assert(k > 0 and k <= gate_up.weight.dim(.expert), "reference top-k must fit the expert count", .{}); + const ids = topk_ids.reshape(.{ .route = tokens * k }); const weights = topk_weights.reshape(.{ .route = tokens * k }).convert(.f32); const hidden = input.reshape(.{ .token = tokens, .in = input.dim(.d) }) .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = tokens, .topk = k, .in = input.dim(.d) }, input.dtype())) @@ -186,15 +176,13 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); - output = local_routes.reshape(.{ .route = tokens * k }).broad(output.shape()).select(output, Tensor.scalar(0, output.dtype())); return output.convert(input.dtype()).convert(.f32) .reshape(.{ .t = tokens, .topk = k, .d = output.dim(.out) }) .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { - stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); - return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ +pub fn fusedExperts(input: Tensor, ids: Tensor, weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { + return reference(input, ids, weights, gate_up, down, .{ .activation = opts.activation, .quantize_input = opts.quantize_input, .routing_weight_placement = opts.routing_weight_placement, diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel index 44b56e08..4057eea3 100644 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -2,8 +2,5 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "tcl", - srcs = [ - "moe.zig", - "tcl.zig", - ], + srcs = ["tcl.zig"], ) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig deleted file mode 100644 index 23fb2274..00000000 --- a/zml/kernels/moe/tcl/moe.zig +++ /dev/null @@ -1,457 +0,0 @@ -//! TCL kernel schedules for Furiosa RNGD fused experts. -const std = @import("std"); - -const builder = @import("dsl/tcl/builder"); - -const zml = @import("../../../zml.zig"); -const tcl = zml.kernel.tcl; -const ActivationKind = @import("../api.zig").ActivationKind; - -pub const Format = enum { bf16, fp8_block128 }; - -/// Dispatch in `tcl.zig` uses the first matching schedule. -/// -/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected -/// expert on the whole batch to reuse its weights, with zero routing weight -/// for unrouted pairs. -/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather -/// weights for each route to avoid the overhead of grouping routes. -/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to -/// reuse weights within each block, then gather results back by route. -pub const Schedule = enum { per_route, blockwise, small_batch }; - -pub const Cfg = struct { - num_tokens: i64, - top_k: i64, - local_num_experts: i64, - hidden_size: i64, - expert_intermediate_size: i64, - weight_scale_dtype: tcl.DType = .bf16, - activation: ActivationKind = .swiglu, -}; - -const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; -const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; - -pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { - if (schedule == .small_batch and format != .fp8_block128) { - @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); - } - return tcl.Kernel(Cfg, .{ - .name = "moe_" ++ @tagName(schedule) ++ switch (format) { - .bf16 => "", - .fp8_block128 => "_fp8", - }, - .inputs = switch (format) { - .bf16 => base_inputs, - .fp8_block128 => base_inputs ++ scale_inputs, - }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, schedule); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, format); - switch (schedule) { - .per_route => try perRoute(b, a, in, cfg.activation, format), - .blockwise => try blockwise(b, a, in, cfg, format), - .small_batch => try fp8SmallBatch(b, a, in, cfg), - } - } - }.run, - }); -} - -const Axes = struct { - T: tcl.Axis, - K: tcl.Axis, - E: tcl.Axis, - D: tcl.Axis, - I: tcl.Axis, - I2: tcl.Axis, - S2: tcl.Axis, - S1: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.num_tokens), - .K = b.axis("K", cfg.top_k), - .E = b.axis("E", cfg.local_num_experts), - .D = b.axis("D", cfg.hidden_size), - .I = b.axis("I", cfg.expert_intermediate_size), - .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), - .S2 = b.axis("S2", 2), - .S1 = b.axis("S1", 1), - }; - } -}; - -fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { - const weight_dtype: tcl.DType = switch (format) { - .bf16 => .bf16, - .fp8_block128 => .f8_e4, - }; - const spec = .{ - .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, - .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, - .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, - }; - if (format == .bf16) { - const t = try b.declareArgs(spec); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; - } - const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); - const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); - const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); - const t = try b.declareArgs(.{ - .x = spec.x, - .ids = spec.ids, - .w = spec.w, - .gate_up = spec.gate_up, - .down = spec.down, - .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, - .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, - }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; -} - -const Args = struct { - x: builder.Tensor, - ids: builder.Tensor, - w: builder.Tensor, - gate_up: builder.Tensor, - down: builder.Tensor, - gate_up_scale: ?builder.Tensor = null, - down_scale: ?builder.Tensor = null, -}; - -const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; - -fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { - if (format == .bf16) return .{}; - const IG = b.axis("IG", @divExact(a.I.size, 128)); - const DG = b.axis("DG", @divExact(a.D.size, 128)); - const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); - var gu_axes: [8]tcl.Axis = undefined; - var dn_axes: [8]tcl.Axis = undefined; - @memcpy(gu_axes[0..batch.len], batch); - @memcpy(dn_axes[0..batch.len], batch); - @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); - @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); - return .{ - .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), - .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), - }; -} - -fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { - var n: usize = 0; - for (axes) |a| { - if (std.mem.eql(u8, a.name, axis.name)) { - storage[n] = group; - n += 1; - storage[n] = lane; - } else storage[n] = a; - n += 1; - } - return storage[0..n]; -} - -/// Block scales multiply each K-block partial before K-block reduction. -/// Only the requested experts are gathered; their bank remains FP8 in HBM. -fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { - if (format == .bf16) { - const op = b.tensorOperation(.{}); - return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); - } - const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); - const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); - const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); - const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); - var x_axes: [16]tcl.Axis = undefined; - var w_axes: [16]tcl.Axis = undefined; - var w2_axes: [16]tcl.Axis = undefined; - var out_axes: [16]tcl.Axis = undefined; - var partial_axes: [16]tcl.Axis = undefined; - const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); - const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); - const out = blockedAxes(to, output, OG, OL, &out_axes); - @memcpy(partial_axes[0..out.len], out); - partial_axes[out.len] = CG; - - // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. - var op = b.tensorOperation(.{}); - const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); - op = b.tensorOperation(.{}); - // Native F32 reciprocal accuracy is qualified separately (within one ULP). - const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); - const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); - op = b.tensorOperation(.{}); - const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); - return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); -} - -/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` -/// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { - var sliced: [4]tcl.Axis = undefined; - var flat: [3]tcl.Axis = undefined; - @memcpy(sliced[0..batch.len], batch); - @memcpy(flat[0..batch.len], batch); - sliced[batch.len] = a.S1; - sliced[batch.len + 1] = a.I; - flat[batch.len] = a.I; - const s = sliced[0 .. batch.len + 2]; - const f = flat[0 .. batch.len + 1]; - const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); - const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // The builder can infer the TCL `Interleaving` tactic for two inputs with - // matching axes, such as `gate` and `up`. That tactic allows exactly one - // binary instruction, so force `.elementwise` for this activation chain. - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const g = op.fetch(gate, .{}); - const activated = if (format == .fp8_block128) blk: { - if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); - - // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal - // BF16 SiLU write, even through as_dram/as_logical. Make its RNE - // rounding observable as integer bits before multiplying by up. - // Remove once the native reproducer preserves the BF16 boundary. - const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const v = op.fetch(bits, .{}); - const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); - const silu = try op.commit(rounded, .{ .dtype = .bf16 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - break :blk op.fetch(silu, .{}); - } else switch (activation) { - .swiglu => g.sigmoid().mulf(g), - .geglu_tanh => geluTanh(g), - else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), - }; - var v = activated.mulf(op.fetch(up, .{})); - if (weights) |w| v = v.mulf(op.fetch(w, .{})); - return op.commit(v, .{ .dtype = .bf16 }); -} - -fn geluTanh(g: builder.Value) builder.Value { - const cube = g.mulf(g).mulf(g); - const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); - return inner.tanh().addf(1.0).mulf(g).mulf(0.5); -} - -/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): -/// Use IO-bound tactics for per-route and FP8 small-batch schedules and -/// compute-bound tactics for blockwise. -/// Keep the compiler's default lowering mode and omit the adaptive-einsum and -/// half-VRF settings removed in TCC 2026.5. -fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { - const config = comptime blk: { - const common = .{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .propagate_sparse_axis_from_op = "None", - }; - const tactics = switch (schedule) { - .blockwise => .{ - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, - .dma_preference = 1.2, - }, - .per_route, .small_batch => .{ - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - }, - }; - // The transaction-simulation setting is only emitted for IO-bound schedules. - const common_info = @typeInfo(@TypeOf(common)).@"struct"; - const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; - const Config = @Struct( - .auto, - null, - common_info.field_names ++ tactics_info.field_names, - common_info.field_types ++ tactics_info.field_types, - common_info.field_attrs ++ tactics_info.field_attrs, - ); - var merged: Config = undefined; - for (common_info.field_names) |field| @field(merged, field) = @field(common, field); - for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); - break :blk merged; - }; - b.compilerConfig(config); -} - -/// Gather the selected experts for every token. -fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - - const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); - const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); - if (format == .fp8_block128) { - const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); - } else { - // The BF16 recipe weights activations and contracts experts together. - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } -} - -/// Rows per expert block: the next power of two of the routes per expert. -fn blockRows(cfg: Cfg) i64 { - const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); -} - -const blocks_per_chunk = 32; - -/// Group routes into expert-major blocks, then run both projections in a -/// device loop over chunks of blocks. -fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. - const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); - op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); - op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); - - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); - { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); - const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); - const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); - const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); -} - -/// Number of experts processed per device-loop iteration, based on the -/// batch's route count (`cfg.num_tokens * cfg.top_k`). -fn expertsPerChunk(cfg: Cfg) i64 { - const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); - return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); -} - -/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. -/// Load each selected expert once and run it for every token. Routing weights -/// scale the F32 down-projection results before summing the experts and steps. -fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { - const chunk = expertsPerChunk(cfg); - const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; - const C = b.axis("C", chunk); - const Go = b.axis("Go", steps); - const J = b.axis("J", steps * chunk); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Each token's routing weight per expert, and the used experts. - const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); - op = b.tensorOperation(.{}); - const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); - // A used expert's rank among them, -1 for the others. - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - - // Slot j holds the used expert of rank j and its tokens' weights; - // slots past the last hold expert 0 with no weight. - const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); - op = b.tensorOperation(.{}); - const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); - // A reduction's condition is read plainly, so broadcast it first. - op = b.tensorOperation(.{}); - const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); - op = b.tensorOperation(.{}); - const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); - const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); - const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); - { - const ids = try b.indexRead(step_experts, loop.iv); - const weights = try b.indexRead(step_gate, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); - const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); - const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); - const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); - op = b.tensorOperation(.{}); - const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); -} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index f05727ce..4b7bcfb3 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -1,165 +1,270 @@ //! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. +//! +//! Few tokens (at most `experts / top_k`): activation stationary, each token +//! gathers the weights of its experts and the down projection contracts over +//! the experts and the intermediate axis at once. +//! +//! More tokens: blockwise. The kernel routes on the device: routes are placed +//! in expert-major blocks of `BB` rows (one-hot, cumulative sums), then a +//! device loop over the used chunks of `BGi` blocks gathers each block's +//! expert weights and runs both projections; outputs are gathered back per +//! route and weighted. const std = @import("std"); +const builder = @import("dsl/tcl/builder"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const kernels = @import("moe.zig"); -const api = @import("../api.zig"); -/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` -/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is -/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with -/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. -pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { - const topk_ids = routing.localIds(); - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); +/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` +/// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). +/// SwiGLU, routing weights applied after the down projection. +pub fn fusedExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); + stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); + stdx.debug.assert(gate_up_linear.quantization == null and down_linear.quantization == null, "TCL MoE takes bf16 experts", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); - stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); - const format = expertFormat(gate_up_linear, down_linear); - const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); - stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); - const gate_up = gate_up_linear.weight; const down = down_linear.weight; - const cfg: kernels.Cfg = .{ - .num_tokens = input.dim(.t), - .top_k = topk_ids.dim(.topk), - .local_num_experts = gate_up.dim(.expert), - .hidden_size = input.dim(.d), - .expert_intermediate_size = down.dim(.dout), - .weight_scale_dtype = switch (format) { - .bf16 => .bf16, - .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), - }, - .activation = std.meta.activeTag(opts.activation), + stdx.debug.assert(input.dtype() == .bf16 and gate_up.dtype() == .bf16 and down.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes bf16 activations and experts, i32 ids", .{}); + const t = input.dim(.t); + const k = topk_ids.dim(.topk); + const cfg: Cfg = .{ + .t = t, + .k = k, + .e = gate_up.dim(.expert), + .d = input.dim(.d), + .i = down.dim(.dout), }; - stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); - const weights = routing.weights.convert(.f32); - const local_weights = switch (format) { - .bf16 => weights, - .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), - }; - return switch (format) { - .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), - .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ - .x = input, - .ids = topk_ids, - .w = local_weights, - .gate_up = gate_up, - .down = down, - .gate_up_scale = s.gate_up, - .down_scale = s.down, - }), - }; + const x = input.reshape(.{ t, cfg.d }); + const ids = topk_ids.reshape(.{ t, k }); + const weights = topk_weights.convert(.f32).reshape(.{ t, k }); + const out_shape = x.shape(); + const y = if (cfg.t * cfg.k <= cfg.e) + ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y + else + Blockwise.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; + return y.reshape(input.shape()); } -const Scales = struct { gate_up: Tensor, down: Tensor }; +pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64 }; -const Format = union(kernels.Format) { - bf16: void, - fp8_block128: Scales, +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.t), + .K = b.axis("K", cfg.k), + .E = b.axis("E", cfg.e), + .D = b.axis("D", cfg.d), + .I = b.axis("I", cfg.i), + .I2 = b.axis("I2", 2 * cfg.i), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } }; -fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { - const scheme = gate_up.quantizationScheme(); - stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); - return switch (scheme orelse { - stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); - return .bf16; - }) { - .fp8_block128 => blk: { - const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; - stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); - break :blk .{ .fp8_block128 = scales }; - }, - else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), - }; +fn args(b: *tcl.Builder, a: Axes) tcl.FinishError!Args { + const t = try b.declareArgs(.{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = .bf16, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = .bf16, .axes = &.{ a.E, a.D, a.I } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; } -fn fp8Scales(linear: zml.nn.Linear) Tensor { - const q = linear.quantization.?; - stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); - stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); - const w = linear.weight; - const scale = q.scales; - stdx.debug.assert(@mod(w.dim(.d), 128) == 0 and @mod(w.dim(.dout), 128) == 0 and scale.dim(.expert) == w.dim(.expert) and scale.dim(.d) * 128 == w.dim(.d) and scale.dim(.dout) * 128 == w.dim(.dout), "TCL MoE FP8 scales must cover 128x128 weight blocks", .{}); - return scale; +const Args = struct { x: builder.Tensor, ids: builder.Tensor, w: builder.Tensor, gate_up: builder.Tensor, down: builder.Tensor }; + +/// `silu(gate) * up` of `h = [batch..., S2, I]` (gate first), times +/// `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // Interleaving requires one binary instruction; SwiGLU chains several. + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + var v = g.sigmoid().mulf(g).mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); } -// Maximum total input tokens per MoE call before padding for the FP8 -// small-batch schedule. It loads each used expert's weights once and reuses -// them across the batch. -// Larger batches fall back to per-route or blockwise dispatch. -const fp8_small_batch_max_tokens = 64; - -fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { - if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { - return run(format, .small_batch, cfg, inputs); - } - if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { - return run(format, .per_route, cfg, inputs); - } - return run(format, .blockwise, cfg, inputs); +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// IO-bound tactics for activation stationary, compute-bound for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, blockwise: bool) void { + if (blockwise) b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + .propagate_sparse_axis_from_op = "None", + }) else b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + .propagate_sparse_axis_from_op = "None", + }); } -fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { - const call_cfg, const prepared = if (schedule == .small_batch) - prepareFp8SmallBatch(cfg, inputs) - else - .{ cfg, prepareRoutedInputs(cfg, inputs) }; +pub const ActivationStationary = tcl.Kernel(Cfg, .{ + .name = "moe_activation_stationary", + .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, false); + const a: Axes = .init(b, cfg); + const in = try args(b, a); + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - const shape = prepared.x.shape(); - var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; + var op = b.tensorOperation(.{}); + const h = try op.commit(op.contract(in.x, gu, &.{ a.T, a.K, a.S2, a.I }), .{}); + // Routing weights scale the activations, so the down projection + // also sums over the experts. + const act = try gated(b, a, h, &.{ a.T, a.K }, in.w); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } + }.run, +}); - if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); - return y.reshape(inputs.x.shape()); +/// Rows per expert block: the next power of two of the routes per expert. +pub fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.t * cfg.k, cfg.e) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); } -fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { - var prepared = inputs; - prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); - const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); - const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); - // These kernels gather every route, and blockwise also scatters each route - // into an expert slot. Give nonlocal routes a valid slot and zero weight. - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); - prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); - prepared.w = valid.select(w, Tensor.scalar(0, .f32)); - return prepared; -} +const blocks_per_chunk = 32; -fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { - // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two - // panics its adder tree ("Fail to cover valid range", 5 tokens padded - // to 8): pad with unrouted tokens. - var padded = cfg; - padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); - const pad = padded.num_tokens - cfg.num_tokens; - var prepared = inputs; - prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); - prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); - prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); - return .{ padded, prepared }; -} +pub const Blockwise = tcl.Kernel(Cfg, .{ + .name = "moe_blockwise", + .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, true); + const a: Axes = .init(b, cfg); + const in = try args(b, a); + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(.bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + op = b.tensorOperation(.{}); + const h = try op.commit(op.contract(chunk, gu, &.{ BGi, BB, a.S2, I }), .{}); + const act = try gated(b, a, h, &.{ BGi, BB }, null); + op = b.tensorOperation(.{}); + const y = try op.commit(op.contract(act, dn, &.{ BGi, BB, D }), .{ .dtype = .bf16 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); + } + }.run, +}); test "tcl fused experts run on furiosa" { - try testFusedExperts(.{ .swiglu = .{} }); - try testFusedExperts(.geglu_tanh); -} - -fn testFusedExperts(activation: zml.moe.Activation) !void { const platform = zml.testing.env(); if (platform.target != .furiosa) return error.SkipZigTest; const allocator = std.testing.allocator; const io = std.testing.io; const bf16 = zml.floats.BFloat16; - // With two routes per token and eight experts, four tokens use per-route - // dispatch and sixteen use blockwise dispatch. + // 2 * 4 <= 8 experts is activation stationary, 16 tokens is blockwise. for ([_]i64{ 4, 16 }) |t| { const k = 2; const e = 8; @@ -171,15 +276,15 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { - return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ - .activation = act, + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { + return fusedExperts(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .{ + .activation = .{ .swiglu = .{} }, .quantize_input = false, .routing_weight_placement = .after_down, }); } }; - var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); + var exe = try zml.module.compile(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, platform, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); @@ -231,12 +336,7 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); } - const gate = switch (activation) { - .swiglu => g / (1 + @exp(-g)), - .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), - else => unreachable, - }; - act[n] = gate * u; + act[n] = g / (1 + @exp(-g)) * u; } for (0..d) |m| { var y: f32 = 0; diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index c74406c3..2bdebdfe 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,15 +5,12 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const Activation = @import("api.zig").Activation; -const ExpertsLayout = @import("api.zig").ExpertsLayout; -const GlobalOptions = @import("api.zig").Options; -const Placement = @import("api.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); +const Placement = @import("moe.zig").RoutingWeightPlacement; const Case = struct { backend: zml.moe.Backend, - activation: Activation = .{ .swiglu = .{} }, + activation: zml.moe.Activation = .{ .swiglu = .{} }, scheme: ?zml.Quantization.Scheme = null, activation_dtype: zml.DataType = .bf16, quantize_input: bool = false, @@ -25,22 +22,15 @@ const Case = struct { width: i64 = 128, intermediate: i64 = 128, experts: i64 = 8, - expert_mesh: *const zml.Mesh, - - fn inputShape(c: Case) Shape { - const shape = Shape.init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); - const spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; - return shape.withPartitioning(c.expert_mesh, spec); - } pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { std.debug.assert(c.backend.isAvailable(platform)); - const x: Tensor = .fromShape(c.inputShape()); - return platform.compileFn(allocator, io, forward, .{ x, c }, .{ .meshes = &.{c.expert_mesh} }); + const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + return platform.compileFn(allocator, io, forward, .{ x, c }, .{}); } pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { - const x: Tensor = .fromShape(c.inputShape()); + const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); var host = try zml.Slice.alloc(allocator, x.shape()); defer host.free(allocator); @@ -62,7 +52,6 @@ const Case = struct { var output: zml.Bufferized(Outputs) = undefined; runner.run(io, .{input}, .{&output}, .{ .wait = true }); defer zml.Buffer.deinitAll(Outputs, &output); - try std.testing.expectEqualDeep(c.inputShape()._sharding, output.actual.shape()._sharding); // BF16 intermediate rounding and different GEMM reduction orders. Every // element must pass; near-zero values use the absolute bound. @@ -179,16 +168,20 @@ fn makeExpertFixture(c: Case) ExpertFixture { return fixture; } -fn interleave(tensor: Tensor, rows: i64) Tensor { - return tensor.reshape(.{ .expert = tensor.dim(0), .projection = 2, .row_block = @divExact(tensor.dim(1), 2 * rows), .row = rows, .column = tensor.dim(2) }) - .transpose(.{ .expert, .row_block, .projection, .row, .column }).reshape(tensor.shape()); +/// Interleave the gate/up halves of the output axis: [E, 2, N/2, K] -> [E, N/2, 2, K]. +fn interleave(tensor: Tensor) Tensor { + const tags = tensor.shape().tags(); + return tensor.withTags(.{ .expert, .out, .in }) + .splitAxis(.out, .{ .projection = 2, .mid = @divExact(tensor.dim(1), 2) }) + .transpose(.{ .expert, .mid, .projection, .in }) + .merge(.{ .out = .{ .mid, .projection } }).withTags(tags); } /// Blackwell block scales: [E, N/128, 4, 32, K/4, 4] -> /// [E, N/128, K/4, 32, 4, 4], retaining the declared [E, N, K] shape. fn swizzleScales(scales: Tensor) Tensor { - return scales.reshape(.{ .expert = scales.dim(0), .row_block = @divExact(scales.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(scales.dim(2), 4), .scale_byte = 4 }) - .transpose(.{ .expert, .row_block, .scale_word, .row_lane, .row_quad, .scale_byte }).reshape(scales.shape()); + return scales.reshape(.{ scales.dim(0), @divExact(scales.dim(1), 128), 4, 32, @divExact(scales.dim(2), 4), 4 }) + .transpose(.{ 0, 1, 4, 3, 2, 5 }).reshape(scales.shape()); } fn packFp4Bytes(weight: Tensor) Tensor { @@ -196,11 +189,10 @@ fn packFp4Bytes(weight: Tensor) Tensor { return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); } -fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { +fn prepareGateUp(tensor: Tensor, layout: zml.moe.ExpertsLayout) Tensor { return switch (layout.gate_up) { .concatenated => tensor, - .row_interleaved_1 => interleave(tensor, 1), - .row_interleaved_8 => interleave(tensor, 8), + .interleaved => interleave(tensor), }; } @@ -218,29 +210,17 @@ fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertF } for ([_]*zml.nn.Linear{ &prepared.gate_up, &prepared.down }) |projection| { - switch (layout.packing) { - .swizzled_scales, .flashinfer_nvfp4 => { - const q = &projection.quantization.?; - q.scales = swizzleScales(q.scales); - q.swizzled_scales = true; - }, - .mxfp4_interleaved8 => { - const q = &projection.quantization.?; - const sf = q.scales; - q.scales = sf.bitCast(.u8) - .reshape(.{ .expert = sf.dim(0), .row_block = @divExact(sf.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(sf.dim(2), 4), .scale_byte = 4 }) - .transpose(.{ .expert, .scale_word, .row_block, .row_lane, .row_quad, .scale_byte }) - .reshape(.{ .expert = sf.dim(0), .scale_word = @divExact(sf.dim(2), 4), .row = sf.dim(1), .scale_byte = 4 }).bitCast(.i32) - .withTags(sf.shape().transpose(.{ 0, 2, 1 })); - }, - .plain => {}, + if (layout.packing == .swizzled_scales or layout.packing == .flashinfer_nvfp4) { + const q = &projection.quantization.?; + q.scales = swizzleScales(q.scales); + q.swizzled_scales = true; } switch (backend) { .cute => { projection.weight = packFp4Bytes(projection.weight); const q = &projection.quantization.?; + q.scales = q.scales.bitCast(.u8); if (scheme == .nvfp4) { - q.scales = q.scales.bitCast(.u8); // CuTe takes the input quantization multiplier and a GEMM // output multiplier that folds in its inverse. const input_scale = q.input_scale.?.asMultiplier(); @@ -283,26 +263,14 @@ fn forward(input: Tensor, c: Case) Outputs { const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); - const options: GlobalOptions = .{ + const options: zml.moe.Options = .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, }; - const data_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; - const expert_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("experts") != null) .parse(.{ .expert = .experts }) else .replicated; - const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, c.expert_mesh, .{ - .in_specs = .{ - .input = data_spec, - .ids = data_spec, - .weights = data_spec, - .gate_up = expert_spec, - .down = expert_spec, - }, - .out_specs = data_spec, - }, options); - // Keep the oracle input replicated so token sharding does not change its GEMM rounding. - const expected = reference.reference(input.replicate(), ids, weights, canonical.gate_up, canonical.down, .{ + const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, options); + const expected = reference.reference(input, ids, weights, canonical.gate_up, canonical.down, .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, @@ -317,7 +285,7 @@ fn forward(input: Tensor, c: Case) Outputs { return .{ .actual = actual, .expected = expected }; } -const gated_activations = [_]Activation{ +const gated_activations = [_]zml.moe.Activation{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .bias = 1 } }, @@ -327,8 +295,8 @@ const gated_activations = [_]Activation{ .geglu, .geglu_tanh, }; -const all_activations = gated_activations ++ [_]Activation{ .relu, .silu, .gelu }; -const clipped_swiglu = [_]Activation{ +const all_activations = gated_activations ++ [_]zml.moe.Activation{ .relu, .silu, .gelu }; +const clipped_swiglu = [_]zml.moe.Activation{ .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .limit = 7 } }, }; @@ -366,7 +334,7 @@ const Format = struct { const Matrix = struct { formats: []const Format = &.{.{}}, - activations: []const Activation, + activations: []const zml.moe.Activation, quantize_inputs: []const bool = &.{false}, placements: []const Placement = &.{.after_down}, biases: []const bool = &.{false}, @@ -378,7 +346,7 @@ const Matrix = struct { } /// Shapes vary fastest, matching the nested loops in the backend tests. - pub fn caseAt(self: Matrix, backend: zml.moe.Backend, expert_mesh: *const zml.Mesh, index: usize) Case { + pub fn caseAt(self: Matrix, backend: zml.moe.Backend, index: usize) Case { std.debug.assert(index < self.count()); var remaining = index; const shape = self.shapes[remaining % self.shapes.len]; @@ -406,7 +374,6 @@ const Matrix = struct { .width = shape.width, .intermediate = shape.intermediate, .experts = shape.experts, - .expert_mesh = expert_mesh, }; } }; @@ -447,7 +414,7 @@ const Pipeline = struct { var index: usize = 0; for (0..matrix.count()) |local_index| { - const c = matrix.caseAt(backend, self.platform.replicated_mesh, local_index); + const c = matrix.caseAt(backend, local_index); if (available) { case_queue.putOne(io, .{ .index = index, .case = c }) catch |err| return switch (err) { error.Closed => {}, @@ -624,7 +591,6 @@ test "Triton MoE prefill launch buckets fit shared memory" { for ([_]i64{ 8, 32, 96, 128, 256, 512, 1024, 2048, 3072, 4096 }) |tokens| { const case: Case = .{ .backend = .triton, - .expert_mesh = platform.replicated_mesh, .tokens = tokens, .width = 256, .intermediate = 128, diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 6e4006fa..3eb457da 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,7 +12,6 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); @@ -1027,7 +1026,7 @@ pub const Combine = struct { topk: i64, expert_offset: i64, experts: i64, - routing_weight_placement: RoutingWeightPlacement = .before_down, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, bn: i64, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); @@ -1093,7 +1092,7 @@ const GemmConfig = struct { // Swiglu clamping. Set to 0 to disable. swiglu_limit: f32, scale_stride: i64, - routing_weight_placement: RoutingWeightPlacement = .before_down, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, bn: i64, bk: i64, }; diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 8ba799a5..9d3a5529 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -6,14 +6,10 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const toDType = zml.kernel.triton.from; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; const kernels = @import("moe.zig"); +const toDType = zml.kernel.triton.from; +const ProjectionLayout = zml.moe.ProjectionLayout; +const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton"); @@ -39,38 +35,109 @@ test { pub fn fusedExperts( input: zml.Tensor, - routing: LocalRouting, + ids: zml.Tensor, + weights: zml.Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: Options, + options: zml.moe.Options, ) zml.Tensor { // HACK(Corentin): only rely on the scheme of gate_up for now. if (gate_up.quantization) |q| switch (q.scheme) { .mxfp4 => if (zml.Compiler.current().platform.target == .cuda) { - return fusedExpertsMxfp4(input, routing, gate_up, down, options); + return fusedExpertsMxfp4(input, ids, weights, gate_up, down, options); }, else => {}, }; - const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; - return fusedExpertsBase(input, routing, gate_up, down, layout, options); + return fusedExpertsBase(input, ids, weights, gate_up, down, options); } pub fn fusedExpertsBase( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, +) zml.Tensor { + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) + .{ .gate_up = .interleaved, .packing = .plain } + else + .{ .gate_up = .concatenated, .packing = .plain }; + + const args: FusedExpertsArgs = .{ + .hidden_states = input, + .gate_up = gate_up, + .down = down, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .activation = opts.activation, + .quantize_input = opts.quantize_input, + .gate_up_layout = layout.gate_up, + .routing_weight_placement = opts.routing_weight_placement, + }; + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (!expert_partition.isSharded()) { + return fusedExpertsImpl(args); + } + + return zml.ops.manualComputation( + (struct { + args: FusedExpertsArgs, + global_num_experts: i64, + + fn call(self: @This(), _: zml.Shape) zml.Tensor { + const local_args = self.args; + const local_num_experts = local_args.gate_up.weight.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + // Map global expert ids to local ids, or -1 for experts outside this partition. + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + var mapped_args = local_args; + mapped_args.expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + + const local_output = fusedExpertsImpl(mapped_args); + const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).call, + .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, + input.shape(), + ); +} + +pub const FusedExpertsArgs = struct { hidden_states: Tensor, - routing_: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - layout: ProjectionLayout, - opts: Options, -) Tensor { - const expert_map_ = routing_.expertMap(); + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + topk_weights: Tensor, + topk_ids: Tensor, + activation: zml.moe.Activation, + expert_map: ?Tensor = null, + /// Use FP8 activations for FP8 weights; false keeps BF16 activations. + quantize_input: bool, + gate_up_layout: ProjectionLayout, + routing_weight_placement: RoutingWeightPlacement, +}; + +pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { + const hidden_states = opts.hidden_states; + const topk_weights = opts.topk_weights; + const topk_ids = opts.topk_ids; const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = gate_up_.quantizationScheme(); - const down_scheme = down_.quantizationScheme(); + const gate_up_scheme = opts.gate_up.quantizationScheme(); + const down_scheme = opts.down.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("triton backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -87,10 +154,10 @@ pub fn fusedExpertsBase( }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); - const down = down_.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); + const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "triton backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -118,13 +185,13 @@ pub fn fusedExpertsBase( stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "triton backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "triton backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "triton backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (expert_map_) |expert_map| + if (opts.expert_map) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "triton backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (expert_map_) |expert_map| + const expert_ids = if (opts.expert_map) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -134,9 +201,9 @@ pub fn fusedExpertsBase( const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = gate_up_.bias, + .bias = opts.gate_up.bias, .input_scale = input_scale, - .weight_scale = gate_up_.quantizationScales(), + .weight_scale = opts.gate_up.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -145,7 +212,7 @@ pub fn fusedExpertsBase( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = opts.activation.applyActivation(gate_up_out, layout); + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -155,9 +222,9 @@ pub fn fusedExpertsBase( const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = down_.bias, + .bias = opts.down.bias, .input_scale = down_input_scale, - .weight_scale = down_.quantizationScales(), + .weight_scale = opts.down.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -494,14 +561,14 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const io = std.testing.io; const platform = zml.testing.env(); const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, activation: Activation) Tensor { - return activation.applyActivation(x, layout); + fn forward(x: Tensor, layout: ProjectionLayout, activation: zml.moe.Activation) Tensor { + return zml.moe.applyActivation(x, activation, layout); } }; const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); const gates = [_]f32{ 0.75, -1.25, 3.5 }; const ups = [_]f32{ 0.875, -2.25, 4.5 }; - for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { var values: [6]zml.floats.BFloat16 = undefined; for (gates, ups, 0..) |gate, up, i| { values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); @@ -510,7 +577,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { - const activation: Activation = .{ .swiglu = .{ .limit = threshold } }; + const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); @@ -666,8 +733,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); const is_gate = switch (layout) { .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), - .row_interleaved_1 => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), - .row_interleaved_8 => columns.divByConst(8).remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), }; const dtype: DataType = if (fp4) .u8 else .bf16; // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. @@ -699,12 +765,13 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { } const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .t = 1, .topk = 2 }) .broad(Shape.init(.{ .t = x.dim(.t), .topk = 2 }, .i32)); - return fusedExpertsBase(x, .{ - .ids = route.addConstant(2), - .weights = route.convert(.f32).addConstant(1).scale(0.25), - .global_num_experts = 8, - .local_num_experts = 8, - }, gate_up, down, layout, .{ + return fusedExpertsImpl(.{ + .hidden_states = x, + .gate_up = gate_up, + .down = down, + .topk_ids = route.addConstant(2), + .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), + .gate_up_layout = layout, .routing_weight_placement = placement, .quantize_input = false, .activation = .{ .swiglu = .{} }, @@ -719,7 +786,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); var input = try zml.Buffer.fromSlice(io, platform, host); defer input.deinit(); - for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); defer exe.deinit(); @@ -799,13 +866,61 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn validateOptions(opts: Options) void { +pub fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "triton backend requires swiglu scale to be null", .{}); } +/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. +/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. +pub fn fusedExpertsMxfp4( + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: zml.moe.Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); + + const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + + const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); + + // Note: we directly forward input sharding to manualComputation + // TODO: improve sharding in attention and moe APIs + // https://github.com/zml/zml/issues/914 + const context: Context = .{ + .input = input, + .ids = ids, + .weights = weights, + .gate_up = gate_up.weight, + .gate_up_scales = gq.scales, + .down = down.weight, + .down_scales = dq.scales, + .global_experts = gate_up.weight.dim(.expert), + .topk = ids.dim(.topk), + .limit = options.activation.swiglu.limit.?, + .routing_weight_placement = options.routing_weight_placement, + .expert_parallel = expert_parallelism, + }; + + return if (expert_parallelism) + zml.ops.manualComputation(Context.body, context, input.shape()) + else + context.body(input.shape()); +} + pub const Config = struct { tokens: i64, hidden: i64, @@ -897,176 +1012,178 @@ fn validateInputs( ); } -/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. -/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. -pub fn fusedExpertsMxfp4( +const Context = struct { input: zml.Tensor, - routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - options: Options, -) zml.Tensor { - validateOptions(options); - stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); - stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.Tensor, + gate_up_scales: zml.Tensor, + down: zml.Tensor, + down_scales: zml.Tensor, + global_experts: i64, + topk: i64, + limit: f32, + routing_weight_placement: RoutingWeightPlacement, + expert_parallel: bool, - const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); - const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + fn body(self: Context, _: zml.Shape) zml.Tensor { + const experts = self.gate_up.dim(.expert); + const hidden = self.down.dim(1); + const intermediate = self.down.dim(2) * 2; + var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); + const tokens = ids.dim(.token); + if (self.expert_parallel) { + const partition_id = zml.ops.partitionId().convert(.i32); + ids = ids.sub(partition_id.scale(experts)); + } + const x = self.input.reshape(.{ tokens, hidden }); + const scales = self.weights.convert(.f32).reshape(.{ tokens, self.topk }); - // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + const cfg: Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = self.global_experts, + .topk = self.topk, + .swiglu_limit = self.limit, + .routing_weight_placement = self.routing_weight_placement, + }; - const topk = routing.ids.dim(.topk); - const experts = gate_up.weight.dim(.expert); - const hidden = down.weight.dim(1); - const intermediate = down.weight.dim(2) * 2; - const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); - const tokens = ids.dim(.token); + cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); + validateInputs(cfg, x, ids, scales, self.gate_up, self.gate_up_scales, self.down, self.down_scales); - const x = input.reshape(.{ tokens, hidden }); - const scales = routing.weights.convert(.f32).reshape(.{ tokens, topk }); + const bn = 16; + const bk = 512; - const cfg: Config = .{ - .tokens = tokens, - .hidden = hidden, - .intermediate = intermediate, - .experts = experts, - .global_experts = routing.global_num_experts, - .topk = topk, - .swiglu_limit = options.activation.swiglu.limit.?, - .routing_weight_placement = options.routing_weight_placement, - }; + const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); - cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); - validateInputs(cfg, x, ids, scales, gate_up.weight, gq.scales, down.weight, dq.scales); - - const bn = 16; - const bk = 512; - - const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); - - // Quantize activations - const q = kernels.Quant.Kernel.call( - .{ .x = x }, - .{ - .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), - .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), - }, - .{ - .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, - .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - const dummy = zml.Tensor.scalar(0, .i32); - var map = dummy; - var perm = dummy; - var sched = dummy; - - if (cfg.tokens != 1) { - const r = kernels.Route.Kernel.call( - .{ .ids = ids }, - .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, - .{ .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, - ); - - const s = kernels.Schedule.Kernel.call( - .{ .ids = ids, .counts = r.counts, .pos = r.pos }, - .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, + // Quantize activations + const q = kernels.Quant.Kernel.call( + .{ .x = x }, .{ - .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .capacity = tile_count, - .group = bn, - }, - .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, + .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), + }, + .{ + .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, + .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, .num_warps = 4, .num_stages = 1, }, ); - map = s.map; - perm = s.perm; - sched = s.sched; + const dummy = zml.Tensor.scalar(0, .i32); + var map = dummy; + var perm = dummy; + var sched = dummy; + + if (cfg.tokens != 1) { + const r = kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, + .{ .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + + const s = kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .capacity = tile_count, + .group = bn, + }, + .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + map = s.map; + perm = s.perm; + sched = s.sched; + } + + // Each narrow tile produces 64 activations: two complete scale groups. + const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; + const m = kernels.GateUp.Kernel.call( + .{ .q = q.q, .s = q.s, .w = self.gate_up.bitCast(.u8), .ws = self.gate_up_scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = self.gate_up_scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), + }, + ); + + const d = kernels.Down.Kernel.call( + .{ .mid = m.mid, .ms = m.ms, .w = self.down.bitCast(.u8), .ws = self.down_scales.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = self.down_scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), + }, + ); + + const result = kernels.Combine.Kernel.call( + .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, + .{ .y = x.shape() }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + }, + .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ).y.reshape(self.input.shape().dims()).withTags(self.input.shape()); + + return if (self.expert_parallel) + zml.ops.allReduce(result, zml.Tensor.add) + else + result; } - - // Each narrow tile produces 64 activations: two complete scale groups. - const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; - const m = kernels.GateUp.Kernel.call( - .{ .q = q.q, .s = q.s, .w = gate_up.weight.bitCast(.u8), .ws = gq.scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, - .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = gq.scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), - }, - ); - - const d = kernels.Down.Kernel.call( - .{ .mid = m.mid, .ms = m.ms, .w = down.weight.bitCast(.u8), .ws = dq.scales.bitCast(.u8), .ids = ids, .sched = sched }, - .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = dq.scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), - }, - ); - - return kernels.Combine.Kernel.call( - .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, - .{ .y = x.shape() }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - }, - .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ).y.reshape(input.shape().dims()).withTags(input.shape()); -} +}; diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig deleted file mode 100644 index 0e288856..00000000 --- a/zml/kernels/moe/weights.zig +++ /dev/null @@ -1,10 +0,0 @@ -const nn = @import("../../nn.zig"); -const Tensor = @import("../../tensor.zig").Tensor; - -pub fn unpackedWeight(linear: nn.Linear) Tensor { - const quantization = linear.quantization orelse return linear.weight; - return if (nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) - nn.unpackFp4(linear.weight, linear.tag, linear.tag) - else - linear.weight; -} diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 587af69f..3e05fed8 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -125,7 +125,7 @@ fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation2(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index ef78cc49..ea4f36ab 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -1,10 +1,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -pub const AttentionOptions = @import("../options.zig").AttentionOptions; -const kernel = @import("attention.zig"); +const paged = @import("../paged_attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); +const kernel = @import("attention.zig"); pub const Options = struct { batch_size: usize, @@ -64,7 +64,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index c0815d1d..aafb8bf1 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -1,7 +1,5 @@ const std = @import("std"); - const zml = @import("zml"); - const cases = @import("cases.zig"); pub const Fixture = struct { diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index afc4298e..af068bad 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -2,6 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); +const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -18,7 +19,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 6645abf1..292e4a9f 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -1,6 +1,7 @@ const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); +const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -17,7 +18,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 12; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md index 8ccc6fd9..5624de0e 100644 --- a/zml/kernels/paged_attention/metal/README.md +++ b/zml/kernels/paged_attention/metal/README.md @@ -10,13 +10,11 @@ transposes queries and caches into the kernels' layouts, launches the the output. Queries and caches are `bf16` or `f16`; other dtypes panic. Causal `bf16` -decode with head dimension 32, 64, 128, 256 or 512 runs `fa_vec_paged`, which -is `bf16` only: a threadgroup holds up to four query heads of a KV head, times -splits of the KV positions, at most 256 threads. Rows split over the positions -are merged by `fa_vec_paged_reduce`; the split count follows the GPU core count -and the number of rows. Everything else, `f16` decode included, runs -`paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, pages of -8, 16 or 32 positions), then `zero_padding_rows`. +decode with head dimension 128, 256 or 512 runs `fa_vec_paged`, which is `bf16` +only, split over four threadgroups and merged by `fa_vec_paged_reduce` at head +dimension 512 past 4096 KV positions. Everything else, `f16` decode included, +runs `paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, +pages of 8, 16 or 32 positions), then `zero_padding_rows`. Causal and non-causal standard masks, left sliding windows, and a custom scale are passed to the kernels. Explicit key indices are rejected, and diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index 2cec68f5..e2d68398 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -3,7 +3,8 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; + +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const paged_attn = @import("paged_attn.zig"); const log = std.log.scoped(.@"zml/attention/metal"); @@ -85,8 +86,8 @@ const DispatchOptions = struct { scale: f32, }; -/// Launches fa_vec_paged for causal bf16 decode and paged_attention_tiled -/// otherwise. +/// Launches fa_vec_paged for causal bf16 decode, split over 4 threadgroups at +/// head_dim 512 past 4096 KV positions, and paged_attention_tiled otherwise. /// `q` and the result are `[tokens, heads, head_dim]`, the caches /// `[pages, block_size, kv_heads, head_dim]`, `block_table` /// `[seqs, max_blocks]`, `seq_lens` `[seqs]` and `query_start_len` @@ -105,7 +106,7 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table const max_blocks: i32 = @intCast(block_table.dim(1)); const vec_decode = opts.is_causal and total_q_tokens == num_seqs and - (head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512) and + (head_dim == 128 or head_dim == 256 or head_dim == 512) and dtype == .bf16; if (vec_decode) { @@ -128,30 +129,35 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table }; const kv_capacity = max_blocks * block_size; - const span = if (opts.sliding_window < 0) kv_capacity else @min(kv_capacity, opts.sliding_window + 32); - const nrows = total_q_tokens * num_heads; - const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), span, coreCount()); + if (head_dim == 512 and kv_capacity > 4096) { + const nwg = 4; + const nrows = total_q_tokens * num_heads; - const cfg: paged_attn.FaVecPaged.Cfg = .{ - .head_dim = head_dim, - .block_size = block_size, - .position_stride = num_kv_heads * head_dim, - .heads = launch.heads, - .splits = launch.splits, - }; - const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); - const out = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = dst }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(total_q_tokens) }, - .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, - .constants = .{ .args = .of(args) }, - }).dst; - if (launch.splits == 1) return out; + const partial = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = zml.Shape.init(.{nrows * (head_dim * nwg + 2 * nwg)}, .f32) }, .{ + .cfg = .{ .head_dim = head_dim, .nsg = 8, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = nwg }, + .grid = .{ nwg, @intCast(num_heads), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, 8, 1 }, + .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, 8), + .constants = .{ .args = .of(args) }, + }).dst; - return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = out, .seq_lens = seq_lens, .cu_seqlens_q = query_start_len }, .{ .dst = q.shape() }, .{ - .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, - .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * paged_attn.FaVecPagedReduce.simdgroups)), 1 }, - .threadgroup = .{ 32, paged_attn.FaVecPagedReduce.simdgroups, 1 }, + return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = partial }, .{ .dst = q.shape() }, .{ + .cfg = .{ .nwg = nwg, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), 1, 1 }, + .threadgroup = .{ 32, nwg, 1 }, + .constants = .{ .args = .of(@as(i32, nrows)) }, + }).dst; + } + + var idx: usize = if (kv_capacity <= 1024) 0 else if (kv_capacity <= 2048) 1 else 2; + if (head_dim == 512 and idx > 1) idx = 1; + const nsg = ([_]i32{ 4, 8, 16 })[idx]; + + return paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = q.shape() }, .{ + .cfg = .{ .head_dim = head_dim, .nsg = nsg, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = 1 }, + .grid = .{ 1, @intCast(num_heads), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, @intCast(nsg), 1 }, + .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, nsg), .constants = .{ .args = .of(args) }, }).dst; } @@ -205,36 +211,3 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, }).out; } - -const VecLaunch = struct { - heads: i32, - splits: i32, - splits_per_threadgroup: i32, -}; - -/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a -/// row's length exit early, so `splits` follows the core and row counts. -fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { - var heads: i32 = @min(group, 4); - while (@rem(group, heads) != 0) heads -= 1; - const want: i32 = if (rows <= 2 * cores) 128 else 64; - const chunks = @divFloor(span + 31, 32); - var splits: i32 = 1; - while (splits * 2 <= @min(want, chunks)) splits *= 2; - return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; -} - -fn coreCount() i32 { - const platform = zml.Compiler.current().platform; - const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse - std.debug.panic("Metal device has no core_count attribute", .{}); - return @intCast(attribute.int64); -} - -test vecLaunch { - try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 2, .splits = 64, .splits_per_threadgroup = 4 }, vecLaunch(256, 2, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 64, .splits_per_threadgroup = 2 }, vecLaunch(512, 4, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 3, .splits = 8, .splits_per_threadgroup = 2 }, vecLaunch(48, 6, 288, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); -} diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 04f4c1ac..20429565 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,8 +3,9 @@ const std = @import("std"); -const metal = @import("../../../kernel/kernel.zig").metal; +const kernel = @import("../../../kernel.zig"); +const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; @@ -17,18 +18,25 @@ const masked_score = -flt_max / 2.0; const max_half = 65504.0; const simd_width = 32; -/// fa_vec_paged: bf16 decode over paged caches. Simdgroup s takes query head -/// s % heads of the threadgroup's KV head group and position split s / heads. +fn pad2(x: i32, n: i32) i32 { + return (x + n - 1) & ~(n - 1); +} + +/// fa_vec_paged: decode attention over paged bf16 caches, one threadgroup +/// of `nsg` simdgroups per (position split, query head, query row). pub const FaVecPaged = struct { pub const Cfg = struct { - /// The key and value head dimension. + /// The key and value head dimension, DK = DV. head_dim: i32, + /// Simdgroups splitting the KV positions. + nsg: i32, /// The page size. block_size: i32, /// The in-page position stride, n_kv * head_dim elements. position_stride: i32, - heads: i32, - splits: i32, + /// Position-split threadgroups. Above 1 the kernel writes f32 partials + /// for FaVecPagedReduce. + nwg: i32, }; pub const Args: metal.Struct = .{ .name = "fa_vec_paged_args", .fields = &.{ @@ -58,10 +66,17 @@ pub const FaVecPaged = struct { .run = run, }); + /// The threadgroup memory of a launch with `nsg` simdgroups. + pub fn threadgroupMemory(head_dim: i32, nsg: i32) u64 { + const padded = pad2(head_dim, 128); + const bytes = (padded + 4 * 32 + 2 * padded) * nsg * 2; + return @intCast(@divFloor(bytes + 15, 16) * 16); + } + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { switch (cfg.head_dim) { - 32, 64, 128, 256, 512 => {}, - else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), + 128, 256, 512 => {}, + else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (128, 256 or 512)", .{cfg.head_dim}), } const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; @@ -74,90 +89,158 @@ pub const FaVecPaged = struct { .seq_lens = int_in, .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, + .shmem_f16 = .{ .threadgroup = .half }, .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, .tgpg = .{ .threadgroups_per_grid = .uint3 }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - .nsg = .{ .simdgroups_per_threadgroup = .ushort }, }); impl(b, cfg, a); } + /// The attention body. fn impl(b: *Builder, cfg: Cfg, a: anytype) void { - const D = cfg.head_dim; - const D4 = @divExact(D, 4); - // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. - const NL = @min(D4, simd_width); - const R = @divExact(simd_width, NL); - const NI = @divExact(D4, NL); - const C = simd_width; + const DK = cfg.head_dim; + const DV = cfg.head_dim; + const NE = 1; + const C = 32; + const NSG = cfg.nsg; const BS = cfg.block_size; const PS = cfg.position_stride; - const HG = cfg.heads; - const NWG = cfg.splits; - const full: metal.LoopOpts = .{ .unroll = .full }; + const NWG = cfg.nwg; - const num_heads = a.args.field("num_heads"); - const group = num_heads.div(a.args.field("n_kv")); - const groups = group.div(HG); const r = a.tgpig.at(2).to(.int); - const tg_head = a.tgpig.at(1).to(.int); - const ikv = tg_head.div(groups); - const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); - const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); - const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); - const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + const iq2 = a.tgpig.at(1).to(.ushort); + const iwg = a.tgpig.at(0).to(.short); + + const DK4 = @divExact(DK, 4); + const DV4 = @divExact(DV, 4); + + const nrows_static = a.tgpg.at(2).to(.long).mul(a.args.field("num_heads").to(.uint)); - // Padding rows: FaVecPagedReduce zeroes them when there are splits. var padding = b.openIf(r.ge(a.cu_seqlens_q.load(a.args.field("num_seqs")))); { - if (NWG == 1) { - const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)); - var loop = b.openFor(a.tiisg.to(.short), D4, simd_width); - { - dst4.store(loop.iv, b.constant(.float4, 0).to(.bfloat4)); - loop.close(); + var first = b.openIf(a.sgitg.eq(0)); + { + const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); + if (NWG == 1) { + const dst4 = a.dst.to(.bfloat4); + var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); + { + const zero = b.constant(.float4, 0).to(.bfloat4); + dst4.store(rid.mul(DV4).add(loop.iv), zero); + loop.close(); + } + } else { + const dst4 = a.dst.to(.float4); + const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); + var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); + { + dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), b.constant(.float4, 0)); + loop.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), 0.0); + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), -flt_max / 2.0); + lane0.close(); + } } + first.close(); } b.return_(); padding.close(); } - const seq = sequenceOf(b, a, r); - const kv_len = seq.kv_len; - const kv_lo = seq.kv_lo; - const ic0_lo = kv_lo.div(C); - const live = if (NWG > 1) liveSplits(b, seq, NWG) else b.lift(@as(i32, 1)); - - if (NWG > 1) { - var idle = b.openIf(iwg.ge(live)); - { - b.return_(); - idle.close(); - } - } - - const pt = a.block_tables.add(seq.idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); - const k_cache = a.k_cache.add(ikv.to(.ulong).mul(D).mul(2)); - const v_cache = a.v_cache.add(ikv.to(.ulong).mul(D).mul(2)); - - const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); - const sub = a.tiisg.div(NL).to(.int); - const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); - const qr = b.array(.float4, @intCast(NI)); - const o = b.array(.float4, @intCast(NI)); - var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); + var search = b.openLoop(.{}); { - const ii = init.iv; - qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); - o.store(ii, b.constant(.float4, 0)); - init.close(); + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + const seq_idx = first_seq.rvalue(); + const q_pos = r.sub(a.cu_seqlens_q.load(seq_idx)); + const query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(a.cu_seqlens_q.load(seq_idx)); + const kv_total = a.seq_lens.load(seq_idx); + const kv_len = kv_total.sub(query_len).add(q_pos).add(1); + const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); + + const pt = a.block_tables.add(seq_idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + + const PK = pad2(DK, 128); + const PK4 = @divExact(PK, 4); + const PV = pad2(DV, 128); + const PV4 = @divExact(PV, 4); + const NW = simd_width; + const NL = @divExact(NW, NE); + const SH = 4 * C; + + const shmem = a.shmem_f16; + const sq4 = shmem.add(0 * PK).to(.half4); + const ss = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float); + const ss4 = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float4); + const sm = shmem.add(a.sgitg.mul(SH)).add(2 * C).add(NSG * PK); + const so4 = shmem.add(b.lift(2).mul(a.sgitg).mul(PV)).add(NSG * PK).add(NSG * SH).to(.float4); + // This lane's output, until the simdgroups merge theirs. + const lane_so4 = so4.add(a.tiisg); + + const q = a.q.add(r.to(.ulong).mul(a.args.field("num_heads").to(.uint)).add(iq2).mul(DK).mul(2)); + const ikv2 = iq2.div(a.args.field("num_heads").div(a.args.field("n_kv"))).to(.short); + const k_cache = a.k_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); + const v_cache = a.v_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); + + const q4 = q.to(.bfloat4); + var load_q = b.openFor(a.tiisg.to(.short), PK4, NW); + { + const i = load_q.iv; + var value = b.openIfElse(i.lt(DK4), .{.half4}); + { + value.yieldThen(.{q4.load(i).to(.half4)}); + } + { + value.yieldElse(.{b.constant(.half4, 0)}); + } + sq4.store(i, value.results[0]); + load_q.close(); + } + var zero_o = b.openFor(b.constant(.short, 0), @divExact(DV4, NL), 1); + { + lane_so4.store(zero_o.iv.mul(NL), b.constant(.float4, 0)); + zero_o.close(); + } + var zero_s = b.openFor(a.tiisg.to(.short), SH / 4, NW); + { + ss4.store(zero_s.iv, b.constant(.float4, 0)); + zero_s.close(); } + b.threadgroupBarrier(.mem_threadgroup); + + // When DV4/NL is a multiple of NW every lane updates the output, so the + // `ty == 0` guard is dropped at comptime. + const all_lanes = @rem(@divExact(DV4, NL), NW) == 0; + const full: metal.LoopOpts = .{ .unroll = .full }; const S = b.local(.float, 0.0); const M = b.local(.float, -flt_max / 2.0); - const ic0 = b.local(.int, ic0_lo.add(iwg)); + const tx = a.tiisg.rem(NL).to(.short); + const ty = a.tiisg.div(NL).to(.short); + + const ic0_lo = kv_lo.div(C); + const ic0 = b.local(.int, ic0_lo.add(iwg.mul(NSG)).add(a.sgitg)); var blocks = b.openLoop(.{}); { const ic = ic0.mul(C); @@ -167,103 +250,175 @@ pub const FaVecPaged = struct { past.close(); } - const s = b.local(.float, 0.0); - var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + const in_kv = ic.add(a.tiisg).lt(kv_len).bitAnd(ic.add(a.tiisg).ge(kv_lo)); + sm.store(a.tiisg, b.ternary(in_kv, b.constant(.half, 0), b.constant(.half, -max_half))); + + const pq4 = sq4.add(tx); + const mqk = b.zeroedArray(.float, C / NE); + var qk = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); { const cc = qk.iv; - const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); - const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(lane); - const d = b.local(.float, 0.0); - var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); + const p = b.min(ic.add(cc), kv_len.sub(1)); + const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); + var dots = b.openForOpts(b.constant(.short, 0), @divExact(DK4, NL), 1, full); { const ii = dots.iv; - d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); + const dot = b.dot(pk4.load(ii.mul(NL)).to(.float4), pq4.load(ii.mul(NL)).to(.float4)); + mqk.store(cc, mqk.load(cc).add(dot)); dots.close(); } - if (R == 1) { - s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); - } else { - var x = d.rvalue(); - var mask: i32 = 1; - while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); - const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); - s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); - } + mqk.store(cc, b.simdSum(mqk.load(cc))); qk.close(); } - const pos = ic.add(a.tiisg.to(.int)); - const in_kv = pos.lt(kv_len).bitAnd(pos.ge(kv_lo)); - const score = b.ternary(in_kv, s.mul(a.args.field("scale")), b.constant(.float, -max_half)); + const score = b.fma(mqk.load(tx), a.args.field("scale"), sm.load(b.lift(NE).mul(tx).add(ty)).to(.float)); + ss.store(b.lift(NE).mul(tx).add(ty), score); + + b.simdgroupBarrier(.mem_threadgroup); const m = M.rvalue(); - M.assign(b.simdMax(b.max(M.rvalue(), score))); + const s = ss.load(a.tiisg); + + M.assign(b.simdMax(b.max(M.rvalue(), s))); + const ms = b.exp(m.sub(M)); - const vs = b.exp(score.sub(M)); + const vs = b.exp(s.sub(M)); + S.assign(S.mul(ms).add(b.simdSum(vs))); - var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); + ss.store(a.tiisg, vs); + + var rescale_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); + var rescale = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); { - o.store(rescale.iv, o.load(rescale.iv).mul(ms)); + const at = rescale.iv.mul(NL); + lane_so4.store(at, lane_so4.load(at).mul(ms)); rescale.close(); } + if (rescale_lanes) |*l| l.close(); - var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + b.simdgroupBarrier(.mem_threadgroup); + + const lo = b.array(.float4, @intCast(@divExact(DV4, NL))); + var zero_lo = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + { + lo.store(zero_lo.iv, b.constant(.float4, 0)); + zero_lo.close(); + } + + const sst = ss.add(ty); + + var pv = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); { const cc = pv.iv; - const row = cc.to(.int).mul(R).add(sub); - const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); - const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(lane); - const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); - var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); + const p = b.min(ic.add(cc), kv_len.sub(1)); + const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); + var dims = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); { const ii = dims.iv; - o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); + const weighted = pv4.load(ii.mul(NL)).to(.float4).mul(b.vec(.float4, .{sst.load(cc.mul(NE))})); + lo.store(ii, lo.load(ii).add(weighted)); dims.close(); } pv.close(); } + var add_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); + var add_o = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + { + const ii = add_o.iv; + const partial = lo.load(ii); + lane_so4.store(ii.mul(NL), lane_so4.load(ii.mul(NL)).add(partial)); + add_o.close(); + } + if (add_lanes) |*l| l.close(); blocks.next(); - ic0.assign(ic0.add(live)); + ic0.assign(ic0.add(NWG * NSG)); blocks.close(); } - var mask: i32 = NL; - while (mask < simd_width) : (mask *= 2) { - var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); - sum.close(); - } + var stats = b.openIf(a.tiisg.eq(0)); + { + ss.store(0, S); + ss.store(1, M); + stats.close(); } - var first = if (R > 1) b.openIf(sub.eq(0)) else null; - if (NWG == 1) { - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + + b.threadgroupBarrier(.mem_threadgroup); + + const rr = b.local(.short, @divFloor(NSG, 2)); + var halving = b.openLoop(.{}); + { + halving.body(rr.gt(0)); + } + { + var group = b.openIf(a.sgitg.lt(rr)); { - dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); - out.close(); + const S0 = ss.load(0); + const S1 = ss.load(rr.mul(@divExact(SH, 2)).add(0)); + + const M0 = ss.load(1); + const M1 = ss.load(rr.mul(@divExact(SH, 2)).add(1)); + + const M_ = b.max(M0, M1); + + const ms0 = b.exp(M0.sub(M_)); + const ms1 = b.exp(M1.sub(M_)); + + const S_ = S0.mul(ms0).add(S1.mul(ms1)); + + var lane0 = b.openIf(a.tiisg.eq(0)); + { + ss.store(0, S_); + ss.store(1, M_); + lane0.close(); + } + + var merge = b.openFor(a.tiisg.to(.short), DV4, NW); + { + const i = merge.iv; + so4.store(i, so4.load(i).mul(ms0).add(so4.load(i.add(rr.mul(PV4))).mul(ms1))); + merge.close(); + } + group.close(); } - } else { - // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). - const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); - dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); - out.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); - dst1.store(b.lift(2).mul(iwg), S); - dst1.store(b.lift(2).mul(iwg).add(1), M); - lane0.close(); + b.threadgroupBarrier(.mem_threadgroup); + rr.assign(rr.shr(1)); + halving.close(); + } + + var first = b.openIf(a.sgitg.eq(0)); + { + const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); + if (NWG == 1) { + const inv_sum = b.ternary(ss.load(0).eq(0.0), 0.0, b.lift(1.0).div(ss.load(0))); + const dst4 = a.dst.to(.bfloat4); + var loop = b.openFor(a.tiisg.to(.short), DV4, NW); + { + const out = so4.load(loop.iv).mul(inv_sum).to(.bfloat4); + dst4.store(rid.mul(DV4).add(loop.iv), out); + loop.close(); + } + } else { + const dst4 = a.dst.to(.float4); + const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); + var loop = b.openFor(a.tiisg.to(.short), DV4, NW); + { + const part = so4.load(loop.iv); + dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), part); + loop.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const sum = ss.load(0); + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), sum); + const max = ss.load(1); + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), max); + lane0.close(); + } } + first.close(); } - if (first) |*scope| scope.close(); } /// The address of position `p`'s bf16 row in `cache`, through the page @@ -275,144 +430,85 @@ pub const FaVecPaged = struct { } }; -/// The sequence of query row `r` and its causal KV range [kv_lo, kv_len). -fn sequenceOf(b: *Builder, a: anytype, r: Value) struct { idx: Value, kv_len: Value, kv_lo: Value } { - const first_seq = b.local(.int, 0); - const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); - var search = b.openLoop(.{}); - { - search.body(first_seq.lt(last_seq)); - } - { - const mid = first_seq.add(last_seq).add(1).div(2); - var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); - { - first_seq.assign(mid); - half.elseBranch(); - } - { - last_seq.assign(mid.sub(1)); - half.close(); - } - search.close(); - } - const idx = first_seq.rvalue(); - const q_pos = r.sub(a.cu_seqlens_q.load(idx)); - const query_len = a.cu_seqlens_q.load(idx.add(1)).sub(a.cu_seqlens_q.load(idx)); - const kv_len = a.seq_lens.load(idx).sub(query_len).add(q_pos).add(1); - const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); - return .{ .idx = idx, .kv_len = kv_len, .kv_lo = kv_lo }; -} - -fn liveSplits(b: *Builder, seq: anytype, splits: i32) Value { - const chunks = seq.kv_len.add(simd_width - 1).div(simd_width).sub(seq.kv_lo.div(simd_width)); - return b.min(splits, chunks); -} - -/// fa_vec_paged_reduce: merges the fa_vec_paged split partials. +/// fa_vec_paged_reduce: merges the `nwg` partials of a split-K fa_vec_paged, +/// one threadgroup per row and head. pub const FaVecPagedReduce = struct { pub const Cfg = struct { - splits: i32, + /// The partials to merge, FaVecPaged's `nwg`. + nwg: i32, /// The head dimension. head_dim: i32, }; - pub const simdgroups = 8; + pub const Args: metal.Struct = .{ .name = "fa_vec_paged_reduce_args", .fields = &.{ + .{ .name = "nrows", .type = .int }, + } }; pub const Kernel = metal.Kernel(Cfg, .{ .name = "fa_vec_paged_reduce", - .inputs = &.{ "htmp", "seq_lens", "cu_seqlens_q" }, + .inputs = &.{"htmp"}, .outputs = &.{"dst"}, .constants = &.{"args"}, .run = run, }); fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; const a = try b.declareArgs(.{ - .args = .{ .constant_struct = FaVecPaged.Args }, + .args = .{ .constant_struct = Args }, .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, - .seq_lens = int_in, - .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, - .tgpg = .{ .threadgroups_per_grid = .uint2 }, + .tgpig = .{ .threadgroup_position_in_grid = .uint }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, }); - const NWG = cfg.splits; - if (NWG > 4 * simd_width) std.debug.panic("fa_vec_paged_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); - const NP = @divFloor(NWG + simd_width - 1, simd_width); + const nwg = cfg.nwg; const DV = cfg.head_dim; - const DV4 = @divExact(DV, 4); - const full: metal.LoopOpts = .{ .unroll = .full }; + const rid = a.tgpig.to(.ulong); + const iwg = a.tiisg.to(.short); + const ss = a.htmp.to(.float).add(a.args.field("nrows").to(.ulong).mul(DV).mul(nwg)); - const rid = a.tgpig.at(0).to(.ulong); - const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); - const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); - - const live = b.local(.int, 0); - var real = b.openIf(r.lt(a.cu_seqlens_q.load(a.args.field("num_seqs")))); + var max = b.openIfElse(iwg.lt(nwg), .{.float}); { - live.assign(liveSplits(b, sequenceOf(b, a, r), NWG)); - real.close(); + max.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(1))}); } - var empty = b.openIf(live.le(0)); { - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst4.store(i, b.constant(.float4, 0).to(.bfloat4)); - lane0.close(); - } - b.return_(); - empty.close(); + max.yieldElse(.{-flt_max / 2.0}); + } + const M = max.results[0]; + var sum = b.openIfElse(iwg.lt(nwg), .{.float}); + { + sum.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(0))}); + } + { + sum.yieldElse(.{0.0}); } - const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); - const Mj = b.array(.float, @intCast(NP)); - const Sj = b.array(.float, @intCast(NP)); - const mx = b.local(.float, -flt_max / 2.0); - var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = load.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - const ok = idx.lt(live); - Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); - Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); - mx.assign(b.max(mx.rvalue(), Mj.load(j))); - load.close(); - } - const m = b.simdMax(mx.rvalue()); - const sum = b.local(.float, 0.0); - var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = weigh.iv; - Mj.store(j, b.exp(Mj.load(j).sub(m))); - sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); - weigh.close(); - } - const S = b.simdSum(sum.rvalue()); + const m = b.simdMax(M); + const ms = b.exp(M.sub(m)); + const S = b.simdSum(sum.results[0].mul(ms)); const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); - const acc = b.local(.float4, b.constant(.float4, 0)); - var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + const DV4 = @divTrunc(DV, 4); + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(nwg)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + var loop = b.openFor(a.sgitg.to(.int), DV4, nwg); { - const j = parts.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - var ok = b.openIf(idx.lt(live)); + const i = loop.iv; + var part_ = b.openIfElse(iwg.lt(nwg), .{.float4}); { - acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); - ok.close(); + part_.yieldThen(.{htmp4.load(i.mul(nwg).add(iwg)).mul(ms)}); } - parts.close(); - } - const v = b.simdSum(acc.rvalue()); - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst4.store(i, v.mul(inv_sum).to(.bfloat4)); - lane0.close(); + { + part_.yieldElse(.{b.constant(.float4, 0)}); + } + const part = part_.results[0]; + const v = b.simdSum(part); + var lane0 = b.openIf(iwg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + loop.close(); } } }; @@ -888,14 +984,13 @@ pub const PagedAttentionTiled = struct { test "the paged attention kernels emit verified modules" { const allocator = std.testing.allocator; - for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { - for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { - allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .block_size = 16, .position_stride = 2 * head_dim, .heads = launch[0], .splits = launch[1] })); - } - for ([_]i32{ 8, 128 }) |splits| { - allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); + for ([_]i32{ 128, 256, 512 }) |head_dim| { + for ([_]i32{ 4, 8, 16 }) |nsg| { + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .nsg = nsg, .block_size = 16, .position_stride = 2 * head_dim, .nwg = 1 })); } } + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = 512, .nsg = 8, .block_size = 16, .position_stride = 1024, .nwg = 4 })); + allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .nwg = 4, .head_dim = 512 })); for ([_]Type{ .half, .bfloat }) |dtype| { allocator.free(try ZeroPaddingRows.Kernel.emit(allocator, .{ .dtype = dtype })); for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index 84716379..42e1861f 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -5,7 +5,7 @@ const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const ragged_attention = @import("ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig deleted file mode 100644 index 35ad4049..00000000 --- a/zml/kernels/paged_attention/options.zig +++ /dev/null @@ -1,36 +0,0 @@ -const Tensor = @import("../../tensor.zig").Tensor; - -pub const AttentionMask = union(enum) { - standard: struct { - /// Restrict attention to keys at or before the query position. - causal: bool = true, - /// Optional left window size, including the current token. - /// Non-causal attention also includes every later valid key. - sliding_window: ?u32 = null, - }, - /// Attend to keys at specific positions. - indices: Tensor, - - /// Attend to every valid key in the sequence. - pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?Tensor = null, -}; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 4a97a38e..81f7ec3e 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,8 +3,6 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); -pub const AttentionMask = @import("options.zig").AttentionMask; -pub const AttentionOptions = @import("options.zig").AttentionOptions; pub const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); @@ -153,6 +151,41 @@ pub const Parameters = union(Backend) { } }; +pub const AttentionMask = union(enum) { + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, + /// Attend to keys at specific positions. + indices: zml.Tensor, + + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?zml.Tensor = null, +}; + pub const KvCache = union(enum) { split: struct { k: zml.Tensor, @@ -199,6 +232,19 @@ pub const KvCache = union(enum) { const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); return .{ page_index, offset }; } + + pub fn getPage( + self: KvCache, + page_index: zml.Tensor, + ) [2]zml.Tensor { + return switch (self) { + .split => |split| .{ + split.k.slice(split.k.axis(.page), .dynSingle(page_index)), + split.v.slice(split.k.axis(.page), .dynSingle(page_index)), + }, + .dense => @panic("TODO"), + }; + } }; pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); @@ -207,7 +253,7 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z _ = k; _ = v; const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; - return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); + return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, partitioning); } const Attention = struct { @@ -234,10 +280,7 @@ const Attention = struct { .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, - .stablehlo => |params| switch (args.kv_cache) { - .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, + .stablehlo => |params| stablehlo.pagedAttention(params, args.q, args.kv_cache, args.opts), .tcl => |params| switch (args.kv_cache) { .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), @@ -435,9 +478,12 @@ test pagedAttention { const parameters: Parameters = .init(.fromBackend(backend_options_args)); const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + const ParameterSpecs = zml.meta.MapRestrict(zml.Tensor, zml.Sharding.Partitioning.Logical).map(Parameters); const partitioning: Partitioning = .{ .in_specs = .{ - .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), + .parameters = switch (parameters) { + inline else => |_, tag| @unionInit(ParameterSpecs, @tagName(tag), .{ .block_table = .replicated, .seq_lens = .replicated, .query_start_len = .replicated }), + }, .q = head_spec, .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, .opts = .{ .mask = .{ .standard = {} }, .sink = null }, diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index 55ee01d5..7ee4df03 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -1,12 +1,12 @@ const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; +const KvCache = @import("../paged_attention.zig").KvCache; +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); pub fn pagedAttention( parameters: triton.paged.Parameters, q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, + kv_cache: KvCache, opts: AttentionOptions, ) zml.Tensor { switch (opts.mask) { @@ -14,12 +14,12 @@ pub fn pagedAttention( .indices => @panic("Sparse attention mask is not supported on StableHLO"), } - const page_size = k.dim(.k_chunk); + const page_size = kv_cache.split.k.dim(.k_chunk); const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); const final_state = zml.ops.@"while"( AttentionLoop, - .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, + .{ .q = padded_q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, .{ // i32 to match triton conventions .slot_id = .scalar(0, .i32), @@ -38,8 +38,7 @@ pub fn pagedAttention( const AttentionLoop = struct { q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, + kv_cache: KvCache, parameters: triton.paged.Parameters, opts: AttentionOptions, @@ -67,7 +66,7 @@ const AttentionLoop = struct { const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); - const page_size_: u32 = @intCast(self.k.dim(.k_chunk)); + const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); return zml.ops.@"if"( struct { @@ -135,10 +134,7 @@ const AttentionLoop = struct { .{ .b, .p }, &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, ); - - const page_axis = self.k.axis(.page); - const active_k = self.k.slice(page_axis, .dynSingle(active_k_page_id)); - const active_v = self.v.slice(page_axis, .dynSingle(active_k_page_id)); + const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); const dtype = self.q.dtype(); const attn_mask: zml.Tensor = attn_mask: { diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index d04b684e..c9993af9 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -3,13 +3,13 @@ //! `Repro:` files are in the reproducer repository, platform/furiosa. const std = @import("std"); -const builder = @import("dsl/tcl/builder"); const stdx = @import("stdx"); +const builder = @import("dsl/tcl/builder"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const AttentionOptions = @import("../options.zig").AttentionOptions; +const paged_attention = @import("../paged_attention.zig"); const triton = @import("../triton/triton.zig"); /// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to @@ -49,7 +49,7 @@ pub const paged = struct { /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. /// Sinks are one logit per query head. - pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { + pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); @@ -67,10 +67,23 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); + if (compiler.meshes[0].numDevices() > 1) return zml.ops.manualComputation(Shard.body, .{ .parameters = parameters, .q = q, .k_cache = k_cache, .v_cache = v_cache, .opts = opts }, q.shape()); return attend(parameters, q, k_cache, v_cache, opts); } - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { + const Shard = struct { + parameters: Parameters, + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + opts: paged_attention.AttentionOptions, + + fn body(self: Shard, _: zml.Shape) Tensor { + return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); + } + }; + + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to @@ -1651,11 +1664,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); const Mod = struct { - pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: AttentionOptions) Tensor { + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged_attention.AttentionOptions) Tensor { return paged.pagedAttention(p, q, k, v, opts); } }; - const opts: AttentionOptions = .{ + const opts: paged_attention.AttentionOptions = .{ .mask = .{ .standard = .{ .causal = case.opts.is_causal, .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, @@ -1729,11 +1742,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), }; defer zml.Buffer.deinitAll(paged.Parameters, &bp); - var opts_d: zml.Bufferized(AttentionOptions) = .{ + var opts_d: zml.Bufferized(paged_attention.AttentionOptions) = .{ .mask = .{ .standard = {} }, .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, }; - defer zml.Buffer.deinitAll(AttentionOptions, &opts_d); + defer zml.Buffer.deinitAll(paged_attention.AttentionOptions, &opts_d); var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); defer result.deinit(); const host = try result.toSliceAlloc(allocator, io); diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index 2a12e012..67416285 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -4,10 +4,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const triton = zml.kernel.triton; -const AttentionOptions = @import("../options.zig").AttentionOptions; +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const kernels = @import("unified_attention.zig"); -const kernels_oneapi = @import("unified_attention_oneapi.zig"); const kernels_sparse = @import("sparse_unified_attention.zig"); +const kernels_oneapi = @import("unified_attention_oneapi.zig"); const log = std.log.scoped(.@"zml/attention/triton"); diff --git a/zml/nn.zig b/zml/nn.zig index eceb316b..a2b582ee 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,13 +2,11 @@ const std = @import("std"); const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; @@ -322,23 +320,13 @@ fn onesGrid(shape: Shape, dt: DataType) Shape { } fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { - // Keep the unmatched-composite marker in the graph for backend lowering. - const ctx = zml.Compiler.current(); - const op = stablehlo.custom_call( - ctx.mlir_ctx, - &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, - &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, - .{ - .call_target_name = "zml$scaled_dot_unmatched", - .has_side_effect = false, - .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, - }, - ctx.location, - ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { - op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); - } - return Tensor._result(out_shape, op.result(0)); + return ops.customCall( + "zml$scaled_dot_unmatched", + .{ in[0], in[1], in[2], in[3] }, + out_shape, + {}, + .{ .has_side_effect = false }, + ); } fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { diff --git a/zml/ops.zig b/zml/ops.zig index f8fdba7e..19d15ec3 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -8,6 +8,18 @@ const platforms = @import("platforms"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); +const manual_computation = @import("ops/manual_computation.zig"); +pub const manualComputation = manual_computation.manualComputation; + +const manual_computation2 = @import("ops/manual_computation2.zig"); +pub const manualComputation2 = manual_computation2.manualComputation2; +pub const ManualPartitioning = manual_computation2.ManualPartitioning; + +test { + _ = manual_computation; + _ = manual_computation2; +} + const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("mem.zig").Bufferized; const Compiler = @import("Compiler.zig"); @@ -16,113 +28,15 @@ const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); -const manual_computation = @import("ops/manual_computation.zig"); -pub const manualComputation = manual_computation.manualComputation; -pub const ManualPartitioning = manual_computation.ManualPartitioning; -pub const PartitionSpecs = manual_computation.PartitionSpecs; -pub const partitionSpecs = manual_computation.partitionSpecs; -pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; - -test { - _ = manual_computation; -} - const log = std.log.scoped(.@"zml/Compiler"); -/// Reduces local tensor values across all processes in the computation. -/// For each tensor index, `func_` combines the values at that index from every -/// process. The result is identical on all processes. -/// -/// Inside a manual computation, `inputs` describes the local tensors at each -/// partition. For example, summing `[1, 2]` and `[3, 4]` from two partitions gives -/// `[4, 6]` at each partition. The local tensor dimensions stay unchanged. -/// -/// * `inputs` is a `Tensor`, a nonempty tuple of tensors, or an `[N]Tensor`. -/// Each input must have matching shapes and dtypes across processes. -/// * `func_` describes how to combine two scalar contributions: -/// - For one input, it must be `fn (left: Tensor, right: Tensor) Tensor`, such -/// as `Tensor.add`. This also applies to a one-element tuple or array. -/// - For multiple inputs, it takes one `ReduceArgs` per input and returns a -/// tuple or array with one scalar tensor per input, in the same order. -/// -/// Returns the same container type as `inputs`, preserving tensor attributes. -/// With one process, returns `inputs` unchanged. -/// -/// The reduction uses one group containing all processes, including mesh replicas. -/// To reduce along one logical mesh axis, use `allReduceOnAxis`. pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@TypeOf(inputs)) { - const count = Compiler.current().meshes[0].numDevices(); - var devices: [Platform.MAX_NUM_DEVICES]i64 = undefined; - for (devices[0..count], 0..) |*device, i| device.* = @intCast(i); - return allReduceGroups(inputs, func_, devices[0..count], count); -} - -/// Reduces local tensor values within process groups defined by `axis` in `mesh`. -/// A group contains processes with the same coordinates on every other logical -/// mesh axis. Separate replicas of the mesh belong to separate groups. Within -/// each group, `func_` combines corresponding elements and the result is identical -/// on all member processes. -/// -/// Use this inside a manual computation to combine partial results along `axis` -/// while preserving the independent computations along the other mesh axes. -/// -/// * `inputs` and `func_` follow `allReduce`'s tensor and reducer contracts. -/// * `mesh` is the logical mesh used by the manual computation and must be -/// included in the compilation's meshes. -/// * `axis` names an axis of that mesh, for example `.experts`. It can be an enum -/// literal or a mesh axis tag. The axis must exist in `mesh`. -/// -/// Returns the same container type with tensor attributes as -/// `inputs`. An axis of size one returns `inputs` unchanged. -/// -/// For a mesh with `.data = 2` and `.experts = 4`, the following sums four local -/// expert contributions independently for each of the two data shards: -/// -/// ```zig -/// const output = allReduceOnAxis(local_output, Tensor.add, mesh, .experts); -/// ``` -/// -/// Each expert shard now has the complete output for its data shard. To expose -/// this result through `manualComputation`, an output spec such as -/// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the -/// result replicated along `.experts`. -pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { - const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical reduction axis"); - const size = mesh.grid.axis_sizes[logical_axis]; - var groups: [Platform.MAX_NUM_DEVICES]i64 = undefined; - var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); - const replicas = mesh.numReplicas(); - for (0..mesh.numDevices()) |device| { - const coords = mesh.grid.logical_coords[device]; - var tile: usize = 0; - var group: usize = 0; - for (0..mesh.logical.axes.len) |dim| { - tile = tile * mesh.grid.axis_sizes[dim] + coords[dim]; - if (dim != logical_axis) group = group * mesh.grid.axis_sizes[dim] + coords[dim]; - } - group = group * replicas + replica_indices[tile]; - replica_indices[tile] += 1; - groups[group * size + coords[logical_axis]] = @intCast(device); - } - return allReduceGroups(inputs, func_, groups[0..mesh.numDevices()], size); -} - -/// Reduces corresponding tensor elements independently within each process group. -/// `inputs`, `func_` and the return value follow `allReduce`'s contract. -/// -/// `groups` identifies processes by their executable partition IDs, as returned -/// by `partitionId()`. Each consecutive `group_size` IDs defines one group. -/// -/// For example, `groups = &.{ 0, 2, 1, 3 }` and `group_size = 2` combine partitions -/// 0 and 2 together, and partitions 1 and 3 together. Both members of each group -/// receive that group's result. Groups of size one return `inputs` unchanged. -fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64, group_size: usize) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); const mlir_ctx = ctx.mlir_ctx; @@ -166,7 +80,8 @@ fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64 else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), }; - if (group_size <= 1) return inputs; + const num_devices = ctx.meshes[0].numDevices(); + if (num_devices <= 1) return inputs; const reducer_block = b: { const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); @@ -215,9 +130,14 @@ fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64 break :b block; }; + var replica_group_storage: [constants.MAX_RANK]i64 = undefined; + for (0..@intCast(num_devices)) |i| { + replica_group_storage[i] = @intCast(i); + } + const replica_group_slice = replica_group_storage[0..@intCast(num_devices)]; const replica_groups_attr = mlir.Attribute.denseElements( - .rankedTensor(&.{ @intCast(groups.len / group_size), @intCast(group_size) }, .int(mlir_ctx, .i64)), - groups, + .rankedTensor(&.{ 1, num_devices }, .int(mlir_ctx, .i64)), + replica_group_slice, ); const handle = ctx.nextChannelId(); @@ -287,20 +207,6 @@ fn AllReduceReturnType(comptime InputsT: type) type { }; } -/// Returns the current process's zero based coordinate on `axis` in `mesh` -/// as a scalar u32. -/// -/// Example: for a mesh {.data = 2, .experts = 2}, each data group has expert coordinates -/// 0 and 1. Use this coordinate inside a manual computation to identify a shard -/// along one axis while `partitionId()` identifies a partition across the whole mesh. -pub fn axisIndex(mesh: *const Sharding.Mesh, axis: anytype) Tensor { - const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical mesh axis"); - var coords: [Platform.MAX_NUM_DEVICES]u32 = undefined; - for (coords[0..mesh.numDevices()], 0..) |*coord, i| coord.* = mesh.grid.logical_coords[i][logical_axis]; - return Tensor.constantTensor(.init(.{ .device = mesh.numDevices() }, .u32), std.mem.sliceAsBytes(coords[0..mesh.numDevices()])) - .gather(.{ .device = partitionId() }, .{}); -} - pub fn partitionId() Tensor { const ctx = Compiler.current(); const op = stablehlo.partition_id(ctx.mlir_ctx, ctx.location).appendTo(ctx.currentScope().block); @@ -1532,15 +1438,7 @@ pub const CuteOps = struct { block: ?[3]i32 = null, zeroed_outputs: []const i32 = &.{}, scalars: []const i64 = &.{}, - symmetric_workspace: ?SymmetricWorkspace = null, output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, - - /// Named per-device workspace shared across CuTe calls and executables. - /// Every rank allocates `bytes`; XLA passes peer addresses and the rank. - pub const SymmetricWorkspace = struct { - name: []const u8, - bytes: i64, - }; }; /// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler @@ -1561,7 +1459,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); } - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; attrs.appendSliceAssumeCapacity(&.{ .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), @@ -1586,13 +1484,6 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso for (opts.scalars, scalar_attrs) |value, *attribute| attribute.* = .int(mlir_ctx, .i64, value); attrs.appendAssumeCapacity(.named(mlir_ctx, "scalars", .array(mlir_ctx, scalar_attrs))); } - if (opts.symmetric_workspace) |workspace| { - stdx.debug.assert(workspace.name.len > 0 and workspace.bytes > 0, "zml.ops.cute({s}): symmetric_workspace needs a name and a positive size", .{opts.name}); - attrs.appendAssumeCapacity(.named(mlir_ctx, "symmetric_workspace", .dict(mlir_ctx, &.{ - .named(mlir_ctx, "name", .string(mlir_ctx, workspace.name)), - .named(mlir_ctx, "bytes", .int(mlir_ctx, .i64, workspace.bytes)), - }))); - } const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); const op = stablehlo.custom_call( @@ -2760,6 +2651,11 @@ pub fn CustomCall( comptime params: struct { /// Name of the custom call, must be unique. name: [:0]const u8, + /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies + /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, + /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output + /// sharding, this setting will avoid the shuffling of the buffer's data. + sharding_aware: bool, /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. has_side_effect: bool, /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. @@ -2793,7 +2689,23 @@ pub fn CustomCall( .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), .compute_on_host = params.compute_on_host, }; - return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); + if (params.sharding_aware) { + return shardingAwareTypedCustomCall( + params.name, + opts, + input_tensors, + output_shapes, + attributes, + ); + } else { + return typedCustomCall( + params.name, + opts, + input_tensors, + output_shapes, + attributes, + ); + } } fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { @@ -2821,6 +2733,44 @@ pub fn CustomCallOutputOperandAliases(I: type, O: type) type { ); } +pub fn shardingAwareTypedCustomCall( + comptime target_name: [:0]const u8, + comptime opts: CustomCallOptions, + input: anytype, + output: anytype, + attributes: anytype, +) ShapeToTensor(@TypeOf(output)) { + const Input = @TypeOf(input); + const Output = @TypeOf(output); + const Attributes = @TypeOf(attributes); + + var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + output_shapes[i] = @field(output, field_name); + } + + const Handler = struct { + input: Input, + attributes: Attributes, + + fn body(self: @This(), sharded_output_shapes: []const Shape) []const Tensor { + return typedCustomCall(target_name, opts, self.input, sharded_output_shapes, self.attributes); + } + }; + const output_tensors = manualComputation( + Handler.body, + .{ .input = input, .attributes = attributes }, + @as([]const Shape, &output_shapes), + ); + + // Convert the slice back to a struct + var out: ShapeToTensor(Output) = undefined; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + @field(out, field_name) = output_tensors[i]; + } + return out; +} + pub fn typedCustomCall( target_name: [:0]const u8, opts: CustomCallOptions, @@ -2835,7 +2785,7 @@ pub fn typedCustomCall( const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); - stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); + stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { @@ -2845,6 +2795,7 @@ pub fn typedCustomCall( } break :b &input_tensors; }, + // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -2860,6 +2811,7 @@ pub fn typedCustomCall( } break :b &output_shapes; }, + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -2924,7 +2876,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.partitioner == .gspmd) { + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -2936,6 +2888,7 @@ pub fn typedCustomCall( } return out; }, + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -2988,7 +2941,7 @@ test customCall { const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2999,17 +2952,9 @@ test customCall { const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); - const Local = struct { - fn body(x: Tensor, output_shape: Shape) Tensor { - return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, - }); - } - }; - const output = manualComputation(Local.body, input, shape, tp, .{ - .in_specs = .parse(.{ ._0 = .model }), - .out_specs = .parse(.{ ._0 = .model }), + const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index a0ec0037..45400206 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); @@ -6,126 +7,44 @@ const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("../Compiler.zig"); -const meta = @import("../meta.zig"); +const DataType = @import("../dtype.zig").DataType; const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; +const zml = struct { + // Nested to disambiguate from stdx.meta + const meta = @import("../meta.zig"); + const testing = @import("../testing.zig"); +}; + const log = std.log.scoped(.@"zml/Compiler"); -/// Partitioning declarations for the tensors in `T`. -/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. -/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes -/// `struct { x: Logical, bias: ?Logical }`. +/// ZML operator describes computation across several devices, +/// but `manualComputation` allows to partition the devices in small group, +/// split the `Tensor` on those partitions, +/// and describes sub-computation that work inside a partition. /// -/// Initialize individual fields to declare different shardings, or use -/// `partitionSpecs` to apply one spec to every tensor in a value. -pub fn PartitionSpecs(comptime T: type) type { - return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); -} - -/// Declares the same sharding for a group of tensors at a `manualComputation` -/// boundary. +/// See: +/// https://github.com/openxla/shardy/blob/main/docs/sdy_dialect.md#sdymanual_computation-sdymanualcomputationop /// -/// ```zig -/// .in_specs = .{ -/// .query = .parse(.{ .h = .model }), -/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), -/// .metadata = partitionSpecs(metadata, .replicated), -/// } -/// ``` +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R` where: +/// - `I` is user defined and contains inputs tensors and any metadata required by the computation +/// - `O` contains the partionned output shapes, types must be `void`, `Shape` or `[]const Shape` for 0, 1 or many outputs +/// - `R` must be `void`, `Tensor, or `[]const Tensor` for 0, 1 or many outputs +/// * `inputs` must be using the same mesh (using .withPartitioning), or be fully replicated +/// * `outputs` contains the global output shapes, +/// and `manualComputation` will pass to `body_fn` the shapes of each local output shard. /// -/// Requires an active compiler and uses its arena for allocations. Use -/// `partitionSpecsAlloc` outside graph construction. Use named struct fields -/// when mixing tensors with non-tensor metadata. -pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { - return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); -} - -/// Builds the same declarations as `partitionSpecs` using an explicit allocator. -pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { - var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; - try meta.mapAlloc(struct { - fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { - return spec_; - } - }.assign, allocator, spec, tensors, &specs); - return specs; -} - -/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. -/// Returns the type of the `partitioning` argument to `manualComputation`. -/// -/// * `in_specs` specifies the sharding of each tensor in the body's first argument. -/// It follows the argument's structure, omitting non-tensor metadata. -/// * `out_specs` specifies the sharding of each shape in the body's second argument. -/// It follows the output shapes' structure and can be omitted for a `void` argument. -/// -/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as -/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across -/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` -/// replicates the entire tensor. The specs must not contain open dimensions. -/// Slice lengths, optional presence and active union tags must match the arguments. -/// -/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single -/// output `Shape`, the partitioning can be declared as: -/// -/// ```zig -/// const partitioning: ManualPartitioning(body) = .{ -/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, -/// .out_specs = .parse(.{ .hidden = .model }), -/// }; -/// ``` -pub fn ManualPartitioning(comptime body_fn: anytype) type { - const Logical = Sharding.Partitioning.Logical; - - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - - const InputSpecs = PartitionSpecs(Inputs); - const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); - - if (OutputSpecs == void) return struct { - in_specs: InputSpecs, - out_specs: void = {}, - }; - - return struct { - in_specs: InputSpecs, - out_specs: OutputSpecs, - }; -} - -/// Describes a computation on local shards of tensors distributed across `mesh`. -/// ZML operations normally describe work on global tensors. The partitioner -/// distributes the inputs according to `in_specs`, and `body_fn` describes the work -/// performed on each shard. The results are exposed to the surrounding graph as -/// global tensors with the sharding declared by `out_specs`. -/// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: -/// - `I` contains the local input tensors and any metadata needed by the body. -/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. -/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes -/// in traversal order. -/// * `inputs` contains the global tensors and metadata. The partitioner reshards -/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. -/// * `output_shapes` contains the global output shapes. The body receives their -/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. -/// * `mesh` is the logical mesh whose axis names are referenced by the specs. -/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. -/// Each partitioned dimension must be divisible by the size of its mesh axis. -/// -/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives -/// the body a dimension of size 8. A replicated dimension keeps its global size. -/// The caller supplies the global output shapes because the body only describes -/// local results: their shapes alone do not determine the global tensor shapes. +/// This depart from the rest of ZML where output shapes are generally inferred, +/// but in this case since `body_fn` explains how to compute parts of the result, +/// it's not clear how to put them back together, +/// hence why the user is asked to explicitly provided the expected output shape. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), output_shapes: stdx.meta.FnParam(body_fn, 1), - mesh: *const Sharding.Mesh, - partitioning: ManualPartitioning(body_fn), ) stdx.meta.FnReturn(body_fn) { const Inputs = stdx.meta.FnParam(body_fn, 0); const OutputShapes = stdx.meta.FnParam(body_fn, 1); @@ -136,42 +55,13 @@ pub fn manualComputation( defer arena_state.deinit(); const arena = arena_state.allocator(); - // The body may return containers borrowed from its inputs or output shapes. - const body_arena = compiler.arena.allocator(); + var owned_inputs: Inputs = inputs; + const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + defer arena.free(flat_input_tensors); - var owned_inputs: Inputs = undefined; - meta.mapAlloc((struct { - fn copy(_: void, tensor: Tensor) Tensor { - return tensor; - } - }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); - - const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); - stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); - - for (flat_input_tensors, in_specs) |tensor, spec| { - tensor._shape = tensor.shape().withPartitioning(mesh, spec); - stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); - } - - var local_output_shapes: OutputShapes = undefined; - meta.mapAlloc((struct { - fn copy(_: void, shape: Shape) Shape { - return shape; - } - }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); - - const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); - const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); - stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); - - const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); - for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { - global.* = local.withPartitioning(mesh, spec); - stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); - local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } + var owned_output_shapes: OutputShapes = output_shapes; + const flat_output_shapes: []*Shape = zml.meta.collectMutPtrs(Shape, arena, &owned_output_shapes) catch @panic("OOM"); + defer arena.free(flat_output_shapes); const WrappedInputs = struct { inputs: *const Inputs, @@ -183,17 +73,15 @@ pub fn manualComputation( outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); } }; var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; - + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &owned_output_shapes }; _ = manualComputationInternal( compiler, arena, - mesh, @typeName(@TypeOf(body_fn)), &wrapped_inputs, flat_input_tensors, @@ -203,7 +91,6 @@ pub fn manualComputation( ) catch |err| switch (err) { error.OutOfMemory => @panic("OOM"), }; - return outputs_struct; } @@ -214,21 +101,27 @@ const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyo noinline fn manualComputationInternal( compiler: *Compiler, arena: std.mem.Allocator, - mesh: *const Sharding.Mesh, functor_name: []const u8, input_struct: *anyopaque, input_tensors: []*Tensor, output_struct: *anyopaque, - global_output_shapes: []const Shape, + output_shapes: []*Shape, local_fn: LocalComputationFn, ) error{OutOfMemory}![]*Tensor { const global_scope = compiler.currentScope(); const mlir_ctx = compiler.mlir_ctx; - log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); + const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { + .mesh => |m| m, + .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {d}: {f} and input {d}: {f}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), + }; + const manual_axes = collectManualAxes(mesh, input_tensors); + log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); const global_input_shapes = try arena.alloc(Shape, input_tensors.len); for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + const global_output_shapes = try arena.alloc(Shape, output_shapes.len); + for (global_output_shapes, output_shapes) |*shape, shape_ptr| shape.* = shape_ptr.*; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -242,17 +135,11 @@ noinline fn manualComputationInternal( const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); for (input_values, input_tensors) |*input_val, tensor_ptr| { - if (compiler.partitioner == .gspmd) { - tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); - } input_val.* = global_scope.value(tensor_ptr.*); } - const manual_block = switch (compiler.partitioner) { - .shardy => mlir.Block.init(input_types, input_locs), - .gspmd => global_scope.block, - }; - errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); + const manual_block = mlir.Block.init(input_types, input_locs); + errdefer manual_block.deinit(); const local_scope = compiler.pushBlock(manual_block); defer local_scope.pop(); @@ -264,14 +151,17 @@ noinline fn manualComputationInternal( }; } + for (output_shapes, global_output_shapes) |shape_ptr, global| { + shape_ptr.* = Sharding.shardedShape(global) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + compiler.manual_computation_depth += 1; defer compiler.manual_computation_depth -= 1; const local_outputs = try local_fn(arena, input_struct, output_struct); stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { - const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); - stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + for (0.., local_outputs, output_shapes) |i, local_tensor, local_shape| { + stdx.debug.assert(local_shape.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, local_shape.*, local_tensor._shape }); } const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); @@ -328,11 +218,95 @@ noinline fn manualComputationInternal( // Returned tensors describe the operation's global output shapes and sharding. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape._sharding = output_shape._sharding; + output_tensor_ptr._shape = output_shape; } return local_outputs; } +const ChosenMesh = union(enum) { + mesh: *const Sharding.Mesh, + incompatible: [2]u32, +}; + +fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ChosenMesh { + var mesh: ?*const Sharding.Mesh = null; + var mesh_source_id: u32 = @intCast(inputs.len); + for (0.., inputs) |i, input| { + if (input.shape()._sharding.mesh) |shape_mesh| { + if (mesh) |m| { + if (m != shape_mesh) + return .{ .incompatible = .{ mesh_source_id, @intCast(i) } }; + } else { + mesh = shape_mesh; + mesh_source_id = @intCast(i); + } + } + } + return .{ .mesh = mesh orelse fallback }; +} + +test chooseCommonMesh { + const fallback: Sharding.Mesh = .initTest(.fallback, .{.all}); + const first: Sharding.Mesh = .initTest(.first, .{ .data, .model }); + const second: Sharding.Mesh = .initTest(.second, .{ .experts, .model }); + const replicated = Tensor.init(.{ .a = 8 }, .f32); + const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); + const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); + + try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); +} + +fn collectManualAxes(mesh: *const Sharding.Mesh, inputs: []const *const Tensor) stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) { + var used: [Sharding.MAX_MESH_RANK]bool = @splat(false); + for (inputs) |input| { + const sharding = input.shape()._sharding; + if (sharding.mesh == null) continue; + std.debug.assert(sharding.mesh.? == mesh); + for (0..input.rank()) |dim| { + if (sharding.partition.get(dim).meshAxis()) |axis| used[axis] = true; + } + } + + var axes: stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) = .empty; + for (mesh.logical.axes.constSlice(), 0..) |axis, i| { + if (used[i]) axes.appendAssumeCapacity(axis); + } + return axes; +} + +test collectManualAxes { + const mesh: Sharding.Mesh = .initTest(.manual, .{ .data, .model, .unused }); + var data = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .data }); + var model = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .model }); + const replicated = Tensor.init(.{ .a = 8 }, .f32); + const open = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .open }); + + const axes = collectManualAxes(&mesh, &.{ &model, &replicated, &data, &model, &open }); + data = replicated; + model = replicated; + try std.testing.expectEqual(@as(usize, 2), axes.len); + try std.testing.expectEqualStrings("data", std.mem.span(axes.get(0))); + try std.testing.expectEqualStrings("model", std.mem.span(axes.get(1))); + try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{ &replicated, &open }).len); + try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{}).len); + + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const expected = shardy.ManualAxesAttr.get(ctx, &.{ + .init(ctx, "data"), + .init(ctx, "model"), + }).attribute(); + try std.testing.expect(shardy_.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); +} + const gspmd = struct { fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); @@ -375,172 +349,54 @@ const gspmd = struct { } }; -test manualComputation { - const zml = @import("../zml.zig"); +const shardy_ = struct { + pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { + var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; + for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); + return shardy.ManualAxesAttr.get(mlir_ctx, attrs.constSlice()).attribute(); + } +}; + +test "manualComputation passes local output shapes to the callback" { const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const mesh = zml.testing.tpMesh(); + const partitions = mesh.numPartitionsForLogicalAxis(.model); + if (partitions == 1) return error.SkipZigTest; - const LocalSum = struct { - x: Tensor, - bias: Tensor, - scale: i32, + var compiler: Compiler = .init(allocator, io, platform, .{ .meshes = &.{mesh} }); + defer compiler.deinit(); + compiler.activate(); + defer compiler.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + const scope = compiler.pushBlock(block); + defer scope.pop(); + + const global_shape = Shape.init(.{ .b = 4, .h = 3 * partitions, .hd = 128 }, .bf16) + .withPartitioning(mesh, .{ .h = .model }); + const local_shape = Shape.init(.{ .b = 4, .h = 3, .hd = 128 }, .bf16); + const input = Tensor.zeroes(global_shape).withPartitioning(mesh, .{ .h = .model }); + var received_shape: ?Shape = null; + const Context = struct { + input: Tensor, + received_shape: *?Shape, fn body(self: @This(), output_shape: Shape) Tensor { - return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); - } - - fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { - return manualComputation( - body, - .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, - x.shape(), - mesh_, - .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, - ); + self.received_shape.* = output_shape; + return self.input; } }; + const output = manualComputation( + Context.body, + .{ .input = input, .received_shape = &received_shape }, + global_shape, + ); - // Each shard sums two elements and broadcasts the result to its local output shape. - const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); - const values = try allocator.alloc(i32, shape.count()); - defer allocator.free(values); - - for (values, 0..) |*value, i| value.* = @intCast(i + 1); - - var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - const actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - - for (actual.constItems(i32), 0..) |value, i| { - const pair = values[i / 2 * 2 ..][0..2]; - try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); - } - - try std.testing.expect(output.shape().eqlWithTags(shape)); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } -} - -test "manualComputation composes across logical and physical meshes" { - const zml = @import("../zml.zig"); - if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - - const Topologies = enum { flat, torus }; - - const Model = struct { - fn increment(x: Tensor, _: Shape) Tensor { - return x.addConstant(1); - } - - fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { - const partitioning: ManualPartitioning(increment) = .{ - .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), - .out_specs = .parse(.{ .x = .model }), - }; - - const first = manualComputation(increment, x, x.shape(), mesh, partitioning); - const converted = first.convert(.f32); - return manualComputation(increment, converted, converted.shape(), mesh, partitioning); - } - }; - - const shape: Shape = .init(.{ .x = 16 }, .i32); - - var values: [16]i32 = undefined; - var expected: [16]f32 = undefined; - for (&values, &expected, 0..) |*value, *result, i| { - value.* = @intCast(i); - result.* = @floatFromInt(i + 2); - } - - for ([_]Topologies{ .flat, .torus }) |topology| { - const platform = try zml.Platform.init(allocator, io, .cpu, .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, - }); - defer platform.deinit(allocator, io); - - // Fold three physical axes into one logical axis, or split one into three. - const logical: Sharding.LogicalMesh = if (topology == .torus) - try .init(.{.model}, &.{.balanced}, &.{8}) - else - try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); - - const mesh = try platform.registerMesh("manual", logical); - - var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); - try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } - } -} - -test "partitionSpecs preserves optional tensors and active cache variants" { - const zml = @import("../zml.zig"); - const Logical = Sharding.Partitioning.Logical; - const Inputs = struct { - layers: []const Tensor, - bias: ?Tensor, - cache: union(enum) { disabled, values: [2]Tensor }, - scale: f32, - }; - - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - - const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); - var inputs: Inputs = .{ - .layers = &.{ tensor, tensor }, - .bias = null, - .cache = .{ .values = .{ tensor, tensor } }, - .scale = 2, - }; - const heads: Logical = .parse(.{ .h = .model }); - const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); - const expected: zml.ops.PartitionSpecs(Inputs) = .{ - .layers = &.{ heads, heads }, - .bias = null, - .cache = .{ .values = .{ heads, heads } }, - }; - try std.testing.expectEqualDeep(expected, specs); - - inputs.layers = &.{}; - inputs.bias = tensor; - inputs.cache = .disabled; - const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); - const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ - .layers = &.{}, - .bias = .replicated, - .cache = .disabled, - }; - try std.testing.expectEqualDeep(expected_replicated, replicated); + try std.testing.expect(received_shape != null); + try std.testing.expect(received_shape.?.eql(local_shape)); + try std.testing.expect(output.shape().eql(global_shape)); + try std.testing.expectEqualDeep(global_shape._sharding, output.shape()._sharding); } diff --git a/zml/ops/manual_computation2.zig b/zml/ops/manual_computation2.zig new file mode 100644 index 00000000..2d3f0aec --- /dev/null +++ b/zml/ops/manual_computation2.zig @@ -0,0 +1,465 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../Compiler.zig"); +const meta = @import("../meta.zig"); +const mlirx = @import("../mlirx.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/Compiler"); + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = meta.MapRestrict(Tensor, Logical).map(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. +pub fn manualComputation2( + comptime body_fn: anytype, + inputs: stdx.meta.FnParam(body_fn, 0), + output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), +) stdx.meta.FnReturn(body_fn) { + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + const Outputs = stdx.meta.FnReturn(body_fn); + + const compiler = Compiler.current(); + var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); + + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + + const WrappedInputs = struct { + inputs: *const Inputs, + output_shapes: *const OutputShapes, + + fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { + const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); + const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); + + outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); + + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + } + }; + + var outputs_struct: Outputs = undefined; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + + _ = manualComputationInternal( + compiler, + arena, + mesh, + @typeName(@TypeOf(body_fn)), + &wrapped_inputs, + flat_input_tensors, + @ptrCast(&outputs_struct), + flat_output_shapes, + WrappedInputs.wrap, + ) catch |err| switch (err) { + error.OutOfMemory => @panic("OOM"), + }; + + return outputs_struct; +} + +const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; + +/// Reified version of manualComputation2. +/// This one is compiled exactly once, user code just need to compile the `collectPtrs` +noinline fn manualComputationInternal( + compiler: *Compiler, + arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, + functor_name: []const u8, + input_struct: *anyopaque, + input_tensors: []*Tensor, + output_struct: *anyopaque, + global_output_shapes: []const Shape, + local_fn: LocalComputationFn, +) error{OutOfMemory}![]*Tensor { + const global_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + + log.debug("manualComputation2({s}) will use mesh {s}", .{ functor_name, mesh.name }); + + const global_input_shapes = try arena.alloc(Shape, input_tensors.len); + for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + + const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); + for (input_types, input_tensors) |*input_ty, tensor_ptr| { + const global = tensor_ptr._shape; + const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); + log.debug("\t- {f} -> {f}", .{ global, local }); + input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); + } + const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); + @memset(input_locs, compiler.unknown_location); + + const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); + for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } + input_val.* = global_scope.value(tensor_ptr.*); + } + + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); + + const local_scope = compiler.pushBlock(manual_block); + defer local_scope.pop(); + for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { + const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above + tensor_ptr.* = switch (compiler.partitioner) { + .shardy => Tensor._result(local, manual_block.argument(i)), + .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), + }; + } + + compiler.manual_computation_depth += 1; + defer compiler.manual_computation_depth -= 1; + + const local_outputs = try local_fn(arena, input_struct, output_struct); + stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation2 body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation2 body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + } + + const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |output_shape, i| { + global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); + } + + const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); + for (local_output_values, local_outputs) |*output_val, tensor_ptr| { + output_val.* = local_scope.value(tensor_ptr.*); + } + + const op: *const mlir.Operation = switch (compiler.partitioner) { + .shardy => op: { + // Shardy creates a sdy.manual_computation block. + _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); + + const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); + for (out_mesh_attrs, global_output_shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); + } + + break :op shardy.manual_computation( + mlir_ctx, + input_values, + global_output_types, + manual_block, + try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), + .get(mlir_ctx, out_mesh_attrs), + try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), + compiler.location, + ).appendTo(global_scope.block); + }, + .gspmd => op: { + // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier + // Shortcut to avoid empty optimization barrier + if (global_output_shapes.len == 0) return &.{}; + + const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); + const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |global_shape, i| { + global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); + } + + break :op stablehlo.optimizationBarrier( + mlir_ctx, + global_values, + global_types, + compiler.location, + ).appendTo(global_scope.block); + }, + }; + + // Returned tensors describe the operation's global output shapes and sharding. + for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { + output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); + output_tensor_ptr._shape._sharding = output_shape._sharding; + } + return local_outputs; +} + +const gspmd = struct { + fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { + const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); + const full_to_shard = stablehlo.custom_call( + mlir_ctx, + &.{scope.value(local_tensor)}, + &.{local_type}, + .{ + .call_target_name = "SPMDFullToShardShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), + }, + }, + location, + ).appendTo(scope.block); + return Tensor._result(local_shape, full_to_shard.result(0)); + } + + fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { + const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); + const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); + + const shard_to_full = stablehlo.custom_call( + compiler.mlir_ctx, + &.{local_value}, + &.{global_type}, + .{ + .call_target_name = "SPMDShardToFullShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), + }, + }, + compiler.location, + ).appendTo(scope.block); + return .{ shard_to_full.result(0), global_type }; + } +}; + +test manualComputation2 { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const mesh = zml.testing.tpMesh(); + + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, + + fn body(self: @This(), output_shape: Shape) Tensor { + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation2( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); + } + }; + + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation2 composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = .parse(.{ .x = .model }), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation2(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation2(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} diff --git a/zml/platform.zig b/zml/platform.zig index 54fb90e4..3b52c08e 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -8,8 +8,6 @@ pub const Target = @import("platforms").Platform; const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; -const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); -const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); @@ -259,12 +257,12 @@ pub const State = union(Target) { furiosa: void, pub const CudaState = struct { - flashinfer_cutlass_moe_runners: ?*flashinfer_cutlass_runtime.Runners = null, + fi_cutlass_moe_runners: ?*zml.moe.cutlass_flashinfer.Runners = null, fn deinit(self: *CudaState) void { - if (self.flashinfer_cutlass_moe_runners) |runners| { + if (self.fi_cutlass_moe_runners) |runners| { runners.deinit(); - self.flashinfer_cutlass_moe_runners = null; + self.fi_cutlass_moe_runners = null; } } }; @@ -383,9 +381,16 @@ pub const Platform = struct { switch (target) { .cuda => { - cuda.initFlashinferCutlass(platform, io) catch |err| { - log.warn("Failed to initialize Flashinfer CUTLASS MoE: {}", .{err}); - }; + if (zml.moe.cutlass_flashinfer.load(arena, io, platform)) { + zml.moe.cutlass_flashinfer.register(platform) catch |err| { + log.warn( + "Failed to register FlashInfer CUTLASS MoE custom calls: {}", + .{err}, + ); + }; + } else |err| { + log.warn("Failed to load FlashInfer CUTLASS MoE: {}", .{err}); + } }, else => {}, } @@ -860,12 +865,6 @@ pub const cuda = struct { } }; - /// SM count of an addressable CUDA device; unavailable attributes return null. - pub fn coreCount(platform: *const zml.Platform, device_index: usize) ?u32 { - const attribute = platform.devices[device_index].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return null; - return std.math.cast(u32, attribute.int64); - } - pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { if (platform.target != .cuda) return null; const devices = platform.pjrt_client.devices(platform.pjrt_api); @@ -878,29 +877,6 @@ pub const cuda = struct { } } else null; } - - fn initFlashinferCutlass(platform: *Platform, io: std.Io) !void { - if (comptime !Target.cuda.isEnabled()) return error.UnsupportedPlatform; - const cc = computeCapability(platform) orelse return error.UnsupportedArchitecture; - - const allocator = platform.arena.allocator(); - const runners = try allocator.create(flashinfer_cutlass_runtime.Runners); - errdefer allocator.destroy(runners); - - try flashinfer_cutlass_runtime.load(allocator, io, cc.sm()); - - runners.* = .{}; - errdefer runners.deinit(); - - try platform.registerFfi(.{ - .name = flashinfer_cutlass_contract.name, - .handler = flashinfer_cutlass_runtime.handler, - .traits = .{ .command_buffer_compatible = true }, - .platform_name = if (flashinfer_cutlass_contract.compute_on_host) "Host" else null, - }); - - platform.state.cuda.flashinfer_cutlass_moe_runners = runners; - } }; pub const rocm = struct { @@ -1015,15 +991,12 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; - const buffers = call_frame.args.buffers(); - if (buffers.len <= 0) return null; - const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = buffers[0]; + const buffer = call_frame.args.buffers()[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -1047,13 +1020,9 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); - defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - - const event = pjrt_buffer.readyEvent(pjrt_api); - defer event.deinit(pjrt_api); - try event.awaitRaw(pjrt_api); + try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 3827fe27..f854f02f 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,337 +2,68 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; +const Dims = Shape.DimsArray; + +const StringBuilder = std.ArrayListUnmanaged(u8); + +pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); +pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); const log = std.log.scoped(.@"zml/safetensors"); -// TODO: consider promoting this to the top level struct -// TODO: Feels like we are missing a mapping file -> tensors that would be helfpful for loading weights -/// Maps tensor names to their shape and positions in different sub files. -pub const TensorRegistry = struct { - arena: std.heap.ArenaAllocator, +const BYTES_HEADER = 8; - tensors: std.StringArrayHashMapUnmanaged(Tensor), - - /// Metadata found in `__metadata__` field of the safetensors header. - /// This is rarely use and the information there is not standardized, so we let user code handle it. - /// Here we just store the plain bytes, reach out if you need this to be already parsed. - metadata: []const u8, - - pub fn init(allocator: std.mem.Allocator) TensorRegistry { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = "", - }; - } - - pub fn deinit(self: *TensorRegistry) void { - self.arena.deinit(); - } - - pub fn fromRepo(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !TensorRegistry { - const file_type, const file = try resolveModelEntrypoint(io, repo); - defer file.close(io); - - var registry: TensorRegistry = .init(allocator); - switch (file_type) { - .safetensors => try registry.parseSafetensors(io, allocator, file), - .index => try registry.parseIndexJson(io, allocator, repo, file), - .full_index => try registry.parseFullIndexJson(io, allocator, repo, file), - .unknown => return error.InvalidPath, - } - return registry; - } - - pub fn fromPath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !TensorRegistry { - const file_type: FileType = .resolve(path); - switch (file_type) { - .safetensors => { - // open directly - const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); - var registry: TensorRegistry = .init(allocator); - try registry.parseSafetensors(io, allocator, file); - return registry; - }, - .full_index, - .index, - => { - // resolve repo as parent folder then open - const repo_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const repo = std.Io.Dir.openDir(.cwd(), io, repo_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); - - var registry: TensorRegistry = .init(allocator); - try switch (file_type) { - .index => registry.parseIndexJson(io, allocator, repo, file), - .full_index => registry.parseFullIndexJson(io, allocator, repo, file), - .safetensors, .unknown => unreachable, - }; - return registry; - }, - .unknown => { - // the path is possibly a folder, let's try that - const repo = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - return try fromRepo(allocator, io, repo); - }, - } - } - - /// Register a tensor, duplicate tensor name and uri. - /// If you create a lot of tensors sharing the same uri, - /// considere calling `registry.tensors` directly and avoid duplicated uris. - pub fn registerTensor(self: *TensorRegistry, tensor: Tensor) !void { - const allocator = self.arena.allocator(); - - const owned_name = try allocator.dupe(u8, tensor.name); - errdefer allocator.free(owned_name); - const owned_uri = try allocator.dupe(u8, tensor.file_uri); - errdefer allocator.free(owned_uri); - - const owned_tensor: Tensor = .{ - .name = owned_name, - .file_uri = owned_uri, - .shape = tensor.shape, - .offset = tensor.offset, - }; - try self.tensors.put(allocator, owned_name, owned_tensor); - } - - pub fn reader( - self: *TensorRegistry, - io: std.Io, - tensor_name: []const u8, - buffer: []u8, - ) !TensorReader { - const tensor = self.tensors.get(tensor_name) orelse { - log.err("Tensor {s} not found in registry", .{tensor_name}); - return error.TensorNotFound; - }; - - return try .init(io, tensor, buffer, .{}); - } - - pub fn totalBytes(self: *TensorRegistry) u64 { - var total: u64 = 0; - - var it = self.tensors.iterator(); - while (it.next()) |entry| { - total += entry.value_ptr.byteSize(); - } - - return total; - } - - /// Parse a given .safetensors file and create entries for each tensor - pub fn parseSafetensors( - self: *TensorRegistry, - io: std.Io, - scratch_allocator: std.mem.Allocator, - file: std.Io.File, - ) !void { - var tmp_state = std.heap.ArenaAllocator.init(scratch_allocator); - defer tmp_state.deinit(); - const tmp_arena = tmp_state.allocator(); - - const self_arena = self.arena.allocator(); - var owned_file_name = try self_arena.alloc(u8, std.fs.max_path_bytes); - const file_name_len = try file.realPath(io, owned_file_name); - // Try to release extra memory, but we don't care if it fails - _ = self_arena.resize(owned_file_name, file_name_len); - owned_file_name = owned_file_name[0..file_name_len]; - - const header_buf = try tmp_arena.alloc(u8, 64 * 1024); - var file_reader: std.Io.File.Reader = file.reader(io, header_buf); - - const json_header_length: u64 = try file_reader.interface.takeInt(u64, .little); - const BYTES_HEADER = 8; - const data_start_offset = BYTES_HEADER + json_header_length; - const json_data = try tmp_arena.alloc(u8, @intCast(json_header_length)); - - try file_reader.interface.readSliceAll(json_data); - const headers: std.json.Value = try std.json.parseFromSliceLeaky( - std.json.Value, - tmp_arena, - json_data, - .{}, - ); - - try self.tensors.ensureUnusedCapacity(self_arena, headers.object.count()); - for (headers.object.keys(), headers.object.values()) |key, value| { - if (std.mem.eql(u8, key, "__metadata__")) { - // value belong to tmp arena, duplicate it into self arena. - var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); - try std.json.fmt(value, .{ .whitespace = .minified }).format(&owned_metadata.writer); - self.metadata = owned_metadata.written(); - continue; - } - - const owned_name = try self_arena.dupe(u8, key); - const tensor: Tensor = try .parse(value.object, owned_file_name, owned_name, data_start_offset); - self.tensors.putAssumeCapacity(owned_name, tensor); - } - } - - pub fn parseIndexJson( - self: *TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - index_json_file: std.Io.File, - ) !void { - var tmp_state: std.heap.ArenaAllocator = .init(gpa); - defer tmp_state.deinit(); - const tmp = tmp_state.allocator(); - - const file_reader_buf = try tmp.alloc(u8, 64 * 1024); - var file_reader: std.Io.File.Reader = index_json_file.reader(io, file_reader_buf); - var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - tmp, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - var safetensors_files: std.StringArrayHashMapUnmanaged(void) = .empty; - try safetensors_files.ensureUnusedCapacity(tmp, 64); - if (index.object.get("weight_map")) |wm| { - for (wm.object.values()) |file_name| { - try safetensors_files.put(tmp, file_name.string, {}); - } - } else { - log.warn("No weight_map attribute found in index", .{}); - } - - for (safetensors_files.keys()) |file_name| { - const file = try repo.openFile(io, file_name, .{}); - defer file.close(io); - try self.parseSafetensors(io, tmp, file); - } - } - - pub fn parseFullIndexJson( - self: *TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - index_json_file: std.Io.File, - ) !void { - var tmp_state: std.heap.ArenaAllocator = .init(gpa); - defer tmp_state.deinit(); - const tmp = tmp_state.allocator(); - - const self_arena = self.arena.allocator(); - - var repo_uri_buf = try tmp.alloc(u8, 1024); - const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; - - // Note: this is relatively inefficient, we read the full file, copy most of it's content, - // then reduce to TensorRegistry format. - // We could update TensorRegistry as we stream the json. - // Zig std json API are a bit awkward around streaming though. - const file_reader_buf = try tmp.alloc(u8, 64 * 1024); - var file_reader = index_json_file.reader(io, file_reader_buf); - var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - tmp, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - var realpaths: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - try realpaths.ensureUnusedCapacity(tmp, 64); - const wm = index.object.get("weight_map") orelse return error.InvalidJson; - - try self.tensors.ensureUnusedCapacity(self_arena, wm.object.count()); - for (wm.object.keys(), wm.object.values()) |key, value| { - const tensor_name = try self_arena.dupe(u8, key); - const object = value.object; - const filename = value.object.get("filename").?.string; - const filename_entry = try realpaths.getOrPut(tmp, filename); - const file_uri = if (filename_entry.found_existing) - filename_entry.value_ptr.* - else uri: { - const uri = try std.fs.path.join(self_arena, &.{ repo_name, filename }); - filename_entry.value_ptr.* = uri; - break :uri uri; - }; - - const tensor_info: Tensor = try .parse(object, file_uri, tensor_name, 0); - self.tensors.putAssumeCapacity(tensor_name, tensor_info); - } - - if (index.object.get("metadata")) |metadata| { - // value belong to tmp arena, duplicate it into self arena. - var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); - try std.json.fmt(metadata, .{ .whitespace = .minified }).format(&owned_metadata.writer); - self.metadata = owned_metadata.written(); - } - } - - pub fn saveAsFullIndexJson( - self: *const TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - out: *std.Io.Writer, - ) !void { - var repo_uri_buf = try gpa.alloc(u8, std.fs.max_path_bytes); - defer gpa.free(repo_uri_buf); - const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; - var total_size: u64 = 0; - for (self.tensors.values()) |value| total_size += value.byteSize(); - - try out.print( - \\{{ - \\"metadata": {{"num_tensors":{d},"total_size":{d}}}, - \\"weight_map": {{ - \\ - , .{ self.tensors.count(), total_size }); - - const n = self.tensors.count(); - for (0.., self.tensors.keys(), self.tensors.values()) |i, key, value| { - std.debug.assert(std.mem.startsWith(u8, value.file_uri, repo_name)); - const local_name = value.file_uri[repo_name.len + 1 ..]; - const end_offset = value.offset + value.byteSize(); - const safetensors_dtype: []const u8 = dt: { - for (safetensors_dtype_to_zml_dtype.keys(), safetensors_dtype_to_zml_dtype.values()) |st_dt, zml_dt| { - if (zml_dt == value.shape._dtype) break :dt st_dt; - } - // Note: we could bitcast to u8, - // but reaching this panic implies the user didn't loaded this TensorRegistry - // from a safetensors file, and manually created the dict. - // In which case, they know what they are doing, - // and can easily insert the bitcast themselves. - std.debug.panic("Safetensors doesn't support data type {}", .{value.shape._dtype}); - }; - const comma = if (i + 1 == n) "" else ","; - try out.print( - \\ "{s}": {{"filename":"{s}","shape":{f},"data_offsets":[{d},{d}],"dtype":"{s}"}}{s} - \\ - , .{ key, local_name, std.json.fmt(value.shape.dims(), .{ .whitespace = .minified }), value.offset, end_offset, safetensors_dtype, comma }); - } - - try out.writeAll( - \\} - \\} - \\ - ); - try out.flush(); - } +pub const FileType = enum { + index, + safetensors, + unknown, }; +pub fn fetchRegistry( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + model: std.Io.File, + model_path: []const u8, +) !TensorRegistry { + const file_type = resolveFiletype(model_path); + + return switch (file_type) { + .index => blk: { + const file_reader_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(file_reader_buf); + + var index_reader = model.reader(io, file_reader_buf); + + var safetensors_index = try parseSafetensorsIndex( + allocator, + &index_reader.interface, + ); + defer safetensors_index.deinit(); + + var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); + + try parseSafetensorsIndexFiles( + allocator, + io, + ®istry, + safetensors_index, + repo, + ); + + break :blk registry; + }, + .safetensors => blk: { + var registry: TensorRegistry = .init(allocator); + + try parseSafetensors(allocator, io, ®istry, model); + + break :blk registry; + }, + else => return error.InvalidPath, + }; +} + pub const TensorReader = struct { tensor: Tensor, file: std.Io.File, @@ -507,30 +238,6 @@ pub const Tensor = struct { return TensorReader.init(io, self, buffer, opts); } - pub fn parse(object: std.json.ObjectMap, file_uri: []const u8, name: []const u8, base_offset: u64) !Tensor { - const shape_field = object.get("shape").?.array; - if (shape_field.items.len > Shape.MAX_RANK) { - log.warn("Can't load tensor {s}, too many dims: {}", .{ name, shape_field.items.len }); - return error.TooManyDimensions; - } - - const offset_field = object.get("data_offsets").?; - const start: u64 = @intCast(offset_field.array.items[0].integer); - const dtype = try stringToDtype(object.get("dtype").?.string); - - var dims: Shape.DimsArray = .empty; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); - } - - return .{ - .file_uri = file_uri, - .name = name, - .shape = .init(dims.slice(), dtype), - .offset = base_offset + start, - }; - } - pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ self.name, @@ -542,100 +249,620 @@ pub const Tensor = struct { } }; -pub const FileType = enum { - full_index, - index, - safetensors, - unknown, +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, - pub fn resolve(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors.full_index.json")) { - return .full_index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { - return .safetensors; - } else { - return .unknown; + tensors: Tensors, + metadata: Metadatas, + + mutex: std.Io.Mutex = .init, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .{}, + }; + } + + pub fn initWithMetadata( + allocator: std.mem.Allocator, + metadata: Metadatas, + ) !TensorRegistry { + var self: TensorRegistry = .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .empty, + }; + + try self.mergeMetadata(metadata); + + return self; + } + + pub fn fromRepo( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + ) !TensorRegistry { + const entrypoint_file, const entrypoint_path = try resolveModelEntrypoint(io, repo); + defer entrypoint_file.close(io); + return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); + } + + pub fn fromPath( + allocator: std.mem.Allocator, + io: std.Io, + path: []const u8, + ) !TensorRegistry { + var repo = try resolveModelRepo(io, path); + defer repo.close(io); + + const entrypoint_file, const entrypoint_path = if (std.mem.endsWith(u8, path, ".safetensors.index.json") or std.mem.endsWith(u8, path, ".safetensors")) + .{ try repo.openFile(io, path, .{ .mode = .read_only }), path } + else + try resolveModelEntrypoint(io, repo); + defer entrypoint_file.close(io); + + return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); + } + + pub fn deinit(self: *TensorRegistry) void { + const allocator = self.arena.allocator(); + self.tensors.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } + + pub fn mergeMetadata( + self: *TensorRegistry, + other: Metadatas, + ) !void { + // self.mutex.lock(); + // defer self.mutex.unlock(); + + const allocator = self.arena.allocator(); + + var it = other.iterator(); + while (it.next()) |entry| { + const key = try allocator.dupe(u8, entry.key_ptr.*); + const value = try entry.value_ptr.*.clone(allocator); + + const gop = try self.metadata.getOrPut(allocator, key); + if (gop.found_existing) { + gop.value_ptr.*.deinit(allocator); + gop.value_ptr.* = value; + log.debug("Overwrote existing metadata key={s} with value={f}", .{ key, value }); + } else { + gop.value_ptr.* = value; + log.debug("Added new metadata key={s} with value={f}", .{ key, value }); + } } } + + pub fn registerTensor( + self: *TensorRegistry, + tensor: Tensor, + ) !void { + const allocator = self.arena.allocator(); + + // self.mutex.lock(); + // defer self.mutex.unlock(); + + var tensor_copy = tensor; + + tensor_copy.name = try allocator.dupe(u8, tensor.name); + tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); + + try self.tensors.put(allocator, tensor_copy.name, tensor_copy); + } + + pub fn reader( + self: *TensorRegistry, + io: std.Io, + tensor_name: []const u8, + buffer: []u8, + ) !TensorReader { + const tensor = self.tensors.get(tensor_name) orelse { + log.err("Tensor {s} not found in registry", .{tensor_name}); + return error.TensorNotFound; + }; + + return try .init(io, tensor, buffer, .{}); + } + + pub fn iterator(self: *TensorRegistry) Tensors.Iterator { + return self.tensors.iterator(); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } }; +pub fn parseSafetensors( + allocator: std.mem.Allocator, + io: std.Io, + registry: *TensorRegistry, + file: std.Io.File, +) !void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const header_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(header_buf); + + var reader = file.reader(io, header_buf); + + var file_uri_buf: [1024]u8 = undefined; + const file_uri_len = try file.realPath(io, &file_uri_buf); + const file_uri = file_uri_buf[0..file_uri_len]; + + const json_header_length: u64 = try reader.interface.takeInt(u64, .little); + const data_start_offset = BYTES_HEADER + json_header_length; + const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); + + try reader.interface.readSliceAll(json_data); + + const headers: std.json.Value = try std.json.parseFromSliceLeaky( + std.json.Value, + arena.allocator(), + json_data, + .{}, + ); + + var it = headers.object.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + + if (std.mem.eql(u8, key, "__metadata__")) { + // Parse metadata into a local arena to avoid concurrent allocations + // against registry.arena (which is not thread-safe). + var local_arena = std.heap.ArenaAllocator.init(allocator); + defer local_arena.deinit(); + + const metas = try parseMetadata(local_arena.allocator(), value); + try registry.mergeMetadata(metas); + + continue; + } + + const shape_field = value.object.get("shape").?.array; + + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); + return error.TooManyDimensions; + } + + const offset_field = value.object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const dtype = try stringToDtype(value.object.get("dtype").?.string); + + var dims: Dims = .empty; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + const tensor: Tensor = .{ + .file_uri = file_uri, + .name = key, + .shape = .init(dims.slice(), dtype), + .offset = data_start_offset + start, + }; + + try registry.registerTensor(tensor); + } +} + +const SafetensorsIndex = struct { + pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); + + arena: std.heap.ArenaAllocator, + + map: Map, + metadata: Metadatas, + + pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .map = .empty, + .metadata = .empty, + }; + } + + pub fn iterator(self: *SafetensorsIndex) Map.Iterator { + return self.map.iterator(); + } + + pub fn deinit(self: *SafetensorsIndex) void { + const allocator = self.arena.allocator(); + self.map.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } +}; + +fn parseSafetensorsIndex( + allocator: std.mem.Allocator, + reader: *std.Io.Reader, +) !SafetensorsIndex { + var safetensors_index: SafetensorsIndex = .init(allocator); + errdefer safetensors_index.deinit(); + + const arena_allocator = safetensors_index.arena.allocator(); + var json_reader: std.json.Reader = .init(arena_allocator, reader); + + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + arena_allocator, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + const weight_map = index.object.get("weight_map"); + + if (weight_map) |wm| { + var it = wm.object.iterator(); + + while (it.next()) |entry| { + const weight_name = entry.key_ptr.*; + const filename = entry.value_ptr.string; + + const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); + + if (!map_entry.found_existing) { + map_entry.value_ptr.* = .empty; + } + + try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + if (index.object.get("metadata")) |metadata_val| { + safetensors_index.metadata = try parseMetadata(arena_allocator, metadata_val); + } + + return safetensors_index; +} + +fn parseSafetensorsIndexFiles( + allocator: std.mem.Allocator, + io: std.Io, + registry: *TensorRegistry, + safetensors_index: SafetensorsIndex, + repo: std.Io.Dir, +) !void { + const filenames = safetensors_index.map.keys(); + for (filenames) |filename| { + const file = try repo.openFile(io, filename, .{}); + defer file.close(io); + try parseSafetensors(allocator, io, registry, file); + } +} + +pub fn resolveFiletype(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path, ".safetensors")) { + return .safetensors; + } else { + return .unknown; + } +} + pub const ModelPathResolutionError = error{ FileNotFound, InvalidPath, } || std.mem.Allocator.Error; -pub fn resolveModelRepo(io: std.Io, path: []const u8) ModelPathResolutionError!std.Io.Dir { - switch (FileType.resolve(path)) { - .full_index, .index, .safetensors => { - // The path is a file, let's open it's folder - const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; +pub fn resolveModelRepo( + io: std.Io, + path: []const u8, +) ModelPathResolutionError!std.Io.Dir { + if (std.mem.endsWith(u8, path, ".safetensors.index.json") or + std.mem.endsWith(u8, path, ".safetensors")) + { + const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, }; + }; - return dir; - }, - .unknown => { - // The path is probably a directory, let's try to open it as such - const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - return dir; - }, + return dir; } -} -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { FileType, std.Io.File } { - const default_entrypoints: []const struct { FileType, []const u8 } = &.{ - .{ .full_index, "model.safetensors.full_index.json" }, - .{ .full_index, "consolidated.safetensors.full_index.json" }, - .{ .index, "model.safetensors.index.json" }, - .{ .index, "consolidated.safetensors.index.json" }, - .{ .safetensors, "model.safetensors" }, - .{ .safetensors, "consolidated.safetensors" }, + const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; }; - for (default_entrypoints) |entrypoint| { - const file_type, const file_name = entrypoint; - const file = repo.openFile(io, file_name, .{ .mode = .read_only }) catch continue; - - return .{ file_type, file }; - } - - return error.FileNotFound; + return dir; } -const safetensors_dtype_to_zml_dtype = std.StaticStringMap(DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E8M0", .f8e8m0 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "F4_E2M1", .f4e2m1 }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, -}); +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, []const u8 } { + const default_entrypoints = .{ + "model.safetensors.index.json", + "consolidated.safetensors.index.json", + "model.safetensors", + "consolidated.safetensors", + }; -fn stringToDtype(safetensor_dtype: []const u8) !DataType { - return safetensors_dtype_to_zml_dtype.get(safetensor_dtype) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_dtype}); + inline for (default_entrypoints) |entrypoint| { + if (repo.openFile(io, entrypoint, .{ .mode = .read_only })) |index_file| { + return .{ index_file, entrypoint }; + } else |_| {} + } + + return ModelPathResolutionError.FileNotFound; +} + +const Metadata = union(enum) { + null: void, + int: i64, + float: f64, + bool: bool, + string: []const u8, + + array_bool: []const bool, + array_int: []const i64, + array_float: []const f64, + array_string: []const []const u8, + + pub const ItemType = enum { + int, + float, + bool, + string, + + pub fn toZigType(comptime kind: ItemType) type { + return switch (kind) { + .int => i64, + .float => f64, + .bool => bool, + .string => []const u8, + }; + } + }; + + pub fn wrap(x: anytype) Metadata { + return switch (@TypeOf(x)) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, + inline f16, f32, f64 => .{ .float = @floatCast(x) }, + bool => .{ .bool = x }, + []const u8 => .{ .string = x }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), + }; + } + + pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { + return switch (@TypeOf(any_slice[0])) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => { + const res = try allocator.alloc(i64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @intCast(val); + return .{ .array_int = res }; + }, + inline f16, f32, f64 => { + const res = try allocator.alloc(f64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @floatCast(val); + return .{ .array_float = res }; + }, + bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, + []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), + }; + } + + pub fn format( + self: Metadata, + writer: *std.Io.Writer, + ) !void { + switch (self) { + .null => _ = try writer.write("null"), + .string => |s| try writer.print("{s}", .{s}), + .bool => |b| try writer.print("{}", .{b}), + .int => |i| try writer.print("{d}", .{i}), + .float => |f| try writer.print("{d}", .{f}), + .array_bool => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{}", .{v}); + } + try writer.writeByte(']'); + }, + .array_int => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_float => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_string => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("\"{s}\"", .{v}); + } + try writer.writeByte(']'); + }, + } + } + + pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { + switch (self) { + .string => |s| allocator.free(s), + .array_bool => |s| allocator.free(s), + .array_int => |s| allocator.free(s), + .array_float => |s| allocator.free(s), + .array_string => |s| { + for (s) |str| allocator.free(str); + allocator.free(s); + }, + else => {}, + } + } + + pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { + return switch (self) { + .null => .null, + .int => |v| .{ .int = v }, + .float => |v| .{ .float = v }, + .bool => |v| .{ .bool = v }, + .string => |s| .{ .string = try allocator.dupe(u8, s) }, + .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, + .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, + .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, + .array_string => |s| blk: { + const new_slice = try allocator.alloc([]const u8, s.len); + errdefer allocator.free(new_slice); + for (s, 0..) |str, i| { + new_slice[i] = try allocator.dupe(u8, str); + } + break :blk .{ .array_string = new_slice }; + }, + }; + } +}; + +fn stringToDtype(safetensor_type: []const u8) !DataType { + const map = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E8M0", .f8e8m0 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "F4_E2M1", .f4e2m1 }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, + }); + + return map.get(safetensor_type) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); return error.UnsupportedDataType; }; } + +fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { + var metadatas: Metadatas = .{}; + var prefix_buf: [1024]u8 = undefined; + var prefix = StringBuilder.initBuffer(&prefix_buf); + + try populateMetadata(allocator, &prefix, val, &metadatas); + + return metadatas; +} + +fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { + const key = prefix.items; + return switch (val) { + .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), + .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), + .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), + .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), + .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), + .array => |v| { + if (v.items.len == 0) return; + if (validSlice(v)) |item_type| { + const data: Metadata = switch (item_type) { + .bool => blk: { + const values = try allocator.alloc(bool, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.bool; + break :blk .{ .array_bool = values }; + }, + .integer => blk: { + const values = try allocator.alloc(i64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.integer; + break :blk .{ .array_int = values }; + }, + .float => blk: { + const values = try allocator.alloc(f64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.float; + break :blk .{ .array_float = values }; + }, + inline .string, .number_string => |tag| blk: { + const values = try allocator.alloc([]const u8, v.items.len); + for (v.items, 0..) |item, i| { + values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); + } + break :blk .{ .array_string = values }; + }, + .null, .array, .object => unreachable, + }; + try metadatas.put(allocator, try allocator.dupe(u8, key), data); + } else { + for (v.items, 0..) |item, i| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + try populateMetadata(allocator, prefix, item, metadatas); + prefix.items.len = old_len; + } + } + }, + .object => |v| { + var obj_iter = v.iterator(); + while (obj_iter.next()) |entry| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.appendSliceAssumeCapacity(entry.key_ptr.*); + try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); + prefix.items.len = old_len; + } + }, + }; +} + +/// We can only create a Zig slice out of json array, if all values +/// in the array have the same type. +fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { + if (v.items.len == 0) return null; + + const item_type: std.meta.Tag(std.json.Value) = v.items[0]; + switch (item_type) { + .null, .array, .object => return null, + else => {}, + } + + for (v.items[1..]) |item| { + if (item != item_type) + return null; + } + + return item_type; +} diff --git a/zml/tensor.zig b/zml/tensor.zig index ee084835..a7782844 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -282,13 +282,7 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { - .cpu, .neuron, .metal => { - // A single memory needs no copy, but the result must still be a distinct - // tensor: `reuseBuffer` donations are keyed by tensor id. - var res = _result(self._shape, self.value()); - res._shape._sharding = self._shape._sharding; - return res; - }, + .cpu, .neuron, .metal => return self, .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, } @@ -4819,7 +4813,7 @@ pub const Tensor = struct { defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation((struct { + ops.manualComputation2((struct { input: Tensor, name: []const u8, diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index b1586afb..551e4bcb 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -380,6 +380,7 @@ pub const Tokenizer = struct { tokenizer: *const Tokenizer, state_storage: []u8, state: *c.iree_tokenizer_decode_state_t, + text: std.Io.Writer.Allocating, fn init(tokenizer: *const Tokenizer, allocator: std.mem.Allocator) !Decoder { var state_size: usize = undefined; @@ -399,12 +400,14 @@ pub const Tokenizer = struct { .tokenizer = tokenizer, .state_storage = state_storage, .state = state.?, + .text = .init(allocator), }; } pub fn deinit(self: *Decoder) void { c.iree_tokenizer_decode_state_deinitialize(self.state); self.tokenizer.allocator.free(self.state_storage); + self.text.deinit(); } pub fn reset(self: *Decoder) !void { @@ -419,33 +422,24 @@ pub const Tokenizer = struct { self.state = state.?; } - pub fn feedOne(self: *Decoder, token_id: u32, out: []u8) ![]u8 { - std.debug.assert(out.len != 0); - var consumed: usize = 0; - var produced: usize = 0; - try checkOk(c.iree_tokenizer_decode_state_feed( - self.state, - tokenIdList(&.{token_id}), - mutableStringView(out), - &consumed, - &produced, - )); - std.debug.assert(consumed == 1); - return out[0..produced]; + pub fn feedOne(self: *Decoder, token_id: u32) ![]const u8 { + self.text.clearRetainingCapacity(); + try self.feedInto(&.{token_id}, &self.text.writer); + return self.text.written(); } - pub fn finalize(self: *Decoder, out: []u8) ![]u8 { - std.debug.assert(out.len != 0); - var produced: usize = 0; - try checkOk(c.iree_tokenizer_decode_state_finalize( - self.state, - mutableStringView(out), - &produced, - )); - return out[0..produced]; + pub fn finalize(self: *Decoder) ![]const u8 { + self.text.clearRetainingCapacity(); + try self.finalizeInto(&self.text.writer); + return self.text.written(); } pub fn decode(self: *Decoder, token_ids: []const u32, out: *std.Io.Writer) !void { + try self.feedInto(token_ids, out); + try self.finalizeInto(out); + } + + fn feedInto(self: *Decoder, token_ids: []const u32, out: *std.Io.Writer) !void { var min_output: usize = @max(token_ids.len * 4, 128); var remaining = token_ids; while (remaining.len > 0) { @@ -480,8 +474,10 @@ pub const Tokenizer = struct { min_output = @max(128, min_output -| produced); remaining = remaining[consumed..]; } + } - min_output = 32; + fn finalizeInto(self: *Decoder, out: *std.Io.Writer) !void { + var min_output: usize = 32; while (true) { var produced: usize = 0; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index fc304b81..6d76a7b7 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -101,18 +101,18 @@ pub const Tokenizer = union(Tokenizers) { }; } - // Only valid until the next decode and while the decoder AND buffer is alive. - pub fn feedOne(self: *Decoder, token_id: u32, buffer: []u8) ![]const u8 { + // Only valid until the next decode and while the decoder is alive. + pub fn feedOne(self: *Decoder, token_id: u32) ![]const u8 { return switch (self.*) { - .iree => |*d| d.feedOne(token_id, buffer), + .iree => |*d| d.feedOne(token_id), inline else => |*d| try d.next(token_id) orelse &.{}, }; } - /// Output is only valid until the next feed/finalize call and while the decoder AND buffer is alive. - pub fn finalize(self: *Decoder, buffer: []u8) ![]const u8 { + /// Output is only valid until the next feed/finalize call and while the decoder is alive. + pub fn finalize(self: *Decoder) ![]const u8 { return switch (self.*) { - .iree => |*v| v.finalize(buffer), + .iree => |*v| v.finalize(), inline else => &.{}, }; } diff --git a/zml/zml.zig b/zml/zml.zig index dafc3a0f..23d12e75 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -26,7 +26,7 @@ pub const Exe = exe.Exe; pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); -pub const kernel = @import("kernel/kernel.zig"); +pub const kernel = @import("kernel.zig"); pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; @@ -81,9 +81,3 @@ pub const KiB = 1024; pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; - -comptime { - // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. - // Otherwise exported functions declared in `bazel` aren't compiled. - std.testing.refAllDecls(bazel); -}