diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index d7d8673c..d3f52b08 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -1363,3 +1363,11 @@ pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u } }).callback, &context), error.InvalidMlirBytecodeVersion); } + +pub fn optimization_barrier(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Value { + return mlir.Operation.make(ctx, "stablehlo.optimization_barrier", .{ + .operands = &.{value}, + .result_type_inference = true, + .location = location, + }); +} diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 2dc2b9f3..2ff6d80c 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1650,10 +1650,10 @@ pub const Location = struct { } pub fn namedFmt(loc: Location, ctx: Context, comptime fmt: [:0]const u8, args: anytype) Location { - var buf: [256]u8 = undefined; + var buf: [512]u8 = undefined; var stream = std.io.fixedBufferStream(&buf); std.fmt.format(stream.writer(), fmt, args) catch { - buf[256 - 3 ..].* = "...".*; + buf[512 - 3 ..].* = "...".*; }; return loc.named(ctx, @ptrCast(stream.getWritten())); } diff --git a/zml/floats.zig b/zml/floats.zig index 230ae97d..08289616 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -83,7 +83,8 @@ fn FloatType(sign_bits: u1, exponent_bits: u8, mantissa_bits: u8, innerT: type) } fn expBias() u8 { - return std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); + if (exponent_bits == 1) return 0; + return (1 << (exponent_bits - 1)) - 1; } pub fn format( diff --git a/zml/module.zig b/zml/module.zig index 453f0cb0..2500b986 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -96,6 +96,7 @@ pub const CompilationContext = struct { _blocks: std.BoundedArray(Block, 64) = .{}, _fn_cache: FnCache = .{}, + _block_name: []const u8 = "", _block_args: TensorToBlockArg = .{}, _unique_id: u64 = 10000, @@ -185,7 +186,16 @@ pub const CompilationContext = struct { } pub fn location(self: *const CompilationContext, src: std.builtin.SourceLocation, comptime name: [:0]const u8, args: anytype) mlir.Location { - return self._mlir_ctx.location(src).namedFmt(self._mlir_ctx, name, args); + return if (self._block_name.len == 0) + self._mlir_ctx.location(src).namedFmt(self._mlir_ctx, name, args) + else if (name.len == 0) + self._mlir_ctx.location(src).namedFmt(self._mlir_ctx, "{s}", .{self._block_name}) + else + self._mlir_ctx.location(src).namedFmt(self._mlir_ctx, "{s}/" ++ name, .{self._block_name} ++ args); + } + + pub fn setBlockName(self: *CompilationContext, block_name: []const u8) void { + self._block_name = block_name; } /// Compiles the given function with the given arguments. @@ -917,13 +927,11 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); - // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); - // setFlag(&options, "xla_gpu_enable_custom_fusions", true); - // setFlags(&options, "xla_gpu_enable_address_computation_fusion", true); - // setFlag(&options, "xla_gpu_enable_dynamic_slice_fusion", true); - // setFlag(&options, "xla_gpu_enable_while_loop_double_buffering", true); + setFlag(&options, "xla_gpu_enable_custom_fusions", true); + setFlag(&options, "xla_gpu_enable_dynamic_slice_fusion", true); + setFlag(&options, "xla_gpu_enable_while_loop_double_buffering", true); // setFlag(&options, "xla_gpu_use_runtime_fusion", true); }, .rocm => { diff --git a/zml/nn.zig b/zml/nn.zig index 7d10a3a5..98812d0b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -161,6 +161,9 @@ pub const RopeOpts = struct { /// - pos_idx: optional tensor which indicates which positions are needed. /// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { + x.getContext().setBlockName("rope"); + defer x.getContext().setBlockName(""); + stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {}", .{x}); const idx = if (pos_idx) |idx| blk: { diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index 854da729..07544ed7 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -37,10 +37,14 @@ fn elementTypeFromDataType(dtype: DataType) [:0]const u8 { } pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { - const ctx = CompilationContext.current(); - const q = q_.transpose(.{ .b, .h, .q, .hd }); - var k = k_.transpose(.{ .b, .h, .k, .hd }); - const v = v_.transpose(.{ .b, .h, .k, .hd }); + var q = if (q_.rank() == 3) q_.insertAxes(0, .{.b}) else q_; + q = q.transpose(.{ .b, .h, .q, .hd }); + + var k = if (k_.rank() == 3) k_.insertAxes(0, .{.b}) else k_; + k = k.transpose(.{ .b, .h, .k, .hd }); + + var v = if (v_.rank() == 3) v_.insertAxes(0, .{.b}) else v_; + v = v.transpose(.{ .b, .h, .k, .hd }); const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); const scale: f32 = if (opts.scale) |_| 1.0 else sqrtHeadDim; @@ -119,6 +123,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { bias = bias.add(attn_mask.broad(bias.shape())); } + const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlirCtx(); const loc = mlir_ctx.location(@src()); const op = dialect.stablehlo.custom_call( @@ -137,6 +142,8 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { }, loc, ); - const result = Tensor._result(q.shape(), op.result(0)).transpose(q_.shape()); + var result = Tensor._result(q.shape(), op.result(0)); + if (q_.rank() == 3) result = result.squeeze(.b); + result = result.transpose(q_.shape()); return result; } diff --git a/zml/ops.zig b/zml/ops.zig index f314e013..f3f44ded 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -777,6 +777,14 @@ pub fn addHostCallback( return Tensor._result(input.shape(), op.result(0)); } +pub fn optimizationBarrier(x: Tensor) Tensor { + const ctx = CompilationContext.current(); + return Tensor._result( + x.shape(), + dialect.stablehlo.optimization_barrier(x.value(), ctx.location(@src(), "", .{})), + ); +} + /// Generalized version of scatter to many inputs. /// See `zml.Tensor.scatterSlices` for documentation on scatter. /// @@ -865,6 +873,7 @@ pub fn scatter( mlir_ctx, input_values.items, &.{indices.value()}, + // TODO transpose updates updates_values.items, update_block, .{ diff --git a/zml/tensor.zig b/zml/tensor.zig index 15543d7f..5be3a685 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -245,7 +245,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.abs(self.getContext().mlirCtx(), self.value(), loc); const dt = switch (self.dtype()) { .c64 => .f32, @@ -290,14 +290,14 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. pub fn countLeadingZeros(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.count_leading_zeros(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. pub fn isFinite(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.is_finite(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape.withDtype(.bool), op.result(0)); } @@ -305,14 +305,14 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. pub fn popcnt(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.popcnt(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the sign of the input Tensor element-wise. pub fn sign(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.sign(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, op.result(0)); } @@ -381,7 +381,7 @@ pub const Tensor = struct { pub fn roundNearestAfz(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.round_nearest_afz(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, op.result(0)); } @@ -390,7 +390,7 @@ pub const Tensor = struct { pub fn roundNearestEven(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.round_nearest_even(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, op.result(0)); } @@ -400,7 +400,7 @@ pub const Tensor = struct { stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {} and {}", .{ re._shape, im._shape }); stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); - const loc = re.getContext().mlirCtx().location(@src()); + const loc = re.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.complex(re.getContext().mlirCtx(), re.value(), im.value(), loc); const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; return _result(re._shape.withDtype(dt), op.result(0)); @@ -421,7 +421,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.real(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -443,7 +443,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.imag(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -966,6 +966,11 @@ pub const Tensor = struct { return binaryOp(@src(), "remainder", dialect.stablehlo.remainder)(self, other); } + /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. + pub fn remainderScalar(self: Tensor, b: anytype) Tensor { + return self.remainder(scalar(b, self.dtype())); + } + /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. pub fn addConstant(self: Tensor, b: anytype) Tensor { return self.add(Tensor.scalar(b, self.dtype())); @@ -994,13 +999,13 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise floor operation of the input Tensor. pub fn floor(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); return _result(self._shape, dialect.stablehlo.floor(self.getContext().mlirCtx(), self.value(), loc).result(0)); } /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. pub fn ceil(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); return _result(self._shape, dialect.stablehlo.ceil(self.getContext().mlirCtx(), self.value(), loc).result(0)); } @@ -1019,14 +1024,14 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const sine_op = dialect.stablehlo.round_nearest_even(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, sine_op.result(0)); } /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.clamp(self.getContext().mlirCtx(), min_.value(), self.value(), max_.value(), loc); return _result(self._shape, op.result(0)); } @@ -1192,7 +1197,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. pub fn sigmoid(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.logistic(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, op.result(0)); } @@ -1678,63 +1683,63 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const negate_op = dialect.stablehlo.negate(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, negate_op.result(0)); } /// Returns a Tensor containing the element-wise cosine of the input Tensor. pub fn cos(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const cosine_op = dialect.stablehlo.cosine(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, cosine_op.result(0)); } /// Returns a Tensor containing the element-wise sine of the input Tensor. pub fn sin(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const sine_op = dialect.stablehlo.sine(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, sine_op.result(0)); } /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. pub fn exp(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.exponential(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. pub fn log(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.log(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise square-root of the input Tensor. pub fn sqrt(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const sqrt_op = dialect.stablehlo.sqrt(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, sqrt_op.result(0)); } /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. pub fn rsqrt(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const rsqrt_op = dialect.stablehlo.rsqrt(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, rsqrt_op.result(0)); } /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. pub fn tanh(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const tanh_op = dialect.stablehlo.tanh(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, tanh_op.result(0)); } /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. pub fn exponentialMinusOne(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const expm1_op = dialect.stablehlo.exponential_minus_one(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, expm1_op.result(0)); } @@ -2165,7 +2170,7 @@ pub const Tensor = struct { } // scoped_log.debug("gatherValues --> {} {any}", .{ res_shape, res_kind.constSlice() }); - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const gather_op = dialect.stablehlo.gather( self.getContext().mlirCtx(), self.value(), @@ -3264,7 +3269,7 @@ pub const Tensor = struct { } } - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.dynamic_update_slice( self.getContext().mlirCtx(), self.value(), @@ -3534,7 +3539,7 @@ pub const Tensor = struct { stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {} and {}", .{ bool_tensor._shape, on_true._shape }); stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {} and {}", .{ bool_tensor._shape, on_false._shape }); - const loc = bool_tensor.getContext().mlirCtx().location(@src()); + const loc = bool_tensor.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.select( bool_tensor.getContext().mlirCtx(), bool_tensor.value(), @@ -3548,7 +3553,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. pub fn not(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); + const loc = self.getContext().location(@src(), "", .{}); const op = dialect.stablehlo.not(self.getContext().mlirCtx(), self.value(), loc); return _result(self._shape, op.result(0)); }