From 6412aa16137b1a0f7950b43e4443a0eb8cb6f2f5 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 26 Nov 2024 18:00:05 +0100 Subject: [PATCH 01/13] better error message for broadcast --- zml/nn.zig | 2 +- zml/tensor.zig | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 86f4bc27..e6135d52 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -775,7 +775,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { var attn_weights = q.dot(k, .{.hd}); // log.debug("attn_weights : {}", .{attn_weights}); // log.debug("attn_mask : {?}", .{attn_mask}); - if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broadcastLeft(attn_weights.shape())); + if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); attn_weights = attn_weights.convert(.f32); if (opts.bias) |bias| { diff --git a/zml/tensor.zig b/zml/tensor.zig index 27ff65dd..adeadbc0 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1849,9 +1849,13 @@ pub const Tensor = struct { /// To avoid use favorise `.broad(shape)` when working with tagged tensors. pub fn broadcast(self: Tensor, output_shape: Shape, axes_: []const i64) Tensor { const res_shape = output_shape.withDtype(self.dtype()); - + stdx.debug.assert(axes_.len == self.rank(), "broadcast expects axes_ to map all axes from self to axes of the output shape, got broadcast({}, {}, {d})", .{ self, output_shape, axes_ }); + for (0.., axes_) |self_ax, other_ax| { + const d = self.dim(self_ax); + stdx.debug.assert(d == 1 or d == output_shape.dim(other_ax), "broadcast expects shape axes to either be 1-sized or to match the target size. got broadcast({}, {}, {d}), error on self axis {} mapping to other axis {}", .{ self, output_shape, axes_, self_ax, other_ax }); + } const result_type = mlir.ext.RankedTensorType.fromShape(self.getContext().mlirCtx(), res_shape).as(mlir.Type).?; - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "broadcast({any}, axes={d})", .{ res_shape, axes_ }); + const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "broadcast({}, {any}, axes={d})", .{ self, res_shape, axes_ }); const broadcast_op = dialect.stablehlo.broadcast_in_dim(self.getContext().mlirCtx(), self.value(), axes_, result_type, loc); return _result(res_shape, broadcast_op.result(0)); From 0e092f4caf3c2589c1710c21c0d658a2510a444a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Nov 2024 11:30:07 +0100 Subject: [PATCH 02/13] fix Tensor.constant for types which aren't valid MLIR attributes --- zml/mlir.zig | 4 ++-- zml/tensor.zig | 21 +++++++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/zml/mlir.zig b/zml/mlir.zig index 1b0c0a5a..38cb104d 100644 --- a/zml/mlir.zig +++ b/zml/mlir.zig @@ -18,7 +18,7 @@ pub const ext = struct { return mlir.RankedTensorType.init(sh.dims(), mlir.ext.Type.fromDType(ctx, sh.dtype())).as(mlir.Type).?; } - pub fn denseElementAttrType(dt: dtype.DataType) mlir.DenseElementsAttributeTypes { + pub fn denseElementAttrType(dt: dtype.DataType) ?mlir.DenseElementsAttributeTypes { return switch (dt) { .bool => .bool, .i8 => .i8, @@ -33,7 +33,7 @@ pub const ext = struct { .f16 => .f16, .f32 => .f32, .f64 => .f64, - inline else => |tag| @panic("Unsupported data type: " ++ @tagName(tag)), + else => null, }; } diff --git a/zml/tensor.zig b/zml/tensor.zig index adeadbc0..584dd127 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -205,6 +205,7 @@ pub const Tensor = struct { pub fn value(self: Tensor) mlir.Value { return self.getContext().getValueAndDonation(self)[0]; } + /// Tell PJRT compiler that memory should be reuse between the two tensors. /// The compiler is already aggressively reusing tensors for intermediate results, /// but this API allows to reuse buffer between input and output arguments @@ -1806,13 +1807,20 @@ pub const Tensor = struct { const singleton_sh = Shape.init(.{}, val.dtype()); const ctx = CompilationContext.current().mlirCtx(); const loc = ctx.location(@src()).namedFmt(ctx, "dims={d}, value={}", .{ sh, val }); - const result_type = mlir.ext.RankedTensorType.fromShape(ctx, singleton_sh); - const elem_type = mlir.ext.denseElementAttrType(val.dtype()); - var constant_op = dialect.stablehlo.constant(ctx, result_type, elem_type, val.constSlice(), loc); + const res_type = mlir.ext.RankedTensorType.fromShape(ctx, singleton_sh); + + var constant_op = if (mlir.ext.denseElementAttrType(val.dtype())) |elem_type| + dialect.stablehlo.constant(ctx, res_type, elem_type, val.constSlice(), loc) + else blk: { + // Not all dtype can be serialized in the IR. If that's not possible, use f32. + const val_f32 = val.as(f32); + break :blk dialect.stablehlo.constant(ctx, res_type, .f32, std.mem.asBytes(&val_f32), loc); + }; + if (sh.rank() > 0) { constant_op = dialect.stablehlo.broadcast_in_dim(ctx, constant_op.result(0), &.{}, mlir.ext.RankedTensorType.fromShape(ctx, sh).as(mlir.Type).?, loc); } - return _result(sh, constant_op.result(0)); + return _result(sh, constant_op.result(0)).convert(val.dtype()); } /// Embeds a buffer with concrete values into an Mlir program. @@ -1820,7 +1828,7 @@ pub const Tensor = struct { const ctx = CompilationContext.current().mlirCtx(); const result_type = mlir.ext.RankedTensorType.fromShape(ctx, val.shape()); const loc = ctx.location(@src()); - const elem_type = mlir.ext.denseElementAttrType(val.dtype()); + const elem_type = mlir.ext.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {}", .{val.shape()}); const constant_op = dialect.stablehlo.constant(ctx, result_type, elem_type, val.data, loc); return _result(val.shape(), constant_op.result(0)); } @@ -2429,7 +2437,7 @@ pub const Tensor = struct { break :blk ax; }; - if (indices.count() == 1) { + if (indices.count() == 1 and !single_coord) { return self.dynamicUpdateSlice1d(updates, coord_axes_.get(0), indices.reshape(.{})); } @@ -3135,6 +3143,7 @@ pub const Tensor = struct { } /// Updates a slice of the input Tensor along a specific axis using the given 'update' Tensor, with a start offset known at runtime. + /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { const placeholder = Tensor.scalar(0, .i32); var start_indices = [_]Tensor{placeholder} ** MAX_RANK; From 71f134b9b3a1ef6e378e5af03627561ebe39785a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Nov 2024 11:31:03 +0100 Subject: [PATCH 03/13] fix small inconsistencies between different versions of sdpa --- zml/nn.zig | 2 +- zml/nn/cuda.zig | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index e6135d52..5b33a508 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -988,7 +988,7 @@ pub fn sdpaChunk(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) PartialSoft var attn_weights = q.dot(k, .{.hd}); // log.debug("attn_weights : {}", .{attn_weights}); // log.debug("attn_mask : {?}", .{attn_mask}); - if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broadcastLeft(attn_weights.shape())); + if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); if (opts.bias) |bias| { attn_weights = attn_weights.add(bias); diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index f00ca762..456b52dc 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -116,16 +116,16 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { var bias = Tensor.constant(Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .k = k.dim(.k) }, q.dtype()), Data.init(q.dtype(), 0)); if (opts.attn_mask) |attn_mask| { - const mask = attn_mask.withTags(.{ .q, .k }).broad(bias.shape()); - bias = bias.add(mask); + bias = bias.add(attn_mask.broad(bias.shape())); } if (opts.bias) |b| { bias = bias.add(b); } - const loc = ctx.mlirCtx().location(@src()); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), + mlir_ctx, &.{ q.value(), k.value(), v.value(), bias.value() }, .{ .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", @@ -135,8 +135,8 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { .output_operand_aliases = &.{}, }, &.{ - mlir.ext.mlirType(ctx.mlirCtx(), q.shape()), - mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(ctx.mlirCtx()).as(mlir.Type).?).as(mlir.Type).?, + mlir.ext.mlirType(mlir_ctx, q.shape()), + mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type).?).asType(), }, loc, ); From e984f3d2fc224d7619340c6301e57dc2c29144e4 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Nov 2024 12:16:13 +0100 Subject: [PATCH 04/13] fix float conversions --- zml/floats.zig | 121 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 93 insertions(+), 28 deletions(-) diff --git a/zml/floats.zig b/zml/floats.zig index ef09fa02..230ae97d 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -1,3 +1,4 @@ +///! Conversion utilities between different Floating point formats. const std = @import("std"); test { @@ -9,12 +10,15 @@ fn allBitsOne(v: anytype) bool { } fn FloatType(sign_bits: u1, exponent_bits: u8, mantissa_bits: u8, innerT: type) type { - return packed struct(std.meta.Int(.unsigned, @intCast(sign_bits + exponent_bits + mantissa_bits))) { + const bit_size = sign_bits + exponent_bits + mantissa_bits; + if (bit_size % 8 != 0) @compileError("FloatType should have a number of bits divisible by 8"); + + return packed struct(std.meta.Int(.unsigned, bit_size)) { const Self = @This(); - mantissa: std.meta.Int(.unsigned, @intCast(mantissa_bits)), - exponent: std.meta.Int(.unsigned, @intCast(exponent_bits)), - sign: std.meta.Int(.unsigned, @intCast(sign_bits)), + mantissa: std.meta.Int(.unsigned, mantissa_bits), + exponent: std.meta.Int(.unsigned, exponent_bits), + sign: std.meta.Int(.unsigned, sign_bits), pub fn zero() Self { return .{ @@ -35,34 +39,51 @@ fn FloatType(sign_bits: u1, exponent_bits: u8, mantissa_bits: u8, innerT: type) /// Lossy conversion from f32, similar to @floatCast pub fn fromF32(f: f32) Self { const vf32: Float32 = @bitCast(f); - const precision_loss = @bitSizeOf(@TypeOf(vf32.mantissa)) - mantissa_bits; + const exp_bias = comptime Self.expBias(); + const exponent = @as(u16, vf32.exponent) + exp_bias -| Float32.expBias(); + const overflow = exponent > std.math.maxInt(std.meta.Int(.unsigned, exponent_bits)); + if (overflow) { + return if (@hasDecl(Self, "inf")) { + return if (vf32.sign == 0) Self.inf() else Self.minusInf(); + } else Self.nan(); + } return .{ .sign = vf32.sign, - .exponent = @intCast(vf32.exponent), - .mantissa = shr(vf32.mantissa, precision_loss), + .exponent = @intCast(exponent), + .mantissa = truncMantissa(vf32.mantissa), }; } /// Lossless conversion to f32. pub fn toF32(self: Self) f32 { var vf32: Float32 = undefined; - const precision_loss = @bitSizeOf(@TypeOf(vf32.mantissa)) - mantissa_bits; + if (@hasDecl(Self, "isInf") and self.isInf()) { + return if (self.sign == 0) std.math.inf(f32) else -std.math.inf(f32); + } vf32 = .{ .sign = self.sign, - .exponent = self.exponent, - .mantissa = @shlExact(@as(@TypeOf(vf32.mantissa), self.mantissa), precision_loss), + .exponent = if (self.exponent == 0) 0 else @intCast(@as(i16, self.exponent) + Float32.expBias() - Self.expBias()), + .mantissa = self.f32Mantissa(), }; return @bitCast(vf32); } - fn truncMantissa(T: type, x: anytype) T { - const off = @bitSizeOf(@TypeOf(x)) - @bitSizeOf(T); + fn truncMantissa(x: anytype) std.meta.FieldType(Self, .mantissa) { + @setRuntimeSafety(false); + const off = @bitSizeOf(@TypeOf(x)) - mantissa_bits; return @intCast(x >> off); } - fn shr(x: anytype, comptime off: u8) std.meta.Int(.unsigned, @bitSizeOf(@TypeOf(x)) - off) { - // @setRuntimeSafety(false); - return @intCast(x >> off); + fn f32Mantissa(self: Self) std.meta.FieldType(Float32, .mantissa) { + @setRuntimeSafety(false); + const f32_mantissa_bits = @bitSizeOf(std.meta.FieldType(Float32, .mantissa)); + + const Res = std.meta.FieldType(Float32, .mantissa); + return @shlExact(@as(Res, self.mantissa), f32_mantissa_bits - mantissa_bits); + } + + fn expBias() u8 { + return std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); } pub fn format( @@ -72,7 +93,11 @@ fn FloatType(sign_bits: u1, exponent_bits: u8, mantissa_bits: u8, innerT: type) writer: anytype, ) !void { _ = options; - try writer.print("{" ++ fmt ++ "}", .{self.toF32()}); + if (fmt.len == 1 and fmt[0] == '_') { + try writer.print("{{ .sign={}, .exp={}, .mantissa={} }}", .{ self.sign, self.exponent, self.mantissa }); + } else { + try writer.print("{" ++ fmt ++ "}", .{self.toF32()}); + } } pub usingnamespace innerT; @@ -124,6 +149,22 @@ pub const Float8E4M3FNUZ = FloatType(1, 4, 3, struct { } }); +test "Float8E4" { + const test_case_e4: TestCase = .{ + .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 64.0, -128 }, + .lossy = &[_]f32{3.02344107628}, + }; + + inline for (.{ + Float8E4M3B11FNUZ, + Float8E4M3FN, + Float8E4M3FNUZ, + }) |Float8T| { + try testCustomFloat(Float8T, test_case_e4); + try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 128.0).toF32()); + } +} + pub const Float8E5M2 = FloatType(1, 5, 2, struct { pub fn nan() Float8E5M2 { return .{ @@ -172,6 +213,16 @@ pub const Float8E5M2FNUZ = FloatType(1, 5, 2, struct { } }); +test "Float8E5" { + const test_case_e5: TestCase = .{ + .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 128.0, -128 }, + .lossy = &[_]f32{3.02344107628}, + }; + inline for (.{ Float8E5M2, Float8E5M2FNUZ }) |Float8T| { + try testCustomFloat(Float8T, test_case_e5); + } +} + pub const BFloat16 = FloatType(1, 8, 7, struct { pub fn nan() BFloat16 { return .{ @@ -215,18 +266,10 @@ test BFloat16 { try std.testing.expectEqual(std.mem.toBytes(BFloat16.inf().neg()), [_]u8{ 0x80, 0xff }); try std.testing.expectEqual(BFloat16.inf(), BFloat16.fromF32(std.math.inf(f32))); - const lossless = [_]f32{ 0, -2, 1.0 / 128.0, -1e64, std.math.inf(f32) }; - for (&lossless) |v| { - try std.testing.expectEqual(v, BFloat16.fromF32(v).toF32()); - } - const lossy = [_]f32{3.02344107628}; - for (&lossy) |x| { - const y = BFloat16.fromF32(x).toF32(); - if (!std.math.approxEqRel(f32, x, y, 1e-2)) { - std.log.err("expected ~{d}, got {d}", .{ x, y }); - return error.TestUnexpectedResult; - } - } + try testCustomFloat(BFloat16, .{ + .lossless = &[_]f32{ 0, -2, 1.0 / 128.0, -1e64, std.math.inf(f32) }, + .lossy = &[_]f32{3.02344107628}, + }); } pub fn floatCast(T: type, x: anytype) T { @@ -235,3 +278,25 @@ pub fn floatCast(T: type, x: anytype) T { else => @floatCast(x.toF32()), }; } + +const TestCase = struct { + lossless: []const f32, + lossy: []const f32, + tolerance: f32 = 1e-2, +}; + +fn testCustomFloat(FloatT: type, test_case: TestCase) !void { + for (test_case.lossless) |x| { + try std.testing.expectEqual(x, FloatT.fromF32(x).toF32()); + } + for (test_case.lossy) |x| { + try expectApproxEqRel(f32, x, FloatT.fromF32(x).toF32(), test_case.tolerance); + } +} + +fn expectApproxEqRel(FloatT: type, x: FloatT, y: FloatT, tolerance: FloatT) !void { + if (!std.math.approxEqRel(f32, x, y, tolerance)) { + std.log.err("expected ~{d}, got {d}", .{ x, y }); + return error.TestUnexpectedResult; + } +} From ddfc9497c296fc8827f72b8e5d715695cdb2de9e Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Nov 2024 14:43:52 +0100 Subject: [PATCH 05/13] f8: add comment --- zml/dtype.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/zml/dtype.zig b/zml/dtype.zig index cfc13cf7..ce030ac2 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -10,6 +10,7 @@ test { pub const DataType = enum(u8) { bool, + // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. f8e4m3b11fnuz, f8e4m3fn, f8e4m3fnuz, From db3675242a6260b4dd9feb760e21f6b0cbb8d246 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Nov 2024 14:43:38 +0100 Subject: [PATCH 06/13] bazelrc: --config=debug --- examples/.bazelrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/.bazelrc b/examples/.bazelrc index 20a4fc47..6d969ccd 100644 --- a/examples/.bazelrc +++ b/examples/.bazelrc @@ -72,7 +72,9 @@ build --repo_env HERMETIC_PYTHON_VERSION=3.11 # Print test errors in the console test --test_output=errors +# --config=debug will used optmized backend, but all the frontend zig code will be compiled in debug mode. build:debug --compilation_mode=opt +build:debug --@rules_zig//zig/settings:mode=debug build:debug --strategy=ZigBuildLib=local build:debug --strategy=ZigBuildObj=local build:debug --strategy=ZigBuildTestLib=local From eb500ace79029f97b4062606c05e5d75bb8fba43 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 26 Nov 2024 15:29:15 +0100 Subject: [PATCH 07/13] make rope accept token_idx directly --- zml/module.zig | 1 + zml/nn.zig | 88 ++++++++++++++++++++++++++++---------------------- zml/tensor.zig | 20 +++++++++--- 3 files changed, 66 insertions(+), 43 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 8c9df46f..5ced2790 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -562,6 +562,7 @@ pub const CompilationContext = struct { self.extractValues(&args, values[function.n_model..]); const op = dialect.func.call(self.mlirCtx(), function.name, values, function.res_types, loc); + // TODO: tags seem to be lost by `callFunc`. var res: stdx.meta.FnResult(func) = undefined; assignResults(op, &res, function.res_shapes); return res; diff --git a/zml/nn.zig b/zml/nn.zig index 5b33a508..7639d590 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -151,46 +151,43 @@ pub const RopeOpts = struct { freq_base: f32 = 10_000, }; -pub const CosSin = [2]Tensor; - /// Rotary position embedding modify queries and keys tensor before compute Q * K in self attention. /// This biases a token to look at token near him. -/// The nice thing with this solution is that you can cache the modified queries and keys directly. +/// The nice thing with rope is that you can cache the modified queries and keys directly. /// See: https://paperswithcode.com/method/rope -pub fn rope(x: Tensor, cos_sin_cache: CosSin, opts: RopeOpts) Tensor { - const cos, const sin = cos_sin_cache; - stdx.debug.assert(x.dim(-1) == 2 * cos.dim(-1), "Couldn't compute rope({}, {}, {})", .{ x, cos, sin }); - // broadcast cos / sin to .{ batch, .seq, .half_dim } +/// +/// Expected shapes of tensor: +/// - x: .{ .s, .hd } where .s is the sequence length and .hd the head dimension +/// - pos_idx: optional tensor which indicates which positions are needed. +/// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. +pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { + stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {}", .{x}); + + const idx = if (pos_idx) |idx| blk: { + stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={}, idx={})", .{ x, idx }); + break :blk idx; + } else blk: { + stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={})", .{x}); + break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); + }; const x_real, const x_imag = splitRealImg(x, opts.impl); - const has_tags = cos.shape().tag(0) != Shape.TagUnknown; - const b_cos = if (has_tags) cos.broad(x_real.shape()) else cos.broadcastLeft(x_real.shape()); - const b_sin = if (has_tags) sin.broad(x_real.shape()) else sin.broadcastLeft(x_real.shape()); + + // compute sin and cos in f32 before downcasting to x type. + const inv_freq = invFreq(x.dim(.hd), opts.freq_base, .f32).withTags(.{.hd}); + const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); + const cos = inv_freq_pos.cos().convert(x.dtype()).broad(x_real.shape()); + const sin = inv_freq_pos.sin().convert(x.dtype()).broad(x_real.shape()); // apply rotation - const y_real = x_real.mul(b_cos).sub(x_imag.mul(b_sin)); - const y_imag = x_real.mul(b_sin).add(x_imag.mul(b_cos)); + const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); + const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); // flatten last dimensions const y = mergeRealImg(y_real, y_imag, opts.impl); - + log.warn("rope({}, {?}) -> {}", .{ x, pos_idx, y }); return y; } -pub fn ropeCosSin(sh: anytype, dtype: DataType, opts: RopeOpts) CosSin { - const shape = Shape.init(sh, dtype); - stdx.debug.assert(shape.rank() == 2, "ropeCosSin({}) shape need to exactly have 2 axes", .{shape}); - const seq_len, const head_dim = .{ shape.dim(0), shape.dim(1) }; - stdx.debug.assert(@mod(head_dim, 2) == 0, "ropeCosSin requires an even head_dim, got {}", .{head_dim}); - - // compute sin and cos in f32 before downcasting to x type. - const inv_freq = invFreq(head_dim, opts.freq_base, .f32); - var inv_freq_pos = Tensor.outer(Tensor.arange(.{ .end = seq_len }, .f32), inv_freq).convert(shape.dtype()); - inv_freq_pos._shape._tags = shape._tags; - const cos = inv_freq_pos.cos(); - const sin = inv_freq_pos.sin(); - return .{ cos, sin }; -} - pub fn splitRealImg(x: Tensor, impl: RopeOpts.Implementation) [2]Tensor { const n = x.dim(-1); @@ -311,16 +308,15 @@ test "real/img" { test "rope" { const platform = zml.testing.env(); - const TestRope = struct { - fn forward(x: Tensor, opts: RopeOpts) Tensor { + const Local = struct { + fn _fwd1(x: Tensor, opts: RopeOpts) Tensor { var input = x; { // Convert input to the requested format const real, const imag = splitRealImg(input, .sequential); input = mergeRealImg(real, imag, opts.impl); } - const cos_sin = ropeCosSin(.{ input.dim(-2), input.dim(-1) }, input.dtype(), opts); - var res = rope(input, cos_sin, opts).squeeze(0); + var res = rope(input, null, opts).squeeze(0); { // Convert back to sequential @@ -329,15 +325,31 @@ test "rope" { } return res; } + + fn _fwd2(x: Tensor, idx: Tensor) [2]Tensor { + return .{ + rope(x, idx, .{ .impl = .interleaved }), + rope(x.gatherValues(.s, idx, .{}), null, .{ .impl = .interleaved }).gatherValues(.s, idx, .{}), + }; + } }; - // x is made such as the interleaved and sequential reps are the same. - // So the two implementations should give the same results. - const x = try zml.Buffer.fromSlice(platform, .{ 1, 5, 4 }, &[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5); - const res1 = try zml.testing.compileAndCall(platform, TestRope.forward, .{ x, RopeOpts{ .impl = .interleaved } }); - const res2 = try zml.testing.compileAndCall(platform, TestRope.forward, .{ x, RopeOpts{ .impl = .sequential } }); + { - try zml.testing.expectClose(res1, res2, 1e-4); + // x is made such as the interleaved and sequential reps are the same. + // So the two implementations should give the same results. + const x = try zml.Buffer.fromSlice(platform, .{ .b = 1, .s = 5, .hd = 4 }, &[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5); + const res1 = try zml.testing.compileAndCall(platform, Local._fwd1, .{ x, RopeOpts{ .impl = .interleaved } }); + const res2 = try zml.testing.compileAndCall(platform, Local._fwd1, .{ x, RopeOpts{ .impl = .sequential } }); + try zml.testing.expectClose(res1, res2, 1e-4); + } + + // { + // const x = try zml.Buffer.fromSlice(platform, .{ .b = 2, .hd = 4 }, &[2 * 4]f32{ 1.0, 0.1, -1.0, -0.5, 1.0, 0.1, -1.0, -0.5 }); + // const idx = try zml.Buffer.fromSlice(platform, .{ .b = 2 }, &[_]i32{ 2, 4 }); + // const res1, const res2 = try zml.testing.compileAndCall(platform, Local._fwd2, .{ x, idx }); + // try zml.testing.expectClose(res1, res2, 1e-4); + // } } /// In neural network we generally care about the relative precision, diff --git a/zml/tensor.zig b/zml/tensor.zig index 584dd127..a44958b2 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1835,15 +1835,25 @@ pub const Tensor = struct { /// Returns a Tensor containing the result of the outer product between the input Tensors. pub fn outer(self: Tensor, other: Tensor) Tensor { - stdx.debug.assert(self.rank() < 2 and other.rank() < 2 and self.rank() + other.rank() != 0, "outer expects tensor ranks to be at most 1, got {} and {}", .{ self.rank(), other.rank() }); - if (self.rank() + other.rank() == 1) { return self.mul(other); } - const dimz = .{ self.dim(0), other.dim(0) }; - const left = self.broadcast(Shape.init(dimz, self.dtype()), &.{0}); - const right = other.broadcast(Shape.init(dimz, other.dtype()), &.{1}); + const other_shape = other.shape(); + var res_shape = self.shape(); + var batching_axes: u8 = 0; + for (0..other.rank()) |ax| { + if (other_shape.tag(ax) != Shape.TagUnknown) { + if (self.shape().hasTag(other_shape.tag(ax))) |batching_ax| { + stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({}, {})", .{ self, other }); + batching_axes += 1; + } + } + + res_shape = res_shape.appendDim(other_shape.dim(ax), other_shape.tag(ax)); + } + const left = self.broad(res_shape); + const right = other.broad(res_shape); return left.mul(right); } From 31e507084828750c4d1739f5e590d06d9e902c4e Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 27 Nov 2024 11:13:13 +0100 Subject: [PATCH 08/13] fixup! make rope accept token_idx directly --- zml/nn.zig | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 7639d590..4ad30514 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -183,9 +183,7 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); // flatten last dimensions - const y = mergeRealImg(y_real, y_imag, opts.impl); - log.warn("rope({}, {?}) -> {}", .{ x, pos_idx, y }); - return y; + return mergeRealImg(y_real, y_imag, opts.impl); } pub fn splitRealImg(x: Tensor, impl: RopeOpts.Implementation) [2]Tensor { From c03448363c19cf59131700df38356ff2058eaef4 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Nov 2024 15:02:13 +0100 Subject: [PATCH 09/13] llama: update to new rope --- examples/llama/llama.zig | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 4416811f..fed40635 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -295,18 +295,15 @@ pub const SelfAttn = struct { const kv_cache = kv_cache_ orelse initKvCache(k.shape()); const seq_len = kv_cache.k.dim(.k); var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), null); - var cos, var sin = zml.nn.ropeCosSin(.{ .s = seq_len, .hd = k.dim(.hd) }, x.dtype(), self.rope_opts); if (token_index) |idx| { - // Note: in Pytorch it would be very inefficient to generate the full ropeCosSin and attn_mask matrices, then slice into it, - // but XLA is able to optimize this correctly. + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. attn_mask = attn_mask.dynamicSlice(.{ .q = .{ .start = idx, .len = 1 } }); - cos = cos.dynamicSlice(.{ .s = .{ .start = idx, .len = 1 } }); - sin = sin.dynamicSlice(.{ .s = .{ .start = idx, .len = 1 } }); } // In self-attention, .s axis is used both for keys and queries. - q = zml.nn.rope(q, .{ cos, sin }, self.rope_opts); - k = zml.nn.rope(k, .{ cos, sin }, self.rope_opts); + q = zml.nn.rope(q, token_index, self.rope_opts); + k = zml.nn.rope(k, token_index, self.rope_opts); q = q.rename(.{ .s = .q }); k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); From c41e161a7dac787781f895b547e1977291d67ad3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Nov 2024 15:06:29 +0100 Subject: [PATCH 10/13] remove unused test --- zml/nn.zig | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 4ad30514..b8d7d935 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -303,11 +303,11 @@ test "real/img" { try testing.expectEqual(20, d_split_interleaved.getValue(i32)); } -test "rope" { +test rope { const platform = zml.testing.env(); const Local = struct { - fn _fwd1(x: Tensor, opts: RopeOpts) Tensor { + fn _fwd(x: Tensor, opts: RopeOpts) Tensor { var input = x; { // Convert input to the requested format @@ -323,31 +323,14 @@ test "rope" { } return res; } - - fn _fwd2(x: Tensor, idx: Tensor) [2]Tensor { - return .{ - rope(x, idx, .{ .impl = .interleaved }), - rope(x.gatherValues(.s, idx, .{}), null, .{ .impl = .interleaved }).gatherValues(.s, idx, .{}), - }; - } }; - { - - // x is made such as the interleaved and sequential reps are the same. - // So the two implementations should give the same results. - const x = try zml.Buffer.fromSlice(platform, .{ .b = 1, .s = 5, .hd = 4 }, &[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5); - const res1 = try zml.testing.compileAndCall(platform, Local._fwd1, .{ x, RopeOpts{ .impl = .interleaved } }); - const res2 = try zml.testing.compileAndCall(platform, Local._fwd1, .{ x, RopeOpts{ .impl = .sequential } }); - try zml.testing.expectClose(res1, res2, 1e-4); - } - - // { - // const x = try zml.Buffer.fromSlice(platform, .{ .b = 2, .hd = 4 }, &[2 * 4]f32{ 1.0, 0.1, -1.0, -0.5, 1.0, 0.1, -1.0, -0.5 }); - // const idx = try zml.Buffer.fromSlice(platform, .{ .b = 2 }, &[_]i32{ 2, 4 }); - // const res1, const res2 = try zml.testing.compileAndCall(platform, Local._fwd2, .{ x, idx }); - // try zml.testing.expectClose(res1, res2, 1e-4); - // } + // x is made such as the interleaved and sequential reps are the same. + // So the two implementations should give the same results. + const x = try zml.Buffer.fromSlice(platform, .{ .b = 1, .s = 5, .hd = 4 }, &[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5); + const res1 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .impl = .interleaved } }); + const res2 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .impl = .sequential } }); + try zml.testing.expectClose(res1, res2, 1e-4); } /// In neural network we generally care about the relative precision, From 8399c3f7564b01e2f2a8a100843d12b179d0ff11 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 25 Nov 2024 10:58:18 +0100 Subject: [PATCH 11/13] WIP --- async/BUILD.bazel | 1 + async/queue.zig | 101 +++++++++++++++++++++++++++++++++++++++++++++ async/zigcoro.zig | 47 +++++++++++++++++++++ stdx/signature.zig | 32 ++++++++------ 4 files changed, 168 insertions(+), 13 deletions(-) create mode 100644 async/queue.zig diff --git a/async/BUILD.bazel b/async/BUILD.bazel index 933b166d..8eb40dc3 100644 --- a/async/BUILD.bazel +++ b/async/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "async", srcs = [ + "queue.zig", "queue_mpsc.zig", ], import_name = "async", diff --git a/async/queue.zig b/async/queue.zig new file mode 100644 index 00000000..aa3dfe58 --- /dev/null +++ b/async/queue.zig @@ -0,0 +1,101 @@ +const std = @import("std"); +const assert = std.debug.assert; + +/// An intrusive queue implementation. The type T must have a field +/// "next" of type `?*T`. +/// +/// For those unaware, an intrusive variant of a data structure is one in which +/// the data type in the list has the pointer to the next element, rather +/// than a higher level "node" or "container" type. The primary benefit +/// of this (and the reason we implement this) is that it defers all memory +/// management to the caller: the data structure implementation doesn't need +/// to allocate "nodes" to contain each element. Instead, the caller provides +/// the element and how its allocated is up to them. +pub fn Intrusive(comptime T: type) type { + return struct { + const Self = @This(); + + /// Head is the front of the queue and tail is the back of the queue. + head: ?*T = null, + tail: ?*T = null, + + /// Enqueue a new element to the back of the queue. + pub fn push(self: *Self, v: *T) void { + assert(v.next == null); + + if (self.tail) |tail| { + // If we have elements in the queue, then we add a new tail. + tail.next = v; + self.tail = v; + } else { + // No elements in the queue we setup the initial state. + self.head = v; + self.tail = v; + } + } + + /// Dequeue the next element from the queue. + pub fn pop(self: *Self) ?*T { + // The next element is in "head". + const next = self.head orelse return null; + + // If the head and tail are equal this is the last element + // so we also set tail to null so we can now be empty. + if (self.head == self.tail) self.tail = null; + + // Head is whatever is next (if we're the last element, + // this will be null); + self.head = next.next; + + // We set the "next" field to null so that this element + // can be inserted again. + next.next = null; + return next; + } + + /// Returns true if the queue is empty. + pub fn empty(self: *const Self) bool { + return self.head == null; + } + }; +} + +test Intrusive { + const testing = std.testing; + + // Types + const Elem = struct { + const Self = @This(); + next: ?*Self = null, + }; + const Queue = Intrusive(Elem); + var q: Queue = .{}; + try testing.expect(q.empty()); + + // Elems + var elems: [10]Elem = .{.{}} ** 10; + + // One + try testing.expect(q.pop() == null); + q.push(&elems[0]); + try testing.expect(!q.empty()); + try testing.expect(q.pop().? == &elems[0]); + try testing.expect(q.pop() == null); + try testing.expect(q.empty()); + + // Two + try testing.expect(q.pop() == null); + q.push(&elems[0]); + q.push(&elems[1]); + try testing.expect(q.pop().? == &elems[0]); + try testing.expect(q.pop().? == &elems[1]); + try testing.expect(q.pop() == null); + + // Interleaved + try testing.expect(q.pop() == null); + q.push(&elems[0]); + try testing.expect(q.pop().? == &elems[0]); + q.push(&elems[1]); + try testing.expect(q.pop().? == &elems[1]); + try testing.expect(q.pop() == null); +} diff --git a/async/zigcoro.zig b/async/zigcoro.zig index 13fdbf33..55f7bd05 100644 --- a/async/zigcoro.zig +++ b/async/zigcoro.zig @@ -5,6 +5,51 @@ const libcoro = @import("libcoro"); const aio = libcoro.asyncio; const queue_mpsc = @import("queue_mpsc.zig"); +pub const Queue = @import("queue.zig").Intrusive; + +pub const Condition = struct { + const CoroResume = struct { + coro: libcoro.Frame, + + fn init() CoroResume { + return .{ .coro = libcoro.xframe() }; + } + + fn func(self: *CoroResume) libcoro.Executor.Func { + return .{ .func = CoroResume.cb, .userdata = self }; + } + + fn cb(ud: ?*anyopaque) void { + const self: *CoroResume = @ptrCast(@alignCast(ud)); + libcoro.xresume(self.coro); + } + }; + + exec: *libcoro.Executor, + waiters: Queue(libcoro.Executor.Func) = .{}, + + pub fn init() Condition { + return .{ .exec = &AsyncThread.current.executor.exec }; + } + + pub fn broadcast(self: *Condition) void { + while (self.waiters.pop()) |waiter| { + self.exec.runSoon(waiter); + } + } + + pub fn signal(self: *Condition) void { + if (self.waiters.pop()) |waiter| self.exec.runSoon(waiter); + } + + pub fn wait(self: *Condition) void { + var res = CoroResume.init(); + var cb = res.func(); + self.waiters.push(&cb); + libcoro.xsuspend(); + } +}; + pub fn Frame(comptime func: anytype) type { const Signature = stdx.meta.FnSignature(func, null); return FrameExx(func, Signature.ArgsT, Signature.ReturnT); @@ -482,6 +527,8 @@ pub const Socket = struct { }; }; +pub const Channel = libcoro.Channel; + pub const Mutex = struct { const VoidChannel = libcoro.Channel(void, .{ .capacity = 1 }); diff --git a/stdx/signature.zig b/stdx/signature.zig index 48aa4e02..d9145f80 100644 --- a/stdx/signature.zig +++ b/stdx/signature.zig @@ -44,22 +44,28 @@ pub fn ArgsTuple(comptime funcT: anytype, comptime argsT: ?type) type { }); } -pub fn FnSignature(comptime func: anytype, comptime argsT: ?type) type { - return FnSignatureX(func, ArgsTuple(@TypeOf(func), argsT)); -} +pub const Signature = struct { + FuncT: type, + ArgsT: type, + ReturnT: type, + ReturnPayloadT: type, + ReturnErrorSet: ?type, +}; -fn FnSignatureX(comptime func: anytype, comptime argsT: type) type { - return struct { - pub const FuncT = @TypeOf(func); - pub const ArgsT = argsT; - pub const ReturnT = @TypeOf(@call(.auto, func, @as(ArgsT, undefined))); - pub const ReturnPayloadT = switch (@typeInfo(ReturnT)) { +pub fn FnSignature(comptime func: anytype, comptime argsT_: ?type) Signature { + const argsT = ArgsTuple(@TypeOf(func), argsT_); + const return_type = @TypeOf(@call(.auto, func, @as(argsT, undefined))); + return Signature{ + .FuncT = @TypeOf(func), + .ArgsT = argsT, + .ReturnT = return_type, + .ReturnPayloadT = switch (@typeInfo(return_type)) { .ErrorUnion => |u| u.payload, - else => ReturnT, - }; - pub const ReturnErrorSet: ?type = switch (@typeInfo(ReturnT)) { + else => return_type, + }, + .ReturnErrorSet = switch (@typeInfo(return_type)) { .ErrorUnion => |u| u.error_set, else => null, - }; + }, }; } From 5a2803c21a68f00b78028cd386625c2a845d0a9e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Nov 2024 19:27:13 +0100 Subject: [PATCH 12/13] WIP --- async/zigcoro.zig | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/async/zigcoro.zig b/async/zigcoro.zig index 55f7bd05..1f8fe93f 100644 --- a/async/zigcoro.zig +++ b/async/zigcoro.zig @@ -527,7 +527,30 @@ pub const Socket = struct { }; }; -pub const Channel = libcoro.Channel; +pub fn Channel(comptime T: type, capacity: usize) type { + return struct { + const Self = @This(); + const Inner = libcoro.Channel(T, .{ .capacity = capacity }); + + inner: Inner, + + pub fn init() Self { + return .{ .inner = Inner.init(&AsyncThread.current.executor.exec) }; + } + + pub fn close(self: *Self) void { + self.inner.close(); + } + + pub fn send(self: *Self, val: T) void { + self.inner.send(val) catch unreachable; + } + + pub fn recv(self: *Self) ?T { + return self.inner.recv(); + } + }; +} pub const Mutex = struct { const VoidChannel = libcoro.Channel(void, .{ .capacity = 1 }); From 032f2d7d596ce9a4006009ed9a0a10d8a986f2c7 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 29 Nov 2024 16:12:14 +0100 Subject: [PATCH 13/13] WIP --- examples/MODULE.bazel | 4 ++-- zml/zml.zig | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index a2ebcc5b..1962efd9 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -100,13 +100,13 @@ filegroup( srcs = ["tokenizer.json"], ) """, - commit = "5206a32e0bd3067aef1ce90f5528ade7d866253f", + commit = "d04e592bb4f6aa9cfee91e2e20afa771667e1d4b", includes = [ "*.safetensors", "model.safetensors.index.json", "tokenizer.json", ], - model = "meta-llama/Meta-Llama-3.1-8B-Instruct", + model = "meta-llama/Meta-Llama-3.1-8B", ) use_repo(huggingface, "Meta-Llama-3.1-8B-Instruct") huggingface.model( diff --git a/zml/zml.zig b/zml/zml.zig index 484e4146..f3a5b2bc 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -33,6 +33,8 @@ pub const call = ops.call; pub const compile = module.compile; pub const compileModel = module.compileModel; pub const compileFn = module.compileFn; +pub const Exe = module.Exe; +pub const ExeWithWeights = module.ExeWithWeights; pub const ops = @import("ops.zig"); pub const tools = struct {