From 360cc860335bb5dea92f8edbabd6bc80498f4b64 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 1 Dec 2025 11:25:03 +0100 Subject: [PATCH 1/6] Initial rework of Context and Platform * Removed `zml.Context` for `zml.init()` and `zml.deinit()` * `zml.Platform` is responsible for loading the PJRT api when initialized. This replace the previous behavior of trying to load all enabled runtimes. --- zml2/BUILD.bazel | 44 ++++ zml2/dtype.zig | 375 +++++++++++++++++++++++++++++++++ zml2/floats.zig | 517 ++++++++++++++++++++++++++++++++++++++++++++++ zml2/main.zig | 15 ++ zml2/pjrtx.zig | 362 ++++++++++++++++++++++++++++++++ zml2/platform.zig | 190 +++++++++++++++++ zml2/posix.h | 2 + zml2/zml.zig | 61 ++++++ 8 files changed, 1566 insertions(+) create mode 100644 zml2/BUILD.bazel create mode 100644 zml2/dtype.zig create mode 100644 zml2/floats.zig create mode 100644 zml2/main.zig create mode 100644 zml2/pjrtx.zig create mode 100644 zml2/platform.zig create mode 100644 zml2/posix.h create mode 100644 zml2/zml.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel new file mode 100644 index 00000000..84eb6dda --- /dev/null +++ b/zml2/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "posix", + hdrs = ["posix.h"], +) + +zig_library( + name = "zml", + srcs = [ + "zml.zig", + "platform.zig", + "pjrtx.zig", + "dtype.zig", + "floats.zig", + ], + zigopts = [ + "-lc", + "-freference-trace=20", + ], + main = "zml.zig", + visibility = ["//visibility:public"], + deps = [ + "//runtimes", + "//mlir", + "//mlir/dialects", + "//pjrt", + "//stdx", + "@rules_zig//zig/runfiles", + ":posix", + ], +) + +zig_binary( + name = "main", + args = [], + data = [], + main = "main.zig", + deps = [ + "//async", + "//zml2:zml", + ], +) diff --git a/zml2/dtype.zig b/zml2/dtype.zig new file mode 100644 index 00000000..b14f483e --- /dev/null +++ b/zml2/dtype.zig @@ -0,0 +1,375 @@ +const std = @import("std"); + +const floats = @import("floats.zig"); + +const C64 = std.math.Complex(f32); +const C128 = std.math.Complex(f64); + +test { + std.testing.refAllDecls(@This()); +} + +pub const DataType = enum(u8) { + bool, + // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. + f4e2m1, + f8e3m4, + f8e4m3, + f8e4m3b11fnuz, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + f8e8m0, + bf16, + f16, + f32, + f64, + i2, + i4, + i8, + i16, + i32, + i64, + u2, + u4, + u8, + u16, + u32, + u64, + c64, + c128, + + pub fn str(self: DataType) [:0]const u8 { + return switch (self) { + inline else => |tag| @tagName(tag), + }; + } + + pub const Class = enum(u8) { + bool, + float, + integer, + complex, + }; + + pub fn class(self: DataType) Class { + return switch (self) { + .bool => .bool, + .f4e2m1, + .f8e3m4, + .f8e4m3, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + .f8e8m0, + .bf16, + .f16, + .f32, + .f64, + => .float, + .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => .integer, + .c64, .c128 => .complex, + }; + } + + pub fn isInteger(self: DataType) bool { + return self.class() == .integer; + } + + pub fn isFloat(self: DataType) bool { + return self.class() == .float; + } + + pub fn isComplex(self: DataType) bool { + return self.class() == .complex; + } + + pub fn fromZigType(comptime T: type) DataType { + return switch (T) { + floats.Float4E2M1 => .f4e2m1, + floats.Float8E3M4 => .f8e3m4, + floats.Float8E4M3 => .f8e4m3, + floats.Float8E4M3B11FNUZ => .f8e4m3b11fnuz, + floats.Float8E4M3FN => .f8e4m3fn, + floats.Float8E4M3FNUZ => .f8e4m3fnuz, + floats.Float8E5M2 => .f8e5m2, + floats.Float8E5M2FNUZ => .f8e5m2fnuz, + floats.Float8E8M0 => .f8e8m0, + floats.BFloat16 => .bf16, + f16 => .f16, + f32 => .f32, + f64 => .f64, + bool => .bool, + i2 => .i2, + i4 => .i4, + i8 => .i8, + i16 => .i16, + i32 => .i32, + i64 => .i64, + u2 => .u2, + u4 => .u4, + u8 => .u8, + u16 => .u16, + u32 => .u32, + u64 => .u64, + C64 => .c64, + C128 => .c128, + else => @compileError("Unsupported Zig type: " ++ @typeName(T)), + }; + } + + pub fn fromSliceElementType(slice: anytype) DataType { + const type_info = @typeInfo(@TypeOf(slice)); + if (type_info != .pointer) { + @compileError("`initFromSlice` expects a slice, got " ++ @tagName(type_info)); + } + + return switch (type_info.pointer.size) { + .slice, .c, .many => DataType.fromZigType(type_info.pointer.child), + .one => b: { + const child_type_info = @typeInfo(type_info.pointer.child); + break :b DataType.fromZigType(child_type_info.array.child); + }, + }; + } + + pub fn toZigType(comptime dtype: DataType) type { + return @FieldType(Data, @tagName(dtype)); + } + + pub fn isSignedInt(dtype: DataType) bool { + return switch (dtype) { + .i4, .i8, .i16, .i32, .i64 => true, + else => false, + }; + } + + pub fn sizeOf(self: DataType) u16 { + return switch (self) { + inline else => |tag| @sizeOf(tag.toZigType()), + }; + } + + pub fn bitSizeOf(self: DataType) u16 { + return switch (self) { + inline else => |tag| @bitSizeOf(tag.toZigType()), + }; + } + + pub fn alignOf(self: DataType) u29 { + return switch (self) { + inline else => |tag| @alignOf(tag.toZigType()), + }; + } + + /// Try to find a type compatible with both dtype. + pub fn resolvePeerType(a: DataType, b: DataType) ?DataType { + if (a == b) { + return a; + } + + // only resolve types in the same class + if (a.class() != b.class()) { + return null; + } + + return if (a.sizeOf() >= b.sizeOf()) a else b; + } + + test resolvePeerType { + try std.testing.expectEqual(DataType.f16.resolvePeerType(.f16), .f16); + try std.testing.expectEqual(DataType.f32.resolvePeerType(.f32), .f32); + + try std.testing.expectEqual(DataType.f16.resolvePeerType(.f32), .f32); + try std.testing.expectEqual(DataType.f32.resolvePeerType(.f16), .f32); + + try std.testing.expectEqual(DataType.f32.resolvePeerType(.f64), .f64); + try std.testing.expectEqual(DataType.f64.resolvePeerType(.f32), .f64); + try std.testing.expectEqual(DataType.f32.resolvePeerType(.i32), null); + + try std.testing.expectEqual(DataType.c64.resolvePeerType(.c128), .c128); + try std.testing.expectEqual(DataType.c128.resolvePeerType(.i32), null); + try std.testing.expectEqual(DataType.c64.resolvePeerType(.f32), null); + } + + pub fn zero(dtype: DataType) Data { + return Data.init(dtype, 0); + } + + pub fn one(dtype: DataType) Data { + return Data.init(dtype, 1); + } + + pub fn minValue(dtype: DataType) Data { + return switch (dtype) { + .bool => .{ .bool = false }, + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).zero), + inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).minus_inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), -std.math.inf(@FieldType(Data, @tagName(tag)))), + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.minInt(@FieldType(Data, @tagName(tag)))), + inline else => |tag| @panic("Unsupported type: " ++ @tagName(tag)), + }; + } + + pub fn maxValue(dtype: DataType) Data { + return switch (dtype) { + .bool => .{ .bool = true }, + inline .f4e2m1, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), + inline .f8e3m4, .f8e4m3, .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), + }; + } + + pub fn constant(dtype: DataType, value: anytype) Data { + return Data.init(dtype, value); + } +}; + +pub const Data = union(DataType) { + bool: bool, + f4e2m1: floats.Float4E2M1, + f8e3m4: floats.Float8E3M4, + f8e4m3: floats.Float8E4M3, + f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, + f8e4m3fn: floats.Float8E4M3FN, + f8e4m3fnuz: floats.Float8E4M3FNUZ, + f8e5m2: floats.Float8E5M2, + f8e5m2fnuz: floats.Float8E5M2FNUZ, + f8e8m0: floats.Float8E8M0, + bf16: floats.BFloat16, + f16: f16, + f32: f32, + f64: f64, + i2: i2, + i4: i4, + i8: i8, + i16: i16, + i32: i32, + i64: i64, + u2: u2, + u4: u4, + u8: u8, + u16: u16, + u32: u32, + u64: u64, + c64: C64, + c128: C128, + + /// Creates `Data` from a `value`. + /// + /// If the `dtype` and `@TypeOf(value)` are incompatible + /// or a cast from `value` to `FieldType(dtype)` would + /// be lossy, a panic occurs. + pub fn init(dtype_: DataType, value: anytype) Data { + const T = @TypeOf(value); + const Ti = @typeInfo(T); + + return switch (dtype_) { + .bool => switch (Ti) { + .bool => .{ .bool = value }, + .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, + else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), + }, + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), + .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), + else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), + }, + inline .f16, .f32, .f64 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Data, @tagName(tag), @floatFromInt(value)), + .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), + else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + .comptime_int => blk: { + const OutT = @FieldType(Data, @tagName(tag)); + if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { + break :blk @unionInit(Data, @tagName(tag), @intCast(value)); + } else { + @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + } + }, + .int => @unionInit(Data, @tagName(tag), @intCast(value)), + else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + .c64 => switch (T) { + C64 => .{ .c64 = value }, + C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + else => @panic("Could not create Data of type c64 from value of type " ++ @typeName(T)), + }, + .c128 => switch (T) { + C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + C128 => .{ .c128 = value }, + else => @panic("Could not create Data of type c128 from value of type " ++ @typeName(T)), + }, + }; + } + + test init { + try std.testing.expectEqual(20.0, Data.init(.f16, 20).f16); + try std.testing.expectEqual(20.5, Data.init(.f16, 20.5).f16); + try std.testing.expectEqual(20, Data.init(.f16, @as(u8, 20)).f16); + try std.testing.expectEqual(-20, Data.init(.f16, @as(i8, -20)).f16); + try std.testing.expectEqual(2000.5, Data.init(.f16, @as(f32, 2000.5)).f16); + + try std.testing.expectEqual(true, Data.init(.bool, true).bool); + + try std.testing.expectEqual(10, Data.init(.u8, 10).u8); + try std.testing.expectEqual(10, Data.init(.u8, @as(u16, 10)).u8); + + try std.testing.expectEqual(10, Data.init(.i8, 10).i8); + try std.testing.expectEqual(10, Data.init(.i8, @as(u16, 10)).i8); + try std.testing.expectEqual(-10, Data.init(.i8, -10).i8); + try std.testing.expectEqual(-10, Data.init(.i8, @as(i16, -10)).i8); + + try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C64.init(1, 2)).c64); + try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C128.init(1, 2)).c64); + try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C128.init(1, 2)).c128); + try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C64.init(1, 2)).c128); + } + + pub fn dtype(self: Data) DataType { + return std.meta.activeTag(self); + } + + pub fn constSlice(data: *const Data) []const u8 { + return switch (data.*) { + inline else => |*value| std.mem.asBytes(value), + }; + } + + pub fn as(self: Data, comptime T: type) T { + // TODO allow more lossless conversions + switch (@typeInfo(T)) { + .bool => return self.bool, + .float => switch (self) { + inline .f16, .f32, .f64 => |v| return @floatCast(v), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), + else => {}, + }, + .int => switch (self) { + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), + else => {}, + }, + else => {}, + } + std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); + } +}; + +pub fn mantissaSize(dtype: DataType) usize { + return switch (dtype) { + .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz => 3, + .f8e5m2, .f8e5m2fnuz => 2, + .f16 => 10, + .bf16 => 7, + .f32 => 23, + .f64 => 52, + else => @panic("Can't get mantissa size for a non-float dtype"), + }; +} diff --git a/zml2/floats.zig b/zml2/floats.zig new file mode 100644 index 00000000..54240485 --- /dev/null +++ b/zml2/floats.zig @@ -0,0 +1,517 @@ +///! Conversion utilities between different Floating point formats. +const std = @import("std"); + +test { + std.testing.refAllDecls(@This()); +} + +fn FloatHelpers(Float: type) type { + const info = @typeInfo(Float); + const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; + if (info != .@"struct" or info.@"struct".backing_integer == null) { + @compileError(err_msg); + } + comptime { + for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { + if (!std.mem.eql(u8, field.name, expected_name)) + @compileError(err_msg); + } + } + + return struct { + const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; + const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; + const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; + const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); + const exp_off: u8 = FloatHelpers(Float32).exp_bias - exp_bias; + + pub const zero: Float = .{ .sign = 0, .exponent = 0, .mantissa = 0 }; + + pub fn neg(x: Float) Float { + return .{ + .sign = x.sign ^ 1, + .exponent = x.exponent, + .mantissa = x.mantissa, + }; + } + + /// Lossy conversion from f32, similar to @floatCast + pub fn fromF32(f: f32) Float { + @setRuntimeSafety(false); + + const vf32: Float32 = @bitCast(f); + const exponent: i16 = @as(i16, vf32.exponent) - exp_off; + const overflow = exponent > std.math.maxInt(@FieldType(Float, "exponent")); + if (overflow) { + @branchHint(.unlikely); + return if (@hasDecl(Float, "inf")) + if (vf32.sign == 0) Float.inf else Float.minus_inf + else + Float.nan; + } + + return if (exponent <= 0) + .{ + .sign = vf32.sign, + .exponent = 0, + .mantissa = shiftMantissa(vf32.mantissa, @intCast(-exponent)), + } + else + .{ + .sign = vf32.sign, + .exponent = @intCast(exponent), + .mantissa = truncMantissa(vf32.mantissa), + }; + } + + /// Lossless conversion to f32. + pub fn toF32(x: Float) f32 { + @setRuntimeSafety(false); + + if (x == zero) return 0.0; + if (isInf(x)) { + @branchHint(.unlikely); + return if (x.sign == 0) std.math.inf(f32) else -std.math.inf(f32); + } + + const vf32: Float32 = if (x.exponent > 0) + .{ + .sign = x.sign, + .exponent = @as(u8, x.exponent) + exp_off, + .mantissa = f32Mantissa(x), + } + else + .{ + .sign = x.sign, + .exponent = exp_off - @clz(x.mantissa), + .mantissa = @as(u23, x.mantissa) << @clz(x.mantissa), + }; + return @bitCast(vf32); + } + + fn truncMantissa(f32_mantissa: u32) @FieldType(Float, "mantissa") { + const rounding_val: u32 = @as(u32, 1) << (f32_mantissa_bits - mantissa_bits - 1); + return @truncate((f32_mantissa + rounding_val) >> (f32_mantissa_bits - mantissa_bits)); + } + + fn shiftMantissa(f32_mantissa: u32, underflow: u8) @FieldType(Float, "mantissa") { + const upper_bit: u32 = @as(u32, 1) << f32_mantissa_bits; + const full_mant32: u32 = f32_mantissa | upper_bit; + // divide the mantissa proportionally to the exponent underflow + const shifted_mant: u32 = full_mant32 >> @truncate(underflow + 1); + return truncMantissa(shifted_mant); + } + + fn f32Mantissa(x: Float) @FieldType(Float32, "mantissa") { + const T = @FieldType(Float32, "mantissa"); + return @as(T, x.mantissa) << f32_mantissa_bits - mantissa_bits; + } + + pub fn formatNumber(x: Float, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + switch (n.mode) { + .binary, .octal, .hex => try writer.print("{{ .sign={}, .exp={}, .mantissa={} }}", .{ x.sign, x.exponent, x.mantissa }), + else => try writer.printFloat(x.toF32(), n), + } + } + }; +} + +pub const Float32 = packed struct(u32) { + mantissa: u23, + exponent: u8, + sign: u1, + + pub const inf: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; + pub const minus_inf = neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +const f32_exp_bias = FloatHelpers(Float32).expBias(); + +pub const Float64 = packed struct(u64) { + mantissa: u52, + exponent: u11, + sign: u1, + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E4M3B11FNUZ = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3B11FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + + pub fn isNan(self: Float8E4M3B11FNUZ) bool { + return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E4M3FN = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + + pub fn isNan(self: Float8E4M3FN) bool { + return self.exponent == nan.exponent and self.mantissa == nan.mantissa; + } + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E4M3FNUZ = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + + pub fn isNan(self: Float8E4M3FNUZ) bool { + return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +test "Float8E4" { + // With 4 bits of exponents power of two can be represented exactly up to 64. + const test_case_e4: TestCase = .{ + .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 64.0, -128, -1.125 / 64.0 }, + .lossy = &[_]f32{ 3.02344107628, 1.0 / 128.0, 1.0 / 512.0 }, + }; + + inline for (.{ + Float8E4M3B11FNUZ, + Float8E4M3FN, + Float8E4M3FNUZ, + }) |Float8T| { + try testCustomFloat(Float8T, test_case_e4); + try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 2048.0).toF32()); + if (@hasDecl(Float8T, "inf")) { + try std.testing.expectEqual(Float8T.inf, Float8T.fromF32(128.0)); + try std.testing.expectEqual(Float8T.inf.neg(), Float8T.fromF32(-128.0)); + } + } +} + +pub const Float8E5M2 = packed struct(u8) { + mantissa: u2, + exponent: u5, + sign: u1, + + pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 1 }; + + pub fn isNan(self: Float8E5M2) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + + pub const inf: Float8E5M2 = .{ + .sign = 0, + .exponent = std.math.maxInt(u5), + .mantissa = 0, + }; + + pub const minus_inf: Float8E5M2 = .neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E5M2FNUZ = packed struct(u8) { + mantissa: u2, + exponent: u5, + sign: u1, + + pub const nan: Float8E5M2FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + + pub fn isNan(self: Float8E5M2FNUZ) bool { + return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +test "Float8E5" { + const test_case_e5: TestCase = .{ + .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 128.0, -128 }, + .lossy = &[_]f32{3.02344107628}, + }; + inline for (.{ Float8E5M2, Float8E5M2FNUZ }) |Float8T| { + try testCustomFloat(Float8T, test_case_e5); + } +} + +pub const BFloat16 = packed struct(u16) { + mantissa: u7, + exponent: u8, + sign: u1, + + pub const nan: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; + + pub fn isNan(self: BFloat16) bool { + return allBitsOne(self.exponent) and self.mantissa != 0; + } + + pub const inf: BFloat16 = .{ + .sign = 0, + .exponent = std.math.maxInt(u8), + .mantissa = 0, + }; + + pub const minus_inf: BFloat16 = .neg(inf); + + // Specialized versions of to/from F32. Since BFloat16 has the same exponent range than F32, + // no overflow/underflow can happen, simplifiying conversion logic. + pub fn toF32(self: BFloat16) f32 { + // Pad the BF16 with zeros 0 + return @bitCast([2]u16{ 0, @bitCast(self) }); + } + + pub fn fromF32(float32: f32) BFloat16 { + var int: u32 = @bitCast(float32); + // Round up if needed. + int += 0x8000; + const parts: [2]u16 = @bitCast(int); + return @bitCast(parts[1]); + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const formatNumber = Helpers.formatNumber; +}; + +test BFloat16 { + // From https://en.wikipedia.org/wiki/Bfloat16_floating-point_format#Examples + try std.testing.expectEqual(BFloat16.fromF32(0), BFloat16{ .sign = 0, .exponent = 0, .mantissa = 0 }); + try std.testing.expectEqual(BFloat16.fromF32(-2), BFloat16{ .sign = 1, .exponent = 127 + 1, .mantissa = 0 }); + try std.testing.expectEqual(BFloat16.fromF32(3.02344107628), BFloat16{ .sign = 0, .exponent = 127 + 1, .mantissa = 66 }); + try std.testing.expectEqual(BFloat16.fromF32(1.0 / 128.0), BFloat16{ .sign = 0, .exponent = 127 - 7, .mantissa = 0 }); + try std.testing.expectEqual(std.mem.toBytes(BFloat16.inf.neg()), [_]u8{ 0x80, 0xff }); + try std.testing.expectEqual(BFloat16.inf, BFloat16.fromF32(std.math.inf(f32))); + + try testCustomFloat(BFloat16, .{ + .lossless = &[_]f32{ 0, -2, 1.0 / 128.0, -1e64, std.math.inf(f32) }, + .lossy = &[_]f32{3.02344107628}, + }); +} + +pub const Float8E4M3 = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3 = @bitCast(0xFF); + + pub fn isNan(self: Float8E4M3) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const inf: Float8E4M3 = .{ + .sign = 0, + .exponent = std.math.maxInt(u4), + .mantissa = 0, + }; + + pub const minus_inf = neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E3M4 = packed struct(u8) { + mantissa: u4, + exponent: u3, + sign: u1, + + pub const nan: Float8E3M4 = @bitCast(0xFF); + + pub fn isNan(self: Float8E3M4) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const inf: Float8E3M4 = .{ + .sign = 0, + .exponent = std.math.maxInt(u3), + .mantissa = 0, + }; + + pub const minus_inf = neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E8M0 = packed struct(u8) { + mantissa: u0 = 0, + exponent: u8, + sign: u0 = 0, + + pub const min_scale: f32 = @bitCast(Float32{ .sign = 0, .exponent = 0, .mantissa = 0b1 << 22 }); + + /// Lossy conversion from f32, similar to @floatCast + pub fn fromF32(f: f32) Float8E8M0 { + const vf32: Float32 = @bitCast(f); + return .{ .exponent = @intCast(vf32.exponent) }; + } + + /// Lossless conversion to f32. + pub fn toF32(x: Float8E8M0) f32 { + if (x.exponent == 0) return min_scale; + const vf32: Float32 = .{ + .sign = 0, + .exponent = x.exponent, + .mantissa = 0, + }; + return @bitCast(vf32); + } + + const Helpers = FloatHelpers(@This()); + pub const formatNumber = Helpers.formatNumber; +}; + +test Float8E8M0 { + try std.testing.expectEqual(Float8E8M0{ .exponent = 127 }, Float8E8M0.fromF32(1.0)); + // try std.testing.expectEqual(5.877472e-39, Float8E8M0.toF32(.{ .exponent = 0})); + + try testCustomFloat(Float8E8M0, .{ + .lossless = &[_]f32{ Float8E8M0.min_scale, 1.0, 64.0, 1.0 / 128.0, std.math.pow(f32, 2.0, 127) }, + .lossy = &[_]f32{1.00001}, + }); +} + +pub const Float4E2M1 = packed struct(u4) { + mantissa: u1, + exponent: u2, + sign: u1, + + pub const nan: Float4E2M1 = @bitCast(@as(u4, 0xF)); + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const formatNumber = Helpers.formatNumber; + + pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + + pub fn toF32(x: Float4E2M1) f32 { + // faster implementation + return values[@as(u4, @bitCast(x))]; + } + + test toF32 { + var to_f32_res: [16]f32 = undefined; + for (&to_f32_res, 0..) |*r, i| { + const x_f4: Float4E2M1 = @bitCast(@as(u4, @intCast(i))); + r.* = x_f4.toF32(); + } + try std.testing.expectEqualSlices(f32, &Float4E2M1.values, &to_f32_res); + } + + test fromF32 { + var from_f32_res: [16]Float4E2M1 = undefined; + for (&from_f32_res, 0..) |*r, i| { + r.* = .fromF32(Float4E2M1.values[i]); + } + try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); + } +}; + +pub fn floatCast(T: type, x: anytype) T { + return switch (T) { + f64, f32, f16 => switch (@TypeOf(x)) { + f64, f32, f16 => @floatCast(x), + else => @floatCast(x.toF32()), + }, + else => switch (@TypeOf(x)) { + f64, f32, f16 => .fromF32(x), + else => .fromF32(x.toF32()), + }, + }; +} + +pub fn isInf(x: anytype) bool { + const Float = @TypeOf(x); + switch (Float) { + f64, f32, f16 => return std.math.isInf(x), + else => {}, + } + if (!@hasDecl(Float, "inf")) return false; + + const FBits = std.meta.Int(.unsigned, @bitSizeOf(Float)); + const remove_sign = ~@as(FBits, 0) >> 1; + return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); +} + +fn allBitsOne(v: anytype) bool { + return v == std.math.maxInt(@TypeOf(v)); +} + +const TestCase = struct { + lossless: []const f32, + lossy: []const f32, + tolerance: f32 = 1e-2, +}; + +fn testCustomFloat(FloatT: type, test_case: TestCase) !void { + for (test_case.lossless) |x| { + try std.testing.expectEqual(x, FloatT.fromF32(x).toF32()); + } + for (test_case.lossy) |x| { + try expectApproxEqRel(f32, x, FloatT.fromF32(x).toF32(), test_case.tolerance); + } +} + +fn expectApproxEqRel(FloatT: type, x: FloatT, y: FloatT, tolerance: FloatT) !void { + if (!std.math.approxEqRel(f32, x, y, tolerance)) { + std.log.err("expected ~{d}, got {d}", .{ x, y }); + return error.TestUnexpectedResult; + } +} diff --git a/zml2/main.zig b/zml2/main.zig new file mode 100644 index 00000000..66d7a6cd --- /dev/null +++ b/zml2/main.zig @@ -0,0 +1,15 @@ +const std = @import("std"); +const async = @import("async"); +const zml = @import("zml"); + +pub fn main() !void { + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + zml.init(); + defer zml.deinit(); + + var platform = try zml.Platform.init(.cpu, .{}); + defer platform.deinit(); +} diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig new file mode 100644 index 00000000..f09ffa1e --- /dev/null +++ b/zml2/pjrtx.zig @@ -0,0 +1,362 @@ +const std = @import("std"); + +const async = @import("async"); +const dialects = @import("mlir/dialects"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +pub const ffi = pjrt.ffi; +pub const Api = pjrt.Api; +pub const ApiError = pjrt.ApiError; +pub const BufferType = pjrt.BufferType; +pub const ClientInitError = pjrt.ClientInitError; +pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; +pub const Device = pjrt.Device; +pub const DeviceDescription = pjrt.DeviceDescription; +pub const Error = pjrt.Error; +pub const ErrorCode = pjrt.ErrorCode; +pub const Executable = pjrt.Executable; +pub const ExecuteContext = pjrt.ExecuteContext; +pub const ExecuteError = ApiError; +pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; +pub const Memory = pjrt.Memory; +pub const MemoryStats = pjrt.MemoryStats; +pub const NamedValue = pjrt.NamedValue; +pub const Profiler = pjrt.Profiler; +pub const SerializeResult = pjrt.SerializeResult; +pub const Stream = pjrt.Stream; + +const zml = struct { + pub const DataType = @import("dtype.zig").DataType; +}; + +const log = std.log.scoped(.zml); + +pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || ApiError; + +fn InnerMixin(comptime innerT: type) type { + return struct { + inline fn inner(self: anytype) if (@typeInfo(@TypeOf(self)).pointer.is_const) *const innerT else *innerT { + return @ptrCast(self); + } + }; +} + +pub const Client = opaque { + const inner = InnerMixin(pjrt.Client).inner; + + pub fn init(api: *const Api, options: []const NamedValue) ClientInitError!*Client { + return @ptrCast(try pjrt.Client.init(api, options)); + } + + pub fn deinit(self: *Client, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn getPlatformName(self: *const Client, api: *const Api) []const u8 { + return self.inner().getPlatformName(api); + } + + pub fn getDevices(self: *const Client, api: *const Api) []const *const Device { + return self.inner().getDevices(api); + } + + pub fn getAddressableDevices(self: *const Client, api: *const Api) []const *const Device { + return self.inner().getAddressableDevices(api); + } + + pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; + } + + pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { + return @ptrCast(try async.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); + } + + pub const CreateViewOfDeviceBufferArgs = pjrt.Client.CreateViewOfDeviceBufferArgs; + pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { + return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); + } + + fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); + defer bytecode.deinit(); + module.op().writeBytecodeWithConfig(&bytecode.writer, .{ .desiredEmitedVersion = 1 }) catch |err| { + log.err("failed to write module bytecode: {}", .{err}); + return switch (err) { + std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + else => |e| e, + }; + }; + + var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); + defer serialized_buffer.deinit(); + + const stablehlo_version = blk: { + if (api.stablehloCurrentVersion()) |requested_version| { + break :blk dialects.stablehlo.stablehloGetSmallerVersion(requested_version, dialects.stablehlo.getCurrentVersion()); + } + break :blk dialects.stablehlo.getMinimumVersion(); + }; + + dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { + log.err("failed to serialize to portable artifact: {}", .{err}); + return switch (err) { + std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + else => |e| e, + }; + }; + + return @ptrCast(try self.inner().compile(api, .{ + .bytecode = serialized_buffer.written(), + .bytecode_format = .mlir, + .compile_options_pb = compile_options_pb, + })); + } + + pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + return try async.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); + } + + pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { + return self.inner().addressableMemories(api); + } + + pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { + for (self.addressableMemories(api)) |mem| { + if (mem.kind(api) == kind) { + return mem; + } + } + return null; + } + + pub const CreateUninitializedBufferArgs = pjrt.Client.CreateUninitializedBufferArgs; + + pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { + return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); + } +}; + +pub const Buffer = opaque { + pub const inner = InnerMixin(pjrt.Buffer).inner; + + pub fn deinit(self: *Buffer, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { + return try self.inner().getDevice(api); + } + + pub fn delete(self: *Buffer, api: *const Api) void { + self.inner().delete(api); + } + + pub fn isDeleted(self: *const Buffer, api: *const Api) bool { + return self.inner().isDeleted(api); + } + + pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { + return self.inner().isOnCpu(api); + } + + pub fn memory(self: *const Buffer, api: *const Api) *const Memory { + return self.inner().memory(api); + } + + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { + return @ptrCast(try self.inner().toHostBuffer(api, dst)); + } + + pub fn getElementType(self: *const Buffer, api: *const Api) zml.DataType { + return dtypeFromBufferType(self.inner().getElementType(api)); + } + + pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { + return self.inner().getDimensions(api); + } + + pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { + return try self.inner().getUnpaddedDimensions(api); + } + + pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { + return try self.inner().getOnDeviceSizeInBytes(api); + } + + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { + return @ptrCast(self.inner().copyToDevice(api, device)); + } + + pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { + return @ptrCast(try self.inner().copyToMemory(api, memory_)); + } + + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { + return @ptrCast(self.inner().getReadyEvent(api)); + } + + pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { + return try self.inner().getOpaqueDeviceMemoryDataPointer(api); + } +}; + +pub const Event = opaque { + pub const inner = InnerMixin(pjrt.Event).inner; + + pub fn deinit(self: *Event, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn isReady(self: *const Event, api: *const Api) bool { + return self.inner().isReady(api); + } + + pub fn getEventError(self: *const Event, api: *const Api) ?*Error { + return self.inner().getEventError(api); + } + + pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { + if (self.isReady(api)) { + return; + } + try self.inner().await(api); + } + + pub fn await(self: *Event, api: *const Api) ApiError!void { + defer self.deinit(api); + + if (self.isReady(api)) { + return; + } + + var ctx = struct { + err: ?*pjrt.Error = null, + event: async.threading.ResetEventSingle = .{}, + }{}; + + try self.inner().onReady(api, &(struct { + fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { + const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); + ctx_.err = err; + ctx_.event.set(); + } + }.call), &ctx); + ctx.event.wait(); + + if (ctx.err) |e| { + defer e.deinit(api); + const err_code = e.getCode(api).toApiError(); + log.err("{t} {s}", .{ err_code, e.getMessage(api) }); + return err_code; + } + } +}; + +pub const LoadedExecutable = opaque { + const inner = InnerMixin(pjrt.LoadedExecutable).inner; + + pub fn deinit(self: *LoadedExecutable, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn delete(self: *LoadedExecutable, api: *const Api) void { + self.inner().delete(api); + } + + pub fn isDeleted(self: *const LoadedExecutable, api: *const Api) bool { + return self.inner().isDeleted(api); + } + + pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []*const Device { + return self.inner().getAddressableDevices(api); + } + + pub const ExecuteArgs = struct { + arguments: []const [*]const *const Buffer, + num_args: usize, + results: []const [*]*Buffer, + events: []?*Event, + non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, + }; + + pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ + .num_args = args.num_args, + .arguments = @ptrCast(args.arguments), + .results = @ptrCast(args.results), + .events = @ptrCast(args.events), + .non_donatable_input_indices = args.non_donatable_input_indices, + .context = args.context, + }); + } + + pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { + return try self.inner().getExecutable(api); + } +}; + +pub const AsyncHostToDeviceTransferManager = opaque { + const inner = InnerMixin(pjrt.AsyncHostToDeviceTransferManager).inner; + + pub fn deinit(self: *AsyncHostToDeviceTransferManager, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn transferData(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, data: []const u8, offset: i64, is_last_transfer: bool) ApiError!*Event { + return @ptrCast(try self.inner().transferData(api, buffer_index, data, offset, is_last_transfer)); + } + + pub fn retrieveBuffer(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!*Buffer { + return @ptrCast(try self.inner().retrieveBuffer(api, buffer_index)); + } + + pub fn device(self: *AsyncHostToDeviceTransferManager, api: *const Api) *Device { + return @ptrCast(self.inner().device(api)); + } + + pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) usize { + return self.inner().bufferCount(api); + } + + pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) usize { + return self.inner().bufferSize(api, buffer_index); + } + + pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: ErrorCode, error_message: []const u8) void { + self.inner().setBufferError(api, buffer_index, error_code, error_message); + } + + pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) void { + return self.inner().addMetadata(api, transfer_metadata); + } +}; + +pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), + }; +} + +pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) zml.DataType { + return switch (pjrt_type) { + .invalid => @panic("Found an invalid pjrt buffer"), + inline else => |tag| @field(zml.DataType, @tagName(tag)), + }; +} + +test bufferTypeFromDtype { + inline for (@typeInfo(zml.DataType).@"enum".fields) |field| { + const dt: zml.DataType = @enumFromInt(field.value); + try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); + } + + inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { + const dt: pjrt.BufferType = @enumFromInt(field.value); + if (dt == .invalid) continue; + try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); + } +} diff --git a/zml2/platform.zig b/zml2/platform.zig new file mode 100644 index 00000000..d942290b --- /dev/null +++ b/zml2/platform.zig @@ -0,0 +1,190 @@ +const std = @import("std"); + +const runtimes = @import("runtimes"); +pub const Target = runtimes.Platform; +const stdx = @import("stdx"); + +const pjrt = @import("pjrtx.zig"); + +const log = std.log.scoped(.zml); + +pub const CompilationOptions = struct { + xla_dump_to: ?[]const u8 = null, + xla_dump_fusion_visualization: bool = false, + xla_dump_hlo_pass_re: ?[]const u8 = null, + sharding_enabled: bool = false, + sharding_axes: stdx.BoundedArray([*:0]const u8, 8) = .{}, + device_memory_size: u64 = 0, +}; + +pub const Platform = struct { + target: Target, + pjrt_api: *const pjrt.Api, + pjrt_client: *pjrt.Client, + + // This make the pjrt struct quite fat, but is only used during compilation. + // TODO: Reconsider having it here, and maybe pass explicitly to compile, + // or create an intermediary struct: + // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` + compilation_options: CompilationOptions = .{}, + + pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; + + pub fn init(target: Target, options: CreateOptions) !Platform { + const api = try runtimes.load(target); + + var named_values_buf: [16]pjrt.NamedValue = undefined; + const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); + const true_num_devices = pjrt_client.getAddressableDevices(api).len; + if (true_num_devices > MAX_NUM_DEVICES) { + log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, true_num_devices, MAX_NUM_DEVICES }); + } + return .{ + .target = target, + .pjrt_api = api, + .pjrt_client = pjrt_client, + .compilation_options = .{}, + }; + } + + pub fn getDevices(self: Platform) []const *const pjrt.Device { + const all_devices = self.pjrt_client.getAddressableDevices(self.pjrt_api); + if (all_devices.len > MAX_NUM_DEVICES) { + return all_devices[0..MAX_NUM_DEVICES]; + } + return all_devices; + } + + pub fn withCompilationOptions(self: Platform, opts: CompilationOptions) Platform { + var res = self; + res.compilation_options = opts; + return res; + } + + pub fn deinit(self: *Platform) void { + self.pjrt_client.deinit(self.pjrt_api); + } + + pub fn memoryForDevice(platform: Platform, memory: pjrt.Memory.Kind, device: *const pjrt.Device) *const pjrt.Memory { + const memory_target: pjrt.Memory.Kind = switch (memory) { + .host_unpinned => switch (platform.target) { + // Cuda doesn't have host_unpinned. + // ROCm doesn't seem to have it either. + // TODO(gwenzek): investigate why it was not forced before. + .cuda, .rocm => .host_pinned, + else => .host_unpinned, + }, + inline else => |t| t, + }; + // TODO measure the cost of this and consider caching. + const device_memories = device.addressableMemories(platform.pjrt_api); + for (device_memories) |m| { + if (memory_target == m.kind(platform.pjrt_api)) { + return m; + } + } + log.err("Platform {t} doesn't have memory {t}", .{ platform.target, memory }); + @panic("Memory kind not found"); + } + + test memoryForDevice { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const memory_fields = @typeInfo(pjrt.Memory.Kind).@"enum".fields; + inline for (memory_fields) |field| { + for (platform.getDevices()) |dev| { + _ = platform.memoryForDevice(@field(pjrt.Memory.Kind, field.name), dev); + } + } + } + + pub fn memoryStats(platform: Platform, device_id: usize) pjrt.MemoryStats { + if (platform.target == .cpu) return .zeroes; + + const device = platform.getDevices()[device_id]; + return device.memoryStats(platform.pjrt_api) catch .zeroes; + } +}; + +pub const CreateOptions = struct { + cpu: Cpu = .{ .device_count = 4 }, + + // bump memory fraction from XLA defaults of 75% to 90%. + // Even on a 8GB GPU it should leave enough space for the Cuda driver + // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 + cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + rocm: struct {} = .{}, + tpu: struct {} = .{}, + neuron: struct {} = .{}, + + pub const Cpu = struct { + device_count: u32, + + fn writeNamedValues(self: Cpu, values: *std.ArrayList(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.from("cpu_device_count", @as(i64, self.device_count))); + } + }; + + pub const Cuda = struct { + allocator: Allocator = .{ .bfc = .{} }, + // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 + // visible_devices: []const i64 = &.{}, + // node_id + // num_nodes + // enable_mock_nccl + // mock_gpu_topology + + pub const Allocator = union(enum) { + /// "Best-Fit with Coalescing" algorithm + bfc: Options, + /// use cudaMallocAsync + async: Options, + /// use raw cuMalloc + platform, + + pub const Options = struct { + preallocate: bool = true, + memory_fraction: f32 = 0.90, + collective_memory_size_mb: u32 = 0, + }; + }; + + fn writeNamedValues(self: Cuda, values: *std.ArrayList(pjrt.NamedValue)) void { + switch (self.allocator) { + .platform => { + values.appendAssumeCapacity(.fromString("allocator", "platform")); + }, + .bfc, .async => |opt| { + values.appendAssumeCapacity(.fromString("allocator", switch (self.allocator) { + .bfc => "bfc", + .async => "cuda_async", + .platform => unreachable, + })); + values.appendAssumeCapacity(.from("preallocate", opt.preallocate)); + if (opt.memory_fraction > 0) { + values.appendAssumeCapacity(.from("memory_fraction", opt.memory_fraction)); + } + if (opt.collective_memory_size_mb > 0) { + const collective = @as(i64, opt.collective_memory_size_mb) * 1024 * 1024; + values.appendAssumeCapacity(.from("collective_memory_size", collective)); + } + }, + } + } + }; + + pub fn toNamedValues(self: CreateOptions, target: Target, out: []pjrt.NamedValue) []pjrt.NamedValue { + var values = std.ArrayList(pjrt.NamedValue).fromOwnedSlice(out); + values.shrinkRetainingCapacity(0); + switch (target) { + .cpu => self.cpu.writeNamedValues(&values), + .cuda => self.cuda.writeNamedValues(&values), + inline else => |t| { + stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); + const options = @field(self, @tagName(t)); + stdx.debug.assertComptime(@sizeOf(@TypeOf(options)) == 0, "zml.platform.CreateOptions.{s} is discarded", .{@tagName(t)}); + }, + } + return values.items; + } +}; diff --git a/zml2/posix.h b/zml2/posix.h new file mode 100644 index 00000000..0d66200c --- /dev/null +++ b/zml2/posix.h @@ -0,0 +1,2 @@ +#include +#include diff --git a/zml2/zml.zig b/zml2/zml.zig new file mode 100644 index 00000000..b1cc842c --- /dev/null +++ b/zml2/zml.zig @@ -0,0 +1,61 @@ +//! Welcome to the ZML API documentation! +//! ZML provides tools to write high level code describing a neural network, +//! compiling it for various accelerators and targets, and executing it. +//! + +// Namespaces +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const runfiles = @import("runfiles"); + +pub const platform = @import("platform.zig"); +pub const Platform = platform.Platform; +pub const Target = platform.Target; +pub const CompilationOptions = platform.CompilationOptions; +pub const floats = @import("floats.zig"); +const dtype = @import("dtype.zig"); +pub const Data = dtype.Data; +pub const DataType = dtype.DataType; +pub const pjrt = @import("pjrtx.zig"); + +var mlir_once = std.once(struct { + fn call() void { + mlir.registerPasses("Transforms"); + } +}.call); + +var runfiles_once = std.once(struct { + fn call_() !void { + if (std.process.hasEnvVarConstant("RUNFILES_MANIFEST_FILE") or std.process.hasEnvVarConstant("RUNFILES_DIR")) { + return; + } + + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + const allocator = arena.allocator(); + defer arena.deinit(); + + var envMap = std.process.EnvMap.init(allocator); + var r = (try runfiles.Runfiles.create(.{ .allocator = allocator })) orelse return; + try r.environment(&envMap); + + var it = envMap.iterator(); + while (it.next()) |entry| { + const keyZ = try allocator.dupeZ(u8, entry.key_ptr.*); + const valueZ = try allocator.dupeZ(u8, entry.value_ptr.*); + _ = c.setenv(keyZ.ptr, valueZ.ptr, 1); + } + } + + fn call() void { + call_() catch @panic("Unable to init runfiles env"); + } +}.call); + +pub fn init() void { + runfiles_once.call(); + mlir_once.call(); +} + +pub fn deinit() void {} From 972c0a4aaa70ef91dafee074467451ce8f2f2d42 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 1 Dec 2025 15:51:38 +0100 Subject: [PATCH 2/6] mlir and dialects modules rework * Complete rework of mlir that uses pointers instead of values for basic types. * Removed unused dialects. --- BUILD.bazel | 4 +- mlir/dialects/BUILD.bazel | 4 - mlir/dialects/arith.zig | 110 - mlir/dialects/dialects.zig | 6 +- mlir/dialects/math.zig | 36 - mlir/dialects/scf.zig | 61 - mlir/dialects/stablehlo/stablehlo.zig | 1578 +++++++------- mlir/dialects/tensor.zig | 36 - mlir/mlir.zig | 2880 ++++++++++++------------- 9 files changed, 2180 insertions(+), 2535 deletions(-) delete mode 100644 mlir/dialects/arith.zig delete mode 100644 mlir/dialects/math.zig delete mode 100644 mlir/dialects/scf.zig delete mode 100644 mlir/dialects/tensor.zig mode change 100755 => 100644 mlir/mlir.zig diff --git a/BUILD.bazel b/BUILD.bazel index 97b2cf13..dfdd43c7 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -5,8 +5,8 @@ zls_completion( visibility = ["//visibility:public"], deps = [ "//async", - "//examples/llama", + #"//examples/llama", "//stdx", - "//zml", + "//zml2:zml", ], ) diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 101f67ff..26cca662 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -3,11 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "dialects", srcs = [ - "arith.zig", "func.zig", - "math.zig", - "scf.zig", - "tensor.zig", ], import_name = "mlir/dialects", main = "dialects.zig", diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig deleted file mode 100644 index 39cbc2a1..00000000 --- a/mlir/dialects/arith.zig +++ /dev/null @@ -1,110 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -pub fn constant(ctx: mlir.Context, value: mlir.Attribute, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "arith.constant", .{ - .attributes = &.{.{ "value", value }}, - .result_type_inference = true, - .location = location, - }); -} - -fn binary_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .location = location, - }); - } - }.call; -} - -fn cast_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Type, mlir.Location) mlir.Operation { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, new_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .results = &.{new_type}, - .location = location, - }); - } - }.call; -} - -pub const addi = binary_fn("arith.addi"); -pub const addf = binary_fn("arith.addf"); -pub const subi = binary_fn("arith.subi"); -pub const subf = binary_fn("arith.subf"); -pub const muli = binary_fn("arith.muli"); -pub const mulf = binary_fn("arith.mulf"); -pub const divsi = binary_fn("arith.divsi"); -pub const divui = binary_fn("arith.divui"); -pub const divf = binary_fn("arith.divf"); -pub const maxnumf = binary_fn("arith.maxnumf"); -pub const maxnumi = binary_fn("arith.maxnumi"); -pub const extsi = cast_fn("arith.extsi"); -pub const extui = cast_fn("arith.extui"); -pub const extf = cast_fn("arith.extf"); -pub const trunci = cast_fn("arith.trunci"); -pub const truncf = cast_fn("arith.truncf"); -pub const fptosi = cast_fn("arith.fptosi"); -pub const fptoui = cast_fn("arith.fptoui"); -pub const sitofp = cast_fn("arith.sitofp"); -pub const uitofp = cast_fn("arith.uitofp"); - -pub const CmpIPredicate = enum { - eq, - ne, - slt, - sle, - sgt, - sge, - ult, - ule, - ugt, - uge, -}; - -pub fn cmpi(ctx: mlir.Context, predicate: CmpIPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "arith.cmpi", .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .attributes = &.{ - .{ "predicate", .int(ctx, .i64, @intFromEnum(predicate)) }, - }, - .location = location, - }); -} - -pub const CmpFPredicate = enum { - false, - oeq, - ogt, - oge, - olt, - ole, - one, - ord, - ueq, - ugt, - uge, - ult, - ule, - une, - uno, - true, -}; - -pub fn cmpf(ctx: mlir.Context, predicate: CmpFPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "arith.cmpf", .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .attributes = &.{ - .{ "predicate", .int(ctx, .i64, @intFromEnum(predicate)) }, - }, - .location = location, - }); -} diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index 47f74d8e..3deac104 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,11 +1,7 @@ const std = @import("std"); -pub const arith = @import("arith.zig"); -pub const func = @import("func.zig"); -pub const math = @import("math.zig"); -pub const scf = @import("scf.zig"); -pub const tensor = @import("tensor.zig"); pub const stablehlo = @import("mlir/dialects/stablehlo"); +pub const func = @import("func.zig"); test { std.testing.refAllDecls(@This()); diff --git a/mlir/dialects/math.zig b/mlir/dialects/math.zig deleted file mode 100644 index 62ef622c..00000000 --- a/mlir/dialects/math.zig +++ /dev/null @@ -1,36 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -const namespace = "math"; - -fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{value}, - .results = &.{value.getType()}, - .location = location, - }); - } - }; -} - -fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{ lhs, rhs }, - .results = &.{}, - .location = location, - }); - } - }; -} - -pub const ipowi = binary_fn("ipowi").call; -pub const fpowi = binary_fn("fpowi").call; -pub const tanh = unary_fn("tanh").call; -pub const sqrt = unary_fn("sqrt").call; -pub const exp = unary_fn("exp").call; -pub const exp2 = unary_fn("exp2").call; -pub const log = unary_fn("log").call; diff --git a/mlir/dialects/scf.zig b/mlir/dialects/scf.zig deleted file mode 100644 index 61fcb189..00000000 --- a/mlir/dialects/scf.zig +++ /dev/null @@ -1,61 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -pub fn ForBody(ExtraArgs: type) type { - return fn (mlir.Context, mlir.Block, ExtraArgs) mlir.Operation; -} - -pub const ForRange = struct { - start: mlir.Value, - end: mlir.Value, - step: mlir.Value, -}; - -pub fn @"for"( - ExtraArgs: type, - ctx: mlir.Context, - range: ForRange, - init_values: []const mlir.Value, - body: ForBody(ExtraArgs), - extra_args: ExtraArgs, - loc: mlir.Location, -) mlir.Operation { - const n_args = init_values.len; - var init_types_buf: [32]mlir.Type = undefined; - var locs_buf: [32]mlir.Location = undefined; - - // The first block argument is the for loop induction variable, - // followed then by all the loop-carried variables. - const init_types = init_types_buf[0 .. n_args + 1]; - const locs = locs_buf[0 .. n_args + 1]; - init_types[0] = range.start.getType(); - locs[0] = loc; - for (1.., init_values) |i, val| { - init_types[i] = val.getType(); - locs[i] = loc; - } - - const block = mlir.Block.init(init_types, locs) catch unreachable; - const yield_op = @call(.auto, body, .{ ctx, block, extra_args }); - std.debug.assert(std.mem.eql(u8, "scf.yield", yield_op.name().str())); - block.appendOperationRecursive(yield_op, .open); - - const for_op = mlir.Operation.make(ctx, "scf.for", .{ - .variadic_operands = &.{ &.{ range.start, range.end, range.step }, init_values }, - .results = init_types[1..], - .blocks = &.{block}, - .location = loc, - .verify = false, - }); - return for_op; -} - -pub fn yield(ctx: mlir.Context, res: []const mlir.Value, loc: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "scf.yield", .{ - .variadic_operands = &.{res}, - .results = &.{}, - .location = loc, - .verify = false, - }); -} diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 7c7c1394..b9165367 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -2,112 +2,106 @@ const std = @import("std"); const c = @import("c"); const mlir = @import("mlir"); +const stringRef = mlir.stringRef; const stdx = @import("stdx"); -pub const abs = functors.unary_fn("stablehlo.abs").call; -pub const cosine = functors.unary_fn("stablehlo.cosine").call; -pub const sine = functors.unary_fn("stablehlo.sine").call; -pub const exponential = functors.unary_fn("stablehlo.exponential").call; -pub const exponential_minus_one = functors.unary_fn("stablehlo.exponential_minus_one").call; -pub const floor = functors.unary_fn("stablehlo.floor").call; -pub const log = functors.unary_fn("stablehlo.log").call; -pub const log_plus_one = functors.unary_fn("stablehlo.log_plus_one").call; -pub const not = functors.unary_fn("stablehlo.not").call; -pub const negate = functors.unary_fn("stablehlo.negate").call; -pub const sqrt = functors.unary_fn("stablehlo.sqrt").call; -pub const tanh = functors.unary_fn("stablehlo.tanh").call; -pub const cbrt = functors.unary_fn("stablehlo.cbrt").call; -pub const ceil = functors.unary_fn("stablehlo.ceil").call; -pub const rsqrt = functors.unary_fn("stablehlo.rsqrt").call; -pub const count_leading_zeros = functors.unary_fn("stablehlo.count_leading_zeros").call; -pub const is_finite = functors.unary_fn("stablehlo.is_finite").call; -pub const logistic = functors.unary_fn("stablehlo.logistic").call; -pub const popcnt = functors.unary_fn("stablehlo.popcnt").call; -pub const sign = functors.unary_fn("stablehlo.sign").call; -pub const real = functors.unary_fn("stablehlo.real").call; -pub const imag = functors.unary_fn("stablehlo.imag").call; +pub const abs = unaryOp("stablehlo.abs").call; +pub const cosine = unaryOp("stablehlo.cosine").call; +pub const sine = unaryOp("stablehlo.sine").call; +pub const exponential = unaryOp("stablehlo.exponential").call; +pub const exponential_minus_one = unaryOp("stablehlo.exponential_minus_one").call; +pub const floor = unaryOp("stablehlo.floor").call; +pub const log = unaryOp("stablehlo.log").call; +pub const log_plus_one = unaryOp("stablehlo.log_plus_one").call; +pub const not = unaryOp("stablehlo.not").call; +pub const negate = unaryOp("stablehlo.negate").call; +pub const sqrt = unaryOp("stablehlo.sqrt").call; +pub const tanh = unaryOp("stablehlo.tanh").call; +pub const cbrt = unaryOp("stablehlo.cbrt").call; +pub const ceil = unaryOp("stablehlo.ceil").call; +pub const rsqrt = unaryOp("stablehlo.rsqrt").call; +pub const count_leading_zeros = unaryOp("stablehlo.count_leading_zeros").call; +pub const is_finite = unaryOp("stablehlo.is_finite").call; +pub const logistic = unaryOp("stablehlo.logistic").call; +pub const popcnt = unaryOp("stablehlo.popcnt").call; +pub const sign = unaryOp("stablehlo.sign").call; +pub const real = unaryOp("stablehlo.real").call; +pub const imag = unaryOp("stablehlo.imag").call; +pub const round_nearest_afz = unaryOp("stablehlo.round_nearest_afz").call; +pub const round_nearest_even = unaryOp("stablehlo.round_nearest_even").call; +pub const tuple = unaryOp("stablehlo.tuple").call; -pub const add = functors.binary_fn("stablehlo.add").call; -pub const multiply = functors.binary_fn("stablehlo.multiply").call; -pub const divide = functors.binary_fn("stablehlo.divide").call; -pub const subtract = functors.binary_fn("stablehlo.subtract").call; -pub const or_ = functors.binary_fn("stablehlo.or").call; -pub const xor = functors.binary_fn("stablehlo.xor").call; -pub const and_ = functors.binary_fn("stablehlo.and").call; -pub const atan2 = functors.binary_fn("stablehlo.atan2").call; -pub const maximum = functors.binary_fn("stablehlo.maximum").call; -pub const minimum = functors.binary_fn("stablehlo.minimum").call; -pub const power = functors.binary_fn("stablehlo.power").call; -pub const remainder = functors.binary_fn("stablehlo.remainder").call; -pub const shift_left = functors.binary_fn("stablehlo.shift_left").call; -pub const shift_right_arithmetic = functors.binary_fn("stablehlo.shift_right_arithmetic").call; -pub const shift_right_logical = functors.binary_fn("stablehlo.shift_right_logical").call; -pub const complex = functors.binary_fn("stablehlo.complex").call; +pub const add = binaryOp("stablehlo.add").call; +pub const multiply = binaryOp("stablehlo.multiply").call; +pub const divide = binaryOp("stablehlo.divide").call; +pub const subtract = binaryOp("stablehlo.subtract").call; +pub const or_ = binaryOp("stablehlo.or").call; +pub const xor = binaryOp("stablehlo.xor").call; +pub const and_ = binaryOp("stablehlo.and").call; +pub const atan2 = binaryOp("stablehlo.atan2").call; +pub const maximum = binaryOp("stablehlo.maximum").call; +pub const minimum = binaryOp("stablehlo.minimum").call; +pub const power = binaryOp("stablehlo.power").call; +pub const remainder = binaryOp("stablehlo.remainder").call; +pub const shift_left = binaryOp("stablehlo.shift_left").call; +pub const shift_right_arithmetic = binaryOp("stablehlo.shift_right_arithmetic").call; +pub const shift_right_logical = binaryOp("stablehlo.shift_right_logical").call; +pub const complex = binaryOp("stablehlo.complex").call; -const functors = struct { - fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); - } - }; - } +pub const bitcast_convert = castOp("stablehlo.bitcast_convert").call; +pub const convert = castOp("stablehlo.convert").call; +pub const reshape = castOp("stablehlo.reshape").call; - pub fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .location = location, - }); - } - }; - } -}; - -pub fn return_(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{&.{value}}, - .verify = false, - .location = location, - }); +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; } -pub fn returns_(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{values}, - .verify = false, - .location = location, - }); +fn unaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .location = location, + }); + } + }; } -pub fn bitcast_convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.bitcast_convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); +pub fn binaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; } -pub fn cholesky(ctx: mlir.Context, value: mlir.Value, lower: bool, location: mlir.Location) mlir.Operation { +pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.cholesky", .{ - .operands = &.{value}, + .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .{ "lower", .i1FromBool(ctx, lower) }, + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower))), }, .location = location, }); } -pub fn clamp(ctx: mlir.Context, min: mlir.Value, value: mlir.Value, max: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Value, max: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.clamp", .{ - .operands = &.{ min, value, max }, + .operands = .{ .flat = &.{ min, value, max } }, .result_type_inference = true, .location = location, }); @@ -119,18 +113,17 @@ pub const DotPrecision = union(enum) { highest, algorithm: DotAlgorithm, - pub fn precisionAttr(self: DotPrecision, ctx: mlir.Context) mlir.Attribute { - const precision = PrecisionAttribute.init(ctx, switch (self) { + pub fn precision(self: DotPrecision) PrecisionAttribute.Precision { + return switch (self) { .fast => .DEFAULT, .high => .HIGH, .highest => .HIGHEST, // When we specify the dot algorithm, we should not specify the precision. .algorithm => .DEFAULT, - }); - return precision.asAttr(); + }; } - pub fn algorithmAttr(self: DotPrecision, ctx: mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { + pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { return switch (self) { .algorithm => |algo| algo.asAttr(ctx, operand_type), else => null, @@ -149,159 +142,172 @@ pub const DotAlgorithm = struct { // bf16_6x: each input is decomposed to 3 bf16 components, then 6 dot operations are done on those components, and the result is accumulated in f32. // not sure where this is available. pub const bf16_6x: DotAlgorithm = .{ - .operand = .bf16, .accumulation = .f32, .component_count = 1, .num_primitive_operations = 6, .allow_imprecise_accumulation = false, }; - - pub fn asAttr(self: DotAlgorithm, ctx: mlir.Context, tensor_type: mlir.RankedTensorType) mlir.Attribute { - const elem_type = tensor_type.getElementType(); - - return mlir.Attribute.wrap(c.stablehloDotAlgorithmGet( - ctx._inner, - elem_type._inner, - elem_type._inner, - self.accumulation.asType(ctx)._inner, - self.component_count, - self.component_count, - self.num_primitive_operations, - self.allow_imprecise_accumulation, - )); - } }; +pub fn dotAlgorithmAttribute(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { + return @ptrCast(c.stablehloDotAlgorithmGet( + ctx.ptr(), + element_type.ptr(), + element_type.ptr(), + mlir.floatType(ctx, dot_algorithm.accumulation).ptr(), + dot_algorithm.component_count, + dot_algorithm.component_count, + dot_algorithm.num_primitive_operations, + dot_algorithm.allow_imprecise_accumulation, + ).ptr); +} + /// General matrix multiplication "a la Einstein sum" /// Note: stablehlo doesn't do type inference for dot_general pub fn dot_general( - ctx: mlir.Context, - lhs: mlir.Value, - rhs: mlir.Value, - result_type: mlir.Type, - location: mlir.Location, + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, opts: struct { lhs_batching_dimensions: []const i64, rhs_batching_dimensions: []const i64, lhs_contracting_dimensions: []const i64, rhs_contracting_dimensions: []const i64, - precision: DotPrecision, + dot_precision: DotPrecision, }, -) mlir.Operation { - const precisions: [2]mlir.Attribute = @splat(opts.precision.precisionAttr(ctx)); - const attributes = [3]mlir.AttrTuple{ - .{ - "dot_dimension_numbers", DotDimensionNumbersAttribute.init(ctx, .{ + location: *const mlir.Location, +) *mlir.Operation { + const precisions: [2]*const mlir.Attribute = @splat(precisionAttribute(ctx, opts.dot_precision.precision())); + const attributes = [3]mlir.NamedAttribute{ + .named( + ctx, + "dot_dimension_numbers", + dotDimensionNumbersAttribute(ctx, .{ .lhs_batching_dimensions = opts.lhs_batching_dimensions, .rhs_batching_dimensions = opts.rhs_batching_dimensions, .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, - }).asAttr(), - }, - .{ "precision_config", .array(ctx, &precisions) }, + }), + ), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &precisions)), // keep algorithm as the last attribute so we can omit it when it's not set. - .{ "algorithm", opts.precision.algorithmAttr(ctx, lhs.getType().as(mlir.RankedTensorType).?) orelse undefined }, + .named(ctx, "algorithm", switch (opts.dot_precision) { + .algorithm => |v| dotAlgorithmAttribute(ctx, v, lhs.type_()), + else => undefined, + }), }; - const n_attributes = if (opts.precision == .algorithm) attributes.len else attributes.len - 1; + const n_attributes = if (opts.dot_precision == .algorithm) attributes.len else attributes.len - 1; return mlir.Operation.make(ctx, "stablehlo.dot_general", .{ - .operands = &.{ lhs, rhs }, - .results = &.{result_type}, + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, .attributes = attributes[0..n_attributes], .location = location, }); } pub fn constant( - ctx: mlir.Context, + ctx: *mlir.Context, dims: []const i64, - elem_type: mlir.DenseElementsAttributeTypes, + elem_type: *const mlir.Type, raw_bytes: []const u8, - location: mlir.Location, -) mlir.Operation { + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.constant", .{ - .operands = &.{}, - .results = &.{.tensor(dims, elem_type.mlirType(ctx))}, - .attributes = &.{.{ "value", .denseElementsFromBytes(ctx, dims, elem_type, raw_bytes) }}, + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, + .attributes = &.{ + .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.get(dims, elem_type).shaped(), raw_bytes)), + }, .location = location, }); } -pub fn convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn broadcast_in_dim(ctx: mlir.Context, operand: mlir.Value, dims: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn broadcast_in_dim( + ctx: *mlir.Context, + operand: *const mlir.Value, + dims: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ - .operands = &.{operand}, - .results = &.{result_type}, + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "broadcast_dimensions", .dense(ctx, .i64, dims) }, + .named(ctx, "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims)), }, .location = location, }); } -pub fn transpose(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location, opts: struct { permutation: []const i64 }) mlir.Operation { +pub fn transpose( + ctx: *mlir.Context, + value: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { permutation: []const i64 }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.transpose", .{ - .operands = &.{value}, - .results = &.{result_type}, + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "permutation", .dense(ctx, .i64, opts.permutation) }, + .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation)), }, .location = location, }); } -pub fn slice(ctx: mlir.Context, operand: mlir.Value, start_indices: []const i64, limit_indices: []const i64, strides: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + start_indices: []const i64, + limit_indices: []const i64, + strides: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.slice", .{ - .operands = &.{operand}, - .results = &.{result_type}, + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "start_indices", .dense(ctx, .i64, start_indices) }, - .{ "limit_indices", .dense(ctx, .i64, limit_indices) }, - .{ "strides", .dense(ctx, .i64, strides) }, + .named(ctx, "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices)), + .named(ctx, "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices)), + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), }, .location = location, }); } -pub fn concatenate(ctx: mlir.Context, inputs: []const mlir.Value, dimension: i64, location: mlir.Location) mlir.Operation { +pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimension: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ - .operands = inputs, + .operands = .{ .flat = inputs }, .result_type_inference = true, .attributes = &.{ - .{ "dimension", .int(ctx, .i64, dimension) }, + .named(ctx, "dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); } -pub fn reshape(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.reshape", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn select(ctx: mlir.Context, condition: mlir.Value, then: mlir.Value, else_: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn select( + ctx: *mlir.Context, + condition: *const mlir.Value, + then: *const mlir.Value, + else_: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.select", .{ - .operands = &.{ condition, then, else_ }, - .results = &.{then.getType()}, + .operands = .{ .flat = &.{ condition, then, else_ } }, + .results = .{ .flat = &.{then.type_()} }, .location = location, }); } pub fn gather( - ctx: mlir.Context, - value: mlir.Value, - indices: mlir.Value, + ctx: *mlir.Context, + value: *const mlir.Value, + indices: *const mlir.Value, slice_sizes: []const i64, - location: mlir.Location, args: struct { offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -311,15 +317,16 @@ pub fn gather( index_vector_dim: i64, indices_are_sorted: bool = false, }, -) mlir.Operation { + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make( ctx, "stablehlo.gather", .{ - .operands = &.{ value, indices }, + .operands = .{ .flat = &.{ value, indices } }, .result_type_inference = true, .attributes = &.{ - .{ "dimension_numbers", GatherDimensionNumbersAttribute.init( + .named(ctx, "dimension_numbers", gatherDimensionNumbersAttribute( ctx, args.offset_dims, args.collapsed_slice_dims, @@ -327,21 +334,15 @@ pub fn gather( args.start_indices_batching_dims, args.start_index_map, args.index_vector_dim, - ).asAttr() }, - .{ "slice_sizes", .dense(ctx, .i64, slice_sizes) }, - .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, + )), + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, slice_sizes)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), }, .location = location, }, ); } -fn elementTypeOrSelf(typ: mlir.Type) mlir.Type { - return if (typ.as(mlir.ShapedType)) |shaped| { - return shaped.elementType(); - } else typ; -} - pub const ScatterArgs = struct { update_window_dims: []const i64, inserted_window_dims: []const i64, @@ -352,9 +353,9 @@ pub const ScatterArgs = struct { indices_are_sorted: bool = false, unique_indices: bool = false, - pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: mlir.Context) mlir.Attribute { - return .{ ._inner = c.stablehloScatterDimensionNumbersGet( - ctx._inner, + pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { + return @ptrCast(c.stablehloScatterDimensionNumbersGet( + ctx.ptr(), @intCast(self.update_window_dims.len), self.update_window_dims.ptr, @intCast(self.inserted_window_dims.len), @@ -366,29 +367,29 @@ pub const ScatterArgs = struct { @intCast(self.scatter_dims_to_operand_dims.len), self.scatter_dims_to_operand_dims.ptr, self.index_vector_dim, - ) }; + ).ptr); } }; pub fn scatter( - ctx: mlir.Context, - inputs: []const mlir.Value, - scatter_indices: []const mlir.Value, - updates: []const mlir.Value, - update_block: mlir.Block, + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + scatter_indices: []const *const mlir.Value, + updates: []const *const mlir.Value, + update_block: *mlir.Block, args: ScatterArgs, - location: mlir.Location, -) mlir.Operation { + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make( ctx, "stablehlo.scatter", .{ - .variadic_operands = &.{ inputs, scatter_indices, updates }, + .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, .blocks = &.{update_block}, .attributes = &.{ - .{ "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx) }, - .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, - .{ "unique_indices", .boolean(ctx, args.unique_indices) }, + .named(ctx, "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), + .named(ctx, "unique_indices", mlir.boolAttribute(ctx, args.unique_indices)), }, .result_type_inference = true, .location = location, @@ -396,163 +397,78 @@ pub fn scatter( ); } -pub fn iota(ctx: mlir.Context, dimension: i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.iota", .{ - .operands = &.{}, - .results = &.{result_type}, + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "iota_dimension", .int(ctx, .i64, dimension) }, + .named(ctx, "iota_dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); } -pub fn reverse(ctx: mlir.Context, operand: mlir.Value, dimensions: []const i64, location: mlir.Location) mlir.Operation { - const result_type = operand.getType(); +pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []const i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.reverse", .{ - .operands = &.{operand}, - .results = &.{result_type}, + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{operand.type_()} }, .attributes = &.{ - .{ "dimensions", .dense(ctx, .i64, dimensions) }, + .named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions)), }, .location = location, }); } -pub fn compare(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, comparison_direction: ComparisonDirection, compare_type: CompareType, location: mlir.Location) mlir.Operation { +pub fn compare( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + comparison_direction: ComparisonDirection.Direction, + compare_type: CompareType.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.compare", .{ - .operands = &.{ lhs, rhs }, + .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .{ "comparison_direction", comparison_direction.asAttr() }, - .{ "compare_type", compare_type.asAttr() }, + .named(ctx, "comparison_direction", comparisonDirection(ctx, comparison_direction)), + .named(ctx, "compare_type", compareType(ctx, compare_type)), }, .location = location, }); } -pub fn reduce( - ctx: mlir.Context, - inputs: []const mlir.Value, - init_values: []const mlir.Value, - dimensions: []const i64, - blkctx: anytype, - blkfn: fn (anytype, mlir.Context, []const mlir.Value, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; - - const block_n_args = inputs.len + init_values.len; - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0..block_n_args]; - var reduce_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type: mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); - reduce_elem_types[i] = arg_type; - reduce_elem_types[inputs.len + i] = arg_type; - } - var block = mlir.Block.open(reduce_elem_types[0..block_n_args], locations) catch unreachable; - { - defer block.close(); - - var block_inputs: [MaxBlockArguments / 2]mlir.Value = undefined; - var block_accs: [MaxBlockArguments / 2]mlir.Value = undefined; - for (0..inputs.len) |i| { - block_inputs[i] = block.argument(i); - block_accs[i] = block.argument(inputs.len + i); - } - _ = blkfn(blkctx, ctx, block_inputs[0..inputs.len], block_accs[0..init_values.len]); - } - - return mlir.Operation.make(ctx, "stablehlo.reduce", .{ - .variadic_operands = &.{ inputs, init_values }, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimensions", .dense(ctx, .i64, dimensions) }, - }, - .location = location, - }); -} - -pub fn sort( - ctx: mlir.Context, - inputs: []const mlir.Value, - dimension: i64, - is_stable: bool, - blkctx: anytype, - compfn: fn (anytype, mlir.Context, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; - - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0 .. inputs.len * 2]; - var sort_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type: mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); - sort_elem_types[i * 2] = arg_type; - sort_elem_types[i * 2 + 1] = arg_type; - } - var block = mlir.Block.init(sort_elem_types[0 .. inputs.len * 2], locations) catch unreachable; - - var block_inputs: [MaxBlockArguments]mlir.Value = undefined; - for (0..inputs.len * 2) |i| { - block_inputs[i] = block.argument(i); - } - _ = compfn(blkctx, ctx, block_inputs[0 .. inputs.len * 2]); - - return mlir.Operation.make(ctx, "stablehlo.sort", .{ - .variadic_operands = &.{inputs}, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimension", .int(ctx, .i64, dimension) }, - .{ "is_stable", .boolean(ctx, is_stable) }, - }, - .location = location, - }); -} - -pub fn dynamic_slice(ctx: mlir.Context, operand: mlir.Value, new_dims: []const i64, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { +pub fn dynamic_slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + new_dims: []const i64, + start_indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.dynamic_slice", .{ - .variadic_operands = &.{ &.{operand}, start_indices }, + .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, .result_type_inference = true, .attributes = &.{ - .{ "slice_sizes", .dense(ctx, .i64, new_dims) }, + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, new_dims)), }, .location = location, }); } -pub fn round_nearest_afz(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_afz", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - -pub fn round_nearest_even(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_even", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - pub const PadOpts = struct { low: []const i64, high: []const i64, interior: []const i64, }; -pub fn pad(ctx: mlir.Context, value: mlir.Value, padding_value: mlir.Value, opts: PadOpts, location: mlir.Location) mlir.Operation { +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, location: *const mlir.Location, opts: PadOpts) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.pad", .{ - .operands = &.{ value, padding_value }, + .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, .attributes = &.{ - .{ "edge_padding_low", .dense(ctx, .i64, opts.low) }, - .{ "edge_padding_high", .dense(ctx, .i64, opts.high) }, - .{ "interior_padding", .dense(ctx, .i64, opts.interior) }, + .named(ctx, "edge_padding_low", mlir.denseArrayAttribute(ctx, .i64, opts.low)), + .named(ctx, "edge_padding_high", mlir.denseArrayAttribute(ctx, .i64, opts.high)), + .named(ctx, "interior_padding", mlir.denseArrayAttribute(ctx, .i64, opts.interior)), }, .location = location, }); @@ -562,18 +478,18 @@ pub const TriangularSolveOpts = struct { left_side: bool, lower: bool, unit_diagonal: bool, - transpose_a: Transpose.Type, + transpose_a: TransposeAttribute.Type, }; -pub fn triangular_solve(ctx: mlir.Context, value: mlir.Value, other: mlir.Value, location: mlir.Location, opts: TriangularSolveOpts) mlir.Operation { +pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, opts: TriangularSolveOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ - .operands = &.{ value, other }, + .operands = .{ .flat = &.{ value, other } }, .result_type_inference = true, .attributes = &.{ - .{ "left_side", .i1FromBool(ctx, opts.left_side) }, - .{ "lower", .i1FromBool(ctx, opts.lower) }, - .{ "unit_diagonal", .i1FromBool(ctx, opts.unit_diagonal) }, - .{ "transpose_a", Transpose.init(ctx, opts.transpose_a).asAttr() }, + .named(ctx, "left_side", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.left_side))), + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.lower))), + .named(ctx, "unit_diagonal", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.unit_diagonal))), + .named(ctx, "transpose_a", transposeAttribute(ctx, opts.transpose_a)), }, .location = location, }); @@ -584,74 +500,132 @@ pub const FftOpts = struct { length: []const i64, }; -pub fn fft(ctx: mlir.Context, value: mlir.Value, location: mlir.Location, opts: FftOpts) mlir.Operation { +pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, opts: FftOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.fft", .{ - .operands = &.{value}, + .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .{ "fft_type", FftType.init(ctx, opts.kind).asAttr() }, - .{ "fft_length", .dense(ctx, .i64, opts.length) }, + .named(ctx, "fft_type", fftType(ctx, opts.kind)), + .named(ctx, "fft_length", mlir.denseArrayAttribute(ctx, .i64, opts.length)), }, .location = location, }); } -pub fn rng(ctx: mlir.Context, a: mlir.Value, b: mlir.Value, shape: mlir.Value, rng_distribution: RngDistribution.Type, location: mlir.Location) mlir.Operation { +pub const RngDistribution = opaque { + const M = mlir.Methods(RngDistribution, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + UNIFORM, + NORMAL, + }; + + pub fn get(ctx: *mlir.Context, value_: Type) *const RngDistribution { + return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + } + + pub fn value(self: *const RngDistribution) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngDistributionAttrGetValue(self.ptr())), + ) orelse unreachable; + } +}; + +pub fn rngDistribution(ctx: *mlir.Context, value: RngDistribution.Type) *const mlir.Attribute { + return @ptrCast(RngDistribution.get(ctx, value)); +} + +pub const RngAlgorithm = opaque { + const M = mlir.Methods(RngAlgorithm, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + DEFAULT, + THREE_FRY, + PHILOX, + }; + + pub fn get(ctx: *mlir.Context, value_: Type) *const RngAlgorithm { + return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + } + + pub fn value(self: *const RngAlgorithm) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngAlgorithmAttrGetValue(self.ptr())), + ) orelse unreachable; + } +}; + +pub fn rngAlgorithm(ctx: *mlir.Context, value: RngAlgorithm.Type) *const mlir.Attribute { + return @ptrCast(RngAlgorithm.get(ctx, value)); +} + +pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng", .{ - .operands = &.{ a, b, shape }, + .operands = .{ .flat = &.{ a, b, shape } }, .result_type_inference = true, .attributes = &.{ - .{ "rng_distribution", RngDistribution.init(ctx, rng_distribution).asAttr() }, + .named(ctx, "rng_distribution", rngDistribution(ctx, rng_distribution)), }, .location = location, }); } -pub fn rng_bit_generator(ctx: mlir.Context, rng_algorithm: RngAlgorithm.Type, initial_state: mlir.Value, res_state_type: mlir.Type, res_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn rng_bit_generator( + ctx: *mlir.Context, + rng_algorithm: RngAlgorithm.Type, + initial_state: *const mlir.Value, + res_state_type: *const mlir.Type, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ - .operands = &.{initial_state}, + .operands = .{ .flat = &.{initial_state} }, .results = &.{ res_state_type, res_type }, .attributes = &.{ - .{ "rng_algorithm", RngAlgorithm.init(ctx, rng_algorithm).asAttr() }, + .named(ctx, "rng_algorithm", rngAlgorithm(ctx, rng_algorithm)), }, .location = location, }); } -pub fn reduce_precision(ctx: mlir.Context, value: mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: mlir.Location) mlir.Operation { +pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ - .operands = &.{value}, + .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .{ "exponent_bits", .int(ctx, .i32, exponent_bits) }, - .{ "mantissa_bits", .int(ctx, .i32, mantissa_bits) }, + .named(ctx, "exponent_bits", mlir.integerAttribute(ctx, .i32, exponent_bits)), + .named(ctx, "mantissa_bits", mlir.integerAttribute(ctx, .i32, mantissa_bits)), }, .location = location, }); } -pub fn dynamic_update_slice(ctx: mlir.Context, operand: mlir.Value, update: mlir.Value, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { +pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, update: *const mlir.Value, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ - .variadic_operands = &.{ &.{operand}, &.{update}, start_indices }, + .operands = .{ .variadic = &.{ &.{operand}, &.{update}, start_indices } }, .result_type_inference = true, .location = location, }); } -pub fn tuple(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.tuple", .{ - .operands = values, - .result_type_inference = true, - .location = location, - }); -} - -pub fn get_tuple_element(ctx: mlir.Context, tuple_value: mlir.Value, index: i64, location: mlir.Location) mlir.Operation { +pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ - .operands = &.{tuple_value}, + .operands = .{ .flat = &.{tuple_value} }, .result_type_inference = true, .attributes = &.{ - .{ "index", .int(ctx, .i32, index) }, + .named(ctx, "index", mlir.integerAttribute(ctx, .i32, index)), }, .location = location, }); @@ -679,46 +653,44 @@ pub const ConvolutionOpts = struct { }; pub fn convolution( - ctx: mlir.Context, - lhs: mlir.Value, - rhs: mlir.Value, + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, opts: ConvolutionOpts, - res_type: mlir.Type, - location: mlir.Location, -) mlir.Operation { - var max_precisions: [2]mlir.Attribute = undefined; + location: *const mlir.Location, +) *mlir.Operation { + var max_precisions: [2]*const mlir.Attribute = undefined; for (opts.precision_config, 0..) |p, i| { - max_precisions[i] = PrecisionAttribute.init(ctx, p).asAttr(); + max_precisions[i] = precisionAttribute(ctx, p); } var window_reversal: [3]i32 = undefined; for (opts.window_reversal, 0..) |w, i| { window_reversal[i] = @intCast(@intFromBool(w)); } return mlir.Operation.make(ctx, "stablehlo.convolution", .{ - .operands = &.{ lhs, rhs }, - .results = &.{res_type}, + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, .attributes = &.{ - .{ "window_strides", .dense(ctx, .i64, opts.window_strides) }, - .{ "padding", .denseElements(ctx, opts.pad_shape, .i64, opts.pad_value) }, - .{ "lhs_dilation", .dense(ctx, .i64, opts.lhs_dilation) }, - .{ "rhs_dilation", .dense(ctx, .i64, opts.rhs_dilation) }, - .{ "window_reversal", .dense(ctx, .bool, window_reversal[0..opts.window_reversal.len]) }, - .{ - "dimension_numbers", ConvDimensionNumbersAttribute.init(ctx, .{ - .input_batch_dimension = opts.input_batch_dimension, - .input_feature_dimension = opts.input_feature_dimension, - .input_spatial_dimensions = opts.input_spatial_dimensions, - .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, - .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, - .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, - .output_batch_dimension = opts.output_batch_dimension, - .output_feature_dimension = opts.output_feature_dimension, - .output_spatial_dimensions = opts.output_spatial_dimensions, - }).asAttr(), - }, - .{ "feature_group_count", .int(ctx, .i64, opts.feature_group_count) }, - .{ "batch_group_count", .int(ctx, .i64, opts.batch_group_count) }, - .{ "precision_config", .array(ctx, &max_precisions) }, + .named(ctx, "window_strides", mlir.denseArrayAttribute(ctx, .i64, opts.window_strides)), + .named(ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(opts.pad_shape, mlir.integerType(ctx, .i64)).shaped(), opts.pad_value)), + .named(ctx, "lhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.lhs_dilation)), + .named(ctx, "rhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.rhs_dilation)), + .named(ctx, "window_reversal", mlir.denseArrayAttribute(ctx, .bool, window_reversal[0..opts.window_reversal.len])), + .named(ctx, "dimension_numbers", convDimensionNumbersAttribute(ctx, .{ + .input_batch_dimension = opts.input_batch_dimension, + .input_feature_dimension = opts.input_feature_dimension, + .input_spatial_dimensions = opts.input_spatial_dimensions, + .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, + .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, + .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, + .output_batch_dimension = opts.output_batch_dimension, + .output_feature_dimension = opts.output_feature_dimension, + .output_spatial_dimensions = opts.output_spatial_dimensions, + })), + .named(ctx, "feature_group_count", mlir.integerAttribute(ctx, .i64, opts.feature_group_count)), + .named(ctx, "batch_group_count", mlir.integerAttribute(ctx, .i64, opts.batch_group_count)), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &max_precisions)), }, .location = location, }); @@ -732,261 +704,350 @@ pub const CustomCallOpts = struct { typed_ffi = 4, }; - call_target_name: [:0]const u8, - has_side_effect: bool, - backend_config: ?mlir.Attribute, + pub const BackendConfig = union(ApiVersion) { + original: []const u8, + status_returning: []const u8, + status_returning_unified: []const u8, + typed_ffi: *const mlir.DictionaryAttribute, + }; + + call_target_name: []const u8, + has_side_effect: ?bool = null, + api_version: ?ApiVersion = null, + backend_config: ?BackendConfig = null, operand_layouts: ?[]const []const usize = null, result_layouts: ?[]const []const usize = null, - output_operand_aliases: []const i64 = &.{}, - additional_attributes: []const mlir.AttrTuple = &.{}, - api_version: ApiVersion, + output_operand_aliases: ?[]const i64 = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, }; -pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { +pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, opts: CustomCallOpts, location: *const mlir.Location) *mlir.Operation { const MAX_OPERANDS = 64; const MAX_RESULTS = 16; - - const backend_config = opts.backend_config orelse mlir.Attribute.string(ctx, ""); - if (@intFromEnum(opts.api_version) < @intFromEnum(CustomCallOpts.ApiVersion.typed_ffi)) { - stdx.debug.assert( - backend_config.isA(mlir.StringAttribute), - "API version < 4 requires a string as backend_config, got {f}", - .{backend_config}, - ); - } else { - stdx.debug.assert( - backend_config.isA(mlir.DictionaryAttribute), - "API version >= 4 requires a dictionary as backend_config, got {f}", - .{backend_config}, - ); - } - - var attrs: stdx.BoundedArray(mlir.AttrTuple, 32) = .{}; - attrs.appendSliceAssumeCapacity(&[_]mlir.AttrTuple{ - .{ "api_version", .int(ctx, .i32, @intFromEnum(opts.api_version)) }, - .{ "call_target_name", .string(ctx, opts.call_target_name) }, - .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, - .{ "backend_config", backend_config }, - }); - - { - var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - for (opts.output_operand_aliases) |alias| { - output_operand_aliases.appendAssumeCapacity( - OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), - ); - } - attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); - } - - const MINOR_TO_MAJOR = blk: { - const MAX_RANK = 8; - var ret: [MAX_RANK]usize = undefined; - for (0..MAX_RANK) |i| { - ret[i] = @intCast(MAX_RANK - i - 1); + const MINOR_TO_MAJOR = comptime blk: { + var ret: [mlir.ShapedType.MAX_RANK]usize = undefined; + for (0..ret.len) |i| { + ret[i] = @intCast(mlir.ShapedType.MAX_RANK - i - 1); } break :blk ret; }; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .{}; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "call_target_name", mlir.stringAttribute(ctx, opts.call_target_name)), + }); + attrs.appendSliceAssumeCapacity(opts.additional_attributes); + + if (opts.has_side_effect) |has_side_effects| { + attrs.appendAssumeCapacity( + .named(ctx, "has_side_effect", mlir.boolAttribute(ctx, has_side_effects)), + ); + } + + const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; + if (opts.api_version orelse backend_config_api_version) |v| { + attrs.appendAssumeCapacity( + .named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v))), + ); + } + if (opts.backend_config) |backend_config| { + attrs.appendAssumeCapacity( + .named(ctx, "backend_config", switch (backend_config) { + .typed_ffi => |v| @ptrCast(v), + inline else => |v| mlir.stringAttribute(ctx, v), + }), + ); + } + + if (opts.output_operand_aliases) |output_operand_aliases| { + var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; + for (output_operand_aliases) |alias| { + buffer.appendAssumeCapacity( + outputOperandAliasAttribute(ctx, .{ .operand_index = alias }), + ); + } + attrs.appendAssumeCapacity( + .named(ctx, "output_operand_aliases", mlir.arrayAttribute(ctx, buffer.constSlice())), + ); + } + + var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .{}; if (opts.operand_layouts) |layouts| { - var operand_layouts: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; for (layouts) |ol| { - operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + ol, + ), + ); } - attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); } else { - const operand_layouts = blk: { - var ret: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; - for (inputs) |input| { - const ranked_type = input.getType().as(mlir.RankedTensorType).?; - const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; - ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); - } - break :blk ret; - }; - attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); + for (inputs) |input| { + const shaped_type = input.type_().isA(mlir.ShapedType).?; + const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + ol, + ), + ); + } } + attrs.appendAssumeCapacity( + .named(ctx, "operand_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + ); + layouts_buffer.clear(); if (opts.result_layouts) |layouts| { - var result_layouts: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; for (layouts) |rl| { - result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + rl, + ), + ); } - attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); } else { - const result_layouts = blk: { - var ret: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - for (res_types) |t| { - const ranked_t = t.as(mlir.RankedTensorType).?; - const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; - ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); - } - break :blk ret; - }; - attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); + for (result_types) |result_type| { + const shaped_type = result_type.isA(mlir.ShapedType).?; + const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + rl, + ), + ); + } } - - attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); + attrs.appendAssumeCapacity( + .named(ctx, "result_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + ); return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ - .operands = inputs, - .results = res_types, + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, .attributes = attrs.constSlice(), .location = location, }); } -pub const DotDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, +pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{value}, .{ + .call_target_name = "CreateBuffer", + .api_version = .typed_ffi, + }, location); +} - pub const is_a_fn = c.stablehloAttributeIsADotDimensionNumbers; - const Self = DotDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ + .call_target_name = "Pin", + .api_version = .typed_ffi, + }, location); +} - pub fn init(ctx: mlir.Context, args: struct { +pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ + .call_target_name = "Unpin", + .api_version = .typed_ffi, + }, location); +} + +pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ + .results = .{ .flat = &.{mlir.rankedTensorType(&.{}, mlir.integerType(ctx, .u32))} }, + .location = location, + }); +} + +pub const DotDimensionNumbersAttribute = opaque { + const M = mlir.Methods(DotDimensionNumbersAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { lhs_batching_dimensions: []const i64, rhs_batching_dimensions: []const i64, lhs_contracting_dimensions: []const i64, rhs_contracting_dimensions: []const i64, - }) Self { - return .{ - ._inner = c.stablehloDotDimensionNumbersGet( - ctx._inner, - @intCast(args.lhs_batching_dimensions.len), - args.lhs_batching_dimensions.ptr, - @intCast(args.rhs_batching_dimensions.len), - args.rhs_batching_dimensions.ptr, - @intCast(args.lhs_contracting_dimensions.len), - args.lhs_contracting_dimensions.ptr, - @intCast(args.rhs_contracting_dimensions.len), - args.rhs_contracting_dimensions.ptr, - ), - }; + }; + + pub fn init(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttribute { + return @ptrCast(c.stablehloDotDimensionNumbersGet( + ctx.ptr(), + @intCast(args.lhs_batching_dimensions.len), + args.lhs_batching_dimensions.ptr, + @intCast(args.rhs_batching_dimensions.len), + args.rhs_batching_dimensions.ptr, + @intCast(args.lhs_contracting_dimensions.len), + args.lhs_contracting_dimensions.ptr, + @intCast(args.rhs_contracting_dimensions.len), + args.rhs_contracting_dimensions.ptr, + ).ptr); } - pub fn getLhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self._inner)); + pub fn getLhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self.ptr())); } - pub fn getLhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self._inner, @intCast(pos)); + pub fn getLhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self._inner)); + pub fn getRhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self.ptr())); } - pub fn getRhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self._inner, @intCast(pos)); + pub fn getRhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getLhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self._inner)); + pub fn getLhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self.ptr())); } - pub fn getLhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self._inner, @intCast(pos)); + pub fn getLhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self._inner)); + pub fn getRhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self.ptr())); } - pub fn getRhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self._inner, @intCast(pos)); + pub fn getRhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } }; -pub const GatherDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, +pub fn dotDimensionNumbersAttribute(ctx: *mlir.Context, args: DotDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(DotDimensionNumbersAttribute.init(ctx, args)); +} - pub const is_a_fn = c.stablehloAttributeIsAGatherDimensionNumbers; - const Self = GatherDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const GatherDimensionNumbersAttribute = opaque { + const M = mlir.Methods(GatherDimensionNumbersAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.eql(c.mlirAttributePrint); pub fn init( - ctx: mlir.Context, + ctx: *mlir.Context, offset_dims: []const i64, collapsed_slice_dims: []const i64, operand_batching_dims: []const i64, start_indices_batching_dims: []const i64, start_index_map: []const i64, index_vector_dim: i64, - ) Self { - return .{ - ._inner = c.stablehloGatherDimensionNumbersGet( - ctx._inner, - @intCast(offset_dims.len), - offset_dims.ptr, - @intCast(collapsed_slice_dims.len), - collapsed_slice_dims.ptr, - @intCast(operand_batching_dims.len), - operand_batching_dims.ptr, - @intCast(start_indices_batching_dims.len), - start_indices_batching_dims.ptr, - @intCast(start_index_map.len), - start_index_map.ptr, - index_vector_dim, - ), - }; + ) *const GatherDimensionNumbersAttribute { + return @ptrCast(c.stablehloGatherDimensionNumbersGet( + ctx.ptr(), + @intCast(offset_dims.len), + offset_dims.ptr, + @intCast(collapsed_slice_dims.len), + collapsed_slice_dims.ptr, + @intCast(operand_batching_dims.len), + operand_batching_dims.ptr, + @intCast(start_indices_batching_dims.len), + start_indices_batching_dims.ptr, + @intCast(start_index_map.len), + start_index_map.ptr, + index_vector_dim, + ).ptr); } - pub fn getOffsetDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self._inner)); + pub fn getOffsetDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self.ptr())); } - pub fn getOffsetDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self._inner, @intCast(pos)); + pub fn getOffsetDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self.ptr(), @intCast(pos)); } - pub fn getCollapsedSliceDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self._inner)); + pub fn getCollapsedSliceDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self.ptr())); } - pub fn getCollapsedSliceDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self._inner, @intCast(pos)); + pub fn getCollapsedSliceDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self._inner)); + pub fn getStartIndexMapSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self.ptr())); } - pub fn getOperandBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self._inner)); + pub fn getOperandBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self.ptr())); } - pub fn getOperandBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self._inner, @intCast(pos)); + pub fn getOperandBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndicesBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self._inner)); + pub fn getStartIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self.ptr())); } - pub fn getStartIndicesBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self._inner, @intCast(pos)); + pub fn getStartIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self._inner, @intCast(pos)); + pub fn getStartIndexMapElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self.ptr(), @intCast(pos)); } - pub fn getIndexVectorDim(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self._inner)); + pub fn getIndexVectorDim(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self.ptr())); } }; -pub const ConvDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, +pub fn gatherDimensionNumbersAttribute( + ctx: *mlir.Context, + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, +) *const mlir.Attribute { + return @ptrCast(GatherDimensionNumbersAttribute.init( + ctx, + offset_dims, + collapsed_slice_dims, + operand_batching_dims, + start_indices_batching_dims, + start_index_map, + index_vector_dim, + )); +} - pub const is_a_fn = c.stablehloAttributeIsAConvDimensionNumbers; - const Self = ConvDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const ConvDimensionNumbersAttribute = opaque { + const M = mlir.Methods(ConvDimensionNumbersAttribute, mlir.Attribute); - pub fn init(ctx: mlir.Context, args: struct { + pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { input_batch_dimension: i64, input_feature_dimension: i64, input_spatial_dimensions: []const i64, @@ -996,106 +1057,116 @@ pub const ConvDimensionNumbersAttribute = struct { output_batch_dimension: i64, output_feature_dimension: i64, output_spatial_dimensions: []const i64, - }) Self { - return .{ - ._inner = c.stablehloConvDimensionNumbersGet( - ctx._inner, - args.input_batch_dimension, - args.input_feature_dimension, - @intCast(args.input_spatial_dimensions.len), - args.input_spatial_dimensions.ptr, - args.kernel_input_feature_dimension, - args.kernel_output_feature_dimension, - @intCast(args.kernel_spatial_dimensions.len), - args.kernel_spatial_dimensions.ptr, - args.output_batch_dimension, - args.output_feature_dimension, - @intCast(args.output_spatial_dimensions.len), - args.output_spatial_dimensions.ptr, - ), - }; + }; + + pub fn init(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttribute { + return @ptrCast(c.stablehloConvDimensionNumbersGet( + ctx.ptr(), + args.input_batch_dimension, + args.input_feature_dimension, + @intCast(args.input_spatial_dimensions.len), + args.input_spatial_dimensions.ptr, + args.kernel_input_feature_dimension, + args.kernel_output_feature_dimension, + @intCast(args.kernel_spatial_dimensions.len), + args.kernel_spatial_dimensions.ptr, + args.output_batch_dimension, + args.output_feature_dimension, + @intCast(args.output_spatial_dimensions.len), + args.output_spatial_dimensions.ptr, + ).ptr); } - pub fn getInputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputBatchDimension(self._inner); + pub fn getInputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetInputBatchDimension(self.ptr()); } - pub fn getInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self._inner); + pub fn getInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self.ptr()); } - pub fn getInputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self._inner)); + pub fn getInputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self.ptr())); } - pub fn getInputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self._inner, @intCast(pos)); + pub fn getInputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getKernelInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self._inner); + pub fn getKernelInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self.ptr()); } - pub fn getKernelOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self._inner); + pub fn getKernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self.ptr()); } - pub fn getKernelSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self._inner)); + pub fn getKernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self.ptr())); } - pub fn getKernelSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self._inner, @intCast(pos)); + pub fn getKernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getOutputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self._inner); + pub fn getOutputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self.ptr()); } - pub fn getOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self._inner); + pub fn getOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self.ptr()); } - pub fn getOutputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self._inner)); + pub fn getOutputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self.ptr())); } - pub fn getOutputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self._inner, @intCast(pos)); + pub fn getOutputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } }; -pub const OutputOperandAliasAttribute = struct { - _inner: c.MlirAttribute, +pub fn convDimensionNumbersAttribute(ctx: *mlir.Context, args: ConvDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(ConvDimensionNumbersAttribute.init(ctx, args)); +} - pub const is_a_fn = c.stablehloAttributeIsAOutputOperandAlias; - pub const asAttr = mlir.Attribute.fromAny(OutputOperandAliasAttribute); - pub const eql = mlir.Attribute.eqlAny(OutputOperandAliasAttribute); +pub const OutputOperandAliasAttribute = opaque { + const M = mlir.Methods(OutputOperandAliasAttribute, c.MlirAttribute); - pub fn init( - ctx: mlir.Context, - output_tuple_indices: []const i64, + pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { + output_tuple_indices: []const i64 = &.{}, operand_index: i64, - operand_tuple_indices: []const i64, - ) OutputOperandAliasAttribute { - return .{ ._inner = c.stablehloOutputOperandAliasGet( - ctx._inner, - @intCast(output_tuple_indices.len), - output_tuple_indices.ptr, - @intCast(operand_index), - @intCast(operand_tuple_indices.len), - operand_tuple_indices.ptr, - ) }; + operand_tuple_indices: []const i64 = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttribute { + return @ptrCast(c.stablehloOutputOperandAliasGet( + ctx.ptr(), + @intCast(args.output_tuple_indices.len), + @ptrCast(args.output_tuple_indices), + @intCast(args.operand_index), + @intCast(args.operand_tuple_indices.len), + @ptrCast(args.operand_tuple_indices), + ).ptr); } }; -pub const PrecisionAttribute = struct { - _inner: c.MlirAttribute, +pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(OutputOperandAliasAttribute.get(ctx, args)); +} - pub const is_a_fn = c.stablehloAttributeIsAPrecisionAttr; - const Self = PrecisionAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const PrecisionAttribute = opaque { + const M = mlir.Methods(PrecisionAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Precision = enum { DEFAULT, @@ -1103,23 +1174,27 @@ pub const PrecisionAttribute = struct { HIGHEST, }; - pub fn init(ctx: mlir.Context, value: Precision) Self { - return .{ ._inner = c.stablehloPrecisionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Precision) *const PrecisionAttribute { + return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Precision { - const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self._inner)); + pub fn getValue(self: *const PrecisionAttribute) Precision { + const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self.ptr())); return std.meta.stringToEnum(Precision, value) orelse unreachable; } }; -pub const ComparisonDirection = struct { - _inner: c.MlirAttribute, +pub fn precisionAttribute(ctx: *mlir.Context, precision: PrecisionAttribute.Precision) *const mlir.Attribute { + return @ptrCast(PrecisionAttribute.init(ctx, precision)); +} - pub const is_a_fn = c.stablehloAttributeIsAComparisonDirectionAttr; - const Self = ComparisonDirection; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const ComparisonDirection = opaque { + const M = mlir.Methods(ComparisonDirection, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Direction = enum { EQ, @@ -1130,23 +1205,27 @@ pub const ComparisonDirection = struct { LT, }; - pub fn init(ctx: mlir.Context, value: Direction) Self { - return .{ ._inner = c.stablehloComparisonDirectionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Direction) *const ComparisonDirection { + return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Direction { - const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self._inner)); + pub fn getValue(self: *const ComparisonDirection) Direction { + const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); return std.meta.stringToEnum(Direction, value) orelse unreachable; } }; -pub const CompareType = struct { - _inner: c.MlirAttribute, +pub fn comparisonDirection(ctx: *mlir.Context, value: ComparisonDirection.Direction) *const mlir.Attribute { + return @ptrCast(ComparisonDirection.init(ctx, value)); +} - pub const is_a_fn = c.stablehloAttributeIsAComparisonTypeAttr; - const Self = CompareType; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const CompareType = opaque { + const M = mlir.Methods(CompareType, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { SIGNED, @@ -1155,23 +1234,27 @@ pub const CompareType = struct { TOTALORDER, }; - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloComparisonTypeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Type) *const CompareType { + return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self._inner)); + pub fn getValue(self: *const CompareType) Type { + const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub const Transpose = struct { - _inner: c.MlirAttribute, +pub fn compareType(ctx: *mlir.Context, value: CompareType.Type) *const mlir.Attribute { + return @ptrCast(CompareType.init(ctx, value)); +} - pub const is_a_fn = c.stablehloAttributeIsATransposeAttr; - const Self = Transpose; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const TransposeAttribute = opaque { + const M = mlir.Methods(TransposeAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsATransposeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { NO_TRANSPOSE, @@ -1179,23 +1262,27 @@ pub const Transpose = struct { ADJOINT, }; - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloTransposeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Type) *const TransposeAttribute { + return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self._inner)); + pub fn getValue(self: *const TransposeAttribute) Type { + const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub const FftType = struct { - _inner: c.MlirAttribute, +pub fn transposeAttribute(ctx: *mlir.Context, value: TransposeAttribute.Type) *const mlir.Attribute { + return @ptrCast(TransposeAttribute.init(ctx, value)); +} - pub const is_a_fn = c.stablehloAttributeIsAFftTypeAttr; - const Self = FftType; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const FftType = opaque { + const M = mlir.Methods(FftType, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { FFT, @@ -1204,114 +1291,45 @@ pub const FftType = struct { IRFFT, }; - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloFftTypeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Type) *const FftType { + return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self._inner)); + pub fn getValue(self: *const FftType) Type { + const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub const RngDistribution = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsARngDistributionAttr; - const Self = RngDistribution; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - UNIFORM, - NORMAL, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloRngDistributionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngDistributionAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const RngAlgorithm = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsARngAlgorithmAttr; - const Self = RngAlgorithm; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - DEFAULT, - THREE_FRY, - PHILOX, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloRngAlgorithmAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngAlgorithmAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub fn stablehloVersionFromCompatibilityRequirement(requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { - const state = struct { - var buf: [32]u8 = undefined; - - fn call(req: c.MlirStablehloCompatibilityRequirement) []u8 { - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - c.stablehloVersionFromCompatibilityRequirement(req, (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); - - return buf[0..stream.pos]; - } - }; - - return state.call(requirement); +pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { + return @ptrCast(FftType.init(ctx, value)); } -pub fn stablehloGetSmallerVersion(version1: []const u8, version2: []const u8) []const u8 { - const Cmp = struct { - v1: []const u8, - v1_is_smaller: bool = undefined, - - pub fn smallerCb(smaller_version: c.MlirStringRef, opaque_cmp: ?*anyopaque) callconv(.c) void { - var cmp: *@This() = @ptrCast(@alignCast(opaque_cmp)); - cmp.v1_is_smaller = std.mem.eql(u8, cmp.v1, smaller_version.data[0..smaller_version.length]); - } - }; - - var cmp_ctx: Cmp = .{ .v1 = version1 }; - const cmp_res = c.stablehloGetSmallerVersion(mlir.stringRef(version1), mlir.stringRef(version2), Cmp.smallerCb, &cmp_ctx); - - std.debug.assert(cmp_res.value != 0); - return if (cmp_ctx.v1_is_smaller) version1 else version2; +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: mlir.StringCallbackCtx = .{ .writer = &writer }; + _ = @call(.auto, streamFn, args ++ .{ mlir.stringCallback, &sctx }); + return writer.buffered(); } -pub fn getCurrentVersion() []const u8 { +pub fn versionFromCompatibilityRequirement(buffer: []u8, requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { + return stringFromStream(buffer, c.stablehloVersionFromCompatibilityRequirement, .{requirement}); +} + +pub fn smallerVersion(version1: []const u8, version2: []const u8) []const u8 { + var buf: [32]u8 = undefined; + const result = stringFromStream(&buf, c.stablehloGetSmallerVersion, .{ stringRef(version1), stringRef(version2) }); + return if (std.mem.eql(u8, result, version1)) version1 else version2; +} + +pub fn currentVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; var str: []const u8 = undefined; var once = std.once(call); fn call() void { - var writer: std.Io.Writer = .fixed(&buf); - c.stablehloGetCurrentVersion(printCallbackNoFail, &writer); - str = writer.buffered(); + str = stringFromStream(&buf, c.stablehloGetCurrentVersion, .{}); } }; @@ -1319,16 +1337,14 @@ pub fn getCurrentVersion() []const u8 { return state.str; } -pub fn getMinimumVersion() []const u8 { +pub fn minimumVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; var str: []const u8 = undefined; var once = std.once(call); fn call() void { - var writer: std.Io.Writer = .fixed(&buf); - c.stablehloGetMinimumVersion(printCallbackNoFail, &writer); - str = writer.buffered(); + str = stringFromStream(&buf, c.stablehloGetMinimumVersion, .{}); } }; @@ -1336,25 +1352,51 @@ pub fn getMinimumVersion() []const u8 { return state.str; } -pub fn serializePortableArtifact( - bytecode: []const u8, - target_version: []const u8, - writer: *std.Io.Writer, -) error{ InvalidMlirBytecodeVersion, WriteFailed }!void { - var writer_err: mlir.WriterWithErr = .{ .writer = writer }; - try mlir.successOr( - c.stablehloSerializePortableArtifactFromStringRef( - mlir.stringRef(bytecode), - mlir.stringRef(target_version), - mlir.WriterWithErr.printCallback, - &writer_err, - ), - error.InvalidMlirBytecodeVersion, +pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromStringRef( + stringRef(bytecode), + stringRef(target_version), + mlir.stringCallback, + &sctx, ); - return try writer_err.check(); + if (sctx.err) |err| { + return err; + } + if (result.value == 0) { + return error.InvalidMlirBytecodeVersion; + } } -fn printCallbackNoFail(mlir_str: c.MlirStringRef, opaque_writer: ?*anyopaque) callconv(.c) void { - const writer: *std.Io.Writer = @ptrCast(@alignCast(opaque_writer)); - writer.writeAll(mlir.fromStringRef(mlir_str)) catch @panic("Failed to write MLIR"); +pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromModule( + module.ptr(), + stringRef(target_version), + mlir.stringCallback, + &sctx, + true, + ); + if (sctx.err) |err| { + return err; + } + if (c.mlirLogicalResultIsFailure(result)) { + return error.InvalidMlirBytecodeVersion; + } +} + +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .flat = &.{&.{value}} }, + .location = location, + .verify = false, + }); +} + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .variadic = &.{values} }, + .verify = false, + .location = location, + }); } diff --git a/mlir/dialects/tensor.zig b/mlir/dialects/tensor.zig deleted file mode 100644 index 8d7de344..00000000 --- a/mlir/dialects/tensor.zig +++ /dev/null @@ -1,36 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -pub fn empty(ctx: mlir.Context, args: struct { - result: mlir.Type, - location: mlir.Location, -}) mlir.Operation { - return mlir.Operation.make(ctx, "tensor.empty", .{ - .results = &.{args.result}, - .location = args.location, - }); -} - -pub fn splat(ctx: mlir.Context, args: struct { - value: mlir.Value, - result: mlir.Type, - location: mlir.Location, -}) mlir.Operation { - return mlir.Operation.make(ctx, "tensor.splat", .{ - .operands = &.{args.value}, - .results = &.{args.result}, - .location = args.location, - }); -} - -pub fn cast(ctx: mlir.Context, args: struct { - source: mlir.Value, - dest: mlir.Type, - location: mlir.Location, -}) mlir.Operation { - return mlir.Operation.make(ctx, "tensor.cast", .{ - .operands = &.{args.source}, - .results = &.{args.dest}, - .location = args.location, - }); -} diff --git a/mlir/mlir.zig b/mlir/mlir.zig old mode 100755 new mode 100644 index a3b9114c..3d3e4b82 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const c = @import("c"); const stdx = @import("stdx"); @@ -8,8 +7,6 @@ const log = std.log.scoped(.mlir); test { std.testing.refAllDeclsRecursive(@This()); - - _ = try Context.init(); } pub const Error = error{ @@ -19,924 +16,1190 @@ pub const Error = error{ MlirUnexpected, /// A resource/executor was not found. NotFound, - /// Bytecode version incompatibility. - InvalidMlirBytecodeVersion, - OutOfMemory, -}; +} || std.mem.Allocator.Error; pub inline fn stringRef(str: []const u8) c.MlirStringRef { return .{ .data = str.ptr, .length = str.len }; } -pub inline fn fromStringRef(str: c.MlirStringRef) []const u8 { +pub inline fn string(str: c.MlirStringRef) []const u8 { // Note: mlir.StringRef need not to be null terminated. return str.data[0..str.length]; } +pub const StringCallbackCtx = struct { + writer: *std.Io.Writer, + err: ?std.Io.Writer.Error = null, +}; + +pub fn stringCallback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { + const ctx: *StringCallbackCtx = @ptrCast(@alignCast(ctx_)); + ctx.writer.writeAll(string(str)) catch |err| { + ctx.err = err; + }; +} + +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: StringCallbackCtx = .{ .writer = &writer }; + _ = @call(.auto, streamFn, args ++ .{ stringCallback, &sctx }); + return writer.buffered(); +} + pub fn registerPasses(comptime passes: []const u8) void { @field(c, "mlirRegister" ++ passes ++ "Passes")(); } -pub fn successOr(res: c.MlirLogicalResult, err: anytype) @TypeOf(err)!void { - return if (res.value == 0) err else {}; +fn isPtrConst(comptime T: type) bool { + return @typeInfo(T).pointer.is_const; } -pub const Registry = struct { - _inner: c.MlirDialectRegistry, +fn CastedPtr(comptime T1: type, comptime T2: type) type { + return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; +} - pub const deinit = helpers.deinit(Registry, c.mlirDialectRegistryDestroy); +pub fn Methods(comptime T: type, comptime NativeT: type) type { + return struct { + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; + pub const Ptr = if (IsPtrConst) *const T else *T; - pub fn init() !Registry { - const registry = c.mlirDialectRegistryCreate(); - return .{ ._inner = .{ .ptr = registry.ptr orelse return Error.MlirUnexpected } }; + pub fn ptr(self: anytype) NativeT { + return .{ .ptr = @ptrCast(@constCast(self)) }; + } + + fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return struct { + fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var ctx: StringCallbackCtx = .{ .writer = writer }; + f(self.ptr(), stringCallback, &ctx); + if (ctx.err) |err| { + return err; + } + } + }.format; + } + + pub fn eql(f: anytype) fn (self: *const T, other: *const T) bool { + return struct { + fn eql(self: *const T, other: *const T) bool { + return f(self.ptr(), other.ptr()); + } + }.eql; + } + + pub fn isA(self: *const T, OtherType: type) ?CastedPtr(@TypeOf(self), OtherType) { + return if (OtherType.isAFn(self.ptr())) @ptrCast(self) else null; + } + }; +} + +pub const DialectRegistry = opaque { + const M = Methods(DialectRegistry, c.MlirDialectRegistry); + + pub const ptr = M.ptr; + + pub fn init() std.mem.Allocator.Error!*DialectRegistry { + return @ptrCast(c.mlirDialectRegistryCreate().ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *DialectRegistry) void { + c.mlirDialectRegistryDestroy(self.ptr()); + } + + pub fn registerDialect(self: *DialectRegistry, comptime name: []const u8) void { + DialectHandle.fromString(name).insertDialect(self); } }; -pub const Context = struct { - _inner: c.MlirContext, - pub const deinit = helpers.deinit(Context, c.mlirContextDestroy); - pub const wrapOr = helpers.wrapOr(Context, c.mlirContextIsNull); +pub const PassManager = opaque { + const M = Methods(PassManager, c.MlirPassManager); - pub fn init() !Context { - return Context.wrapOr(c.mlirContextCreate()) orelse Error.MlirUnexpected; + pub const ptr = M.ptr; + + pub fn init(ctx: *Context) *PassManager { + return @ptrCast(c.mlirPassManagerCreate(ctx.ptr()).ptr); } - pub fn initWithRegistry(registry: Registry, threadingEnabled: bool) !Context { - return Context.wrapOr( - c.mlirContextCreateWithRegistry(registry._inner, threadingEnabled), - ) orelse Error.InvalidMlir; + pub fn deinit(self: *const PassManager) void { + c.mlirPassManagerDestroy(self.ptr()); + } + + pub fn asOpPassManager(self: *PassManager) *OpPassManager { + return @ptrCast(c.mlirPassManagerGetAsOpPassManager(self.ptr()).ptr); + } + + pub fn runOnOp(self: *const PassManager, op: *Operation) !void { + const result = c.mlirPassManagerRunOnOp(self.ptr(), op.ptr()); + if (result.value == 0) { + return error.MlirUnexpected; + } + } + + pub const EnableIRPrintingOpts = struct { + printBeforeAll: bool = false, + printAfterAll: bool = false, + printModuleScope: bool = false, + printAfterOnlyOnChange: bool = false, + printAfterOnlyOnFailure: bool = false, + flags: Operation.PrintFlags = .{}, + treePrintingPath: []const u8 = "", + }; + + pub fn enableIRPrinting(self: *PassManager, opts: EnableIRPrintingOpts) void { + const flags = opts.flags.createNative(); + defer c.mlirOpPrintingFlagsDestroy(flags); + + c.mlirPassManagerEnableIRPrinting( + self.ptr(), + opts.printBeforeAll, + opts.printAfterAll, + opts.printModuleScope, + opts.printAfterOnlyOnChange, + opts.printAfterOnlyOnFailure, + flags, + stringRef(opts.treePrintingPath), + ); + } + +}; + +pub const OpPassManager = opaque { + const M = Methods(OpPassManager, c.MlirOpPassManager); + + pub const ptr = M.ptr; + pub const format = M.format(c.mlirPrintPassPipeline); + + pub fn addPass(self: *OpPassManager, pass: *Pass) void { + c.mlirOpPassManagerAddOwnedPass(self.ptr(), pass.ptr()); + } + + pub fn addPasses(self: *OpPassManager, passes: []const *Pass) void { + for (passes) |pass| { + self.addPass(pass); + } + } + + pub fn addPipeline(self: *OpPassManager, pipeline_name: []const u8) !void { + var buffer: [1024]u8 = undefined; + const result = stringFromStream(&buffer, c.mlirOpPassManagerAddPipeline, .{ self.ptr(), stringRef(pipeline_name) }); + if (result.len != 0) { + log.err("mlir error: {s}", .{result}); + return error.MlirUnexpected; + } + } +}; + +pub const Pass = opaque { + const M = Methods(Pass, c.MlirPass); + + pub const ptr = M.ptr; + + pub fn create(comptime name: @TypeOf(.enum_literal)) *Pass { + return @ptrCast(@field(c, "mlirCreate" ++ @tagName(name))().ptr); + } +}; + +pub const Context = opaque { + const M = Methods(Context, c.MlirContext); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirContextEqual); + + pub const InitOptions = struct { + registry: ?*const DialectRegistry = null, + threading: bool = false, + }; + + pub fn init(opts: InitOptions) std.mem.Allocator.Error!*Context { + const ctx = if (opts.registry) |r| c.mlirContextCreateWithRegistry( + r.ptr(), + opts.threading, + ) else c.mlirContextCreateWithThreading(opts.threading); + return @ptrCast(ctx.ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *Context) void { + c.mlirContextDestroy(self.ptr()); } pub fn setMultiThreading(self: *Context, enabled: bool) void { - c.mlirContextEnableMultithreading(self._inner, enabled); + c.mlirContextEnableMultithreading(self.ptr(), enabled); } - pub fn appendDialectRegistry(self: *Context, registry: Registry) void { - c.mlirContextAppendDialectRegistry(self._inner, registry._inner); + pub fn appendDialectRegistry(self: *Context, registry: *const DialectRegistry) void { + c.mlirContextAppendDialectRegistry(self.ptr(), registry.ptr()); } pub fn loadAllAvailableDialects(self: *Context) void { - c.mlirContextLoadAllAvailableDialects(self._inner); + c.mlirContextLoadAllAvailableDialects(self.ptr()); } - pub fn numRegisteredDialects(self: Context) usize { - return @intCast(c.mlirContextGetNumRegisteredDialects(self._inner)); + pub fn numRegisteredDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumRegisteredDialects(self.ptr())); } - pub fn numLoadedDialects(self: Context) usize { - return @intCast(c.mlirContextGetNumLoadedDialects(self._inner)); + pub fn numLoadedDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumLoadedDialects(self.ptr())); } - pub fn isRegisteredOperation(self: Context, op: [:0]const u8) bool { - return c.mlirContextIsRegisteredOperation(self._inner, stringRef(op)); + pub fn numThreads(self: *const Context) usize { + return @intCast(c.mlirContextGetNumThreads(self.ptr())); } - pub fn location(self: Context, src: std.builtin.SourceLocation) Location { - return Location.fromSrc(self, src); + pub fn isRegisteredOperation(self: *const Context, op: []const u8) bool { + return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); + } + + pub fn format(self: *const Context, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ + @typeName(Context), + self.numThreads(), + self.numRegisteredDialects(), + self.numLoadedDialects(), + }); } }; -pub const Module = struct { - _inner: c.MlirModule, +pub const Location = opaque { + const M = Methods(Location, c.MlirLocation); - pub const deinit = helpers.deinit(Module, c.mlirModuleDestroy); - pub const wrapOr = helpers.wrapOr(Module, c.mlirModuleIsNull); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirLocationEqual); + pub const format = M.format(c.mlirLocationPrint); - pub fn init(loc: Location) Module { - return .{ ._inner = c.mlirModuleCreateEmpty(loc._inner) }; + pub fn fromSrc(ctx: *Context, src: std.builtin.SourceLocation) *const Location { + return fileLineCol( + ctx, + src.file, + src.line, + src.column, + ); } - pub fn parse(ctx: Context, source: [:0]const u8) !Module { - return Module.wrapOr( - c.mlirModuleCreateParse(ctx._inner, stringRef(source)), - ) orelse Error.InvalidMlir; + pub fn fileLineCol(ctx: *Context, file: []const u8, line: usize, column: usize) *const Location { + return @ptrCast(c.mlirLocationFileLineColGet( + ctx.ptr(), + stringRef(file), + @intCast(line), + @intCast(column), + ).ptr); } - pub fn fromOperation(operation: Operation) Module { - return .{ ._inner = c.mlirModuleFromOperation(operation._inner) }; + pub fn callSite(callee: *const Location, caller: *const Location) *const Location { + return @ptrCast(c.mlirLocationCallSiteGet(callee.ptr(), caller.ptr()).ptr); } - pub fn context(self: Module) Context { - return .{ ._inner = c.mlirModuleGetContext(self._inner) }; + pub fn fused(ctx: *Context, locs: []const *const Location, metadata: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFusedGet( + ctx.ptr(), + @intCast(locs.len), + @ptrCast(locs), + metadata.ptr(), + ).ptr); } - pub fn getBody(self: Module) Block { - return .{ ._inner = c.mlirModuleGetBody(self._inner) }; + pub fn named(self: *const Location, ctx: *Context, loc_name: []const u8) *const Location { + return @ptrCast(c.mlirLocationNameGet(ctx.ptr(), stringRef(loc_name), self.ptr()).ptr); } - pub fn op(self: Module) Operation { - return .{ ._inner = c.mlirModuleGetOperation(self._inner) }; + pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { + var buf: [256]u8 = undefined; + return loc.named(loc, ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { + buf[buf.len - 3 ..].* = "...".*; + break :blk &buf; + }); } - pub fn hash(self: Module, hasher: *std.hash.XxHash64) void { - return self.op().hash(hasher); + pub fn fromAttribute(attribute: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFromAttribute(attribute.ptr()).ptr); + } + + pub fn unknown(ctx: *Context) *const Location { + return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); } }; -pub const PassManager = struct { - _inner: c.MlirPassManager, +pub const Type = opaque { + const M = Methods(Type, c.MlirType); - pub const deinit = helpers.deinit(PassManager, c.mlirPassManagerDestroy); - pub const wrapOr = helpers.wrapOr(PassManager, c.mlirPassManagerIsNull); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; - pub fn init(ctx: Context) !PassManager { - return PassManager.wrapOr( - c.mlirPassManagerCreate(ctx._inner), - ) orelse Error.MlirUnexpected; - } - - pub fn initOnOperation(ctx: Context, op: [:0]const u8) !PassManager { - return PassManager.wrapOr( - c.mlirPassManagerCreateOnOperation(ctx._inner, stringRef(op)), - ) orelse Error.MlirUnexpected; - } - - pub fn asOpPassManager(self: PassManager) OpPassManager { - return .{ ._inner = c.mlirPassManagerGetAsOpPassManager(self._inner) }; - } - - // TODO mlirPassManagerEnableIRPrinting - // pub fn enableIRPrinting(self: *PassManager) void {} - - pub fn runOnOp(self: *PassManager, op: Operation) error{InvalidMlir}!void { - if (c.mlirPassManagerRunOnOp(self._inner, op._inner).value == 0) { - return Error.InvalidMlir; - } + pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { + return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); } }; -fn _mlir_passpipeline_error(err: c.MlirStringRef, ctx: ?*anyopaque) callconv(.c) void { - _ = ctx; - std.debug.print(">>ERROR: {s}\n", .{err.data}); +pub const IndexType = opaque { + const M = Methods(IndexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAIndex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context) *const IndexType { + return @ptrCast(c.mlirIndexTypeGet(ctx.ptr()).ptr); + } +}; + +pub fn indexType(ctx: *Context) *const Type { + return @ptrCast(IndexType.get(ctx)); } -pub const OpPassManager = struct { - _inner: c.MlirOpPassManager, - - pub fn addPipeline(self: *OpPassManager, pipeline: [:0]const u8) error{OutOfMemory}!void { - if (c.mlirOpPassManagerAddPipeline( - self._inner, - stringRef(pipeline), - &_mlir_passpipeline_error, - null, - ).value == 0) { - return Error.OutOfMemory; - } - } -}; - -pub const Identifier = struct { - _inner: c.MlirIdentifier, - - pub fn get(ctx: Context, str_: [:0]const u8) Identifier { - return .{ ._inner = c.mlirIdentifierGet(ctx._inner, stringRef(str_)) }; - } - - pub fn context(self: Identifier) Context { - return .{ ._inner = c.mlirIdentifierGetContext(self._inner) }; - } - - pub fn str(self: Identifier) []const u8 { - return fromStringRef(c.mlirIdentifierStr(self._inner)); - } - - pub fn equals(self: Identifier, other: Identifier) bool { - return c.mlirIdentifierEqual(self._inner, other._inner); - } -}; - -pub const AttrTuple = struct { [:0]const u8, Attribute }; - -pub const Attribute = struct { - _inner: c.MlirAttribute, - - pub const dump = helpers.dump(Attribute, c.mlirAttributeDump); - pub const eql = helpers.eql(Attribute, c.mlirAttributeEqual); - pub const format = helpers.format(Attribute, c.mlirAttributePrint); - pub const wrapOr = helpers.wrapOr(Attribute, c.mlirAttributeIsNull); - - pub fn wrap(c_attr: c.MlirAttribute) Attribute { - return .{ ._inner = c_attr }; - } - - pub fn parse(ctx: Context, attr: [:0]const u8) !Attribute { - return Attribute.wrapOr( - c.mlirAttributeParseGet(ctx._inner, stringRef(attr)), - ) orelse Error.InvalidMlir; - } - - pub fn fromAny(SpecificAttr: type) fn (x: SpecificAttr) Attribute { - return struct { - fn cast(x: SpecificAttr) Attribute { - return .{ ._inner = x._inner }; - } - }.cast; - } - - pub fn isA(self: Attribute, SpecificAttr: type) bool { - return SpecificAttr.is_a_fn(self._inner); - } - - // utilities function to built common attributes. - // All attributes are upcasted to the Attribute type, making it easier to chain construct, - // but losing type information. - - pub fn null_() Attribute { - return .wrap(c.mlirAttributeGetNull()); - } - - pub fn string(ctx: Context, str: []const u8) Attribute { - return StringAttribute.init(ctx, str).asAttr(); - } - - pub fn type_(t: Type) Attribute { - return TypeAttribute.init(t).asAttr(); - } - - pub fn unit(ctx: Context) Attribute { - return .wrap(c.mlirUnitAttrGet(ctx._inner)); - } - - pub fn boolean(ctx: Context, value: bool) Attribute { - return BoolAttribute.init(ctx, value).asAttr(); - } - - pub fn i1FromBool(ctx: Context, value: bool) Attribute { - return IntegerAttribute(.i1).init(ctx, @intFromBool(value)).asAttr(); - } - - pub fn int(ctx: Context, comptime int_type: IntegerTypes, value: i64) Attribute { - return IntegerAttribute(int_type).init(ctx, value).asAttr(); - } - - pub fn float(ctx: Context, comptime float_type: FloatTypes, value: f64) Attribute { - return FloatAttribute(float_type).init(ctx, value).asAttr(); - } - - pub fn array(ctx: Context, attrs: []const Attribute) Attribute { - return ArrayAttribute.init(ctx, attrs).asAttr(); - } - - pub fn dense(ctx: Context, comptime dt: DenseArrayTypes, values: []const dt.ZigType()) Attribute { - return DenseArrayAttribute(dt).init(ctx, values).asAttr(); - } - - /// Use a tensor as an attribute. - /// The tensor is specified by dims, dtype and a flat slice of values. - pub fn denseElements(ctx: Context, dims: []const i64, comptime dt: DenseElementsAttributeTypes, values: []const dt.ZigType()) Attribute { - return DenseElementsAttribute(dt).init(.tensor(dims, dt.mlirType(ctx)), values).asAttr(); - } - - pub fn denseElementsFromBytes(ctx: Context, dims: []const i64, dt: DenseElementsAttributeTypes, raw_bytes: []const u8) Attribute { - const shape: Type = .tensor(dims, dt.mlirType(ctx)); - return .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( - shape._inner, - @intCast(raw_bytes.len), - raw_bytes.ptr, - ) }; - } - - pub fn symbol(ctx: Context, flat_name: [:0]const u8) Attribute { - return FlatSymbolRefAttribute.init(ctx, flat_name).asAttr(); - } - - pub fn named(attr: Attribute, ctx: Context, name: [:0]const u8) NamedAttribute { - return NamedAttribute.named(ctx, name, attr); - } - - pub fn dict(ctx: Context, named_attrs: []const AttrTuple) Attribute { - var attr_buf: [32]NamedAttribute = undefined; - stdx.debug.assert(named_attrs.len <= attr_buf.len, ".dict helper only support up to {} attribute, got {}. You will need to call mlir.DictionaryAttribute directly", .{ attr_buf.len, named_attrs.len }); - - const attrs = attr_buf[0..named_attrs.len]; - for (attrs, named_attrs) |*attr, tuple| { - attr.* = .named(ctx, tuple[0], tuple[1]); - } - - return DictionaryAttribute.init(ctx, attrs).asAttr(); - } - - pub fn eqlAny(Attr: type) fn (Attr, Attr) bool { - return struct { - fn eql(a: Attr, b: Attr) bool { - return a.asAttr().eql(b.asAttr()); - } - }.eql; - } -}; - -pub const NamedAttribute = extern struct { - _inner: c.MlirNamedAttribute, - - pub fn wrap(c_named_attribute: c.MlirNamedAttribute) NamedAttribute { - return @bitCast(c_named_attribute); - } - - pub fn named(ctx: Context, name: [:0]const u8, attr: Attribute) NamedAttribute { - return .{ ._inner = .{ - .name = c.mlirIdentifierGet(ctx._inner, stringRef(name)), - .attribute = attr._inner, - } }; - } - - pub fn init(name: Identifier, attr: Attribute) NamedAttribute { - return .{ ._inner = .{ - .name = name._inner, - .attribute = attr._inner, - } }; - } -}; - -pub const StringAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAString; - pub const asAttr = Attribute.fromAny(StringAttribute); - pub const eql = Attribute.eqlAny(StringAttribute); - - pub fn init(ctx: Context, str: []const u8) StringAttribute { - return .{ ._inner = c.mlirStringAttrGet(ctx._inner, stringRef(str)) }; - } - - pub fn value(self: StringAttribute) []const u8 { - return fromStringRef(c.mlirStringAttrGetValue(self._inner)); - } -}; - -pub const BoolAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsABool; - pub const asAttr = Attribute.fromAny(BoolAttribute); - pub const eql = Attribute.eqlAny(BoolAttribute); - - pub fn init(ctx: Context, value_: bool) BoolAttribute { - return .{ ._inner = c.mlirBoolAttrGet(ctx._inner, if (value_) 1 else 0) }; - } - - pub fn value(self: BoolAttribute) bool { - return c.mlirBoolAttrGetValue(self._inner); - } -}; - -pub const TypeAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAType; - pub const eql = Attribute.eqlAny(TypeAttribute); - - pub fn init(type_: Type) TypeAttribute { - return .{ ._inner = c.mlirTypeAttrGet(type_._inner) }; - } - - pub fn typ(self: TypeAttribute) Type { - return .{ ._inner = c.mlirAttributeGetType(self._inner) }; - } - - pub const asAttr = Attribute.fromAny(TypeAttribute); -}; - -pub const ArrayAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAArray; - pub const asAttr = Attribute.fromAny(ArrayAttribute); - pub const eql = Attribute.eqlAny(ArrayAttribute); - - pub fn init(ctx: Context, attrs: []const Attribute) ArrayAttribute { - return .{ ._inner = c.mlirArrayAttrGet(ctx._inner, @intCast(attrs.len), @ptrCast(attrs.ptr)) }; - } - - pub fn size(self: ArrayAttribute) usize { - return @intCast(c.mlirArrayAttrGetNumElements(self._inner)); - } - - pub fn get(self: ArrayAttribute, index: usize) Attribute { - return .{ ._inner = c.mlirArrayAttrGetElement(self._inner, @intCast(index)) }; - } -}; - -pub fn IntegerAttribute(comptime it: IntegerTypes) type { - const ZigType, const getter = comptime switch (it) { - .i1, .i2, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, - .si4, .si8, .si16, .si32, .si64 => .{ i64, c.mlirIntegerAttrGetValueSInt }, - .u2, .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, - .unknown => @compileError("IntegerAttribute(unknown)"), - }; - - return struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAInteger; - - pub const IntegerTypeType = IntegerType(it); - const IntAttr = @This(); - - pub const asAttr = Attribute.fromAny(IntAttr); - pub const eql = Attribute.eqlAny(IntAttr); - - pub fn init(ctx: Context, value: i64) IntAttr { - return .{ ._inner = c.mlirIntegerAttrGet( - IntegerType(it).init(ctx)._inner, - value, - ) }; - } - - pub fn get(value: IntAttr) ZigType { - return @intCast(getter(value._inner)); - } - }; -} - -pub fn FloatAttribute(comptime ft: FloatTypes) type { - return struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAFloat; - const FloatAttr = @This(); - pub const asAttr = Attribute.fromAny(FloatAttr); - - pub fn init(ctx: Context, value: f64) FloatAttr { - return .{ ._inner = c.mlirFloatAttrDoubleGet( - ctx._inner, - FloatType(ft).init(ctx)._inner, - value, - ) }; - } - - pub fn get(value: FloatAttr) f64 { - return c.mlirFloatAttrGetValueDouble(value._inner); - } - }; -} - -pub const DenseArrayTypes = enum { - bool, - i8, - i16, - i32, - i64, - f32, - f64, - - pub fn ZigType(comptime dt: DenseArrayTypes) type { - return switch (dt) { - .bool => i32, - .i8 => i8, - .i16 => i16, - .i32 => i32, - .i64 => i64, - .f32 => f32, - .f64 => f64, - }; - } -}; - -pub fn DenseArrayAttribute(comptime dt: DenseArrayTypes) type { - const _is_a_fn, const get_fn, const get_element_fn = switch (dt) { - .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, - .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, - .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, - .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, - .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, - .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, - .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, - }; - - return struct { - _inner: c.MlirAttribute, - const Attr = @This(); - const ElementType = dt; - const ElementTypeZig = dt.ZigType(); - - pub const asAttr = Attribute.fromAny(Attr); - pub const eql = Attribute.eqlAny(Attr); - pub const is_a_fn = _is_a_fn; - - pub fn init(ctx: Context, values: []const ElementTypeZig) Attr { - return .{ ._inner = get_fn(ctx._inner, @intCast(values.len), @ptrCast(values.ptr)) }; - } - - pub fn get(self: Attr, pos: usize) ElementTypeZig { - return get_element_fn(self._inner, @intCast(pos)); - } - - pub fn len(self: Attr) usize { - return @intCast(c.mlirDenseArrayGetNumElements(self._inner)); - } - }; -} - -pub const DenseElementsAttributeTypes = enum { - bool, +pub const IntegerTypes = enum { + i1, + i2, + i4, i8, i16, i32, i64, + i128, + u1, + u2, + u4, u8, u16, u32, u64, + u128, + si1, + si4, + si8, + si16, + si32, + si64, + si128, + + pub fn bitwidth(self: IntegerTypes) usize { + return switch (self) { + .i1, .u1, .si1 => 1, + .i2, .u2 => 2, + .i4, .u4, .si4 => 4, + .i8, .u8, .si8 => 8, + .i16, .u16, .si16 => 16, + .i32, .u32, .si32 => 32, + .i64, .u64, .si64 => 64, + .i128, .u128, .si128 => 128, + }; + } + + pub fn signedness(self: IntegerTypes) IntegerType.Signedness { + return switch (self) { + .i1, .i2, .i4, .i8, .i16, .i32, .i64, .i128 => .signless, + .si1, .si4, .si8, .si16, .si32, .si64, .si128 => .signed, + .u1, .u2, .u4, .u8, .u16, .u32, .u64, .u128 => .unsigned, + }; + } +}; + +pub const IntegerType = opaque { + pub const Signedness = enum { + signless, + signed, + unsigned, + }; + + const M = Methods(IntegerType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { + return exact(ctx, it.bitwidth(), it.signedness()); + } + + fn exact(ctx: *Context, bitwidth: usize, sign: Signedness) *const IntegerType { + return @ptrCast(switch (sign) { + .signless => c.mlirIntegerTypeGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .signed => c.mlirIntegerTypeSignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .unsigned => c.mlirIntegerTypeUnsignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + }); + } +}; + +pub fn integerType(ctx: *Context, it: IntegerTypes) *const Type { + return @ptrCast(IntegerType.get(ctx, it)); +} + +pub const FloatTypes = enum { + f4e2m1fn, + f6e2m3fn, + f6e3m2fn, + f8e3m4, + f8e4m3, + f8e4m3b11fnuz, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + f8e8m0fnu, bf16, f16, f32, f64, - index, - - pub fn ZigType(comptime dt: DenseElementsAttributeTypes) type { - return switch (dt) { - .bool => bool, - .i8 => i8, - .i16 => i16, - .i32 => i32, - .i64 => i64, - .u8 => u8, - .u16 => u16, - .u32 => u32, - .u64 => u64, - .bf16 => u16, - .f16 => f16, - .f32 => f32, - .f64 => f64, - .index => usize, - }; - } - - pub fn mlirType(dt: DenseElementsAttributeTypes, ctx: Context) Type { - return switch (dt) { - .bool => .int(ctx, .i1), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .u8 => .int(ctx, .u8), - .u16 => .int(ctx, .u16), - .u32 => .int(ctx, .u32), - .u64 => .int(ctx, .u64), - .bf16 => .float(ctx, .bf16), - .f16 => .float(ctx, .f16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .index => .index(ctx), - }; - } + // f80, + // f128, + tf32, }; -pub fn DenseElementsAttribute(comptime dt: DenseElementsAttributeTypes) type { - return struct { - _inner: c.MlirAttribute, +pub fn FloatType(comptime ft: FloatTypes) type { + const _isAFn, const getter = switch (ft) { + .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, + .f6e2m3fn => .{ c.mlirTypeIsAFloat6E2M3FN, c.mlirFloat6E2M3FNTypeGet }, + .f6e3m2fn => .{ c.mlirTypeIsAFloat6E3M2FN, c.mlirFloat6E3M2FNTypeGet }, + .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, + .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, + .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, + .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, + .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, + .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, + .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, + .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, + .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, + .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, + .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, + .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, + .tf32 => .{ c.mlirTypeIsATF32, c.mlirTF32TypeGet }, + }; - const Attr = @This(); + return opaque { + const Self = @This(); + const M = Methods(Self, c.MlirType); - pub const is_a_fn = c.mlirAttributeIsADenseElements; - pub const asAttr = Attribute.fromAny(Attr); - pub const eql = Attribute.eqlAny(Attr); + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); - pub fn init(shaped_type: Type, slice: []const dt.ZigType()) Attr { - const raw_bytes = std.mem.sliceAsBytes(slice); - const attr: Attr = .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( - shaped_type._inner, - @intCast(raw_bytes.len), - @ptrCast(raw_bytes.ptr), - ) }; - std.debug.assert(attr._inner.ptr != null); - return attr; - } - - pub fn len(self: Attr) usize { - return @intCast(c.mlirElementsAttrGetNumElements(self._inner)); - } - - pub fn items(self: Attr) []const dt.ZigType() { - const raw_bytes: [*]const u8 = c.mlirDenseElementsAttrGetRawData(self._inner) orelse unreachable; - const ptr: [*]const dt.ZigType() = @ptrCast(@alignCast(raw_bytes)); - // Note the mlir API returns us the number of elements, not the number of bytes, - // that's why we track the element type at comptime to allow items to work. - return ptr[0..self.len()]; - } - - pub fn bytes(self: Attr) []const u8 { - return std.mem.sliceAsBytes(self.items()); + pub fn get(ctx: *Context) *const Self { + return @ptrCast(getter(ctx.ptr()).ptr); } }; } -pub const FlatSymbolRefAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAFlatSymbolRef; - const Self = FlatSymbolRefAttribute; - pub const eql = Attribute.eqlAny(Self); +pub fn floatType(ctx: *Context, ft: FloatTypes) *const Type { + return switch (ft) { + inline else => |v| @ptrCast(FloatType(v).get(ctx)), + }; +} - pub fn init(ctx: Context, str: [:0]const u8) Self { - return .{ ._inner = c.mlirFlatSymbolRefAttrGet(ctx._inner, stringRef(str)) }; +pub const ComplexTypes = enum { + c64, + c128, +}; + +pub const ComplexType = opaque { + const M = Methods(ComplexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAComplex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context, complex_type: ComplexTypes) *const ComplexType { + return @ptrCast(c.mlirComplexTypeGet(switch (complex_type) { + .c64 => floatType(ctx, .f32).ptr(), + .c128 => floatType(ctx, .f64).ptr(), + }).ptr); + } +}; + +pub fn complexType(ctx: *Context, complex_type: ComplexTypes) *const Type { + return @ptrCast(ComplexType.get(ctx, complex_type)); +} + +pub const Attribute = opaque { + const M = Methods(Attribute, c.MlirAttribute); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, attr: []const u8) Error!*const Attribute { + return @ptrCast(c.mlirAttributeParseGet(ctx.ptr(), stringRef(attr)).ptr orelse return Error.InvalidMlir); } - pub fn value(self: Self) []const u8 { - return fromStringRef(c.mlirFlatSymbolRefAttrGetValue(self._inner)); + pub fn context(self: *const Context) *Context { + return @ptrCast(c.mlirAttributeGetContext(self.ptr()).ptr); } - pub const asAttr = Attribute.fromAny(Self); + pub fn dialect(self: *const Context) *const Dialect { + return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); + } +}; + +pub const StringAttribute = opaque { + const M = Methods(StringAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAString; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, str: []const u8) *const StringAttribute { + return @ptrCast(c.mlirStringAttrGet(ctx.ptr(), stringRef(str)).ptr); + } + + pub fn value(self: *const StringAttribute) []const u8 { + return string(c.mlirStringAttrGetValue(self.ptr())); + } +}; + +pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { + return @ptrCast(StringAttribute.init(ctx, str)); +} + +pub const IntegerAttribute = opaque { + const M = Methods(IntegerAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { + return @ptrCast(c.mlirIntegerAttrGet( + IntegerType.get(ctx, it).ptr(), + @intCast(value_), + ).ptr); + } + + pub fn value(self: *const IntegerAttribute, comptime T: type) T { + const getValue = switch (@typeInfo(T).int.signedness) { + .signed => c.mlirIntegerAttrGetValueInt, + .unsigned => c.mlirIntegerAttrGetValueUInt, + }; + return @intCast(getValue(self.ptr())); + } +}; + +pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, it, value_)); +} + +pub const BoolAttribute = opaque { + const M = Methods(BoolAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsABool; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, value_: bool) *const BoolAttribute { + return @ptrCast(c.mlirBoolAttrGet(ctx.ptr(), @intFromBool(value_)).ptr); + } + + pub fn value(self: *const BoolAttribute) bool { + return c.mlirBoolAttrGetValue(self.ptr()); + } +}; + +pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { + return @ptrCast(BoolAttribute.init(ctx, value)); +} + +pub const TypeAttribute = opaque { + const M = Methods(TypeAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(type_: *const Type) *const TypeAttribute { + return @ptrCast(c.mlirTypeAttrGet(type_.ptr()).ptr); + } + + pub fn value(self: *const TypeAttribute) *const Type { + return @ptrCast(c.mlirTypeAttrGetValue(self.ptr()).ptr); + } +}; + +pub fn typeAttribute(type_: *const Type) *const Attribute { + return @ptrCast(TypeAttribute.init(type_)); +} + +pub const Dialect = struct { + const M = Methods(Dialect, c.MlirDialect); + + pub const ptr = M.ptr; + + pub fn context(self: *const Dialect) *Context { + return @ptrCast(c.mlirDialectGetContext(self.ptr()).ptr); + } + + pub fn namespace(self: *const Dialect) []const u8 { + return string(c.mlirDialectGetNamespace(self.ptr())); + } + + pub fn format(self: *const Dialect, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(Dialect), self.namespace() }); + } +}; + +pub const DialectHandle = opaque { + const M = Methods(DialectHandle, c.MlirDialectHandle); + + pub const ptr = M.ptr; + + pub fn format(self: *const DialectHandle, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(DialectHandle), self.namespace() }); + } + + pub fn namespace(self: *const DialectHandle) []const u8 { + return string(c.mlirDialectHandleGetNamespace(self.ptr())); + } + + pub fn insertDialect(self: *const DialectHandle, registry: *DialectRegistry) void { + c.mlirDialectHandleInsertDialect(self.ptr(), registry.ptr()); + } + + pub fn registerDialect(self: *const DialectHandle, ctx: *Context) void { + c.mlirDialectHandleRegisterDialect(self.ptr(), ctx.ptr()); + } + + pub fn loadDialect(self: *const DialectHandle, ctx: *Context) *Dialect { + return @ptrCast(c.mlirDialectHandleLoadDialect(self.ptr(), ctx.ptr()).ptr); + } + + pub fn fromString(comptime namespace_: []const u8) *const DialectHandle { + return @ptrCast(@field(c, "mlirGetDialectHandle__" ++ namespace_ ++ "__")().ptr); + } +}; + +pub const Identifier = opaque { + const M = Methods(Identifier, c.MlirIdentifier); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirIdentifierEqual); + + pub fn context(self: *const Identifier) *Context { + return @ptrCast(c.mlirIdentifierGetContext(self.ptr()).ptr); + } + + pub fn get(ctx: *Context, str_: []const u8) *const Identifier { + return @ptrCast(c.mlirIdentifierGet(ctx.ptr(), stringRef(str_)).ptr); + } + + pub fn str(self: *const Identifier) []const u8 { + return string(c.mlirIdentifierStr(self.ptr())); + } + + pub fn format(self: *const Identifier, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .str = \"{s}\" }}", .{ @typeName(Identifier), self.str() }); + } +}; + +pub const Module = opaque { + const M = Methods(Module, c.MlirModule); + + pub const ptr = M.ptr; + + pub fn init(loc: *const Location) *Module { + return @ptrCast(@constCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr)); + } + + pub fn deinit(self: *Module) void { + c.mlirModuleDestroy(self.ptr()); + } + + pub fn format(self: *const Module, writer: *std.Io.Writer) !void { + return self.operation().format(writer); + } + + pub fn fromOperation(op: *Operation) *Module { + return @ptrCast(@constCast(c.mlirModuleFromOperation(op.ptr()).ptr)); + } + + pub fn operation(self: *const Module) *Operation { + return @ptrCast(c.mlirModuleGetOperation(self.ptr()).ptr); + } + + pub fn parse(ctx: *Context, source: []const u8) Error!*Module { + return @ptrCast(@constCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir)); + } + + pub fn context(self: *const Module) *Context { + return @ptrCast(c.mlirModuleGetContext(self.ptr()).ptr); + } + + pub fn body(self: *const Module) *Block { + return @ptrCast(c.mlirModuleGetBody(self.ptr()).ptr); + } +}; + +pub const Block = opaque { + const M = Methods(Block, c.MlirBlock); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirBlockEqual); + pub const format = M.format(c.mlirBlockPrint); + + pub fn init(args: []const *const Type, locs: []const *const Location) *Block { + return @ptrCast(c.mlirBlockCreate(@intCast(args.len), @ptrCast(args), @ptrCast(locs)).ptr); + } + + pub fn deinit(self: *Block) void { + std.debug.assert(self.parentOperation() == null); + c.mlirBlockDestroy(self.ptr()); + } + + pub fn argument(self: *const Block, index: usize) *const Value { + return @ptrCast(c.mlirBlockGetArgument(self.ptr(), @intCast(index)).ptr); + } + + pub fn numArguments(self: *const Block) usize { + return @intCast(c.mlirBlockGetNumArguments(self.ptr())); + } + + pub fn addArgument(self: *Block, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockAddArgument(self.ptr(), type_.ptr(), loc.ptr()).ptr); + } + + pub fn insertArgument(self: *Block, index: usize, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockInsertArgument(self.ptr(), @intCast(index), type_.ptr(), loc.ptr()).ptr); + } + + pub fn appendOwnedOperation(self: *Block, op: *Operation) void { + c.mlirBlockAppendOwnedOperation(self.ptr(), op.ptr()); + } + + pub fn appendOwnedOperations(self: *Block, ops: []const *Operation) void { + for (ops) |op| { + self.appendOwnedOperation(op); + } + } + + pub fn parentOperation(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); + } + + pub fn detach(self: *Block) void { + c.mlirBlockDetach(self.ptr()); + } + + pub fn terminator(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetTerminator(self.ptr()).ptr); + } +}; + +pub const NamedAttribute = struct { + inner: c.MlirNamedAttribute, + + pub fn ptr(self: NamedAttribute) c.MlirNamedAttribute { + return self.inner; + } + + pub fn name(self: NamedAttribute) *const Identifier { + return @ptrCast(self.inner.name.ptr); + } + + pub fn attribute(self: NamedAttribute) *const Attribute { + return @ptrCast(self.inner.attribute.ptr); + } + + pub fn named(ctx: *Context, name_: []const u8, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = Identifier.get(ctx, name_).ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn init(name_: *const Identifier, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = name_.ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn format(self: NamedAttribute, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s}{{ .{s} = \"{f}\" }}", .{ + @typeName(NamedAttribute), + self.name().str(), + self.attribute(), + }); + } +}; + +pub const DictionaryAttribute = opaque { + const M = Methods(DictionaryAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADictionary; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attributes: []const NamedAttribute) *const DictionaryAttribute { + return @ptrCast(c.mlirDictionaryAttrGet( + ctx.ptr(), + @intCast(attributes.len), + @ptrCast(attributes), + ).ptr); + } + + pub fn size(self: *const DictionaryAttribute) usize { + return @intCast(c.mlirDictionaryAttrGetNumElements(self.ptr())); + } + + pub fn get(self: *const DictionaryAttribute, pos: usize) NamedAttribute { + return .{ .inner = c.mlirDictionaryAttrGetElement(self.ptr(), @intCast(pos)) }; + } + + pub fn getByName(self: *const DictionaryAttribute, name: []const u8) ?*const Attribute { + return @ptrCast(c.mlirDictionaryAttrGetElementByName(self.ptr(), stringRef(name)).ptr); + } +}; + +pub fn dictionaryAttribute(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { + return @ptrCast(DictionaryAttribute.init(ctx, attributes)); +} + +pub const Value = opaque { + const M = Methods(Value, c.MlirValue); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + pub const isA = M.isA; + + pub fn type_(val: *const Value) *const Type { + return @ptrCast(c.mlirValueGetType(val.ptr()).ptr); + } + + pub fn setType(self: *Value, type__: *const Type) void { + c.mlirValueSetType(self.ptr(), type__.ptr()); + } + + pub fn firstUse(self: *const Value) *const OpOperand { + return @ptrCast(c.mlirValueGetFirstUse(self.ptr()).ptr); + } + + pub fn replaceAllUsesWith(self: *const Value, with: *const Value) void { + c.mlirValueReplaceAllUsesOfWith(self.ptr(), with.ptr()); + } +}; + +pub const OpOperand = opaque { + const M = Methods(OpOperand, c.MlirOpOperand); + + pub const ptr = M.ptr; + + pub fn owner(self: *const OpOperand) *Operation { + return @ptrCast(c.mlirOpOperandGetOwner(self.ptr()).ptr); + } + + pub fn number(self: *const OpOperand) usize { + return @intCast(c.mlirOpOperandGetOperandNumber(self.ptr())); + } + + pub fn nextUse(self: *const OpOperand) ?*const OpOperand { + return @ptrCast(c.mlirOpOperandGetNextUse(self.ptr()).ptr); + } +}; + +pub const OpResult = opaque { + const M = Methods(OpResult, c.MlirValue); + + pub const isAFn = c.mlirValueIsAOpResult; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn owner(self: *const OpResult) *Operation { + return @ptrCast(c.mlirOpResultGetOwner(self.ptr()).ptr); + } + + pub fn resultNumber(self: *const OpResult) usize { + return @intCast(c.mlirOpResultGetResultNumber(self.ptr())); + } +}; + +pub const BlockArgument = opaque { + const M = Methods(BlockArgument, c.MlirValue); + + pub const isAFn = c.mlirValueIsABlockArgument; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn block(arg: *const BlockArgument) *Block { + return @ptrCast(c.mlirBlockArgumentGetOwner(arg.ptr()).ptr); + } + + pub fn number(self: *const BlockArgument) usize { + return @intCast(c.mlirBlockArgumentGetArgNumber(self.ptr())); + } + + pub fn setType(self: *BlockArgument, type_: *const Type) void { + c.mlirBlockArgumentSetType(self.ptr(), type_.ptr()); + } }; pub const OperationState = struct { - _inner: c.MlirOperationState, + inner: c.MlirOperationState, - const Self = OperationState; - - pub fn init(name: [:0]const u8, loc: Location) Self { - return .{ ._inner = c.mlirOperationStateGet(stringRef(name), loc._inner) }; + pub fn init(name: []const u8, location: *const Location) OperationState { + return .{ .inner = c.mlirOperationStateGet(stringRef(name), location.ptr()) }; } - pub fn addResult(self: *Self, type_: Type) void { - c.mlirOperationStateAddResults(&self._inner, 1, &[_]c.MlirType{type_._inner}); + pub fn addResults(self: *OperationState, results: []const *const Type) void { + c.mlirOperationStateAddResults(&self.inner, @intCast(results.len), @ptrCast(results)); } - pub fn addResults(self: *Self, types: []const Type) void { - c.mlirOperationStateAddResults(&self._inner, @intCast(types.len), @ptrCast(types.ptr)); + pub fn addOperands(self: *OperationState, operands: []const *const Value) void { + c.mlirOperationStateAddOperands(&self.inner, @intCast(operands.len), @ptrCast(operands)); } - pub fn addOperand(self: *Self, value: Value) void { - c.mlirOperationStateAddOperands(&self._inner, 1, &[_]c.MlirValue{value._inner}); + pub fn addOwnedRegions(self: *OperationState, regions: []const *Region) void { + c.mlirOperationStateAddOwnedRegions(&self.inner, @intCast(regions.len), @ptrCast(regions)); } - pub fn addOperands(self: *Self, values: []const Value) void { - c.mlirOperationStateAddOperands(&self._inner, @intCast(values.len), @ptrCast(values.ptr)); + pub fn addSuccessors(self: *OperationState, successors: []*const Block) void { + c.mlirOperationStateAddSuccessors(&self.inner, @intCast(successors.len), @ptrCast(successors)); } - pub fn addRegion(self: *Self, region: *Region) void { - c.mlirOperationStateAddOwnedRegions(&self._inner, 1, &[_]c.MlirRegion{region._inner}); + pub fn addAttributes(self: *OperationState, attributes: []const NamedAttribute) void { + c.mlirOperationStateAddAttributes(&self.inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); } - pub fn addRegions(self: *Self, regions: []const Region) void { - c.mlirOperationStateAddOwnedRegions(&self._inner, @intCast(regions.len), @ptrCast(regions.ptr)); - } - - pub fn addAttribute(self: *Self, ctx: Context, name: [:0]const u8, attr: Attribute) void { - c.mlirOperationStateAddAttributes(&self._inner, 1, @ptrCast(&.{ - .{ - .name = Identifier.get(ctx, name)._inner, - .attribute = attr._inner, - }, - })); - } - - pub fn addAttributeRaw(self: *Self, name: Identifier, attr: Attribute) void { - c.mlirOperationStateAddAttributes(&self._inner, 1, @ptrCast(&.{ - .{ - .name = name._inner, - .attribute = attr._inner, - }, - })); - } - - pub fn addAttributes(self: *Self, attributes: []const NamedAttribute) void { - c.mlirOperationStateAddAttributes(&self._inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); - } - - pub fn resultTypeInference(self: *Self, enabled: bool) void { - self._inner.enableResultTypeInference = enabled; + pub fn enableResultTypeInference(self: *OperationState) void { + c.mlirOperationStateEnableResultTypeInference(&self.inner); } }; -pub const DictionaryAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsADictionary; - pub const asAttr = Attribute.fromAny(DictionaryAttribute); - pub const eql = Attribute.eqlAny(DictionaryAttribute); +pub const Operation = opaque { + const M = Methods(Operation, c.MlirOperation); - pub fn init(ctx: Context, attributes: []const NamedAttribute) DictionaryAttribute { - return .{ ._inner = c.mlirDictionaryAttrGet( - ctx._inner, - @intCast(attributes.len), - @ptrCast(attributes.ptr), - ) }; + const MAX_SEGMENTS = 32; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirOperationEqual); + pub const format = M.format(c.mlirOperationPrint); + + pub fn init(state: *OperationState) !*Operation { + defer state.* = undefined; + return @ptrCast(c.mlirOperationCreate(&state.inner).ptr orelse return Error.InvalidMlir); } - pub fn size(self: DictionaryAttribute) usize { - return @intCast(c.mlirDictionaryAttrGetNumElements(self._inner)); + pub fn deinit(self: *Operation) void { + std.debug.assert(self.block() == null); + c.mlirOperationDestroy(self.ptr()); } - pub fn get(self: DictionaryAttribute, pos: usize) NamedAttribute { - return .wrap(c.mlirDictionaryAttrGetElement(self._inner, @bitCast(pos))); + pub fn parse(ctx: *Context, source: []const u8, name_: []const u8) Error!*Operation { + return @ptrCast(c.mlirOperationCreateParse(ctx.ptr(), stringRef(source), stringRef(name_)).ptr orelse return Error.InvalidMlir); } - pub fn getByName(self: DictionaryAttribute, name: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirDictionaryAttrGetElementByName(self._inner, stringRef(name))); - } -}; - -pub const Operation = struct { - const Self = Operation; - _inner: c.MlirOperation, - - pub const dump = helpers.dump(Operation, c.mlirOperationDestroy); - pub const deinit = helpers.deinit(Operation, c.mlirOperationDestroy); - pub const wrapOr = helpers.wrapOr(Operation, c.mlirOperationIsNull); - pub const eql = helpers.eql(Operation, c.mlirOperationEqual); - - pub fn init(state: *OperationState) !Self { - return Self.wrapOr(c.mlirOperationCreate(&state._inner)) orelse Error.InvalidMlir; + pub fn name(self: *const Operation) []const u8 { + return string(c.mlirOperationGetName(self.ptr())); } - pub fn make(ctx: Context, op_name: [:0]const u8, args: struct { - operands: ?[]const Value = null, - variadic_operands: ?[]const []const Value = null, - tt_variadic_operands: ?[]const []const Value = null, - results: ?[]const Type = null, - variadic_results: ?[]const []const Type = null, - result_type_inference: ?bool = null, - n_regions: usize = 0, - attributes: ?[]const AttrTuple = null, - blocks: ?[]const Block = null, - verify: bool = true, - location: Location, - }) Self { - var state = OperationState.init(op_name, args.location); - std.debug.assert(!(args.operands != null and args.variadic_operands != null)); - if (args.operands) |operands| { - state.addOperands(operands); - } else if (args.variadic_operands) |operands_segments| { - const MAX_SEGMENTS = 32; - var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + pub fn clone(self: *const Operation) *Operation { + return @ptrCast(c.mlirOperationClone(self.ptr()).ptr); + } - for (operands_segments) |operands| { - state.addOperands(operands); - segments.appendAssumeCapacity(@intCast(operands.len)); - } - state.addAttribute(ctx, "operandSegmentSizes", .denseElements(ctx, &.{@intCast(segments.len)}, .i32, segments.constSlice())); - } else if (args.tt_variadic_operands) |operands_segments| { - // stablehlo and triton seems to disagree on the expected type of operandSegmentSizes, let's fix that. - const MAX_SEGMENTS = 32; - var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + pub fn block(self: *const Operation) ?*Block { + return @ptrCast(c.mlirOperationGetBlock(self.ptr()).ptr); + } - for (operands_segments) |operands| { - state.addOperands(operands); - segments.appendAssumeCapacity(@intCast(operands.len)); - } - state.addAttribute(ctx, "operandSegmentSizes", .dense(ctx, .i32, segments.constSlice())); - } - if (args.result_type_inference) |enable| { - state.resultTypeInference(enable); - } - std.debug.assert(!(args.results != null and args.variadic_results != null)); - if (args.results) |results| { - state.addResults(results); - } else if (args.variadic_results) |result_segments| { - for (result_segments) |results| { - state.addResults(results); - } - } - for (0..args.n_regions) |_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - state.addRegion(®ion_); - } - if (args.attributes) |attrs| { - for (attrs) |attr| { - state.addAttributeRaw( - Identifier.get(ctx, attr[0]), - attr[1], - ); - } - } - if (args.blocks) |blocks_| { - for (blocks_) |block_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - region_.appendBlock(block_); - state.addRegion(®ion_); - } - } + pub fn appendTo(self: *Operation, block_: *Block) *Operation { + block_.appendOwnedOperation(self); + return self; + } - const new_op = Operation.init(&state) catch { - @panic("Failed to create MLIR operation"); + pub fn parent(self: *const Operation) ?*Operation { + return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); + } + + pub fn removeFromParent(self: *Operation) void { + c.mlirOperationRemoveFromParent(self.ptr()); + } + + pub fn numOperands(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumOperands(self.ptr())); + } + + pub fn operand(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetOperand(self.ptr(), @intCast(index)).ptr); + } + + pub fn setOperand(self: *Operation, index: usize, value: *const Value) void { + c.mlirOperationSetOperand(self.ptr(), @intCast(index), value.ptr()); + } + + pub fn numResults(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumResults(self.ptr())); + } + + pub fn result(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetResult(self.ptr(), @intCast(index)).ptr); + } + + pub fn nextInBlock(self: *const Operation) *const Operation { + return @ptrCast(c.mlirOperationGetNextInBlock(self.ptr()).ptr); + } + + pub fn context(self: *const Operation) *Context { + return @ptrCast(c.mlirOperationGetContext(self.ptr()).ptr); + } + + pub fn setAttributeByName(self: *Operation, name_: []const u8, attribute: *const Attribute) void { + c.mlirOperationSetAttributeByName(self.ptr(), stringRef(name_), attribute.ptr()); + } + + pub const PrintFlags = struct { + elide_large_elements_attrs: ?usize = null, + debug_info: bool = false, + debug_info_pretty_form: bool = true, + print_generic_op_form: bool = false, + use_local_scope: bool = false, + assume_verified: bool = false, + + const Ctx = struct { + self: *const Operation, + flags: PrintFlags, }; + + fn createNative(self: PrintFlags) c.MlirOpPrintingFlags { + const flags = c.mlirOpPrintingFlagsCreate(); + if (self.elide_large_elements_attrs) |v| { + c.mlirOpPrintingFlagsElideLargeElementsAttrs(flags, @intCast(v)); + } + c.mlirOpPrintingFlagsEnableDebugInfo( + flags, + self.debug_info, + self.debug_info_pretty_form, + ); + if (self.print_generic_op_form) { + c.mlirOpPrintingFlagsPrintGenericOpForm(flags); + } + if (self.use_local_scope) { + c.mlirOpPrintingFlagsUseLocalScope(flags); + } + if (self.assume_verified) { + c.mlirOpPrintingFlagsAssumeVerified(flags); + } + return flags; + } + }; + + fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + const flags = fctx.flags.createNative(); + defer c.mlirOpPrintingFlagsDestroy(flags); + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationPrintWithFlags(fctx.self.ptr(), flags, stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } + } + + pub fn fmt(self: *const Operation, flags: PrintFlags) std.fmt.Alt(PrintFlags.Ctx, formatWithFlags) { + return .{ .data = .{ + .self = self, + .flags = flags, + } }; + } + + pub const MakeArgs = struct { + pub const AttrTuple = struct { []const u8, *const Attribute }; + + pub const Operands = union(enum) { + flat: []const *const Value, + variadic: []const []const *const Value, + }; + + pub const ResultTypes = union(enum) { + flat: []const *const Type, + variadic: []const []const *const Type, + }; + + operands: ?Operands = null, + results: ?ResultTypes = null, + result_type_inference: bool = false, + attributes: ?[]const NamedAttribute = null, + blocks: ?[]const *Block = null, + verify: bool = true, + location: ?*const Location = null, + }; + + pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { + var state: OperationState = .init(name_, args.location orelse .unknown(ctx)); + if (args.result_type_inference) { + state.enableResultTypeInference(); + } + if (args.operands) |operands| switch (operands) { + .flat => |v| state.addOperands(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_operands| { + state.addOperands(segment_operands); + sizes.appendAssumeCapacity(@intCast(segment_operands.len)); + } + state.addAttributes(&.{ + .named(ctx, "operandSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + }); + }, + }; + if (args.results) |results| switch (results) { + .flat => |v| state.addResults(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_results| { + state.addResults(segment_results); + sizes.appendAssumeCapacity(@intCast(segment_results.len)); + } + state.addAttributes(&.{ + .named(ctx, "resultSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + }); + }, + }; + if (args.attributes) |attrs| { + state.addAttributes(attrs); + } + if (args.blocks) |blocks| { + const region = Region.init(); + for (blocks) |block_| { + region.appendOwnedBlock(block_); + } + state.addOwnedRegions(&.{region}); + } + const new_op = try Operation.init(&state); + errdefer new_op.deinit(); if (args.verify and new_op.verify() == false) { - log.err("Failed to verify MLIR operation:\n{f}", .{new_op.mlirFormatter(.{ .debug_info = true })}); - @panic("Failed to verify MLIR operation"); + log.err("Failed to verify MLIR operation:\n{f}", .{ + new_op.fmt(.{ + .print_generic_op_form = true, + .debug_info = true, + }), + }); + return Error.InvalidMlir; } return new_op; } - pub fn initParse(ctx: Context, str: [:0]const u8) !Self { - return Self.wrapOr( - c.mlirOperationCreateParse(ctx._inner, stringRef(str), stringRef("pouet")), - ) orelse Error.InvalidMlir; + pub fn make(ctx: *Context, op_name: []const u8, args: MakeArgs) *Operation { + return try_make(ctx, op_name, args) catch { + @panic("Failed to create MLIR operation"); + }; } - pub fn clone(self: Self) !Self { - return Self.wrapOr( - c.mlirOperationClone(self._inner), - ) orelse Error.InvalidMlir; + pub fn verify(self: *const Operation) bool { + return c.mlirOperationVerify(self.ptr()); } - pub fn name(self: Self) Identifier { - return .{ ._inner = c.mlirOperationGetName(self._inner) }; - } - - pub fn removeFromParent(self: *Self) void { - c.mlirOperationRemoveFromParent(self._inner); - } - - pub fn numOperands(self: Self) usize { - return @intCast(c.mlirOperationGetNumOperands(self._inner)); - } - - pub fn operand(self: Self, index: usize) Value { - return .{ ._inner = c.mlirOperationGetOperand(self._inner, @intCast(index)) }; - } - - pub fn setOperand(self: *Self, index: usize, value: Value) void { - c.mlirOperationSetOperand(self._inner, @intCast(index), value._inner); - } - - pub fn numResults(self: Self) usize { - return @intCast(c.mlirOperationGetNumResults(self._inner)); - } - - pub fn result(self: Self, index: usize) Value { - return .{ ._inner = c.mlirOperationGetResult(self._inner, @intCast(index)) }; - } - - pub fn nextInBlock(self: Self) Self { - return .{ ._inner = c.mlirOperationGetNextInBlock(self._inner) }; - } - - // pub fn previousInBlock(self: Self) Self { - // return .{ ._inner = c.mlirOperationGetPrevInBlock(self._inner) }; - // } - - pub fn block(self: Self) ?Block { - return Block.wrapOr(c.mlirOperationGetBlock(self._inner)); - } - - pub fn parent(self: Self) ?Self { - return Self.wrapOr(c.mlirOperationGetParentOperation(self._inner)); - } - - pub fn region(self: Self, index: usize) Region { - return .{ ._inner = c.mlirOperationGetRegion(self._inner, @intCast(index)) }; - } - - pub fn context(self: Self) Context { - return .{ ._inner = c.mlirOperationGetContext(self._inner) }; - } - - pub fn writeBytecode(self: Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - var writer_with_err: WriterWithErr = .{ .writer = writer }; - c.mlirOperationWriteBytecode( - self._inner, - WriterWithErr.printCallback, - &writer_with_err, - ); - return writer_with_err.check(); - } - - pub fn writeBytecodeWithConfig(self: Self, writer: *std.Io.Writer, config: struct { - desiredEmitedVersion: ?i64 = null, - }) error{ InvalidMlirBytecodeVersion, WriteFailed }!void { - const cfg = c.mlirBytecodeWriterConfigCreate(); - defer c.mlirBytecodeWriterConfigDestroy(cfg); - if (config.desiredEmitedVersion) |v| { - c.mlirBytecodeWriterConfigDesiredEmitVersion(cfg, v); - } - - var writer_with_err: WriterWithErr = .{ .writer = writer }; - try successOr(c.mlirOperationWriteBytecodeWithConfig( - self._inner, - cfg, - &WriterWithErr.printCallback, - &writer_with_err, - ), error.InvalidMlirBytecodeVersion); - return writer_with_err.check(); - } - - /// Enable a full dump of the IR. - /// Usage `std.log.debug("{}", .{ module.op().mlirFormatter(.{}) }); - pub fn mlirFormatter(self: Operation, flags: OpPrintingFlags) MlirFormatter { - return .{ .op = self, .flags = flags }; - } - - const MlirFormatter = struct { - op: Operation, - flags: OpPrintingFlags, - - pub fn format(self: MlirFormatter, writer: *std.Io.Writer) !void { - try self.op.print(writer, self.flags); - } - }; - - pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) std.Io.Writer.Error!void { - const pflags = flags.create(); - defer c.mlirOpPrintingFlagsDestroy(pflags); - - var writer_err: WriterWithErr = .{ .writer = writer }; - c.mlirOperationPrintWithFlags(self._inner, pflags, WriterWithErr.printCallback, &writer_err); - return writer_err.check(); - } - - pub fn verify(self: Self) bool { - return c.mlirOperationVerify(self._inner); - } - - pub fn getLocation(self: Self) Location { - return .{ ._inner = c.mlirOperationGetLocation(self._inner) }; + pub fn location(self: *const Operation) *const Location { + return @ptrCast(c.mlirOperationGetLocation(self.ptr()).ptr); } pub const WalkOrder = enum(c.MlirWalkOrder) { @@ -950,7 +1213,7 @@ pub const Operation = struct { skip = c.MlirWalkResultSkip, }; - pub fn walk(self: Self, order: WalkOrder, ctx: anytype, walkfn: fn (ctx: anytype, op: Operation) WalkResult) void { + pub fn walk(self: *Operation, order: WalkOrder, ctx: anytype, walkfn: fn (ctx_: @TypeOf(ctx), op: *Operation) WalkResult) void { var inner_ctx = .{ .ctx = ctx }; const ContextType = @TypeOf(inner_ctx); @@ -959,7 +1222,7 @@ pub const Operation = struct { (struct { pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @intFromEnum(walkfn(inner_ctx_.ctx, .{ ._inner = op })); + return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, @@ -967,792 +1230,383 @@ pub const Operation = struct { ); } - pub fn getAttribute(self: Self, pos: usize) NamedAttribute { - return .{ ._inner = c.mlirOperationGetAttribute(self._inner, @intCast(pos)) }; - } + pub const BytecodeWriterConfig = struct { + desired_emit_version: ?usize = null, - pub fn getAttributeByName(self: Self, name_: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirOperationGetAttributeByName(self._inner, stringRef(name_))); - } - - pub fn setAttributeByName(self: Self, name_: [:0]const u8, attr: Attribute) void { - c.mlirOperationSetAttributeByName(self._inner, stringRef(name_), attr._inner); - } - - pub fn removeAttributeByName(self: Self, name_: [:0]const u8) bool { - return c.mlirOperationRemoveAttributeByName(self._inner, stringRef(name_)); - } - - /// Hash the canonicalized IR, without debug information that can change across builds. - pub fn hash(op: Operation, hasher: *std.hash.XxHash64) void { - // Note: before we where using op.writeBytecode(writer), - // but it crashes on some inputs, notably for unused variables. - // So we use the text representation of the mlir. - // See https://github.com/zml/zml/issues/97. - const flags = OpPrintingFlags.create(.{ .debug_info = false }); - defer c.mlirOpPrintingFlagsDestroy(flags); - - c.mlirOperationPrintWithFlags( - op._inner, - flags, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const _hasher: *std.hash.XxHash64 = @ptrCast(@alignCast(ctx_)); - _hasher.update(str.data[0..str.length]); - } - }).callback, - hasher, - ); - } -}; - -pub const OpPrintingFlags = struct { - elide_large_elements_attrs: ?usize = null, - debug_info: bool = false, - debug_info_pretty_form: bool = true, - print_generic_op_form: bool = false, - use_local_scope: bool = false, - assume_verified: bool = false, - - pub fn create(self: OpPrintingFlags) c.MlirOpPrintingFlags { - const pflags = c.mlirOpPrintingFlagsCreate(); - if (self.elide_large_elements_attrs) |v| { - c.mlirOpPrintingFlagsElideLargeElementsAttrs(pflags, @intCast(v)); - } - c.mlirOpPrintingFlagsEnableDebugInfo(pflags, self.debug_info, self.debug_info_pretty_form); - if (self.print_generic_op_form) { - c.mlirOpPrintingFlagsPrintGenericOpForm(pflags); - } - if (self.use_local_scope) { - c.mlirOpPrintingFlagsUseLocalScope(pflags); - } - if (self.assume_verified) { - c.mlirOpPrintingFlagsAssumeVerified(pflags); - } - return pflags; - } -}; - -pub const OpOperand = struct { - _inner: c.MlirOpOperand, - pub const wrapOr = helpers.wrapOr(OpOperand, c.mlirOpOperandIsNull); - - pub fn owner(self: OpOperand) Operation { - return .{ ._inner = c.mlirOpOperandGetOwner(self._inner) }; - } - - pub fn number(self: OpOperand) usize { - return @intCast(c.mlirOpOperandGetOperandNumber(self._inner)); - } - - pub fn nextUse(self: OpOperand) ?OpOperand { - return wrapOr(c.mlirOpOperandGetNextUse(self._inner)); - } -}; - -pub const Region = struct { - _inner: c.MlirRegion, - - pub const eql = helpers.eql(Region, c.mlirRegionEqual); - pub const deinit = helpers.deinit(Region, c.mlirRegionDestroy); - pub const wrapOr = helpers.wrapOr(Region, c.mlirRegionIsNull); - - const Self = Region; - - pub fn init() !Self { - return Self.wrapOr(c.mlirRegionCreate()) orelse Error.InvalidMlir; - } - - pub fn appendBlock(self: *Self, block: Block) void { - c.mlirRegionAppendOwnedBlock(self._inner, block._inner); - } - - pub fn insertBlock(self: *Self, index: isize, block: Block) void { - c.mlirRegionInsertOwnedBlock(self._inner, index, block._inner); - } - - pub fn insertBlockBefore(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockBefore(self._inner, reference._inner, block._inner); - } - - pub fn insertBlockAfter(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockAfter(self._inner, reference._inner, block._inner); - } - - pub fn firstBlock(self: Self) Block { - return .{ ._inner = c.mlirRegionGetFirstBlock(self._inner) }; - } -}; - -pub const Value = struct { - _inner: c.MlirValue, - - pub const dump = helpers.dump(Value, c.mlirValueDump); - pub const eql = helpers.eql(Value, c.mlirValueEqual); - pub const format = helpers.format(Value, c.mlirValuePrint); - pub const wrapOr = helpers.wrapOr(Value, c.mlirValueIsNull); - - pub fn getType(val: Value) Type { - return .{ ._inner = c.mlirValueGetType(val._inner) }; - } - - pub fn setType(val: *Value, typ: Type) void { - c.mlirValueSetType(val._inner, typ._inner); - } - - pub fn firstUse(val: Value) OpOperand { - return .{ ._inner = c.mlirValueGetFirstUse(val._inner) }; - } - - pub fn replaceAllUsesWith(val: Value, with: Value) void { - c.mlirValueReplaceAllUsesOfWith(val._inner, with._inner); - } - - pub fn owner(val: Value) Operation { - return .{ ._inner = c.mlirOpResultGetOwner(val._inner) }; - } - - pub fn isABlockArgument(val: Value) bool { - return c.mlirValueIsABlockArgument(val._inner); - } - - pub fn isAOpResult(val: Value) bool { - return c.mlirValueIsAOpResult(val._inner); - } - - pub const Kind = union(enum) { - block_argument: BlockArgument, - op_result: Operation, - null, + const Ctx = struct { + self: *const Operation, + config: ?BytecodeWriterConfig, + }; }; - pub fn kind(val: Value) Kind { - if (val.isAOpResult()) { - return .{ .op_result = val.owner() }; - } - if (val.isABlockArgument()) { - return .{ .block_argument = .{ ._inner = val._inner } }; - } - // From MLIR docs: - // https://mlir.llvm.org/doxygen/classmlir_1_1Value.html#details - // > An SSA value is either a BlockArgument or the result of an operation. - return .null; - } -}; + const WriteBytecodeError = error{InvalidMlirBytecodeVersion} || std.Io.Writer.Error; -pub const BlockArgument = struct { - _inner: c.MlirValue, - - pub fn block(arg: BlockArgument) Block { - return .{ ._inner = c.mlirBlockArgumentGetOwner(arg._inner) }; - } - - pub fn number(arg: BlockArgument) usize { - return @bitCast(c.mlirBlockArgumentGetArgNumber(arg._inner)); - } - - pub fn format(self: BlockArgument, writer: *std.Io.Writer) !void { - const value = Value{ ._inner = self._inner }; - return value.format(writer); - } -}; - -pub const Type = struct { - _inner: c.MlirType, - - pub const dump = helpers.dump(Type, c.mlirTypeDump); - pub const eql = helpers.eql(Type, c.mlirTypeEqual); - pub const format = helpers.format(Type, c.mlirTypePrint); - pub const wrapOr = helpers.wrapOr(Type, c.mlirTypeIsNull); - - pub fn parse(ctx: Context, str: [:0]const u8) !Type { - return Type.wrapOr( - c.mlirTypeParseGet(ctx._inner, stringRef(str)), - ) orelse Error.InvalidMlir; - } - - pub fn as(generic: Type, SpecificType: type) ?SpecificType { - if (@hasDecl(SpecificType, "is_a_fn")) { - return if (SpecificType.is_a_fn(generic._inner)) - .{ ._inner = generic._inner } - else - null; - } - @compileError("Mlir subclass of type need `is_a_fn` attribute: " ++ @typeName(SpecificType)); - } - - pub fn fromAny(SpecificType: type) fn (x: SpecificType) Type { - stdx.debug.assertComptime(@hasDecl(SpecificType, "asType"), "Type.fromAny expects a type subclass, got: {}. Missing `asAttr` declaration.", .{SpecificType}); - return struct { - fn cast(x: SpecificType) Type { - return .{ ._inner = x._inner }; + pub fn writeBytecode(self: *const Operation, config: ?BytecodeWriterConfig, writer: *std.Io.Writer) WriteBytecodeError!void { + if (config) |config_| { + const bc_config = c.mlirBytecodeWriterConfigCreate(); + defer c.mlirBytecodeWriterConfigDestroy(bc_config); + if (config_.desired_emit_version) |version| { + c.mlirBytecodeWriterConfigDesiredEmitVersion(bc_config, @intCast(version)); } - }.cast; - } + var sctx: StringCallbackCtx = .{ .writer = writer }; + const logical_result = c.mlirOperationWriteBytecodeWithConfig(self.ptr(), bc_config, stringCallback, &sctx); - pub fn eqlAny(SpecificType: type) fn (SpecificType, SpecificType) bool { - return struct { - fn eql(a: SpecificType, b: SpecificType) bool { - return a.asType().eql(b.asType()); + if (logical_result.value == 0) { + return WriteBytecodeError.InvalidMlirBytecodeVersion; } - }.eql; + + if (sctx.err) |err| { + return err; + } + return; + } + + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationWriteBytecode(self.ptr(), stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } } - pub fn index(ctx: Context) Type { - return IndexType.init(ctx).asType(); + fn fmtBytecodeImpl(ctx: BytecodeWriterConfig.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + try ctx.self.writeBytecode(ctx.config, writer); } - pub fn int(ctx: Context, int_type: IntegerTypes) Type { - return switch (int_type) { - .unknown => @panic("Unknown integer type"), - inline else => |t| IntegerType(t).init(ctx).asType(), - }; - } - - pub fn float(ctx: Context, float_type: FloatTypes) Type { - return switch (float_type) { - inline else => |t| FloatType(t).init(ctx).asType(), - }; - } - - pub fn complex(ctx: Context, complex_type: ComplexTypes) Type { - return switch (complex_type) { - .unknown => @panic("Unknown complex type can't be created like this"), // What's the point ? - inline else => |t| ComplexType(t).init(ctx).asType(), - }; - } - - pub fn tuple(ctx: Context, types: []const Type) Type { - return (TupleType.init(ctx, types) catch unreachable).asType(); - } - - pub fn function(ctx: Context, args: []const Type, results: []const Type) Type { - return (FunctionType.init(ctx, args, results) catch unreachable).asType(); - } - - pub fn tensor(dimensions: []const i64, elem_type: Type) Type { - return RankedTensorType.init(dimensions, elem_type).asType(); + pub fn fmtBytecode(self: *const Operation, config: ?BytecodeWriterConfig) std.fmt.Alt(BytecodeWriterConfig.Ctx, fmtBytecodeImpl) { + return .{ .data = .{ + .self = self, + .config = config, + } }; } }; -pub const IndexType = struct { - _inner: c.MlirType, +pub const Region = opaque { + const M = Methods(Region, c.MlirRegion); - pub const asType = Type.fromAny(IndexType); - pub const eql = Type.eqlAny(IndexType); - pub const format = helpers.format(IndexType, c.mlirTypePrint); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirRegionEqual); - pub fn init(ctx: Context) IndexType { - return .{ ._inner = c.mlirIndexTypeGet(ctx._inner) }; + pub fn init() *Region { + return @ptrCast(c.mlirRegionCreate().ptr); + } + + pub fn deinit(self: *Region) void { + c.mlirRegionDestroy(self.ptr()); + } + + pub fn appendOwnedBlock(self: *Region, block: *Block) void { + c.mlirRegionAppendOwnedBlock(self.ptr(), block.ptr()); + } + + pub fn insertOwnedBlock(self: *Region, index: usize, block: *Block) void { + c.mlirRegionInsertOwnedBlock(self.ptr(), @intCast(index), block.ptr()); + } + + pub fn insertOwnedBlockBefore(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockBefore(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn insertOwnedBlockAfter(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockAfter(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn firstBlock(self: *const Region) ?*Block { + return @ptrCast(c.mlirRegionGetFirstBlock(self.ptr()).ptr); + } + + pub fn takeBody(self: *Region, from: *Region) void { + c.mlirRegionTakeBody(self.ptr(), from.ptr()); + } + + pub fn nextInOperation(self: *const Region) *Region { + return @ptrCast(c.mlirRegionGetNextInOperation(self.ptr()).ptr); } }; -pub const IntegerTypes = enum { - i1, - i2, - i4, +pub const ShapedType = opaque { + const M = Methods(ShapedType, c.MlirType); + + pub const MAX_RANK = 16; + + pub const isAFn = c.mlirTypeIsAShaped; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn elementType(self: *const ShapedType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const ShapedType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const ShapedType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } +}; + +pub const RankedTensorType = opaque { + const M = Methods(RankedTensorType, c.MlirType); + + pub const isAFn = c.mlirTypeIsARankedTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(dimensions: []const i64, elemType: *const Type, encoding: ?*Attribute) *const RankedTensorType { + return @ptrCast(c.mlirRankedTensorTypeGet( + @intCast(dimensions.len), + @ptrCast(dimensions), + elemType.ptr(), + if (encoding) |e| e.ptr() else c.mlirAttributeGetNull(), + ).ptr); + } + + pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return get( + dims.constSlice(), + other.elementType(), + ); + } + + pub fn elementType(self: *const RankedTensorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const RankedTensorType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const RankedTensorType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn shaped(self: *const RankedTensorType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); +} + +pub fn rankedTensorTypeWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); +} + +pub const DenseElementsAttribute = opaque { + const M = Methods(DenseElementsAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADenseElements; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(shaped_type: *const ShapedType, values: anytype) *const DenseElementsAttribute { + const bytes = std.mem.sliceAsBytes(values); + return @ptrCast(c.mlirDenseElementsAttrRawBufferGet( + shaped_type.ptr(), + @intCast(bytes.len), + @ptrCast(bytes), + ).ptr orelse unreachable); + } +}; + +pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: anytype) *const Attribute { + return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); +} + +pub const DenseArrayTypes = enum { + bool, i8, i16, i32, i64, - si4, - si8, - si16, - si32, - si64, - u2, - u4, - u8, - u16, - u32, - u64, - - unknown, -}; - -pub fn IntegerType(comptime it: IntegerTypes) type { - const Config = switch (it) { - .i1 => .{ 1, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i2 => .{ 2, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i4 => .{ 4, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i8 => .{ 8, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i16 => .{ 16, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i32 => .{ 32, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i64 => .{ 64, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .si4 => .{ 4, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si8 => .{ 8, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si16 => .{ 16, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si32 => .{ 32, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si64 => .{ 64, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .u2 => .{ 2, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u4 => .{ 4, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u8 => .{ 8, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u16 => .{ 16, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u32 => .{ 32, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u64 => .{ 64, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .unknown => .{ 0, null, null }, - }; - - return struct { - _inner: c.MlirType, - - const Int = @This(); - pub const is_a_fn = switch (it) { - .unknown => c.mlirTypeIsAInteger, - else => typeIsAIntegerExact, - }; - - pub const asType = Type.fromAny(Int); - pub const eql = Type.eqlAny(Int); - pub const format = helpers.format(Int, c.mlirTypePrint); - - fn typeIsAIntegerExact(typ: c.MlirType) callconv(.c) bool { - const bit_width = Config[0]; - const is_sign = Config[2]; - return c.mlirTypeIsAInteger(typ) and (c.mlirIntegerTypeGetWidth(typ) == bit_width) and is_sign(typ); - } - - pub const BitWidth = Config[0]; - - pub const init = if (it != .unknown) struct { - pub fn init(ctx: Context) Int { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner, BitWidth) }; - } - }.init else {}; - }; -} - -pub const FloatTypes = enum { - f4e2m1fn, - f8e3m4, - f8e4m3, - f8e4m3b11fnuz, - f8e4m3fn, - f8e4m3fnuz, - f8e5m2, - f8e5m2fnuz, - f8e8m0fnu, - bf16, - f16, f32, f64, - pub fn asType(self: FloatTypes, ctx: Context) Type { - return switch (self) { - inline else => |ft| FloatType(ft).init(ctx).asType(), + fn ZigType(comptime ElementType: @This()) type { + return switch (ElementType) { + .bool => i32, + .i8 => i8, + .i16 => i16, + .i32 => i32, + .i64 => i64, + .f32 => f32, + .f64 => f64, }; } }; -pub fn FloatType(comptime ft: FloatTypes) type { - const Config: struct { - *const fn (c.MlirType) callconv(.c) bool, - *const fn (c.MlirContext) callconv(.c) c.MlirType, - } = switch (ft) { - .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, - .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, - .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, - .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, - .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, - .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, - .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, - .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, - .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, - .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, - .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, - .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, - .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, +pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { + const ElementTypeZig = ElementType.ZigType(); + + const _isAFn, const getFn, const getElementFn = switch (ElementType) { + .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, + .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, + .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, + .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, + .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, + .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, + .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, }; return struct { - _inner: c.MlirType, - const Self = @This(); - pub const is_a_fn = Config[0]; + const M = Methods(@This(), c.MlirAttribute); - pub const asType = Type.fromAny(Self); - pub const eql = Type.eqlAny(Self); - pub const format = helpers.format(Self, c.mlirTypePrint); + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); - pub fn init(ctx: Context) Self { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner) }; + pub fn init(ctx: *Context, values: []const ElementTypeZig) *const Self { + return @ptrCast(getFn(ctx.ptr(), @intCast(values.len), @ptrCast(values)).ptr); + } + + pub fn element(self: *const Self, pos: usize) ElementTypeZig { + return getElementFn(self.ptr(), @intCast(pos)); + } + + pub fn numElements(self: *const Self) usize { + return @intCast(c.mlirDenseArrayGetNumElements(self.ptr())); } }; } -pub const ComplexTypes = enum { - c64, - c128, - - unknown, -}; - -pub fn ComplexType(comptime ct: ComplexTypes) type { - return struct { - _inner: c.MlirType, - const Complex = @This(); - - fn mlirC64TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF32TypeGet(ctx)); - } - - fn mlirC128TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF64TypeGet(ctx)); - } - - fn mlirTypeIsAC64(typ: c.MlirType) callconv(.c) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF32(element_type); - } - - fn mlirTypeIsAC128(typ: c.MlirType) callconv(.c) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF64(element_type); - } - - const Config = switch (ct) { - .c64 => .{ mlirTypeIsAC64, mlirC64TypeGet }, - .c128 => .{ mlirTypeIsAC128, mlirC128TypeGet }, - .unknown => .{ c.mlirTypeIsAComplex, null }, - }; - - fn typeIsAUnknownComplex(typ: c.MlirType) callconv(.c) bool { - return c.mlirTypeIsAComplex(typ); - } - - pub const is_a_fn = Config[0]; - - pub const asType = Type.fromAny(Complex); - pub const eql = Type.eqlAny(Complex); - pub const format = helpers.format(Complex, c.mlirTypePrint); - pub const ComplexTypeType: ComplexTypes = ct; - - pub const init = if (ct != .unknown) struct { - pub fn init(ctx: Context) Complex { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner) }; - } - }.init else {}; - }; +pub fn denseArrayAttribute(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { + return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); } -pub const TupleType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsATuple; +pub const ArrayAttribute = opaque { + const M = Methods(ArrayAttribute, c.MlirAttribute); - pub fn init(ctx: Context, elements: []const Type) !TupleType { - const tuple_type = c.mlirTupleTypeGet( - ctx._inner, - @intCast(elements.len), - @ptrCast(elements.ptr), - ); - return .{ ._inner = .{ .ptr = tuple_type.ptr orelse return error.InvalidMlir } }; + pub const isAFn = c.mlirAttributeIsAArray; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attrs: []const *const Attribute) *const ArrayAttribute { + return @ptrCast(c.mlirArrayAttrGet(ctx.ptr(), @intCast(attrs.len), @ptrCast(attrs)).ptr); } - pub fn getNumTypes(self: TupleType) usize { - return @intCast(c.mlirTupleTypeGetNumTypes(self._inner)); + pub fn element(self: *const ArrayAttribute, index: usize) Attribute { + return @ptrCast(c.mlirArrayAttrGetElement(self.ptr(), @intCast(index)).ptr); } - pub fn getElementType(self: TupleType, index: usize) Type { - return .{ ._inner = c.mlirTupleTypeGetType(self._inner, @intCast(index)) }; + pub fn numElements(self: *const ArrayAttribute) usize { + return @intCast(c.mlirArrayAttrGetNumElements(self.ptr())); } - - pub const asType = Type.fromAny(TupleType); }; -pub const FunctionType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsAFunction; - pub const asType = Type.fromAny(FunctionType); - pub const eql = Type.eqlAny(FunctionType); +pub fn arrayAttribute(ctx: *Context, attrs: []const *const Attribute) *const Attribute { + return @ptrCast(ArrayAttribute.init(ctx, attrs)); +} - pub fn init(ctx: Context, args: []const Type, results: []const Type) !FunctionType { - const func_type = c.mlirFunctionTypeGet( - ctx._inner, +pub const FunctionType = opaque { + const M = Methods(FunctionType, c.MlirType); + pub const isAFn = c.mlirTypeIsAFunction; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(ctx: *Context, args: []const *const Type, results: []const *const Type) *const FunctionType { + return @ptrCast(c.mlirFunctionTypeGet( + ctx.ptr(), @intCast(args.len), @ptrCast(args.ptr), @intCast(results.len), @ptrCast(results.ptr), - ); - return .{ ._inner = .{ .ptr = func_type.ptr orelse return error.InvalidMlir } }; + ).ptr); } }; -pub const RankedTensorType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsARankedTensor; - pub const asType = Type.fromAny(RankedTensorType); - pub const eql = Type.eqlAny(RankedTensorType); - pub const format = helpers.format(RankedTensorType, c.mlirTypePrint); - - pub fn init(dimensions: []const i64, elemType: Type) RankedTensorType { - return .{ ._inner = c.mlirRankedTensorTypeGet( - @intCast(dimensions.len), - @ptrCast(dimensions.ptr), - elemType._inner, - c.mlirAttributeGetNull(), - ) }; - } - - pub fn getElementType(self: RankedTensorType) Type { - return .{ ._inner = c.mlirShapedTypeGetElementType(self._inner) }; - } - - pub fn getRank(self: RankedTensorType) usize { - return @intCast(c.mlirShapedTypeGetRank(self._inner)); - } - - pub fn getDimension(self: RankedTensorType, dim: usize) i64 { - return c.mlirShapedTypeGetDimSize(self._inner, @intCast(dim)); - } -}; - -pub const Dialect = struct { - _inner: c.MlirDialect, - - pub fn getContext(self: Dialect) Context { - return .{ ._inner = c.mlirDialectGetContext(self._inner) }; - } - - pub fn getNamespace(self: Dialect) []const u8 { - return fromStringRef(c.mlirDialectGetNamespace(self._inner)); - } -}; - -pub const DialectHandle = struct { - _inner: c.MlirDialectHandle, - - pub fn getNamespace(self: DialectHandle) []const u8 { - return fromStringRef(c.mlirDialectHandleGetNamespace(self._inner)); - } - - pub fn insertDialect(self: DialectHandle, registry: Registry) void { - c.mlirDialectHandleInsertDialect(self._inner, registry._inner); - } - - pub fn registerDialect(self: DialectHandle, ctx: Context) void { - c.mlirDialectHandleRegisterDialect(self._inner, ctx._inner); - } - - pub fn loadDialect(self: DialectHandle, ctx: Context) Dialect { - return .{ ._inner = c.mlirDialectHandleLoadDialect(self._inner, ctx._inner) }; - } - - pub fn fromString(comptime namespace: []const u8) DialectHandle { - return .{ ._inner = @field(c, "mlirGetDialectHandle__" ++ namespace ++ "__")() }; - } -}; - -pub const Location = struct { - _inner: c.MlirLocation, - - pub const eql = helpers.eql(Location, c.mlirLocationEqual); - pub const format = helpers.format(Location, c.mlirLocationPrint); - - pub fn fromSrc(ctx: Context, src: std.builtin.SourceLocation) Location { - return .{ ._inner = c.mlirLocationFileLineColGet( - ctx._inner, - stringRef(src.file), - @intCast(src.line), - @intCast(src.column), - ) }; - } - - pub fn fileLineCol(ctx: Context, file: []const u8, line: usize, column: usize) Location { - return .{ ._inner = c.mlirLocationFileLineColGet( - ctx._inner, - stringRef(file), - @intCast(line), - @intCast(column), - ) }; - } - - pub fn callSite(callee: Location, caller: Location) Location { - return .{ ._inner = c.mlirLocationCallSiteGet(callee._inner, caller._inner) }; - } - - pub fn fused(ctx: Context, locations: []const Location, metadata: Attribute) Location { - return .{ ._inner = c.mlirLocationFusedGet( - ctx._inner, - @intCast(locations.len), - @ptrCast(locations.ptr), - metadata._inner, - ) }; - } - - pub fn named(loc: Location, ctx: Context, loc_name: []const u8) Location { - return .{ ._inner = c.mlirLocationNameGet(ctx._inner, stringRef(loc_name), loc._inner) }; - } - - pub fn namedFmt(loc: Location, ctx: Context, comptime fmt: [:0]const u8, args: anytype) Location { - var buf: [256]u8 = undefined; - var writer: std.Io.Writer = .fixed(&buf); - writer.print(fmt, args) catch { - buf[256 - 3 ..].* = "...".*; - }; - return loc.named(ctx, writer.buffered()); - } - - pub fn unknown(ctx: Context) Location { - return .{ ._inner = c.mlirLocationUnknownGet(ctx._inner) }; - } -}; - -pub const Block = struct { - _inner: c.MlirBlock, - - pub const wrapOr = helpers.wrapOr(Block, c.mlirBlockIsNull); - pub const deinit = helpers.deinit(Block, c.mlirBlockDestroy); - pub const eql = helpers.eql(Block, c.mlirBlockEqual); - - pub fn init(args: []const Type, locs: []const Location) !Block { - const block = Block.wrapOr( - c.mlirBlockCreate(@intCast(args.len), @ptrCast(args.ptr), @ptrCast(locs.ptr)), - ); - return block orelse error.InvalidMlir; - } - - pub fn argument(self: Block, index: usize) Value { - return .{ ._inner = c.mlirBlockGetArgument(self._inner, @intCast(index)) }; - } - - pub fn numArguments(self: Block) usize { - return @intCast(c.mlirBlockGetNumArguments(self._inner)); - } - - pub fn addArgument(self: *Block, typ: Type, loc: Location) Value { - return .{ ._inner = c.mlirBlockAddArgument(self._inner, typ._inner, loc._inner) }; - } - - pub fn insertArgument(self: *Block, index: usize, typ: Type, loc: Location) Value { - return .{ ._inner = c.mlirBlockInsertArgument(self._inner, @intCast(index), typ._inner, loc._inner) }; - } - - pub fn equals(self: Block, other: Block) bool { - return c.mlirBlockEqual(self._inner, other._inner); - } - - pub fn appendOperation(self: Block, op: Operation) void { - c.mlirBlockAppendOwnedOperation(self._inner, op._inner); - } - - pub fn appendOperations(self: *Block, ops: []const Operation) void { - for (ops) |op| { - c.mlirBlockAppendOwnedOperation(self._inner, op._inner); - } - } - - pub const RecursiveOpts = enum { open, hermetic }; - - pub fn appendValueRecursive(self: Block, value: Value, opt: RecursiveOpts) void { - switch (value.kind()) { - .op_result => |parent_op| self.appendOperationRecursive(parent_op, opt), - .block_argument => |arg| { - // Hermetic blocks are not allowed to use arguments from other blocks. - stdx.debug.assert(opt == .open or self.eql(arg.block()), "Can't add {f} from {*} block to {*} block", .{ arg, arg.block()._inner.ptr, self._inner.ptr }); - }, - .null => @panic("InvalidMlir"), - } - } - - pub fn appendOperationRecursive(self: Block, op: Operation, opt: RecursiveOpts) void { - if (op.block()) |prev_block| { - // Hermetic blocks are not allowed to reference values from other blocks. - stdx.debug.assert(opt == .open or self.equals(prev_block), "Can't add {} from {*} block to {*} block", .{ op, prev_block._inner.ptr, self._inner.ptr }); - return; - } - for (0..op.numOperands()) |i| { - self.appendValueRecursive(op.operand(i), opt); - } - self.appendOperation(op); - } -}; - -pub const helpers = struct { - pub fn eql(T: type, equal_fn: fn (@FieldType(T, "_inner"), @FieldType(T, "_inner")) callconv(.c) bool) fn (T, T) bool { - return struct { - fn eql(a: T, b: T) bool { - return equal_fn(a._inner, b._inner); - } - }.eql; - } - - pub fn deinit(T: type, deinit_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (*T) void { - return struct { - fn deinit(a: *T) void { - deinit_fn(a._inner); - a.* = undefined; - } - }.deinit; - } - - pub fn dump(T: type, dump_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (T) void { - return struct { - fn dump(a: T) void { - return dump_fn(a._inner); - } - }.dump; - } - - pub fn isNull(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (T) bool { - return struct { - fn isNull(a: T) bool { - return is_null_fn(a._inner); - } - }.isNull; - } - - pub fn format( - Any: type, - print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void, - ) fn (Any, *std.Io.Writer) std.Io.Writer.Error!void { - return struct { - pub fn format(self: Any, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try callPrintFn(Any, self, print_fn, writer); - } - }.format; - } - - pub fn wrapOr(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (@FieldType(T, "_inner")) ?T { - return struct { - fn wrapOr(inner: @FieldType(T, "_inner")) ?T { - if (is_null_fn(inner)) return null; - return .{ ._inner = inner }; - } - }.wrapOr; - } -}; - -pub const MlirStrCallback = fn (c.MlirStringRef, ?*anyopaque) callconv(.c) void; - -pub fn callPrintFn( - T: type, - value: T, - print_fn: fn (@FieldType(T, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void, - writer: *std.Io.Writer, -) std.Io.Writer.Error!void { - var writer_with_err: WriterWithErr = .{ .writer = writer }; - print_fn(value._inner, &WriterWithErr.printCallback, &writer_with_err); - return writer_with_err.check(); +pub fn functionType(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { + return @ptrCast(FunctionType.init(ctx, args, results)); } -pub const WriterWithErr = struct { - writer: *std.Io.Writer, - err: ?std.Io.Writer.Error = null, +pub const FlatSymbolRefAttribute = opaque { + const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); - pub fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { - var ctx: *WriterWithErr = @ptrCast(@alignCast(opaque_ctx)); - if (ctx.err) |_| return; - ctx.writer.writeAll(fromStringRef(mlir_str)) catch |err| { - ctx.err = err; - return; - }; + pub const isAFn = c.mlirAttributeIsAFlatSymbolRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, symbol_: []const u8) *const FlatSymbolRefAttribute { + return @ptrCast(c.mlirFlatSymbolRefAttrGet(ctx.ptr(), stringRef(symbol_)).ptr); } - pub fn check(self: WriterWithErr) std.Io.Writer.Error!void { - if (self.err) |err| return err; + pub fn symbol(self: *const FlatSymbolRefAttribute) []const u8 { + return string(c.mlirFlatSymbolRefAttrGetValue(self.ptr())); } }; + +pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { + return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); +} + +pub const UnitAttribute = opaque { + const M = Methods(UnitAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAUnit; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn get(ctx: *Context) *const UnitAttribute { + return @ptrCast(c.mlirUnitAttrGet(ctx.ptr()).ptr); + } +}; + +pub fn unitAttribute(ctx: *Context) *const Attribute { + return @ptrCast(UnitAttribute.get(ctx)); +} + +pub const MemRefType = opaque { + const M = Methods(MemRefType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { + return @ptrCast(c.mlirMemRefTypeGet( + element_type.ptr(), + @intCast(shape.len), + @ptrCast(shape), + if (layout) |l| l.ptr() else .{}, + if (memory_space) |m| m.ptr() else .{}, + ).ptr); + } + + pub fn fromShaped(other: *const ShapedType) *const MemRefType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return init( + other.elementType(), + dims.constSlice(), + null, + null, + ); + } + + pub fn shaped(self: *const MemRefType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { + return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); +} From 0e2eb1f92c82b848e67cbe2d643922cf46750481 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 2 Dec 2025 18:00:48 +0100 Subject: [PATCH 3/6] Ported almost all previous Tensor operation to new mlir api * Tests are disabled for now and marked with TODOs * Missing operations are also marked with TODOs --- zml2/BUILD.bazel | 4 + zml2/dtype.zig | 290 +-- zml2/meta.zig | 776 ++++++++ zml2/mlirx.zig | 80 + zml2/shape.zig | 1097 +++++++++++ zml2/tensor.zig | 4510 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 6613 insertions(+), 144 deletions(-) create mode 100644 zml2/meta.zig create mode 100644 zml2/mlirx.zig create mode 100644 zml2/shape.zig create mode 100644 zml2/tensor.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 84eb6dda..90c95102 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -14,6 +14,10 @@ zig_library( "pjrtx.zig", "dtype.zig", "floats.zig", + "shape.zig", + "tensor.zig", + "mlirx.zig", + "meta.zig", ], zigopts = [ "-lc", diff --git a/zml2/dtype.zig b/zml2/dtype.zig index b14f483e..cb8af137 100644 --- a/zml2/dtype.zig +++ b/zml2/dtype.zig @@ -137,7 +137,7 @@ pub const DataType = enum(u8) { } pub fn toZigType(comptime dtype: DataType) type { - return @FieldType(Data, @tagName(dtype)); + return @FieldType(Value, @tagName(dtype)); } pub fn isSignedInt(dtype: DataType) bool { @@ -195,171 +195,171 @@ pub const DataType = enum(u8) { try std.testing.expectEqual(DataType.c64.resolvePeerType(.f32), null); } - pub fn zero(dtype: DataType) Data { - return Data.init(dtype, 0); + pub fn zero(dtype: DataType) Value { + return .init(dtype, 0); } - pub fn one(dtype: DataType) Data { - return Data.init(dtype, 1); + pub fn one(dtype: DataType) Value { + return .init(dtype, 1); } - pub fn minValue(dtype: DataType) Data { + pub fn minValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = false }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).zero), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).minus_inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), -std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.minInt(@FieldType(Data, @tagName(tag)))), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).zero), + inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).minus_inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), -std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.minInt(@FieldType(Value, @tagName(tag)))), inline else => |tag| @panic("Unsupported type: " ++ @tagName(tag)), }; } - pub fn maxValue(dtype: DataType) Data { + pub fn maxValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = true }, - inline .f4e2m1, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e3m4, .f8e4m3, .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), + inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.maxInt(@FieldType(Value, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), }; } - pub fn constant(dtype: DataType, value: anytype) Data { - return Data.init(dtype, value); + pub fn constant(dtype: DataType, value: anytype) Value { + return .init(dtype, value); } -}; -pub const Data = union(DataType) { - bool: bool, - f4e2m1: floats.Float4E2M1, - f8e3m4: floats.Float8E3M4, - f8e4m3: floats.Float8E4M3, - f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, - f8e4m3fn: floats.Float8E4M3FN, - f8e4m3fnuz: floats.Float8E4M3FNUZ, - f8e5m2: floats.Float8E5M2, - f8e5m2fnuz: floats.Float8E5M2FNUZ, - f8e8m0: floats.Float8E8M0, - bf16: floats.BFloat16, - f16: f16, - f32: f32, - f64: f64, - i2: i2, - i4: i4, - i8: i8, - i16: i16, - i32: i32, - i64: i64, - u2: u2, - u4: u4, - u8: u8, - u16: u16, - u32: u32, - u64: u64, - c64: C64, - c128: C128, + pub const Value = union(DataType) { + bool: bool, + f4e2m1: floats.Float4E2M1, + f8e3m4: floats.Float8E3M4, + f8e4m3: floats.Float8E4M3, + f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, + f8e4m3fn: floats.Float8E4M3FN, + f8e4m3fnuz: floats.Float8E4M3FNUZ, + f8e5m2: floats.Float8E5M2, + f8e5m2fnuz: floats.Float8E5M2FNUZ, + f8e8m0: floats.Float8E8M0, + bf16: floats.BFloat16, + f16: f16, + f32: f32, + f64: f64, + i2: i2, + i4: i4, + i8: i8, + i16: i16, + i32: i32, + i64: i64, + u2: u2, + u4: u4, + u8: u8, + u16: u16, + u32: u32, + u64: u64, + c64: C64, + c128: C128, - /// Creates `Data` from a `value`. - /// - /// If the `dtype` and `@TypeOf(value)` are incompatible - /// or a cast from `value` to `FieldType(dtype)` would - /// be lossy, a panic occurs. - pub fn init(dtype_: DataType, value: anytype) Data { - const T = @TypeOf(value); - const Ti = @typeInfo(T); + /// Creates `Data` from a `value`. + /// + /// If the `dtype` and `@TypeOf(value)` are incompatible + /// or a cast from `value` to `FieldType(dtype)` would + /// be lossy, a panic occurs. + pub fn init(dtype_: DataType, value: anytype) Value { + const T = @TypeOf(value); + const Ti = @typeInfo(T); - return switch (dtype_) { - .bool => switch (Ti) { - .bool => .{ .bool = value }, - .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, - else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), - }, - inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), - else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), - }, - inline .f16, .f32, .f64 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @floatFromInt(value)), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { - .comptime_int => blk: { - const OutT = @FieldType(Data, @tagName(tag)); - if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { - break :blk @unionInit(Data, @tagName(tag), @intCast(value)); - } else { - @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); - } + return switch (dtype_) { + .bool => switch (Ti) { + .bool => .{ .bool = value }, + .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, + else => @panic("Could not create Value of type bool from value of type " ++ @typeName(T)), }, - .int => @unionInit(Data, @tagName(tag), @intCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - .c64 => switch (T) { - C64 => .{ .c64 = value }, - C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - else => @panic("Could not create Data of type c64 from value of type " ++ @typeName(T)), - }, - .c128 => switch (T) { - C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - C128 => .{ .c128 = value }, - else => @panic("Could not create Data of type c128 from value of type " ++ @typeName(T)), - }, - }; - } - - test init { - try std.testing.expectEqual(20.0, Data.init(.f16, 20).f16); - try std.testing.expectEqual(20.5, Data.init(.f16, 20.5).f16); - try std.testing.expectEqual(20, Data.init(.f16, @as(u8, 20)).f16); - try std.testing.expectEqual(-20, Data.init(.f16, @as(i8, -20)).f16); - try std.testing.expectEqual(2000.5, Data.init(.f16, @as(f32, 2000.5)).f16); - - try std.testing.expectEqual(true, Data.init(.bool, true).bool); - - try std.testing.expectEqual(10, Data.init(.u8, 10).u8); - try std.testing.expectEqual(10, Data.init(.u8, @as(u16, 10)).u8); - - try std.testing.expectEqual(10, Data.init(.i8, 10).i8); - try std.testing.expectEqual(10, Data.init(.i8, @as(u16, 10)).i8); - try std.testing.expectEqual(-10, Data.init(.i8, -10).i8); - try std.testing.expectEqual(-10, Data.init(.i8, @as(i16, -10)).i8); - - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C64.init(1, 2)).c64); - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C128.init(1, 2)).c64); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C128.init(1, 2)).c128); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C64.init(1, 2)).c128); - } - - pub fn dtype(self: Data) DataType { - return std.meta.activeTag(self); - } - - pub fn constSlice(data: *const Data) []const u8 { - return switch (data.*) { - inline else => |*value| std.mem.asBytes(value), - }; - } - - pub fn as(self: Data, comptime T: type) T { - // TODO allow more lossless conversions - switch (@typeInfo(T)) { - .bool => return self.bool, - .float => switch (self) { - inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), - else => {}, - }, - .int => switch (self) { - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), - else => {}, - }, - else => {}, + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatFromInt(value))), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatCast(value))), + else => @panic("Could not create Value of type bf16 from value of type " ++ @typeName(T)), + }, + inline .f16, .f32, .f64 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @floatFromInt(value)), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @floatCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + .comptime_int => blk: { + const OutT = @FieldType(Value, @tagName(tag)); + if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { + break :blk @unionInit(Value, @tagName(tag), @intCast(value)); + } else { + @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + } + }, + .int => @unionInit(Value, @tagName(tag), @intCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + .c64 => switch (T) { + C64 => .{ .c64 = value }, + C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + else => @panic("Could not create Value of type c64 from value of type " ++ @typeName(T)), + }, + .c128 => switch (T) { + C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + C128 => .{ .c128 = value }, + else => @panic("Could not create Value of type c128 from value of type " ++ @typeName(T)), + }, + }; } - std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); - } + + test init { + try std.testing.expectEqual(20.0, Value.init(.f16, 20).f16); + try std.testing.expectEqual(20.5, Value.init(.f16, 20.5).f16); + try std.testing.expectEqual(20, Value.init(.f16, @as(u8, 20)).f16); + try std.testing.expectEqual(-20, Value.init(.f16, @as(i8, -20)).f16); + try std.testing.expectEqual(2000.5, Value.init(.f16, @as(f32, 2000.5)).f16); + + try std.testing.expectEqual(true, Value.init(.bool, true).bool); + + try std.testing.expectEqual(10, Value.init(.u8, 10).u8); + try std.testing.expectEqual(10, Value.init(.u8, @as(u16, 10)).u8); + + try std.testing.expectEqual(10, Value.init(.i8, 10).i8); + try std.testing.expectEqual(10, Value.init(.i8, @as(u16, 10)).i8); + try std.testing.expectEqual(-10, Value.init(.i8, -10).i8); + try std.testing.expectEqual(-10, Value.init(.i8, @as(i16, -10)).i8); + + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C64.init(1, 2)).c64); + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C128.init(1, 2)).c64); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C128.init(1, 2)).c128); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C64.init(1, 2)).c128); + } + + pub fn dtype(self: Value) DataType { + return std.meta.activeTag(self); + } + + pub fn constSlice(data: *const Value) []const u8 { + return switch (data.*) { + inline else => |*value| std.mem.asBytes(value), + }; + } + + pub fn as(self: Value, comptime T: type) T { + // TODO allow more lossless conversions + switch (@typeInfo(T)) { + .bool => return self.bool, + .float => switch (self) { + inline .f16, .f32, .f64 => |v| return @floatCast(v), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), + else => {}, + }, + .int => switch (self) { + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), + else => {}, + }, + else => {}, + } + std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); + } + }; }; pub fn mantissaSize(dtype: DataType) usize { @@ -373,3 +373,5 @@ pub fn mantissaSize(dtype: DataType) usize { else => @panic("Can't get mantissa size for a non-float dtype"), }; } + +pub const Data = DataType.Value; diff --git a/zml2/meta.zig b/zml2/meta.zig new file mode 100644 index 00000000..891a27ec --- /dev/null +++ b/zml2/meta.zig @@ -0,0 +1,776 @@ +const std = @import("std"); +const testing = std.testing; + +const stdx = @import("stdx"); +const FnParam = stdx.meta.FnParam; +const asSlice = stdx.meta.asSlice; + +test { + std.testing.refAllDecls(@This()); +} + +/// Visit a given type `T` and replace all fields containing `From` by fields containing `To`. +pub fn MapType(From: type, To: type) type { + return struct { + pub fn map(T: type) type { + switch (T) { + To => return To, + ?To => return ?To, + From => return To, + *From => return *To, + ?From => return ?To, + else => {}, + } + + return switch (@typeInfo(T)) { + .@"struct" => |struct_infos| { + const fields = struct_infos.fields; + var same: bool = true; + var struct_fields: [fields.len]std.builtin.Type.StructField = undefined; + for (struct_fields[0..], fields) |*struct_field, field| { + if (!field.is_comptime) { + const R = map(field.type); + if (R == field.type) { + struct_field.* = field; + } else { + struct_field.* = .{ + .name = field.name, + .type = R, + .default_value_ptr = null, + .is_comptime = field.is_comptime, + .alignment = @alignOf(R), + }; + same = false; + // Handle the case `field: ?Tensor = null` + // Generic handling of default value is complicated, + // it would require to call the callback at comptime. + if (R == ?To) { + struct_field.default_value_ptr = &@as(R, null); + } + } + } else { + struct_field.* = field; + } + } + if (same) return T; + return @Type(.{ .@"struct" = .{ + .layout = .auto, + .fields = struct_fields[0..], + .decls = &.{}, + .is_tuple = struct_infos.is_tuple, + } }); + }, + .@"union" => |union_info| { + const fields = union_info.fields; + var same: bool = true; + var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; + for (union_fields[0..], fields) |*union_field, field| { + const R = map(field.type); + if (R == field.type) { + union_field.* = field; + } else { + union_field.* = .{ + .name = field.name, + .type = R, + .alignment = @alignOf(R), + }; + same = false; + } + } + if (same) return T; + return @Type(.{ .@"union" = .{ + .layout = .auto, + .tag_type = union_info.tag_type, + .fields = union_fields[0..], + .decls = &.{}, + } }); + }, + .array => |arr_info| [arr_info.len]map(arr_info.child), + .pointer => |ptr_info| switch (ptr_info.size) { + .slice => if (ptr_info.is_const) + []const map(ptr_info.child) + else + []map(ptr_info.child), + .one => if (ptr_info.is_const) + *const map(ptr_info.child) + else + *map(ptr_info.child), + else => T, + }, + .optional => |opt_info| ?map(opt_info.child), + else => T, + }; + } + }; +} + +test MapType { + const A = struct { a: u32 }; + const B = struct { b: u32 }; + + const A2B = MapType(A, B); + + const StructA = struct { some: []const A, one: A, maybe: ?A, other: u32 }; + const struct_b = A2B.map(StructA){ + .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } }, + .maybe = null, + .one = .{ .b = 2 }, + .other = 43, + }; + _ = struct_b; + + const UnionA = union { some: []const A, one: A, maybe: ?A, other: u32 }; + const union_b = [_]A2B.map(UnionA){ + .{ .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } } }, + .{ .one = .{ .b = 2 } }, + .{ .maybe = null }, + .{ .other = 43 }, + }; + _ = union_b; +} + +/// Given a callback: `fn(Ctx, From) To`, recursively visits the given `from` struct +/// and calls the callback when it finds a `From` element, and writes it to the `to` struct. +/// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. +/// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. +/// For example it can convert from a comptime array to a runtime slice. +/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. +/// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. +/// +/// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. +/// Any `To` struct inside `from` will be copied over to the target. +pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam(cb, 0), from: anytype, to: anytype) !void { + // TODO: handle tuple to slice conversion + // TODO: handle error bubbling up + const From = FnParam(cb, 1); + const To = stdx.meta.FnResult(cb); + const FromStruct = @TypeOf(from); + const type_info_to_ptr = @typeInfo(@TypeOf(to)); + if (type_info_to_ptr != .pointer) { + stdx.debug.compileError("convertType is expecting a mutable `to` argument but received: {}", .{@TypeOf(to)}); + } + const ToStruct = type_info_to_ptr.pointer.child; + const type_info_to = @typeInfo(ToStruct); + + if (FromStruct == From) { + // We have an issues with `Tensor` -> `Shape` -> `Tensor` conversion when compiling ZML functions where one argument is a Shape itself. + // Normally we should call `cb` on all `Shape`. + // But the "ShapeOf" struct will have more Shape than need on the output. + // So here we take a hint from the receiving object. + // If the target is indeed a Tensor, use the callback, but if the target is `Shape` just copy it over. + if (ToStruct != To and FromStruct == ToStruct) { + to.* = from; + } else { + to.* = @call(.auto, cb, .{ ctx, from }); + } + return; + } + + if (FromStruct == To) { + to.* = from; + return; + } + + if (@sizeOf(ToStruct) == 0) return; + + switch (type_info_to) { + .@"struct" => |info| inline for (info.fields) |field| { + if (field.is_comptime or @sizeOf(field.type) == 0) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + switch (field_type_info) { + // .pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), + .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field.name)) { + try mapAlloc( + cb, + allocator, + ctx, + @field(from, field.name), + &@field(to, field.name), + ); + } else if (field.default_value_ptr) |_| { + @field(to, field.name) = null; + } else { + stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); + }, + else => @field(to, field.name) = @field(from, field.name), + } + }, + .@"union" => { + switch (from) { + inline else => |_, tag| { + to.* = @unionInit(ToStruct, @tagName(tag), undefined); + try mapAlloc(cb, allocator, ctx, @field(from, @tagName(tag)), &@field(to, @tagName(tag))); + }, + } + }, + .array => for (from, to) |f, *t| { + try mapAlloc(cb, allocator, ctx, f, t); + }, + .pointer => |ptr_info| switch (ptr_info.size) { + .one => switch (type_info_to_ptr.pointer.size) { + // pointer to array -> slice promotion + .slice => { + const items = try allocator.alloc(type_info_to_ptr.pointer.child, from.len); + for (from, items) |f, *t| { + try mapAlloc(cb, allocator, ctx, f, t); + } + to.* = items; + }, + else => try mapAlloc(cb, allocator, ctx, from.*, to.*), + }, + .slice => { + const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); + for (from, items) |f, *t| { + try mapAlloc(cb, allocator, ctx, f, t); + } + to.* = items; + }, + else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), + }, + .optional => if (from) |f| { + to.* = @as(@typeInfo(type_info_to_ptr.pointer.child).optional.child, undefined); + try mapAlloc(cb, allocator, ctx, f, &(to.*.?)); + } else { + to.* = null; + }, + .int, .float, .@"enum" => to.* = from, + else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), + } +} + +test mapAlloc { + const B = struct { b: u8 }; + const A = struct { + a: u8, + pub fn convert(_: void, a: @This()) B { + return .{ .b = a.a }; + } + }; + + const Empty = struct {}; + + const AA = struct { + field: A, + array: [2]A, + slice: []const A, + other: u8, + // We want to allow conversion from comptime to runtime, because Zig type inference works like this. + comptime static_val: u8 = 8, + comptime static_slice: [2]A = .{ .{ .a = 11 }, .{ .a = 12 } }, + field_with_empty: struct { A, Empty }, + }; + const BB = struct { + field: B, + array: [2]B, + slice: []const B, + other: u8, + static_val: u8, + static_slice: []B, + field_with_empty: struct { B, Empty }, + }; + + const aa: AA = .{ + .field = .{ .a = 4 }, + .array = .{ .{ .a = 5 }, .{ .a = 6 } }, + .other = 7, + .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, + .field_with_empty = .{ .{ .a = 9 }, .{} }, + }; + var bb: BB = undefined; + + try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); + defer testing.allocator.free(bb.slice); + defer testing.allocator.free(bb.static_slice); + + try testing.expectEqual(4, bb.field.b); + try testing.expectEqual(5, bb.array[0].b); + try testing.expectEqual(6, bb.array[1].b); + try testing.expectEqual(7, bb.other); + try testing.expectEqual(8, bb.static_val); + try testing.expectEqual(9, bb.slice[0].b); + try testing.expectEqual(10, bb.slice[1].b); + try testing.expectEqual(11, bb.static_slice[0].b); + try testing.expectEqual(12, bb.static_slice[1].b); +} + +/// Visit a given type `T` and: +/// * replace all fields containing `From` by fields containing `To` +/// * drop all fields not containing any `From`. +/// The returned type will contains only `To` making it easy for the compiler to produce compact layout. +/// Used by `zml.Bufferized` to strip compile time arguments from a model struct. +pub fn MapRestrict(From: type, To: type) type { + return struct { + pub fn map(T: type) type { + switch (T) { + From => return To, + ?From => return ?To, + *From => return *To, + *const From => return *const To, + []From => return []To, + []const From => return []const To, + else => {}, + } + + if (!Contains(T, From)) return void; + + return switch (@typeInfo(T)) { + .@"struct" => |struct_infos| { + // We know that at least one of the struct field contains a From. + // We map each field individually. Fields without From and comptime fields are removed. + const fields = struct_infos.fields; + var num_fields: usize = 0; + + var struct_fields: [fields.len]std.builtin.Type.StructField = undefined; + for (fields) |field| { + if (!field.is_comptime and Contains(field.type, From)) { + const R = map(field.type); + if (R == field.type) { + struct_fields[num_fields] = field; + } else { + const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; + struct_fields[num_fields] = .{ + .name = name, + .type = R, + .default_value_ptr = null, + .is_comptime = false, + .alignment = @alignOf(R), + }; + // Handle the case `field: ?Tensor = null` + // Generic handling of default value is not possible. + if (R == ?To) { + struct_fields[num_fields].default_value_ptr = &@as(R, null); + } + } + num_fields += 1; + } + } + if (num_fields == 0) return void; + return @Type(.{ .@"struct" = .{ + .layout = .auto, + .fields = struct_fields[0..num_fields], + .decls = &.{}, + .is_tuple = struct_infos.is_tuple, + } }); + }, + .@"union" => |union_info| { + // We know that at least one of the union field contains a From. + // We map each field individually. Fields without From, are replaced by "void". + const fields = union_info.fields; + var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; + for (0.., fields) |i, field| { + const FT = map(field.type); + union_fields[i] = .{ + .name = field.name, + .type = FT, + .alignment = @alignOf(FT), + }; + } + return @Type(.{ .@"union" = .{ + .layout = .auto, + .tag_type = union_info.tag_type, + .fields = union_fields[0..], + .decls = &.{}, + } }); + }, + .array => |arr_info| [arr_info.len]map(arr_info.child), + .pointer => |ptr_info| switch (ptr_info.size) { + .slice => if (ptr_info.is_const) + []const map(ptr_info.child) + else + []map(ptr_info.child), + .one => if (ptr_info.is_const) + *const map(ptr_info.child) + else + *map(ptr_info.child), + .many => if (ptr_info.is_const) + [*]const map(ptr_info.child) + else + [*]map(ptr_info.child), + .c => if (ptr_info.is_const) + [*c]map(ptr_info.child) + else + [*c]map(ptr_info.child), + }, + .optional => |opt_info| ?map(opt_info.child), + else => T, + }; + } + }; +} + +test MapRestrict { + const A = struct { a: u32 }; + const B = struct { b: u32 }; + + const A2B = MapRestrict(A, B); + + const StructA = struct { some: []const A, one: A, maybe: ?A, other: u32 }; + const struct_b = A2B.map(StructA){ + .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } }, + .maybe = null, + .one = .{ .b = 2 }, + // Note how struct_b doesn't even have a .other field now. + }; + _ = struct_b; + + const UnionA = union { some: []const A, one: A, maybe: ?A, other: u32 }; + const union_b = [_]A2B.map(UnionA){ + .{ .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } } }, + .{ .one = .{ .b = 2 } }, + .{ .maybe = null }, + // Note how union_b.other is void now. + .{ .other = {} }, + }; + _ = union_b; +} + +/// Recursively visit the given struct and calls the callback for each K found. +/// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. +pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { + const Callback = @TypeOf(cb); + const Ptr = @TypeOf(v); + const type_info_v = @typeInfo(Ptr); + if (type_info_v != .pointer) { + stdx.debug.compileError("zml.meta.visit({}) is expecting a pointer/slice input, but received: {}", .{ Callback, Ptr }); + } + const ptr_info = type_info_v.pointer; + const Child = ptr_info.child; + + const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { + .pointer => |info| .{ info.child, !info.is_const }, + else => stdx.debug.compileError("zml.meta.visit is expecting a callback with a pointer as second argument but found {}", .{FnParam(cb, 1)}), + }; + // Abort if v doesnt' contain any K. + if (comptime !Contains(Ptr, K)) return; + + // Handle simple cases. + switch (Ptr) { + *const K, *K => return cb(ctx, v), + *const ?K, *?K => return if (v.*) |*val| cb(ctx, val) else {}, + []const K, []K => { + for (v) |*v_elem| cb(ctx, v_elem); + return; + }, + else => {}, + } + + // Handle stdx.BoundedArray that contains uninitalized data. + if (@typeInfo(Child) == .@"struct" and @hasDecl(Child, "constSlice") and @hasDecl(Child, "slice")) { + return visit(cb, ctx, if (mutating_cb) v.slice() else v.constSlice()); + } + + // Recursively visit fields of v. + switch (ptr_info.size) { + .one => switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + switch (field_type_info) { + .pointer => visit(cb, ctx, @field(v, field.name)), + .array, .optional, .@"union", .@"struct" => visit(cb, ctx, &@field(v, field.name)), + else => {}, + } + }, + .array => |_| for (v) |*elem| visit(cb, ctx, elem), + .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .@"union" => switch (v.*) { + inline else => |*v_field| visit(cb, ctx, v_field), + }, + else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), + }, + .slice => { + for (v) |*v_elem| { + switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + if (field_type_info == .pointer) { + visit(cb, ctx, @field(v_elem, field.name)); + } else { + visit(cb, ctx, &@field(v_elem, field.name)); + } + }, + .array => |_| for (v) |*elem| visit(cb, ctx, elem), + .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .@"union" => switch (v_elem.*) { + inline else => |*v_field| visit(cb, ctx, v_field), + }, + else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), + } + } + }, + .many, .c => stdx.debug.compileError("zml.meta.visit({}) doesn't support [*] style pointers, got: {}", .{ Callback, Ptr }), + } +} + +test visit { + const Attr = struct { data: usize }; + const OtherAttr = struct { other: []const u8 }; + const NestedAttr = struct { nested: Attr }; + const NestedAttrOptional = struct { nested: ?Attr }; + const SimpleStruct = struct { prop: Attr }; + const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; + const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; + + const LocalContext = struct { result: usize }; + + { + var context: LocalContext = .{ .result = 0 }; + const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *const Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 } }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *const Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(3, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + const prop5: stdx.BoundedArray(Attr, 8) = .{ + .buffer = @splat(.{ .data = 4 }), + .len = 2, + }; + const container: NestedTypesStruct = .{ + .prop1 = .{ .data = 1 }, + .prop2 = .{ .other = "hello" }, + .prop3 = .{ .nested = .{ .data = 2 } }, + .prop4 = .{ .nested = .{ .data = 3 } }, + .prop5 = prop5, // 4 will be counted twice. + }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *const Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(14, context.result); + } +} + +pub fn count(T: type, value: anytype) u32 { + var counter: u32 = 0; + visit(struct { + pub fn cb(res: *u32, _: *const T) void { + res.* += 1; + } + }.cb, &counter, value); + return counter; +} + +pub fn first(T: type, value: anytype) T { + var res: ?T = null; + visit(struct { + pub fn cb(res_ptr: *?T, x: *const T) void { + if (res_ptr.* == null) res_ptr.* = x.*; + } + }.cb, &res, &value); + return res.?; +} + +/// Given a `fn([]const T, Args) T` and a slice of values, +/// will combine all values in one value. +/// Only T elements of values will be looked at. +/// This only works for simple types, in particular `zip` doesn't follow pointers. +/// Which means that zip only allocate temp memory, and nothing need to be freed after the call. +pub fn zip(comptime func: anytype, allocator: std.mem.Allocator, values: anytype, args: anytype) error{OutOfMemory}!asSlice(@TypeOf(values)) { + const sliceT = @typeInfo(FnParam(func, 0)); + const T = sliceT.pointer.child; + const V = asSlice(@TypeOf(values)); + if (V == T) { + return @call(.auto, func, .{values} ++ args); + } + // const fn_args + + return switch (@typeInfo(V)) { + .pointer => stdx.debug.compileError("zip only accept by value arguments. Received: {}", .{V}), + .@"struct" => |struct_info| { + var out: V = values[0]; + inline for (struct_info.fields) |f| { + if (f.is_comptime) continue; + if (@typeInfo(f.type) == .pointer) { + stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); + } + var fields = try allocator.alloc(f.type, values.len); + defer allocator.free(fields); + for (values, 0..) |val, i| { + fields[i] = @field(val, f.name); + } + @field(out, f.name) = try zip(func, allocator, fields, args); + } + return out; + }, + .array => |arr_info| { + if (@typeInfo(arr_info.child) == .pointer) { + stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); + } + var out: V = undefined; + var slice = try allocator.alloc(arr_info.child, values.len); + defer allocator.free(slice); + for (&out, 0..) |*o, j| { + for (values, 0..) |val, i| { + slice[i] = val[j]; + } + o.* = try zip(func, allocator, slice, args); + } + return out; + }, + .@"union", .optional => stdx.debug.compileError("zip doesn't yet support {}", .{V}), + else => values[0], + }; +} + +test zip { + const A = struct { a: u8, b: [2]u8 }; + const a0: A = .{ .a = 1, .b = .{ 2, 3 } }; + const a1: A = .{ .a = 4, .b = .{ 5, 6 } }; + + const Sum = struct { + pub fn call(x: []const u8) u8 { + var res: u8 = 0; + for (x) |xx| res += xx; + return res; + } + }; + const a_sum: A = try zip(Sum.call, testing.allocator, &[_]A{ a0, a1 }, .{}); + try testing.expectEqual(A{ .a = 5, .b = .{ 7, 9 } }, a_sum); +} + +/// Given a func(X) -> Y or a func(Ctx, X) -> Y, +/// finds all X in the given object, and write the result of func(X) into an arraylist. +pub fn collectAlloc( + func: anytype, + func_ctx: _CollectCtx(func), + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]stdx.meta.FnSignature(func, null).ReturnT { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); + + const CollectAllocCtx = struct { + func_ctx: _CollectCtx(func), + allocator: std.mem.Allocator, + out: std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT) = .empty, + oom: bool = false, + + fn cb(ctx: *@This(), val: *const _CollectArg(func)) void { + if (ctx.oom) return; + const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); + ctx.out.append(ctx.allocator, res) catch { + ctx.oom = true; + }; + } + }; + var context = CollectAllocCtx{ .func_ctx = func_ctx, .allocator = allocator }; + visit(CollectAllocCtx.cb, &context, obj); + if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); +} + +/// Given a func(X) -> Y or a func(Ctx, X) -> Y, +/// finds all X in the given object, and write the result of func(X) into a given slice. +/// Asserts that the number of X found is equal to the slice len. +pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnResult(func)) void { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); + const LocalContext = struct { + func_ctx: _CollectCtx(func), + out: @TypeOf(out), + idx: usize = 0, + }; + var context = LocalContext{ .func_ctx = func_ctx, .out = out }; + visit((struct { + fn cb(ctx: *LocalContext, val: *const _CollectArg(func)) void { + if (ctx.idx >= ctx.out.len) return; + + const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); + ctx.out[ctx.idx] = res; + ctx.idx += 1; + } + }).cb, &context, obj); + std.debug.assert(context.idx == context.out.len); +} + +fn _CollectCtx(func: anytype) type { + const params = @typeInfo(@TypeOf(func)).@"fn".params; + if (params.len == 1) return void; + return params[0].type orelse @compileError("anytype not supported in collect"); +} + +fn _CollectArg(func: anytype) type { + const params = @typeInfo(@TypeOf(func)).@"fn".params; + return params[params.len - 1].type orelse @compileError("anytype not supported in collect"); +} + +pub fn Contains(Haystack: type, T: type) bool { + switch (Haystack) { + T, ?T => return true, + *T, ?*T => return true, + *const T, ?*const T => return true, + []const T, ?[]const T => return true, + anyopaque => return false, + else => {}, + } + + return switch (@typeInfo(Haystack)) { + .@"struct" => |info| { + inline for (info.fields) |field| { + if (!field.is_comptime and Contains(field.type, T)) + return true; + } + return false; + }, + .@"union" => |info| { + inline for (info.fields) |field| { + if (Contains(field.type, T)) + return true; + } + return false; + }, + .array => |info| Contains(info.child, T), + .pointer => |info| Contains(info.child, T), + .optional => |info| Contains(info.child, T), + .vector => |info| Contains(info.child, T), + else => false, + }; +} diff --git a/zml2/mlirx.zig b/zml2/mlirx.zig new file mode 100644 index 00000000..2224b88d --- /dev/null +++ b/zml2/mlirx.zig @@ -0,0 +1,80 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +const dtype = @import("dtype.zig"); +const Shape = @import("shape.zig").Shape; + +const mlirx = @This(); + +pub const Type = struct { + pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { + return switch (dt) { + .bool => mlir.integerType(ctx, .i1), + .f4e2m1 => mlir.floatType(ctx, .f4e2m1fn), + .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), + .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), + .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), + .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), + .bf16 => mlir.floatType(ctx, .bf16), + .f16 => mlir.floatType(ctx, .f16), + .f32 => mlir.floatType(ctx, .f32), + .f64 => mlir.floatType(ctx, .f64), + .i4 => mlir.integerType(ctx, .i4), + .i8 => mlir.integerType(ctx, .i8), + .i16 => mlir.integerType(ctx, .i16), + .i32 => mlir.integerType(ctx, .i32), + .i64 => mlir.integerType(ctx, .i64), + .u4 => mlir.integerType(ctx, .u4), + .u8 => mlir.integerType(ctx, .u8), + .u16 => mlir.integerType(ctx, .u16), + .u32 => mlir.integerType(ctx, .u32), + .u64 => mlir.integerType(ctx, .u64), + .c64 => mlir.complexType(ctx, .c64), + .c128 => mlir.complexType(ctx, .c128), + }; + } + + // TODO(Corentin): Maybe remove that as it now requires a mlir context. + // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. + pub fn toDType(mlir_type: *const mlir.Type) dtype.DataType { + const mapping = .{ + .{ .bool, mlir.IntegerType(.i1) }, + + .{ .f8e4m3b11fnuz, mlir.FloatType(.f8e4m3b11fnuz) }, + .{ .f8e4m3fn, mlir.FloatType(.f8e4m3fn) }, + .{ .f8e4m3fnuz, mlir.FloatType(.f8e4m3fnuz) }, + .{ .f8e5m2, mlir.FloatType(.f8e5m2) }, + .{ .f8e5m2fnuz, mlir.FloatType(.f8e5m2fnuz) }, + .{ .bf16, mlir.FloatType(.bf16) }, + .{ .f16, mlir.FloatType(.f16) }, + .{ .f32, mlir.FloatType(.f32) }, + .{ .f64, mlir.FloatType(.f64) }, + + .{ .i4, mlir.IntegerType(.i4) }, + .{ .i8, mlir.IntegerType(.i8) }, + .{ .i16, mlir.IntegerType(.i16) }, + .{ .i32, mlir.IntegerType(.i32) }, + .{ .i64, mlir.IntegerType(.i64) }, + + .{ .u4, mlir.IntegerType(.u4) }, + .{ .u8, mlir.IntegerType(.u8) }, + .{ .u16, mlir.IntegerType(.u16) }, + .{ .u32, mlir.IntegerType(.u32) }, + .{ .u64, mlir.IntegerType(.u64) }, + + .{ .c64, mlir.ComplexType(.c64) }, + .{ .c128, mlir.ComplexType(.c128) }, + }; + + inline for (mapping) |entry| { + const dt, const mlirT = entry; + if (mlirT.isAFn(mlir_type._inner)) { + return dt; + } + } + + std.debug.panic("Could not convert mlir.Type to DataType: {}", .{mlir_type}); + } +}; diff --git a/zml2/shape.zig b/zml2/shape.zig new file mode 100644 index 00000000..564f33b5 --- /dev/null +++ b/zml2/shape.zig @@ -0,0 +1,1097 @@ +const std = @import("std"); +const testing = std.testing; +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const DataType = @import("dtype.zig").DataType; + +const EnumLiteral = @TypeOf(.enum_literal); + +const log = std.log.scoped(.shape); + +test { + std.testing.refAllDecls(Shape); +} + +/// Represent the shape of a tensor. +pub const Shape = struct { + pub const MAX_RANK: u8 = 8; + + pub const Tag = [*:0]const u8; + pub const TagUnknown = "_".ptr; + const TagLast = "last".ptr; + + pub const DimsArray = stdx.BoundedArray(i64, MAX_RANK); + pub const TagsArray = stdx.BoundedArray(Tag, MAX_RANK); + pub const AxesArray = stdx.BoundedArray(u3, MAX_RANK); + pub const ShardingInfo = @Vector(MAX_RANK, bool); + + const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** MAX_RANK }; + + _dtype: DataType, + _dims: DimsArray = .{}, + _tags: TagsArray = UnknownTags, + _sharding_info: ShardingInfo = @splat(false), + + pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { + const T = @TypeOf(v); + + if (T == Shape) { + return .{ v._dims, v._tags }; + } + + if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger)) { + var dims_ = DimsArray.init(0) catch unreachable; + var tags_ = TagsArray.init(0) catch unreachable; + for (v) |d| { + dims_.appendAssumeCapacity(@intCast(d)); + tags_.appendAssumeCapacity(TagUnknown); + } + return .{ dims_, tags_ }; + } + + if (comptime stdx.meta.isStruct(T)) { + var dims_: DimsArray = .{}; + var tags_: TagsArray = .{}; + inline for (std.meta.fields(T)) |field| { + const fv = @field(v, field.name); + if (comptime stdx.meta.isInteger(field.type)) { + dims_.appendAssumeCapacity(@intCast(fv)); + } else if (@TypeOf(fv) == EnumLiteral and comptime isAutoDim(fv)) { + dims_.appendAssumeCapacity(-1); + } else { + stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); + } + if (comptime stdx.meta.isTuple(T)) { + tags_.appendAssumeCapacity(TagUnknown); + } else { + tags_.appendAssumeCapacity(toTag(field)); + } + } + + return .{ dims_, tags_ }; + } + + stdx.debug.compileError("expected a dimension tuple eg '.{{ .a = 10, .b = 20}}' or '.{{ 10, 20 }}', got {}", .{T}); + } + + test parseDimensions { + const ref = Shape.init(.{ .a = 0, .b = 1 }, .f32); + const dims_, const tags_ = parseDimensions(.{ .a = 0, .b = 1 }); + + try testing.expectEqualSlices(i64, ref.dims(), dims_.constSlice()); + try testing.expectEqualSlices(Tag, ref.tags(), tags_.constSlice()); + } + + pub fn parseAxes(self: Shape, v: anytype) struct { AxesArray, TagsArray } { + const T = @TypeOf(v); + + if (T == Shape) { + return self.parseAxes(self.tags()); + } + + var axes_ = AxesArray.init(0) catch unreachable; + var tags_ = TagsArray.init(0) catch unreachable; + + if (comptime stdx.meta.isSliceOfAny(T, isAxisConvertible)) { + for (v) |d| { + axes_.appendAssumeCapacity(self.axis(d)); + tags_.appendAssumeCapacity(self.tag(d)); + } + return .{ axes_, tags_ }; + } + + if (comptime stdx.meta.isTupleOfAny(T, isAxisConvertible)) { + inline for (std.meta.fields(T)) |field| { + axes_.appendAssumeCapacity(self.axis(@field(v, field.name))); + tags_.appendAssumeCapacity(self.tag(@field(v, field.name))); + } + return .{ axes_, tags_ }; + } + + stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); + } + + pub fn parseTags(v: anytype) TagsArray { + const T = @TypeOf(v); + stdx.debug.assertComptime(stdx.meta.isTupleOf(T, EnumLiteral), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); + var tags_ = TagsArray.init(0) catch unreachable; + inline for (v) |field| { + tags_.appendAssumeCapacity(toTag(field)); + } + return tags_; + } + + /// Create a shape from a struct literal, eg: + /// Shape.init(.{ .h = 1024, .w = 512, .c = 3 }); + /// Shape.init(.{ 1024, 512, 3 }); + pub fn init(dimz: anytype, dt: DataType) Shape { + var res: Shape = .{ ._dtype = dt }; + res._dims, res._tags = parseDimensions(dimz); + return res; + } + + pub fn scalar(dt: DataType) Shape { + return .{ ._dtype = dt }; + } + + /// Creates a Shape with dims set to `.{0, 1, 2, ..., rank-1}`. + pub fn range(rank_: usize, dt: DataType) Shape { + var res: Shape = .{ ._dtype = dt }; + for (0..rank_) |i| { + res._dims.append(@intCast(i)) catch { + stdx.debug.panic("Too many dimensions! Max: {d}, passed: {d}", .{ res._dims.capacity(), rank_ }); + }; + res._tags.append(TagUnknown) catch unreachable; + } + return res; + } + + pub fn dtype(self: Shape) DataType { + return self._dtype; + } + + pub fn rank(self: Shape) u4 { + self.ensureDimsAndTagsAreSync(); + return @intCast(self._dims.len); + } + + pub fn dim(self: Shape, ax: anytype) i64 { + self.ensureDimsAndTagsAreSync(); + return self._dims.get(self.axis(ax)); + } + + pub fn dims(self: *const Shape) []const i64 { + self.ensureDimsAndTagsAreSync(); + return self._dims.constSlice(); + } + + fn isAxisConvertible(comptime T: type) bool { + return stdx.meta.isInteger(T) or isTagConvertible(T); + } + + fn isTagConvertible(comptime T: type) bool { + return switch (T) { + EnumLiteral => true, + std.builtin.Type.StructField => true, + Tag => true, + else => false, + }; + } + + fn toTag(v: anytype) Tag { + const T = @TypeOf(v); + return switch (T) { + EnumLiteral => @tagName(v).ptr, + std.builtin.Type.StructField => v.name.ptr, + Tag => v, + else => stdx.debug.compileError("Shape tag should be an EnumLiteral, a Shape.Tag or a StructField, got {}", .{T}), + }; + } + + inline fn ensureDimsAndTagsAreSync(self: Shape) void { + stdx.debug.assert(self._dims.len == self._tags.len, "Tags and dims have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + } + + pub fn tag(self: Shape, ax: anytype) Tag { + self.ensureDimsAndTagsAreSync(); + return self._tags.get(self.axis(ax)); + } + + /// Returns a printable name for a given axis. + /// Either the tag itself, or a digit if it's not tagged. + pub fn debugTag(self: Shape, ax: usize) []const u8 { + const t = self.tag(ax); + if (t != TagUnknown) return std.mem.span(t); + + return "01234567"[ax .. ax + 1]; + } + + pub fn setTag(self: Shape, ax: anytype, tag_: anytype) Shape { + var res = self; + res._tags.set(self.axis(ax), toTag(tag_)); + return res; + } + + pub fn tags(self: *const Shape) []const Tag { + self.ensureDimsAndTagsAreSync(); + return self._tags.constSlice(); + } + + pub fn hasTag(self: Shape, tag_: anytype) ?u3 { + return self.axisFromTagMaybe(toTag(tag_)); + } + + pub fn hasTags(self: Shape, tagz: anytype) bool { + const T = @TypeOf(tagz); + + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + for (tagz) |t| { + if (self.hasTag(t) == null) { + return false; + } + } + return true; + } + + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + inline for (tagz) |t| { + if (self.hasTag(t) == null) { + return false; + } + } + return true; + } + + stdx.debug.compileError("Expected tuple of tags, got {any}", .{T}); + } + + pub fn isFullyTagged(self: Shape) bool { + for (self._tags.constSlice()) |t| { + if (t == TagUnknown) return false; + } + return true; + } + + pub fn axis(self: Shape, axis_: anytype) u3 { + self.ensureDimsAndTagsAreSync(); + + const T = @TypeOf(axis_); + if (comptime stdx.meta.isInteger(T)) { + return self.axisFromInt(@intCast(axis_)); + } + + if (comptime isTagConvertible(T)) { + return self.axisFromTag(toTag(axis_)); + } + + stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}); + } + + pub fn axes(self: Shape, axes_: anytype) AxesArray { + self.ensureDimsAndTagsAreSync(); + + const T = @TypeOf(axes_); + + if (T == Shape) { + return self.axes(axes_.tags()); + } + + var res = AxesArray.init(0) catch unreachable; + + if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger) or stdx.meta.isSliceOf(T, Tag)) { + for (axes_) |ax| { + res.appendAssumeCapacity(self.axis(ax)); + } + return res; + } + + if (comptime stdx.meta.isStruct(T)) { + inline for (std.meta.fields(T)) |field| { + res.appendAssumeCapacity(self.axis(@field(axes_, field.name))); + } + return res; + } + + stdx.debug.compileError("axes expects an int-tuple or a tuple of enum literal, got {}", .{T}); + } + + fn axisFromInt(self: Shape, a: isize) u3 { + const rk: i8 = self.rank(); + if (a < -rk or a > rk) { + stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); + } + return if (a < 0) + @intCast(a + rk) + else + @intCast(a); + } + + fn axisFromTagMaybe(self: Shape, t: Tag) ?u3 { + if (t == TagUnknown) return null; + + if (axisFromLiteralInt(t)) |ax| return ax; + + if (@inComptime()) { + // At comptime two duplicated strings may have two different representations + const t_bytes: []const u8 = std.mem.span(t); + for (self.tags(), 0..) |self_tag, ax| { + if (std.mem.eql(u8, t_bytes, std.mem.span(self_tag))) { + return @truncate(ax); + } + } + return null; + } + + // But at runtime the comptime strings have been deduplicated and ptr match is enough. + if (std.mem.indexOfScalar(Tag, self.tags(), t)) |ax| { + return @truncate(ax); + } + return null; + } + + /// Handle .{ ._0 = x } syntax. + fn axisFromLiteralInt(t: Tag) ?u3 { + // match .{ '_', '0-9', null } + if (t[0] == '_' and t[1] >= '0' and t[1] < '8' and t[2] == 0) { + return @intCast(t[1] - '0'); + } + return null; + } + + fn axisFromTag(self: Shape, d: Tag) u3 { + stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); + return self.axisFromTagMaybe(d) orelse { + stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); + }; + } + + test axis { + try testing.expectEqual(1, Shape.init(.{ 5, 2 }, .f32).axis(1)); + try testing.expectEqual(1, Shape.init(.{ 5, 2 }, .f32).axis(-1)); + try testing.expectEqual(1, Shape.init(.{ .a = 5, .b = 2 }, .f32).axis(.b)); + } + + /// The number of element inside a Tensor described by this shape. + pub fn count(self: Shape) usize { + var res: i64 = 1; + for (self.dims()) |d| { + stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{self}); + res *= d; + } + return @intCast(res); + } + + /// Total size in bytes needed to represent this shape. + pub fn byteSize(self: Shape) usize { + return self.dtype().sizeOf() * self.count(); + } + + /// Compares the two shapes described, ignoring tagging. + pub fn eql(self: Shape, other: Shape) bool { + return std.mem.eql(i64, self.dims(), other.dims()) and self.dtype() == other.dtype(); + } + + /// Compares the two shapes described, ignoring tagging and dtype. + pub fn eqlDims(self: Shape, other: Shape) bool { + return std.mem.eql(i64, self.dims(), other.dims()); + } + + /// Compares the two shapes described including tags. + pub fn eqlWithTags(self: Shape, other: Shape) bool { + return self.eql(other) and std.mem.eql(Tag, self.tags(), other.tags()) and self.dtype() == other.dtype(); + } + + /// Format the shape. + /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" + /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" + pub fn format(self: Shape, writer: *std.Io.Writer) !void { + _ = try writer.writeByte('{'); + + var need_comma = false; + for (self.dims(), 0..) |d, i| { + if (need_comma) try writer.writeByte(','); + const t = self.tag(i); + if (t != TagUnknown) { + try writer.print("{s}={d}", .{ t, d }); + } else { + try writer.print("{d}", .{d}); + } + if (self._sharding_info[i]) { + try writer.writeByte('!'); + } + need_comma = true; + } + if (need_comma) try writer.writeByte(','); + _ = try writer.write(@tagName(self.dtype())); + _ = try writer.writeByte('}'); + } + + /// Broadcasts a Tensor to the given shape, extending dimensions if needed. + pub fn canBroadcastTo(self: Shape, other: Shape) bool { + // Already the right shape + if (std.mem.eql(i64, self.dims(), other.dims())) return true; + + if (std.mem.eql(Tag, self.tags(), other.tags())) { + for (0..self.rank()) |i| { + if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; + } + return true; + } + + // Non ambiguous broadcasting + // TODO: broad is error prone because of this: + // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } + if (self.rank() == 0 or self.rank() == other.rank()) { + for (0..self.rank()) |i| { + if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; + } + return true; + } + + for (self.dims(), self.tags()) |d, t| { + // TODO this is also wrong when the axes are in different order in the two shapes. + const other_ax = other.hasTag(t) orelse return false; + if (d != 1 and d != other.dim(other_ax)) return false; + } + return true; + } + + pub fn reshape(self: Shape, new_shape_: anytype) Shape { + var new_shape: Shape = .{ ._dtype = self.dtype() }; + new_shape._dims, new_shape._tags = parseDimensions(new_shape_); + new_shape.inferMissingAxis(self.count()); + stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); + return new_shape; + } + + fn inferMissingAxis(self: *Shape, n_: usize) void { + stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); + + const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; + // We can't use `self.count()` yet cause we have negative dims. + var tmp_count: i64 = 1; + for (self.dims()) |d| { + if (d > 0) { + tmp_count *= d; + } + } + const n: i64 = @intCast(n_); + // Abort, `reshape` will panic with more context. + if (@mod(n, tmp_count) != 0) { + return; + } + self._dims.set(inferred_ax, @divExact(n, tmp_count)); + } + + test reshape { + const x = Shape.init(.{ 2, 5, 3 }, .f32); + { + const res = x.reshape(.{ .auto, 3 }); + try testing.expectEqualSlices(i64, &.{ 10, 3 }, res.dims()); + } + { + const res = x.reshape(.{ 10, -1 }); + try testing.expectEqualSlices(i64, &.{ 10, 3 }, res.dims()); + } + { + const res = x.reshape(.{-1}); + try testing.expectEqualSlices(i64, &.{30}, res.dims()); + } + } + + pub fn setDim(self: Shape, ax: anytype, d: i64) Shape { + var res = self; + res._dims.set(self.axis(ax), d); + return res; + } + + pub const set = setDim; + + fn isAutoDim(v: anytype) bool { + return toTag(v) == toTag(.auto); + } + + fn isDynDim(v: anytype) bool { + return toTag(v) == toTag(.dyn); + } + + /// Inserts one ore more axes with the given dimensions, before the given axis. + /// Negative axis is interpreted wrt the current shape. + /// `.last` axis can be used to insert at the end (ie to append). + /// ``` + /// .{10, 11, 12}.insert(1, 2) -> .{10, 2, 11, 12} + /// .{10, 11, 12}.insert(-1, 2) -> .{10, 11, 2, 12} + /// .{10, 11, 12}.insert(.last, 2) -> .{10, 11, 12, 2} + /// ``` + pub fn insert(self: Shape, axis_: anytype, dimz: anytype) Shape { + const dims_, const tags_ = parseDimensions(dimz); + const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + self.rank() + else + self.axis(axis_); + + var res = self; + res._dims.insertSlice(ax, dims_.constSlice()) catch unreachable; + res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + return res; + } + + test insert { + try testing.expectEqualSlices(i64, &.{ 10, 1, 11, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).insert(1, .{1}).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12, 1, 13 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).insert(-1, .{1}).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12, 13, 1 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).insert(.last, .{1}).dims()); + } + + pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { + stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); + + const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + self.rank() + else + self.axis(axis_); + + var res = self; + res._dims.insert(ax, d) catch unreachable; + res._tags.insert(ax, toTag(tag_)) catch unreachable; + return res; + } + + pub fn append(self: Shape, v: anytype) Shape { + var res = self; + const dims_, const tags_ = parseDimensions(v); + res._dims.appendSliceAssumeCapacity(dims_.constSlice()); + res._tags.appendSliceAssumeCapacity(tags_.constSlice()); + return res; + } + + test append { + try testing.expectEqualSlices( + i64, + &.{ 10, 11, 12, 1 }, + Shape.init(.{ 10, 11, 12 }, .f32).append(.{1}).dims(), + ); + + try testing.expect( + Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32).eqlWithTags( + Shape.init(.{ .a = 10, .b = 11 }, .f32).append(.{ .c = 12 }), + ), + ); + } + + pub fn appendDim(self: Shape, d: i64, tag_: ?Tag) Shape { + var res = self; + res._dims.appendAssumeCapacity(d); + res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); + return res; + } + + pub fn remove(self: Shape, axis_: anytype) Shape { + var res = self; + const a = self.axis(axis_); + _ = res._dims.orderedRemove(a); + _ = res._tags.orderedRemove(a); + return res; + } + + pub const drop = remove; + + test remove { + try std.testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); + try std.testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); + } + + pub fn removeMany(self: Shape, axes_: anytype) Shape { + var to_remove = self.axes(axes_); + if (to_remove.len == 0) return self; + std.mem.sort(u3, to_remove.slice(), {}, std.sort.asc(u3)); + + var sh: Shape = self; + const rk = self.rank(); + var res_ax: u32 = 0; + for (0..rk) |ax| { + if (std.mem.indexOfScalar(u3, to_remove.constSlice(), @intCast(ax))) |_| { + continue; + } + + sh._dims.buffer[res_ax] = self._dims.buffer[ax]; + sh._tags.buffer[res_ax] = self._tags.buffer[ax]; + res_ax += 1; + } + sh._dims.len = rk - to_remove.len; + sh._tags.len = rk - to_remove.len; + return sh; + } + + test removeMany { + try std.testing.expectEqualSlices( + i64, + &.{12}, + Shape.init(.{ 10, 11, 12 }, .f32).removeMany(.{ 0, 1 }).dims(), + ); + try std.testing.expectEqualSlices( + i64, + &.{ 10, 11 }, + Shape.init(.{ 10, 11, 12, 13 }, .f32).removeMany(.{ -1, -2 }).dims(), + ); + } + + pub fn transpose(self: Shape, permutations: anytype) Shape { + std.debug.assert(self.rank() == permutations.len); + const permutations_ = self.axes(permutations); + var res = self; + for (permutations_.constSlice(), 0..) |permutation, i| { + res._dims.set(i, self.dim(permutation)); + res._tags.set(i, self.tag(permutation)); + } + return res; + } + + test transpose { + try testing.expect( + Shape.init(.{ 12, 11, 10 }, .f32).eql( + Shape.init(.{ 10, 11, 12 }, .f32).transpose(.{ 2, 1, 0 }), + ), + ); + + try testing.expect( + Shape.init(.{ .a = 10, .c = 12, .b = 11, .d = 13 }, .f32).eqlWithTags( + Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32).transpose(.{ 0, 2, 1, 3 }), + ), + ); + } + + /// Tag each ax of this shape with tags from a tuple. + pub fn withTags(self: Shape, tagz: anytype) Shape { + const T = @TypeOf(tagz); + + if (T == Shape) { + return self.withTags(tagz.tags()); + } + + var res = self; + + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {any}", .{ self, tagz }); + for (tagz, 0..) |tag_, i| { + res._tags.set(i, toTag(tag_)); + } + return res; + } + + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, tagz }); + inline for (tagz, 0..) |tag_, i| { + res._tags.set(i, toTag(tag_)); + } + return res; + } + + stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); + } + + test withTags { + { + const tagged = Shape.init(.{ 0, 1 }, .f32).withTags(.{ .a, .b }); + try testing.expectEqual(0, tagged.axis(.a)); + try testing.expectEqual(1, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2 }, .f32).withTags(.{ ._, .a, .b }); + try testing.expectEqual(1, tagged.axis(.a)); + try testing.expectEqual(2, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2, 3 }, .f32).withTags(.{ ._, ._, .a, .b }); + try testing.expectEqual(2, tagged.axis(.a)); + try testing.expectEqual(3, tagged.axis(.b)); + } + } + + /// Tag the last axes of this shape with tags from a tuple. + pub fn withPartialTags(self: Shape, tagz: anytype) Shape { + const T = @TypeOf(tagz); + + if (T == Shape) { + return self.withPartialTags(tagz.tags()); + } + + var res = self; + + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {any}", .{ self, tagz }); + for (tagz, self.rank() - tagz.len..) |tag_, i| { + res._tags.set(i, toTag(tag_)); + } + return res; + } + + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {}", .{ self, tagz }); + inline for (tagz, self.rank() - tagz.len..) |tag_, i| { + res._tags.set(i, toTag(tag_)); + } + return res; + } + + stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); + } + + test withPartialTags { + { + const tagged = Shape.init(.{ 0, 1 }, .f32).withPartialTags(.{ .a, .b }); + try testing.expectEqual(0, tagged.axis(.a)); + try testing.expectEqual(1, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2 }, .f32).withPartialTags(.{ .a, .b }); + try testing.expectEqual(1, tagged.axis(.a)); + try testing.expectEqual(2, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2, 3, 4 }, .f32).withPartialTags(.{ .a, .b }); + try testing.expectEqual(3, tagged.axis(.a)); + try testing.expectEqual(4, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2, 3, 4, 5, 6 }, .f32).withPartialTags(.{ .a, .b, .c }); + try testing.expectEqual(4, tagged.axis(.a)); + try testing.expectEqual(5, tagged.axis(.b)); + try testing.expectEqual(6, tagged.axis(.c)); + } + } + + pub fn withDtype(self: Shape, dt: DataType) Shape { + var res = self; + res._dtype = dt; + return res; + } + + pub fn withSharding(self: Shape, axes_: anytype) Shape { + var res = self; + // Reset sharding. + res._sharding_info = @splat(false); + for (self.axes(axes_).constSlice()) |ax| { + res._sharding_info[ax] = true; + } + return res; + } + + /// Renames some of the tags in this shape. + /// Shape.init(.{ .a = 10, .b = 20 }).rename(.{ .b = .batch }); // .{ .a = 10, .batch = 20 }; + pub fn rename(self: Shape, renames: anytype) Shape { + const T = @TypeOf(renames); + stdx.debug.assertComptime(stdx.meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); + var res = self; + inline for (std.meta.fields(T)) |field| { + const new_field = @field(renames, field.name); + stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); + res._tags.set(self.axis(field), toTag(new_field)); + } + return res; + } + + test rename { + { + const tagged = Shape.init(.{ .a = 0, .b = 1 }, .f32).rename(.{ .a = .x, .b = .y }); + try testing.expectEqual(0, tagged.dim(.x)); + try testing.expectEqual(1, tagged.dim(.y)); + } + { + const tagged = Shape.init(.{ .a = 0, .b = 1, .c = 2 }, .f32).rename(.{ .a = .x, .c = .z }); + try testing.expectEqual(0, tagged.dim(.x)); + try testing.expectEqual(1, tagged.dim(.b)); + try testing.expectEqual(2, tagged.dim(.z)); + } + } + + pub fn computeStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + const rk = self.rank(); + var strides: stdx.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; + if (rk == 0) return strides; + + const V = @Vector(MAX_RANK, i64); + const rank_mask = std.simd.iota(u8, MAX_RANK) < @as(@Vector(MAX_RANK, u8), @splat(rk)); + // For each axis compute the product of all following dimensions + // and the element size in bytes. + var d: V = @bitCast(self._dims.buffer); + d = @select(i64, rank_mask, d, @as(V, @splat(1))); + d = std.simd.shiftElementsLeft(d, 1, self.dtype().sizeOf()); + d = std.simd.prefixScan(.Mul, -1, d); + + strides.buffer = @bitCast(d); + return strides; + } + + /// Returns the permutation needed to transpose this shape + /// so that the given axes are contiguous. + pub fn contiguousPerm(self: Shape, axes_: anytype) AxesArray { + const axes__, _ = self.parseAxes(axes_); + var perms = AxesArray.init(0) catch unreachable; + for (0..self.rank()) |i| { + if (std.mem.indexOfScalar(u3, axes__.constSlice(), @intCast(i))) |_| { + continue; + } + perms.appendAssumeCapacity(@intCast(i)); + } + perms.appendSliceAssumeCapacity(axes__.constSlice()); + return perms; + } + + test contiguousPerm { + const abc = Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32); + try testing.expect( + Shape.init(.{ .b = 11, .c = 12, .a = 10 }, .f32).eqlWithTags( + abc.transpose(abc.contiguousPerm(.{.a}).constSlice()), + ), + ); + try testing.expect( + Shape.init(.{ .c = 12, .b = 11, .a = 10 }, .f32).eqlWithTags( + abc.transpose(abc.contiguousPerm(.{ .b, .a }).constSlice()), + ), + ); + const abcd = Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32); + try testing.expect( + Shape.init(.{ .a = 10, .c = 12, .b = 11, .d = 13 }, .f32).eqlWithTags( + abcd.transpose(abcd.contiguousPerm(.{ .b, .d }).constSlice()), + ), + ); + + const abcde = Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13, .e = 14 }, .f32); + try testing.expect( + Shape.init(.{ .a = 10, .b = 11, .d = 13, .c = 12, .e = 14 }, .f32).eqlWithTags( + abcde.transpose(abcde.contiguousPerm(.{ .b, .d, .c, .e }).constSlice()), + ), + ); + } + + /// Splits the given axis in several axes. + /// eg: `Shape.init(.{ .a = 10, .b = 3 }).split(.a, .{.a1 = 5, .a2 = 2}); -> .{.a1 = 5, .a2 = 2, .b = 3}` + /// The number of elements in the split shape must match the number of element + /// in the target axis. + pub fn splitAxis(self: Shape, axis_: anytype, split_shape_: anytype) Shape { + const ax = self.axis(axis_); + const dims_, const tags_ = parseDimensions(split_shape_); + var new_shape = self; + new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; + new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; + new_shape.inferMissingAxis(self.count()); + return new_shape; + } + + test splitAxis { + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = 5, .a2 = 2 }), + ), + ); + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = .auto, .a2 = 2 }), + ), + ); + } + + pub fn splitAxes(self: Shape, axes_: anytype) Shape { + const T = @TypeOf(axes_); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); + + var res = self; + inline for (std.meta.fields(T)) |field| { + res = res.splitAxis(field, @field(axes_, field.name)); + } + return res; + } + + test splitAxes { + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxes(.{ .a = .{ .a1 = 5, .a2 = .auto } }), + ), + ); + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxes(.{ .a = .{ .a1 = 5, .a2 = .auto } }), + ), + ); + } + + /// Merge the given axes into one axis. + /// eg: `Shape.init(.{.a1 = 5, .a2 = 2, .b = 3}).merge(.{ .a = .{ .a1, .a2 }); -> .{ .a = 10, .b = 3 }` + pub fn mergeAxis(self: Shape, axis_: anytype, axes_: anytype) Shape { + const axes__ = self.axes(axes_); + + const first_axis = axes__.get(0); + const last_axis = axes__.get(axes__.len - 1); + + var new_dim: i64 = 1; + for (axes__.constSlice(), first_axis..) |ax, counter| { + new_dim *= self.dim(ax); + stdx.debug.assert(ax == counter, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); + } + + var new_shape = self; + new_shape._dims.set(first_axis, new_dim); + new_shape._dims.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.dims()[last_axis + 1 ..]) catch unreachable; + new_shape._tags.set(first_axis, if (comptime isTagConvertible(@TypeOf(axis_))) toTag(axis_) else TagUnknown); + new_shape._tags.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.tags()[last_axis + 1 ..]) catch unreachable; + return new_shape; + } + + test mergeAxis { + try testing.expect( + Shape.init(.{ .a = 10, .b = 3, .c = 4 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3, .c = 4 }, .f32).mergeAxis(.a, .{ .a1, .a2 }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 5, .c = 6 }, .f32).eqlWithTags( + Shape.init(.{ .a = 5, .b1 = 2, .b2 = 3 }, .f32).mergeAxis(.c, .{ .b1, .b2 }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxis(.a, .{ toTag(.a1), toTag(.a2) }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxis(toTag(.a), @as([]const Tag, &.{ toTag(.a1), toTag(.a2) })), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxis(.a, @as([]const usize, &.{ 0, 1 })), + ), + ); + } + + pub fn mergeAxes(self: Shape, axes_: anytype) Shape { + const T = @TypeOf(axes_); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); + + var res = self; + inline for (std.meta.fields(T)) |field| { + stdx.debug.assertComptime(stdx.meta.isTupleOfAny(field.type, isAxisConvertible) or stdx.meta.isSliceOfAny(field.type, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{field.type}); + res = res.mergeAxis(field, @field(axes_, field.name)); + } + return res; + } + + test mergeAxes { + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = .{ .a1, .a2 } }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = .{ toTag(.a1), toTag(.a2) } }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = .{ 0, 1 } }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = @as([]const usize, &.{ 0, 1 }) }), + ), + ); + } + + fn intersectTags(a: []const Tag, b: []const Tag) TagsArray { + var res = TagsArray.init(0) catch unreachable; + for (a) |tag_| { + if (std.mem.indexOfScalar(Tag, b, tag_)) { + res.appendAssumeCapacity(tag_); + } + } + return res; + } + + pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, MAX_RANK), TagsArray } { + const V = @TypeOf(v); + + var vals_: stdx.BoundedArray(T, MAX_RANK) = .{}; + var tags_: TagsArray = .{}; + + if (comptime stdx.meta.isSliceOf(V, T)) { + for (v) |d| { + vals_.appendAssumeCapacity(d); + } + return .{ vals_, tags_ }; + } + + if (comptime stdx.meta.isStruct(V)) { + const fields = std.meta.fields(V); + stdx.debug.assertComptime(fields.len <= MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, MAX_RANK }); + inline for (fields) |field| { + const fv = @field(v, field.name); + vals_.appendAssumeCapacity(fv); + + if (!comptime stdx.meta.isTuple(V)) { + tags_.appendAssumeCapacity(toTag(field)); + } + } + return .{ vals_, tags_ }; + } + + stdx.debug.compileError("parseStruct expects struct or tuple, got {}", .{V}); + } + + test parseStruct { + const vals_, const tags_ = parseStruct(f32, .{ .a = 0.1, .b = 1.2 }); + + try testing.expectEqualSlices(f32, &.{ 0.1, 1.2 }, vals_.constSlice()); + try testing.expectEqualSlices(Tag, &.{ "a".ptr, "b".ptr }, tags_.constSlice()); + } + + /// Parses a struct literal into a list of options for each axes. + pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, MAX_RANK) { + const V = @TypeOf(options); + + var res: stdx.BoundedArray(T, MAX_RANK) = .{}; + if (comptime stdx.meta.isSliceOf(V, T)) { + stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); + for (options) |d| { + res.appendAssumeCapacity(d); + } + } + + if (comptime stdx.meta.isStruct(V)) { + for (0..self.rank()) |_| res.appendAssumeCapacity(default); + const fields = std.meta.fields(V); + stdx.debug.assertComptime(fields.len <= MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, MAX_RANK, fields.len }); + inline for (fields) |field| { + const a = self.axis(field); + res.buffer[a] = @field(options, field.name); + } + return res; + } + + stdx.debug.compileError("parseStruct expects struct or tuple, got {}", .{V}); + } + + test parseAxesOptions { + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .u8); + const scaling = shape.parseAxesOptions(f32, .{ .b = 1.2, .a = 0.1 }, 1.0); + + try testing.expectEqualSlices(f32, &.{ 0.1, 1.2, 1.0 }, scaling.constSlice()); + } + + test "comptimeShape" { + comptime { + const s = Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); + try std.testing.expectEqual(3, s.rank()); + try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try std.testing.expectEqual(1, s.axis(.b)); + } + + // comptime only the shape + { + const s = comptime Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); + try std.testing.expectEqual(3, s.rank()); + try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try std.testing.expectEqual(1, s.axis(.b)); + } + } + + pub fn outer(self: Shape, other: Shape) Shape { + var res_shape = self; + var batching_axes: u8 = 0; + for (0..other.rank()) |ax| { + if (other.tag(ax) != Shape.TagUnknown) { + if (self.hasTag(other.tag(ax))) |batching_ax| { + stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({f}, {f})", .{ self, other }); + batching_axes += 1; + } + } + + res_shape = res_shape.appendDim(other.dim(ax), other.tag(ax)); + } + return res_shape; + } +}; diff --git a/zml2/tensor.zig b/zml2/tensor.zig new file mode 100644 index 00000000..c74955c0 --- /dev/null +++ b/zml2/tensor.zig @@ -0,0 +1,4510 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); +const meta = @import("meta.zig"); + +const Shape = @import("shape.zig").Shape; +const mlir = @import("mlir"); +const mlirx = @import("mlirx.zig"); +const dialects = @import("mlir/dialects"); + +const DataType = @import("dtype.zig").DataType; +const CompilationContext = struct {}; + +pub const Tensor = struct { + var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; + pub const MAX_RANK = Shape.MAX_RANK; + + id: usize, + auto_broadcast: bool = false, + _shape: Shape, + _value: ?*const mlir.Value = null, + + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { + try writer.print("Tensor({f})", .{self._shape}); + } + + /// Returns the shape of a Tensor. + pub fn shape(self: Tensor) Shape { + return self._shape; + } + + /// Returns the datatype of a Tensor. + pub fn dtype(self: Tensor) DataType { + return self._shape.dtype(); + } + + /// Returns the rank of a Tensor. + pub fn rank(self: Tensor) u4 { + return self._shape.rank(); + } + + /// Returns the number of element of a Tensor. + pub fn count(self: Tensor) usize { + return self._shape.count(); + } + + /// Returns the size in bytes of a Tensor. + pub fn byteSize(self: Tensor) usize { + return self._shape.byteSize(); + } + + /// Internal use + /// + /// Creates a tensor from a Shape and an mlir.Value. + pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { + const res: Tensor = .{ + ._shape = sh, + ._value = val, + .id = Tensor.current_id.fetchAdd(1, .seq_cst), + }; + + if (builtin.mode == .Debug) { + // Check that the MLIR value actually have the same shape. + const other = fromMlirValue(val); + stdx.debug.internalAssert(sh.eql(other._shape), "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); + } + + return res; + } + + /// Creates a Tensor from a mlir.Value + /// + /// The shape is derived from the type of the mlir.Value. + pub fn fromMlirValue(val: mlir.Value) Tensor { + const ranked_tensor = val.getType().as(mlir.RankedTensorType).?; + const n = ranked_tensor.getRank(); + + stdx.debug.assert(n <= MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, MAX_RANK }); + + var sh: Shape = .{ ._dtype = mlirx.Type.toDType(ranked_tensor.getElementType()) }; + for (0..n) |i| { + sh._dims.appendAssumeCapacity(ranked_tensor.getDimension(i)); + } + sh._tags.resize(n) catch unreachable; + + return .{ ._shape = sh, ._value = val, .id = Tensor.current_id.fetchAdd(1, .seq_cst) }; + } + + /// Returns the dimension of axis 'axis_'. + /// + /// 'axis_' can be an integer or a tag. + pub fn dim(self: Tensor, axis_: anytype) i64 { + return self._shape.dim(axis_); + } + + /// Returns the dimensions of a Tensor as a slice. + pub fn dims(self: *const Tensor) []const i64 { + return self._shape.dims(); + } + + /// Returns the index of axis 'axis_'. + /// + /// 'axis_' can be an integer or a tag. + pub fn axis(self: Tensor, axis_: anytype) u3 { + return self._shape.axis(axis_); + } + + /// Returns the indices of each of the given axes. + /// + /// 'axis_' can be an integer or a tag. + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, Tensor.MAX_RANK) { + return self._shape.axes(axes_); + } + + /// Returns a Tensor tagged with the tags in 'tagz'. + pub fn withTags(self: Tensor, tagz: anytype) Tensor { + var res = self; + res._shape = self._shape.withTags(tagz); + return res; + } + + /// Returns a Tensor tagged partially with the tags in 'tagz'. + /// + /// If 'tagz' is of length n, the n last dimensions of the Tensor will be tagged. + pub fn withPartialTags(self: Tensor, tagz: anytype) Tensor { + var res = self; + res._shape = self._shape.withPartialTags(tagz); + return res; + } + + // TODO(Corentin) + pub fn withSharding(self: Tensor, axes_: anytype) Tensor { + _ = self; // autofix + _ = axes_; // autofix + @panic("TODO"); + } + + // TODO(Corentin) + //pub fn toMemory(self: Tensor, kind: Memory) Tensor { + // @panic("TODO"); + //} + + /// Returns a Tensor with new tag names. + pub fn rename(self: Tensor, renames: anytype) Tensor { + var res = self; + res._shape = self._shape.rename(renames); + return res; + } + + pub fn renameAxis(self: Tensor, ax: i8, name: @TypeOf(.enum_literal)) Tensor { + var res = self; + res._shape._tags.set(self.axis(ax), @tagName(name).ptr); + return res; + } + + /// Returns the mlir.Value associated with the Tensor. + /// + /// This will fail if used outside of a compilation context. + pub fn value(self: Tensor) *const mlir.Value { + if (self.getContext().currentScope().id_to_argument.get(self.id)) |argument_index| { + return self.getContext().currentScope().block.argument(argument_index); + } else if (self._value) |v| { + return v; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + } + + /// Tell PJRT compiler that memory should be reuse between the two tensors. + /// The compiler is already aggressively reusing tensors for intermediate results, + /// but this API allows to reuse buffer between input and output arguments + /// of a given function. + /// Note this is visible from the outside. The caller of a function with donations + /// is not allowed to reuse the donated input buffer after the call. + /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. + pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { + const compilation_context = CompilationContext.current(); + const scope = compilation_context.currentScope(); + if (scope.id_to_argument.get(origin.id)) |argument_index| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = argument_index; + } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = origin_donation; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + return self; + } + + /// Returns a Tensor containing the absolute value of each element of the input Tensor. + pub fn abs(self: Tensor) Tensor { + const op = dialects.stablehlo.abs(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const dt = switch (self.dtype()) { + .c64 => .f32, + .c128 => .f64, + else => self.dtype(), + }; + + return _result(self._shape.withDtype(dt), op.result(0)); + } + + /// Returns a Tensor whose elements have been bitcast to a target datatype. + /// + /// The Tensor shape needs to be compatible with the target datatype. + pub fn bitCast(self: Tensor, dt: DataType) Tensor { + const src_bit_size = self.dtype().bitSizeOf(); + const tgt_bit_size = dt.bitSizeOf(); + var res_shape = if (src_bit_size == tgt_bit_size) + self._shape + else if (src_bit_size > tgt_bit_size) gt: { + const new_dim = std.math.divExact(u16, src_bit_size, tgt_bit_size) catch std.debug.panic("bitcast expects target datatype to be a multiple of source datatype when upcasting, got {} (bitsize of {}) and {} (bitsize of {})", .{ self.dtype(), src_bit_size, dt, tgt_bit_size }); + var res = self._shape; + res = res.append(.{ .bitcast = new_dim }); + break :gt res; + } else lt: { + // several contiguous elements of self maps to one element of the result + stdx.debug.assert(self.dim(-1) * src_bit_size == tgt_bit_size, "bitcast expects elements of the input tensor last dimension to map to one element of the target datatype, got {0} elements (bitsize of {0}x{1}={2}) and {3} (bitsize of {4})", .{ self.dim(-1), src_bit_size, self.dim(-1) * src_bit_size, dt, tgt_bit_size }); + break :lt self._shape.remove(-1); + }; + + res_shape = res_shape.withDtype(dt); + + const op = dialects.stablehlo.bitcast_convert( + self.getContext().mlirCtx(), + self.value(), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(res_shape, op.result(0)); + } + + /// Returns the given tensor as one contiguous buffer of bytes. + pub fn bytes(self: Tensor) Tensor { + return self.bitCast(.u8).flatten().withTags(.{.bytes}); + } + + /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. + pub fn countLeadingZeros(self: Tensor) Tensor { + const op = dialects.stablehlo.count_leading_zeros(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. + pub fn isFinite(self: Tensor) Tensor { + const op = dialects.stablehlo.is_finite(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape.withDtype(.bool), op.result(0)); + } + + /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. + pub fn popcnt(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); + const op = dialects.stablehlo.popcnt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the sign of the input Tensor element-wise. + pub fn sign(self: Tensor) Tensor { + const op = dialects.stablehlo.sign(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. + /// + /// See https://pytorch.org/docs/stable/generated/torch.fmod.html for more details. + pub fn fmod(self: Tensor, divisor: f32) Tensor { + return self.remainder(scalar(divisor, .f32).broadcast(self._shape, &.{})); + } + + // TODO(Corentin) + //test fmod { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; + // const expectations: [2][6]f32 = .{ .{ -1.0, -0.0, -1.0, 1.0, 0.0, 1.0 }, .{ 1.0000, 0.5000, 0.0000, 1.0000, 0.5000, -0.5000 } }; + // const divisors: [2]f32 = .{ 2, -1.5 }; + + // inline for (inputs, expectations, divisors) |i, e, d| { + // const input = try zml.Buffer.fromSlice(platform, .{6}, &i); + // const output = try zml.testing.compileAndCall(platform, Tensor.fmod, .{ input, d }); + + // try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{6}, &e), output, 1e-4); + // } + //} + + /// Returns a Tensor containing the element-wise left-shift operation of 'self' by 'other'. + pub fn shiftLeft(self: Tensor, other: Tensor) Tensor { + return binaryOp("shiftLeft", dialects.stablehlo.shift_left)(self, other); + } + + /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. + pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { + return binaryOp("shiftRightArithmetic", dialects.stablehlo.shift_right_arithmetic)(self, other); + } + + /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. + pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { + return binaryOp("shiftRightLogical", dialects.stablehlo.shift_right_logical)(self, other); + } + + /// Returns the Cholesky decomposition of the input Tensor. + /// + /// 'lower' controls the form of the output Tensor. The output will be lower-triangular if 'lower' is true + /// and upper-triangular otherwise. + pub fn cholesky(self: Tensor, lower: bool) Tensor { + stdx.debug.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); + + const op = dialects.stablehlo.cholesky(mlirCtx(), self.value(), lower, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Solves the system of linear equations formed by the input tensors. + pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialects.stablehlo.TriangularSolveOpts) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); + stdx.debug.assert(self.rank() <= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank and be <= 2, got {} and {}", .{ self.rank(), other.rank() }); + + const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties away from zero, of the input Tensor. + pub fn roundNearestAfz(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); + + const op = dialects.stablehlo.round_nearest_afz(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties towards the even integer, of the input Tensor. + pub fn roundNearestEven(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); + + const op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor of complex number converted from a pair of real and imaginary Tensors. + pub fn complex(re: Tensor, im: Tensor) Tensor { + stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); + stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); + + const op = dialects.stablehlo.complex(mlirCtx(), re.value(), im.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; + return _result(re._shape.withDtype(dt), op.result(0)); + } + + /// Returns a Tensor containing the element-wise real part of the input Tensor. + /// + /// Tensor type can float or complex. + pub fn real(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isComplex() or self.dtype().isFloat(), "real expects tensor type to be a float or a complex, got {}", .{self.dtype()}); + + if (self.dtype().isFloat()) { + return self; + } + + const dt: DataType = switch (self.dtype()) { + .c64 => .f32, + .c128 => .f64, + else => unreachable, + }; + const op = dialects.stablehlo.real(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape.withDtype(dt), op.result(0)); + } + + /// Returns a Tensor containing the element-wise imaginary part of the input Tensor. + /// + /// Tensor type can float or complex. + pub fn imag(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isFloat() or self.dtype().isComplex(), "imag expects tensor type to be a float or a complex, got {}", .{self.dtype()}); + + // Real tensors don't have imaginary part. + if (self.dtype().isFloat()) { + return Tensor.constant(self._shape, self.dtype().zero()); + } + + const dt: DataType = switch (self.dtype()) { + .bf16, .f16, .f32, .f64 => self.dtype(), + .c64 => .f32, + .c128 => .f64, + else => unreachable, + }; + const op = dialects.stablehlo.imag(mlirCtx(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape.withDtype(dt), op.result(0)); + } + + /// Returns the Fast Fourier Transform (FFT) of the input Tensor. + pub fn fft(self: Tensor, opts: dialects.stablehlo.FftOpts) Tensor { + // TODO: support tagged API. + + stdx.debug.assert(1 <= opts.length.len and opts.length.len <= 3, "fft expects 'opts.length' length to be between 1 and 3 (inclusive), got {}", .{opts.length.len}); + stdx.debug.assert(opts.length.len <= self.rank(), "fft expects 'opts.length' length to be less than tensor rank, got {} and {}", .{ opts.length.len, self.rank() }); + + const sh = switch (opts.kind) { + .FFT, .IFFT => blk: { + stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); + + break :blk self._shape; + }, + .RFFT => blk: { + stdx.debug.assert(self.dtype() == .f32 or self.dtype() == .f64, "fft({}) expects tensor type to be f32 or f64, got {}", .{ opts, self.dtype() }); + stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); + + const dt: DataType = switch (self.dtype()) { + .f32 => .c64, + else => .c128, + }; + const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1), 2) + 1); + break :blk shape_.withDtype(dt); + }, + .IRFFT => blk: { + stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); + stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({any}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); + + const dt: DataType = switch (self.dtype()) { + .c64 => .f32, + else => .f64, + }; + const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1) - 1, 2)); + break :blk shape_.withDtype(dt); + }, + }; + + const op = dialects.stablehlo.fft(mlirCtx(), self.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(sh, op.result(0)); + } + + pub const Rng = struct { + _state: Tensor, + algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, + + pub fn init() Rng { + return .{ ._state = .init(Shape.init(.{2}, .u64)) }; + } + + // TODO(Corentin) + //pub fn initBuffer(platform: Platform, seed: u128) !Bufferized(Rng) { + // return .{ + // ._state = try zml.Buffer.fromBytes(platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), + // }; + //} + + /// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. + /// + /// The given Rng state should not be used anymore (or you'll get the same numbers again). + /// The output is guaranteed to be deterministic function of `self` Rng state, + /// but it is not guaranteed to be deterministic between implementations. + pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { + const op = dialects.stablehlo.rng_bit_generator( + mlirCtx(), + self.algorithm, + self._state.value(), + mlir.rankedTensorType(self._state.dims(), mlirx.Type.fromDType(mlirCtx(), self._state.dtype())), + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; + } + + fn update(self: Rng, new_state: mlir.Value) Rng { + return .{ + ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), + .algorithm = self.algorithm, + }; + } + + /// Returns a Tensor of the given shape, filled with uniformly sampled floating point numbers from an interval, + /// and a new Rng state. + /// + /// https://en.wikipedia.org/wiki/Continuous_uniform_distribution + pub fn uniform( + self: Rng, + shape_: Shape, + opts: struct { min: f64 = 0, max: f64 = 1 }, + ) struct { Rng, Tensor } { + const dt = if (shape_.dtype().isFloat()) shape_.dtype() else .f32; + + const mantissa_bit_count = @import("dtype.zig").mantissaSize(dt); + const bit_count: usize = dt.bitSizeOf(); + const rng_bit_count = if (mantissa_bit_count < 8) 8 else bit_count; + const uint_dtype: DataType = switch (bit_count) { + 8 => .u8, + 16 => .u16, + 32 => .u32, + 64 => .u64, + else => stdx.debug.panic("uniform don't support non-byte aligned dtype. Got: {f}", .{shape_}), + }; + + const rng, const bits = self.bitGenerator(shape_.withDtype(uint_dtype)); + + // Erase bits outside of mantissa. + var float_bits = bits.shiftRightLogical(scalar(rng_bit_count - mantissa_bit_count, uint_dtype)); + + // Set exponent bits to represent e^0 (eg 127 for f32). + float_bits = float_bits.logical(.OR, scalar(1, dt).bitCast(uint_dtype)); + + // float_bits now uniformly represents number in [1, 2[ range. + // Let's convert to floats, and subtract one to go to [0, 1[ range. + var floats = float_bits.bitCast(dt).sub(scalar(1, dt)); + floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); + + // Convert back to integer if needed. + return .{ rng, floats.convert(shape_.dtype()) }; + } + + //test uniform { + // const zml = @import("zml.zig"); + // const Stats = struct { + // const Stats = @This(); + + // mean: Tensor, + // variance: Tensor, + // min: Tensor, + // max: Tensor, + + // pub fn uniformStats( + // rand: Rng, + // shape_: Shape, + // opts: struct { min: f64, max: f64 }, + // ) struct { Rng, Stats } { + // const rng, const data = rand.uniform(shape_, .{ .min = opts.min, .max = opts.max }); + // const mean_ = data.mean(0); + // const variance = data.sub(mean_.broad(data.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + // return .{ rng, .{ + // .mean = mean_, + // .variance = variance, + // .min = data.min(0), + // .max = data.max(0), + // } }; + // } + // }; + + // const platform = zml.testing.env(); + // // Compute stats over a uniform distribution on [-2, 10]. + // const rand, const stats = try zml.testing.compileAndCallWithTensors( + // platform, + // Stats.uniformStats, + // .{ Rng.shape(), zml.Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, + // .{try Rng.init(platform, 1234)}, + // ); + + // // Check the Rng state has been modified. + // try std.testing.expect(try rand._state.getValue(u128) != 1234); + + // // Check the mean and variance are close to theoritical values. + // const mean_ = try stats.mean.getValue(f32); + // try std.testing.expectApproxEqAbs(4, mean_, 0.03); + + // const variance = try stats.variance.getValue(f32); + // try std.testing.expectApproxEqAbs(12.0 * 12.0 / 12.0, variance, 0.01); + + // // Check that no value is outside of the interval + // // and we have samples close to the edges. + // const min_ = try stats.min.getValue(f32); + // try std.testing.expect(min_ >= -2); + // try std.testing.expectApproxEqAbs(-2, min_, 0.05); + + // const max_ = try stats.max.getValue(f32); + // try std.testing.expect(max_ < 10); + // try std.testing.expectApproxEqAbs(10, max_, 0.05); + //} + + /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a normal distribution. + /// + /// Note: this uses stablehlo.rng which is deprecated. + /// https://github.com/openxla/stablehlo/blob/main/rfcs/20240503-opset-deprecations.md + pub fn normal(sh: Shape, opts: struct { mean: f64 = 0, stddev: f64 = 1 }) Tensor { + stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); + + const a = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.mean)); + const b = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.stddev)); + const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); + const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(sh, op.result(0)); + } + + /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. + /// + /// Often used in ML because of the reparametrization tricks. + /// Sampling from a gumbel distribution is equivalent to sample + /// from a softmax distribution, but doesn't require to compute the sum of exponentials. + /// https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks + /// See `sampleTokens` for a practical use case. + /// Note: we only implement the μ=0, β=1 version. + pub fn gumbel(self: Rng, shape_: Shape) struct { Rng, Tensor } { + const rand, const u = self.uniform( + // Always use .f32 to have a big enough mantissa. + shape_.withDtype(.f32), + // We don't want 0 to be sampled otherwise `log` will return -inf. + .{ .min = std.math.floatEps(f32), .max = 1 }, + ); + return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; + } + + //test gumbel { + // const zml = @import("zml.zig"); + // const Stats = struct { + // const Stats = @This(); + + // mean: Tensor, + // variance: Tensor, + // actual_dist: Tensor, + + // pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { + // const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); + // const rng, const data = rand.gumbel(s); + // const flat = data.flattenAll(); + // const mean_ = flat.mean(0); + // const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + + // // Test out the gumbel reparametrization trick + // var x = target_dist.log().withTags(.{.d}).broad(s); + // x = x.add(data); + // const samples = x.argMax(.d).indices.squeeze(.d); + + // // count 0, 1, 2 and 3 in samples: + // // - map 0 to 1, 1 to 2**16, 2 to 2**32, 3 to N**58 + // // - sum in u64 + // // - split to [4]u16 + // const powers = blk: { + // var powers: [4]u64 = undefined; + // for (&powers, 0..) |*p, i| p.* = std.math.pow(u64, 2, i * 16); + // break :blk powers; + // }; + // const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); + // const counts = values.gatherValues(.d, samples, .{}).sum(.n).bitCast(.u16); + // const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); + // return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; + // } + // }; + + // const platform = zml.testing.env(); + // const tgt_dist = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; + // const rand, const stats = try zml.testing.compileAndCallWithTensors( + // platform, + // Stats.gumbelStats, + // .{ Rng.shape(), zml.Shape.init(.{tgt_dist.len}, .f32) }, + // .{ try Rng.init(platform, 1234), try .fromArray(platform, tgt_dist) }, + // ); + // // Check the Rng state has been modified. + // try std.testing.expect(try rand._state.getValue(i128) != 1234); + + // // Check the mean and variance are close to theoritical values. + // const mean_ = try stats.mean.getValue(f32); + // try std.testing.expectApproxEqAbs(0.5772, mean_, 0.02); + + // const variance = try stats.variance.getValue(f32); + // const pi = std.math.pi; + // try std.testing.expectApproxEqAbs(pi * pi / 6.0, variance, 0.03); + + // // Check the distribution obtained with the gumbel trick matches the target distribution. + // const actual_dist = try stats.actual_dist.getValue([4]f32); + // scoped_log.debug("tgt_dist: {d}, actual_dist: {d}", .{ tgt_dist, actual_dist }); + // for (tgt_dist, actual_dist) |tgt, actual| { + // // We normalize tgt_dist to make it a well formed distribution. + // // We didn't do it before calling gumbel, because the gumbel trick + // // doesn't require normalized distributions as input. + // try std.testing.expectApproxEqAbs(tgt / 10.0, actual, 0.05); + // } + //} + }; + + /// Returns a Tensor containing the element-wise conversion to another floating point type. + pub fn reducePrecision(self: Tensor, exponent_bits: i32, mantissa_bits: i32) Tensor { + stdx.debug.assert(self.dtype().isFloat(), "reducePrecision expects tensor type to be a float, got {}", .{self.dtype()}); + stdx.debug.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); + stdx.debug.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); + + const op = dialects.stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts, loc: mlir.Location) Tensor { + _ = loc; // autofix + stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); + const N = self.rank(); + stdx.debug.guard(opts.window_strides.len == N - 2, @src()); + for (opts.window_strides) |s| stdx.debug.guard(0 < s, @src()); + stdx.debug.guard(opts.lhs_dilation.len == N - 2, @src()); + for (opts.lhs_dilation) |d| stdx.debug.guard(0 < d, @src()); + stdx.debug.guard(opts.rhs_dilation.len == N - 2, @src()); + for (opts.rhs_dilation) |d| stdx.debug.guard(0 < d, @src()); + stdx.debug.guard(opts.window_reversal.len == N - 2, @src()); + stdx.debug.guard(@rem(self.dim(opts.input_batch_dimension), opts.batch_group_count) == 0, @src()); + stdx.debug.guard(@rem(self.dim(opts.input_feature_dimension), opts.feature_group_count) == 0, @src()); + stdx.debug.guard(opts.input_spatial_dimensions.len == N - 2, @src()); + stdx.debug.guard(opts.input_batch_dimension != opts.input_feature_dimension, @src()); + stdx.debug.guard(0 <= opts.input_batch_dimension and opts.input_batch_dimension < N, @src()); + stdx.debug.guard(0 <= opts.input_feature_dimension and opts.input_feature_dimension < N, @src()); + for (opts.input_spatial_dimensions, 0..) |d, i| { + stdx.debug.guard(d != opts.input_batch_dimension, @src()); + stdx.debug.guard(d != opts.input_feature_dimension, @src()); + stdx.debug.guard(0 <= d and d < N, @src()); + if (i < opts.input_spatial_dimensions.len - 1) continue; + stdx.debug.guard(std.mem.indexOfScalar(i64, opts.input_spatial_dimensions[i + 1 ..], d) == null, @src()); + } + stdx.debug.guard(other.dim(opts.kernel_input_feature_dimension) == @divTrunc(self.dim(opts.input_feature_dimension), opts.feature_group_count), @src()); + stdx.debug.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.batch_group_count) == 0, @src()); + stdx.debug.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.feature_group_count) == 0, @src()); + stdx.debug.guard(opts.kernel_spatial_dimensions.len == N - 2, @src()); + stdx.debug.guard(opts.kernel_input_feature_dimension != opts.kernel_output_feature_dimension, @src()); + stdx.debug.guard(0 <= opts.kernel_input_feature_dimension and opts.kernel_input_feature_dimension < N, @src()); + stdx.debug.guard(0 <= opts.kernel_output_feature_dimension and opts.kernel_output_feature_dimension < N, @src()); + for (opts.kernel_spatial_dimensions, 0..) |d, i| { + stdx.debug.guard(d != opts.kernel_input_feature_dimension, @src()); + stdx.debug.guard(d != opts.kernel_output_feature_dimension, @src()); + stdx.debug.guard(0 <= d and d < N, @src()); + if (i < opts.kernel_spatial_dimensions.len - 1) continue; + stdx.debug.guard(std.mem.indexOfScalar(i64, opts.kernel_spatial_dimensions[i + 1 ..], d) == null, @src()); + } + stdx.debug.guard(opts.output_spatial_dimensions.len == N - 2, @src()); + stdx.debug.guard(opts.output_batch_dimension != opts.output_feature_dimension, @src()); + stdx.debug.guard(0 <= opts.output_batch_dimension and opts.output_batch_dimension < N, @src()); + stdx.debug.guard(0 <= opts.output_feature_dimension and opts.output_feature_dimension < N, @src()); + for (opts.output_spatial_dimensions, 0..) |d, i| { + stdx.debug.guard(d != opts.output_batch_dimension, @src()); + stdx.debug.guard(d != opts.output_feature_dimension, @src()); + stdx.debug.guard(0 <= d and d < N, @src()); + if (i < opts.output_spatial_dimensions.len - 1) continue; + stdx.debug.guard(std.mem.indexOfScalar(i64, opts.output_spatial_dimensions[i + 1 ..], d) == null, @src()); + } + stdx.debug.guard(0 < opts.feature_group_count, @src()); + stdx.debug.guard(0 < opts.batch_group_count, @src()); + stdx.debug.guard(opts.feature_group_count == 1 or opts.batch_group_count == 1, @src()); + var used_opts = opts; + used_opts.pad_shape = &.{ @intCast(N - 2), 2 }; + used_opts.precision_config = &.{ .DEFAULT, .DEFAULT }; + + var new_shape = self._shape; + var res_dim: i64 = undefined; + + for (0..N) |i| { + if (i == @as(usize, @intCast(opts.output_batch_dimension))) { + res_dim = @divTrunc(self.dim(opts.input_batch_dimension), opts.batch_group_count); + } else if (i == @as(usize, @intCast(opts.output_feature_dimension))) { + res_dim = other.dim(opts.kernel_output_feature_dimension); + } else { + // calculate spatial dimension value + const spatial_dim: usize = std.mem.indexOfScalar(i64, opts.output_spatial_dimensions, @as(i64, @intCast(i))).?; + const lhs_dim = opts.input_spatial_dimensions[spatial_dim]; + const rhs_dim = opts.kernel_spatial_dimensions[spatial_dim]; + const dilated_input_shape_lhs_dim: i64 = if (self.dim(lhs_dim) == 0) 0 else (self.dim(lhs_dim) - 1) * opts.lhs_dilation[spatial_dim] + 1; + const left_pad_value, const right_pad_value = if (opts.pad_value.len == 1) + .{ opts.pad_value[0], opts.pad_value[0] } + else + .{ opts.pad_value[2 * spatial_dim], opts.pad_value[2 * spatial_dim + 1] }; + const padded_input_shape_lhs_dim = left_pad_value + dilated_input_shape_lhs_dim + right_pad_value; + const dilated_window_shape_lhs_dim: i64 = if (other.dim(rhs_dim) == 0) 0 else (other.dim(rhs_dim) - 1) * opts.rhs_dilation[spatial_dim] + 1; + const is_empty_window_lhs_dim = padded_input_shape_lhs_dim == 0 or dilated_window_shape_lhs_dim > padded_input_shape_lhs_dim; + res_dim = if (is_empty_window_lhs_dim) 0 else @divTrunc(padded_input_shape_lhs_dim - dilated_window_shape_lhs_dim, opts.window_strides[spatial_dim]) + 1; + } + + new_shape = new_shape.set(i, res_dim); + } + + // inferred shape '[1, 256, 1, 12008]' is incompatible with return type of operation '[1, 256, 1, 11978]' + const op = dialects.stablehlo.convolution( + mlirCtx(), + self.value(), + other.value(), + used_opts, + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(new_shape, op.result(0)); + } + + /// Returns a Tensor containing the result of the 1D convolution of 'input' by 'kernel'. + pub fn conv1d( + input: Tensor, + kernel: Tensor, + opts: struct { + window_strides: i64 = 1, + padding: []const i64 = &.{ 0, 0 }, + lhs_dilation: i64 = 1, + rhs_dilation: i64 = 1, + window_reversal: bool = false, + input_batch_dimension: i64 = 0, + input_feature_dimension: i64 = 1, + input_spatial_dimensions: i64 = 2, + kernel_output_feature_dimension: i64 = 0, + kernel_input_feature_dimension: i64 = 1, + kernel_spatial_dimensions: i64 = 2, + output_batch_dimension: i64 = 0, + output_feature_dimension: i64 = 1, + output_spatial_dimensions: i64 = 2, + feature_group_count: i64 = 1, + batch_group_count: i64 = 1, + }, + ) Tensor { + return input.convolution(kernel, .{ + .window_strides = &.{opts.window_strides}, + .pad_value = opts.padding, + .lhs_dilation = &.{opts.lhs_dilation}, + .rhs_dilation = &.{opts.rhs_dilation}, + .window_reversal = &.{opts.window_reversal}, + .input_batch_dimension = opts.input_batch_dimension, + .input_feature_dimension = opts.input_feature_dimension, + .input_spatial_dimensions = &.{opts.input_spatial_dimensions}, + .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, + .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, + .kernel_spatial_dimensions = &.{opts.kernel_spatial_dimensions}, + .output_batch_dimension = opts.output_batch_dimension, + .output_feature_dimension = opts.output_feature_dimension, + .output_spatial_dimensions = &.{opts.output_spatial_dimensions}, + .feature_group_count = opts.feature_group_count, + .batch_group_count = opts.batch_group_count, + }, .unknown(mlirCtx())); + } + + /// Returns a Tensor containing the result of the 2D convolution of 'input' by 'kernel'. + /// Defaults values correspond to a (B, C_in, W, H) image, (C_out, C_in, W, H) kernel weights and (B, C_out, W, H) output. + pub fn conv2d( + input: Tensor, + kernel: Tensor, + opts: struct { + window_strides: []const i64 = &.{ 1, 1 }, + padding: []const i64 = &.{ 0, 0, 0, 0 }, + lhs_dilation: []const i64 = &.{ 1, 1 }, + rhs_dilation: []const i64 = &.{ 1, 1 }, + window_reversal: []const bool = &.{ false, false }, + input_batch_dimension: i64 = 0, + input_feature_dimension: i64 = 1, + input_spatial_dimensions: []const i64 = &.{ 2, 3 }, + kernel_input_feature_dimension: i64 = 1, + kernel_output_feature_dimension: i64 = 0, + kernel_spatial_dimensions: []const i64 = &.{ 2, 3 }, + output_batch_dimension: i64 = 0, + output_feature_dimension: i64 = 1, + output_spatial_dimensions: []const i64 = &.{ 2, 3 }, + feature_group_count: i64 = 1, + batch_group_count: i64 = 1, + }, + ) Tensor { + return input.convolution(kernel, .{ + .window_strides = opts.window_strides, + .pad_value = opts.padding, + .lhs_dilation = opts.lhs_dilation, + .rhs_dilation = opts.rhs_dilation, + .window_reversal = opts.window_reversal, + .input_batch_dimension = opts.input_batch_dimension, + .input_feature_dimension = opts.input_feature_dimension, + .input_spatial_dimensions = opts.input_spatial_dimensions, + .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, + .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, + .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, + .output_batch_dimension = opts.output_batch_dimension, + .output_feature_dimension = opts.output_feature_dimension, + .output_spatial_dimensions = opts.output_spatial_dimensions, + .feature_group_count = opts.feature_group_count, + .batch_group_count = opts.batch_group_count, + }, .unknown(mlirCtx())); + } + + /// Returns a Tensor containing the element-wise addition of the input Tensors. + pub fn add(self: Tensor, other: Tensor) Tensor { + return binaryOp("add", dialects.stablehlo.add)(self, other); + } + + /// Returns a Tensor containing the element-wise subtraction of the input Tensors. + pub fn sub(self: Tensor, other: Tensor) Tensor { + return binaryOp("subtract", dialects.stablehlo.subtract)(self, other); + } + + /// Returns a Tensor containing the element-wise multiplication of the input Tensors. + pub fn mul(self: Tensor, other: Tensor) Tensor { + return binaryOp("mul", dialects.stablehlo.multiply)(self, other); + } + + /// Returns a Tensor containing the element-wise division of the input Tensors. + pub fn div(self: Tensor, other: Tensor) Tensor { + return binaryOp("div", dialects.stablehlo.divide)(self, other); + } + + /// Returns a Tensor containing the element-wise exponentiation of the input Tensors. + pub fn pow(self: Tensor, other: Tensor) Tensor { + return binaryOp("pow", dialects.stablehlo.power)(self, other); + } + + /// Returns a Tensor containing the element-wise maximum operation of the input Tensors. + pub fn maximum(self: Tensor, other: Tensor) Tensor { + return binaryOp("maximum", dialects.stablehlo.maximum)(self, other); + } + + /// Returns a Tensor containing the element-wise minimum operation of the input Tensors. + pub fn minimum(self: Tensor, other: Tensor) Tensor { + return binaryOp("minimum", dialects.stablehlo.minimum)(self, other); + } + + /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. + pub fn remainder(self: Tensor, other: Tensor) Tensor { + return binaryOp("remainder", dialects.stablehlo.remainder)(self, other); + } + + /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. + pub fn addConstant(self: Tensor, b: anytype) Tensor { + return self.add(.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. + pub fn divByConst(self: Tensor, b: anytype) Tensor { + return self.div(.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise power of the input Tensor by a constant. + pub fn powByConst(self: Tensor, b: anytype) Tensor { + return self.pow(.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. + pub fn scale(self: Tensor, val: anytype) Tensor { + return self.mul(.scalar(val, self.dtype())); + } + + pub const LogicalOp = enum { OR, XOR, AND }; + + /// Returns a Tensor containing the element-wise logical operation of the input Tensors. + pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { + return switch (logical_op) { + .OR => binaryOp("or", dialects.stablehlo.or_)(self, other), + .XOR => binaryOp("xor", dialects.stablehlo.xor)(self, other), + .AND => binaryOp("and", dialects.stablehlo.and_)(self, other), + }; + } + + /// Returns a Tensor containing the element-wise floor operation of the input Tensor. + pub fn floor(self: Tensor) Tensor { + return _result(self._shape, dialects.stablehlo.floor(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); + } + + /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. + pub fn ceil(self: Tensor) Tensor { + return _result(self._shape, dialects.stablehlo.ceil(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); + } + + /// Returns a Tensor containing the element-wise conversion to another type. + pub fn convert(self: Tensor, to: DataType) Tensor { + if (to == self.dtype()) { + return self; + } + + const res_type = mlir.rankedTensorType(self.shape().dims(), mlirx.Type.fromDType(mlirCtx(), to)); + const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape.withDtype(to), op.result(0)); + } + + // TODO(Corentin) + //test convert { + // const floats = @import("floats.zig"); + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // // f4e2m1 + // { + // const x = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + // var x_f4: [x.len]floats.Float4E2M1 = undefined; + // for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); + + // const x_d = try zml.Buffer.fromArray(platform, x); + // const x_f4_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f4e2m1 }); + + // const x_f4_xla = x_f4_xla_d.getValue(@TypeOf(x_f4)); + // errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); + // try std.testing.expectEqualDeep(x_f4, x_f4_xla); + // } + + // // f8e3m4 + // { + // const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; + // var x_f8e3: [x.len]floats.Float8E3M4 = undefined; + // for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + + // const x_d = try zml.Buffer.fromArray(platform, x); + // const x_f8e3_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f8e3m4 }); + + // const x_f8e3_xla = x_f8e3_xla_d.getValue(@TypeOf(x_f8e3)); + // errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); + // try std.testing.expectEqualDeep(x_f8e3, x_f8e3_xla); + // } + //} + + /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. + pub fn round(self: Tensor) Tensor { + const round_op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, round_op.result(0)); + } + + /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. + pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { + const op = dialects.stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + // TODO(Corentin) + /// See torch.matmul + //pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { + // return @import("torch.zig").matmul(lhs, rhs); + //} + + /// Matrix multiplication, where contracting axes are specified using their tags. + /// eg dot(.{ .a, .b, .c }, .{ .a, .c, .d }, .c) -> .{ .a, .b, .d } + /// Axes with the same tag on both sides, and which aren't contracting, + /// are considered "batching axes". + pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { + const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); + const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); + + var batching_axes: stdx.BoundedArray([2]i8, Shape.MAX_RANK) = .{}; + for (0..lhs.rank()) |lhs_tag_index| { + const lhs_tag = lhs.shape().tag(lhs_tag_index); + if (lhs_tag == Shape.toTag(args)) continue; + if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { + batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); + } + } + return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); + } + + // TODO(Corentin) + //test dot { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .c = 20 }, .{ .c = 20 }, .{.c}, .{} }, + // .{ + // .{ .a = 20, .b = 21, .c = 22 }, + // .{ .a = 20, .d = 23, .c = 22 }, + // .{.c}, + // .{ .a = 20, .b = 21, .d = 23 }, + // }, + // .{ + // .{ .a = 20, .b = 21, .c = 22 }, + // .{ .c = 22, .d = 23, .e = 24 }, + // .{.c}, + // .{ .a = 20, .b = 21, .d = 23, .e = 24 }, + // }, + // .{ + // .{ .a = 20, .b = 21, .c = 22 }, + // .{ .c = 22, .d = 23, .a = 20 }, + // .{ .c, .a }, + // .{ .b = 21, .d = 23 }, + // }, + // }) |testcase| { + // const x_shape, const y_shape, const ctr, const z_shape = testcase; + // const x = Tensor.constant(x_shape, .{ .f32 = 0.0 }); + // const y = Tensor.constant(y_shape, .{ .f32 = 0.0 }); + // const z = x.dot(y, ctr); + + // try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), z.shape()); + // } + //} + + /// Generalized matrix multiplication of two tensors along the specified axes. + /// In this version batching dimensions need to be explicitly specified. + /// The result shape is made of (batching_axes ++ lhs_result_axes ++ rhs_result_axes. + /// Where "result axes" are non-contracting, non-batching axes of each input tensor. + pub fn dotGeneral( + lhs: Tensor, + rhs: Tensor, + contracting_axes: []const [2]i8, + batching_axes: []const [2]i8, + ) Tensor { + stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); + + const Axes = stdx.BoundedArray(i64, MAX_RANK); + + var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + // Validate batching axes + var lhs_batching_axes: Axes = .{}; + var rhs_batching_axes: Axes = .{}; + for (batching_axes) |b_axes| { + const l, const r = b_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + var t = lhs._shape.tag(l); + if (t == Shape.TagUnknown) t = rhs._shape.tag(r); + res_shape = res_shape.appendDim(lhs._shape.dim(l), t); + lhs_batching_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_batching_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + // Validate contracting axes + var lhs_contracting_axes: Axes = .{}; + var rhs_contracting_axes: Axes = .{}; + for (contracting_axes) |c_axes| { + const l, const r = c_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + // Result shape is obtained by concatenating batching dimensions, (already done) + // then dimensions from lhs axes that aren't contracting nor batching, + // then dimensions from rhs axes that aren't contracting nor batching. + for (0..lhs.rank()) |l| { + if (std.mem.indexOfScalar(i64, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, lhs_batching_axes.constSlice(), @intCast(l))) |_| { + continue; + } + res_shape = res_shape.appendDim(lhs._shape.dim(l), lhs._shape.tag(l)); + } + for (0..rhs.rank()) |r| { + if (std.mem.indexOfScalar(i64, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, rhs_batching_axes.constSlice(), @intCast(r))) |_| { + continue; + } + res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); + } + + const mlir_ctx = lhs.getContext().mlir_ctx; + const loc = mlir.Location.unknown(mlirCtx()); + const op = dialects.stablehlo.dot_general( + mlir_ctx, + lhs.value(), + rhs.value(), + mlir.rankedTensorType(res_shape.dims(), mlir.Type.fromDType(mlir_ctx, res_shape.dtype())), + .{ + .lhs_batching_dimensions = lhs_batching_axes.constSlice(), + .rhs_batching_dimensions = rhs_batching_axes.constSlice(), + .lhs_contracting_dimensions = lhs_contracting_axes.constSlice(), + .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), + .dot_precision = .fast, + }, + loc, + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. + pub fn sigmoid(self: Tensor) Tensor { + const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + pub const logistic = sigmoid; + + /// Returns a Tensor containing the ReLU activation function applied to each element of the input Tensor. + pub fn relu(self: Tensor) Tensor { + return self.maximum(.constant(self.dims(), self.dtype().zero())); + } + + /// Returns a Tensor containing the leaky-ReLU activation function applied to each element of the input Tensor. + /// + /// LeakyReLU(x) = max(0,x) + negative_slope * min(0,x) + /// ref: https://paperswithcode.com/method/leaky-relu + pub fn leakyReLU(self: Tensor, negative_slope: f32) Tensor { + const below_zero = self.scale(negative_slope).minimum(.scalar(0, self.dtype())); + return self.relu().add(below_zero); + } + + // TODO(Corentin) + //test leakyReLU { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const input = try zml.Buffer.fromSlice(platform, .{2}, &[_]f32{ -0.6884, 1.6795 }); + // const res = try zml.testing.compileAndCall(platform, leakyReLU, .{ input, 0.1 }); + + // const expectation = zml.HostBuffer.fromArray(&[2]f32{ -0.0688, 1.6795 }); + // try zml.testing.expectClose(expectation, res, 1e-4); + //} + + /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. + pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor) Tensor { + const sigmoid_tensor = self.mul(.constant(self._shape, DataType.Value.init(self.dtype(), beta))).sigmoid(); + const one_minus_sigmoid_tensor: Tensor = .constant(self._shape, .init(self.dtype(), 1)).sub(sigmoid_tensor); + + return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.matmul(self).add(b))); + } + + /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. + /// + /// We use an approximation of the erf function using tanh: + /// gelu(x) ≃ 0.5 * x * (1 + tanh(sqrt(2 / pi) * (x + 0.044715 * x^3))) + /// see: https://paperswithcode.com/method/gelu + pub fn gelu(x: Tensor) Tensor { + const scaled_x_cube = x.mul(x).mul(x).scale(0.044715); + const beta = std.math.sqrt(2.0 / std.math.pi); + const tanh_ = x.add(scaled_x_cube).scale(beta).tanh(); + return tanh_.addConstant(1).mul(x).scale(0.5); + } + + /// Returns a Tensor containing an approximation of the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. + /// + /// It's an even more crude approximation than gelu. + pub fn quickGelu(x: Tensor) Tensor { + return x.scale(1.702).sigmoid().mul(x); + } + + /// Returns a Tensor containing the Sigmoid Linear Unit (SiLU) activation function applied to each element of the input Tensor. + /// + /// silu(x) = x σ(x) + pub fn silu(x: Tensor) Tensor { + return x.mul(.sigmoid(x)); + } + + /// Computes softmax along the given axis. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) + pub fn softmax(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + const max_val = self.max(a); + const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), self.dtype())); + + const exp_diff_max = self.sub(max_val).convert(.f32).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a)).convert(self.dtype()); + + // If a row is full -inf return full 0 instead of full nan, + // this fix attention when mask hides a full row. + return row_mask.broad(self.shape()).select(res, .scalar(0, self.dtype())); + } + + /// Computes softmax, but adds a bias to the sum of exponentiel. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) + pub fn softmaxBiased(self: Tensor, axis_: anytype, bias: ?Tensor) Tensor { + const a = self.axis(axis_); + + if (bias == null) return self.softmax(axis_); + const b = bias.?.convert(self.dtype()).broad(self.shape().setDim(a, 1)); + const max_val: Tensor = maximum(self.max(a), b); + const exp_diff_max = self.sub(max_val).exp(); + const bias_diff_max = b.sub(max_val).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a).add(bias_diff_max)); + + // The bias means that denominator won't be 0, we don't need to handle that case. + return res; + } + + /// Returns a Tensor containing the log of the sum of exponential over the given axis. + pub fn logSumExp(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + // stabilization: shift `self` by it's max value before passing to exponent. + const max_ = self.max(a); + const log_sum_exp = log(sum(exp(self.sub(max_.broad(self._shape))), a)); + // restore the shift again + return max_.add(log_sum_exp); + } + + // TODO(Corentin) + /// Returns a Tensor containing the sum of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. + //pub fn sum(self: Tensor, axis_: anytype) Tensor { + // const a = self.axis(axis_); + // return ops.reduce( + // struct { + // pub fn acc(x: Tensor, res: Tensor) Tensor { + // return res.add(x.convert(res.dtype())); + // } + // }.acc, + // self, + // Tensor.scalar(0, self.dtype()), + // &.{a}, + // ); + //} + + /// Returns a Tensor containing the mean of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. + pub fn mean(self: Tensor, axis_: anytype) Tensor { + return self.sum(axis_).divByConst(self.dim(axis_)); + } + + // TODO(Corentin) + /// Returns a Tensor containing the cumulative sum of elements over the given axis. + /// Output shape is the same as input shape. + /// [0, 1, 0, 1, 0, 0, 1, 1].cumulativeSum(0) -> [0, 1, 1, 2, 2, 2, 3, 4] + /// The last value contains the sum of all element in the array. + //pub fn cumulativeSum(self: Tensor, axis_: anytype) Tensor { + // const rk = self.rank(); + // const a = self.axis(axis_); + + // const ones = [_]i64{1} ** MAX_RANK; + // var window_dimensions = ones; + // window_dimensions[a] = self.dim(a); + // var padding = [_][2]i64{.{ 0, 0 }} ** MAX_RANK; + // padding[a] = .{ self.dim(a) - 1, 0 }; + + // return ops.reduceWindow( + // Tensor.add, + // self, + // Tensor.scalar(0, self.dtype()), + // .{ + // .base_dilations = ones[0..rk], + // .window_dilations = ones[0..rk], + // .window_strides = ones[0..rk], + // .window_dimensions = window_dimensions[0..rk], + // .padding = padding[0..rk], + // }, + // ); + //} + + //test cumulativeSum { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _cumsum(input: Tensor) Tensor { + // const x = input.withPartialTags(.{.n}); + // const y = x.cumulativeSum(.n); + // // Check that tags are propagated + // std.debug.assert(y.shape().eqlWithTags(x.shape())); + // return y; + // } + // }; + + // const x = try zml.Buffer.fromArray( + // platform, + // [2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } }, + // ); + // const res = try zml.testing.compileAndCall(platform, Local._cumsum, .{x}); + // try std.testing.expectEqual( + // [2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, + // try res.getValue([2][5]f32), + // ); + //} + + /// Returns a transposed Tensor computed using the given axes. + pub fn transpose(self: Tensor, axes_: anytype) Tensor { + const axes__ = self.axes(axes_).constSlice(); + const default_perm = [MAX_RANK]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; + const no_op = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + + const permutation: []const i64 = if (axes__.len == 0) + default_perm[MAX_RANK - self.rank() ..] + else + toI64(axes__); + + stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, MAX_RANK + 2)] }); + + if (std.mem.eql(i64, permutation, no_op[0..self.rank()])) { + return self; + } + + const res_shape = self._shape.transpose(permutation); + if (transposeIsJustAReshape(self.shape(), permutation)) { + return self.reshape(res_shape); + } + + const op = dialects.stablehlo.transpose( + mlirCtx(), + self.value(), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .{ .permutation = toI64(permutation) }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + pub fn swapAxes(self: Tensor, a: anytype, b: anytype) Tensor { + if (self.axis(a) == self.axis(b)) return self; + var perm: Shape.AxesArray = .{}; + for (0..self.rank()) |i| { + perm.appendAssumeCapacity(@intCast(i)); + } + perm.set(self.axis(a), self.axis(b)); + perm.set(self.axis(b), self.axis(a)); + return self.transpose(perm.constSlice()); + } + + /// Returns a Tensor with the given axis unflattened. + /// + /// unflatten((d0, d1, axis_m, d3), 2, n) -> (d0, d1, n, d2_m, d3) + pub fn unflatten(self: Tensor, axis_: i8, n: i64) Tensor { + // TODO: move to torch.zig, this equivalent to `spitAxis` + stdx.debug.assert(self.rank() < Tensor.MAX_RANK, "unflatten expects input tensor rank to be less than {}, got {}", .{ Tensor.MAX_RANK, self.rank() }); + + const a = if (axis_ >= 0) self.axis(axis_) else self.axis(axis_) + 1; + const new_dim = std.math.divExact(i64, self.dim(a), n) catch std.debug.panic("unflatten expects chosen dimension to be divisible by 'n' but {} is not divisible by {}", .{ self.dim(a), n }); + const new_shape = self._shape.set(a, n).insert(a + 1, .{ ._ = new_dim }); + + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), + self.value(), + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + mlirx.tensorType(self.getContext().mlirCtx(), new_shape), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(new_shape, reshaped_val.result(0)); + } + + /// Splits the given axis in several axes. + /// eg: `Tensor.init(.{ .a = 10, .b = 3 }).split(.a, .{.a1 = 5, .a2 = 2});` + /// The number of elements in the split shape must match the number of element + /// in the target axis. + pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { + const new_shape = self._shape.splitAxis(ax, split_shape); + + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), + self.value(), + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(new_shape, reshaped_val.result(0)); + } + + /// Merges two or more contiguous axes into one axis. + pub fn merge(self: Tensor, merges_: anytype) Tensor { + return self.reshape(self._shape.mergeAxes(merges_)); + } + + /// Merges two or more non-contiguous axes into one axis. + /// Will make a transpose if needed. + /// .{ .a, .b, .c }.mergeTranspose(.{ .a, .c }, .ac) -> .{ .b, .ac } + pub fn mergeTranspose(self: Tensor, axes_: anytype, merged: @TypeOf(.enum_literal)) Tensor { + const cont = self.contiguous(axes_); + return cont.reshape(cont._shape.mergeAxis(merged, axes_)); + } + + /// Transposes the input Tensor, such has the given axes end up in contiguous position. + /// .{ .a, .b, .c, .d }.contiguous(.{ .c, .a }) -> .{ .b, .d, .c, .a } + pub fn contiguous(self: Tensor, axes_: anytype) Tensor { + const perm = self._shape.contiguousPerm(axes_); + return self.transpose(perm.constSlice()); + } + + pub fn flatten(self: Tensor) Tensor { + return self.reshape(.{self.count()}); + } + + pub const Slice = struct { + start: i64 = 0, + end: i64 = to_the_end, + step: i32 = 1, + singleton: bool = false, + + const full = .{ .start = 0, .end = to_the_end, .step = 1 }; + + pub fn single(offset: i64) Slice { + return .{ .start = offset, .end = offset + 1, .singleton = true }; + } + + pub fn absolute(s: Slice, d: i64) Slice { + const start = if (s.start < 0) d + s.start else s.start; + const end = if (s.end == to_the_end) d else if (s.end < 0) d + s.end else s.end; + const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; + stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f}", .{ s, d, res }); + return res; + } + + const to_the_end = std.math.maxInt(i64); + + pub fn format(self: Slice, writer: *std.Io.Writer) !void { + if (self.singleton) { + try writer.print("[{d}]", .{self.start}); + } else if (self.end == to_the_end and self.step == 1) { + try writer.print("[{d}..]", .{self.start}); + } else if (self.step == 1) { + try writer.print("[{d}..{d}]", .{ self.start, self.end }); + } else { + try writer.print("[{d}..{d}:{d}]", .{ self.start, self.end, self.step }); + } + } + }; + + /// Slices the input Tensor over the given axis using the given parameters. + pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { + var slices = [_]Slice{.{}} ** MAX_RANK; + slices[self.axis(axis_)] = s; + return self.slice(slices[0..self.rank()]); + } + + /// Slices the input Tensor using the given parameters. + pub fn slice(self: Tensor, slices: []const Slice) Tensor { + var start_indices: [MAX_RANK]i64 = undefined; + var strides: [MAX_RANK]i64 = undefined; + var limit_indices: [MAX_RANK]i64 = undefined; + var res_shape: Shape = self._shape; + + for (slices, 0..) |s, a| { + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); + + const args: Slice = s.absolute(self.dim(a)); + start_indices[a] = args.start; + limit_indices[a] = args.end; + strides[a] = args.step; + res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); + } + + const result_type = mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())); + const slice_op = dialects.stablehlo.slice( + mlirCtx(), + self.value(), + start_indices[0..self.rank()], + limit_indices[0..self.rank()], + strides[0..self.rank()], + result_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + var res = _result(res_shape, slice_op.result(0)); + var to_remove: Shape.AxesArray = .{}; + for (slices, 0..) |s, a| { + if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); + } + return res.reshape(res_shape.removeMany(to_remove.constSlice())); + } + + // TODO(Corentin) + //test slice { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + + // // Wrap slice1d to hide the anytype in the signature. + // const Local = struct { + // pub fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { + // return input.slice1d(ax, slice_); + // } + // }; + + // { + // const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, 0, .{ .end = 1 } }); + // try std.testing.expectEqual([5]f32{ 0, 1, 2, 3, 4 }, try res.getValue([5]f32)); + // } + // { + // const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, 1, .{ .start = 1, .step = 2 } }); + // try std.testing.expectEqual([4]f32{ 1, 3, 6, 8 }, try res.getValue([4]f32)); + // } + // { + // const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, -1, .{ .start = -2 } }); + // try std.testing.expectEqual([4]f32{ 3, 4, 8, 9 }, try res.getValue([4]f32)); + // } + //} + + inline fn wrapIndex(self: Tensor, axis_: usize, idx: i64) i64 { + return if (idx < 0) self.dim(axis_) + idx else idx; + } + + pub fn choose1d(self: Tensor, axis_: anytype, i: i64) Tensor { + return self.slice1d(axis_, .single(i)); + } + + pub fn choose(self: Tensor, offsets: anytype) Tensor { + const off, const tags = Shape.parseDimensions(offsets); + var slices = [_]Slice{.{}} ** MAX_RANK; + for (off.constSlice(), tags.constSlice()) |o, t| { + const ax = self.axis(t); + slices[ax] = .single(o); + } + return self.slice(slices[0..self.rank()]); + } + + /// Concatenates the input Tensors along the given axis. + pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { + if (tensors.len == 1) return tensors[0]; + stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); + var buffer: [32]mlir.Value = undefined; + std.debug.assert(tensors.len <= buffer.len); + std.debug.assert(tensors.len > 0); + const a = tensors[0].axis(axis_); + // TODO(Corendos): Check that tensor axes match. + + var concatenated_dim: i64 = 0; + for (tensors, 0..) |t, i| { + buffer[i] = t.value(); + concatenated_dim += t.dim(a); + } + + const res_shape = tensors[0]._shape.set(a, concatenated_dim); + const op = dialects.stablehlo.concatenate(mlirCtx(), buffer[0..tensors.len], a, .unknown(mlirCtx())).appendTo(currentBlock()); + // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); + return _result(res_shape, op.result(0)); + } + + /// Concatenates the input Tensors along a new axis. The Tensors must have the same shape. + /// For x, y, z of shape .{ .a = 10, .b = 11, .c = 12 }: + /// - Tensor.stack(&.{x, y, z}, .b, .layers) -> .{ .a, .layers, .b, .c } + /// - Tensor.stack(&.{x, y, z}, 1, .layers) -> .{ .a, .layers, .b, .c } + /// - Tensor.stack(&.{x, y, z}, .last, .layers) -> .{ .a, .b, .c, .layers } + pub fn stack(tensors: []const Tensor, axis_: anytype, tag: anytype) Tensor { + // Note: we could ask the compilation context for some memory instead of stack allocating + stdx.debug.assert(tensors.len <= 32, "stack only supports up to 32 tensors, got {}", .{tensors.len}); + + const shape0 = tensors[0]._shape; + const res_shape = shape0.insertTag(axis_, 1, tag); + + for (tensors[1..]) |tensor| { + stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); + } + + var reshaped: [32]Tensor = undefined; + for (tensors, 0..) |tensor, i| { + reshaped[i] = tensor.reshape(res_shape); + } + + // Be careful here: we need to resolve ax before calling concatenate, + // because we added an axis, so all + const ax = if (@TypeOf(axis_) == @TypeOf(.enum_literal) and axis_ == .last) + shape0.rank() + else + shape0.axis(axis_); + + return Tensor.concatenate(reshaped[0..tensors.len], ax); + } + + /// Repeats a Tensor several times along the given axis. + /// + /// * repeat1d(x, axis, 4) = concat(&.{x, x, x, x}, axis); + /// * repeat1d([0, 1, 2, 3], 0, 2) = [0, 1, 2, 3, 0, 1, 2, 3] + pub fn repeat1d(self: Tensor, axis_: anytype, n_rep: u63) Tensor { + if (n_rep == 1) { + return self; + } + + const a = self.axis(axis_); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); + + const broadshape = self._shape.insert(a, .{n_rep}); + const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a); + + return self.broadcast(broadshape, repeat_dims.dims()).reshape(res_shape); + } + + /// Repeats a Tensor several times along the given axes. + pub fn repeat(self: Tensor, n_reps: []const u63) Tensor { + // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); + stdx.debug.assert(n_reps.len == self.rank(), "repeat expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); + + var res = self; + for (n_reps, 0..) |n_rep, a| { + if (n_rep == 1) continue; + + res = res.repeat1d(a, n_rep); + } + return res; + } + + // TODO(Corentin) + //test repeat1d { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + // return x.repeat1d(axis_, n_reps); + // } + // }; + + // { + // const inputs: [3]u8 = .{ 1, 2, 3 }; + // const expectations: [6]u8 = .{ 1, 2, 3, 1, 2, 3 }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 0, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + // { + // const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + // const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 1, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + //} + + /// Repeats in line each value along the given axis. + /// + /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] + /// This is equivalent to repeat(ax+1) unless ax is the last axis. + pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u63) Tensor { + const a = self.axis(axis_); + const broadshape = self._shape.insert(a + 1, .{n_rep}); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); + + const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); + } + + /// Repeats in line each value along the given axes. + pub fn stutter(self: Tensor, n_reps: []const u63) Tensor { + // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); + stdx.debug.assert(n_reps.len == self.rank(), "stutter expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); + + var res = self; + for (n_reps, 0..) |n_rep, a| { + if (n_rep == 1) continue; + res = res.stutter1d(@intCast(a), n_rep); + } + return res; + } + + // TODO(Corentin) + //test stutter1d { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + // return x.stutter1d(axis_, n_reps); + // } + // }; + + // { + // const inputs: [3]u8 = .{ 1, 2, 3 }; + // const expectations: [6]u8 = .{ 1, 1, 2, 2, 3, 3 }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + // { + // const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + // const expectations: [2][6]u8 = .{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 1, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + // { + // const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + // const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + //} + + /// Returns a Tensor containing the element-wise negation of the input Tensor. + pub fn negate(self: Tensor) Tensor { + const negate_op = dialects.stablehlo.negate(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, negate_op.result(0)); + } + + /// Returns a Tensor containing the element-wise cosine of the input Tensor. + pub fn cos(self: Tensor) Tensor { + const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, cosine_op.result(0)); + } + + /// Returns a Tensor containing the element-wise sine of the input Tensor. + pub fn sin(self: Tensor) Tensor { + const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, sine_op.result(0)); + } + + /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. + pub fn exp(self: Tensor) Tensor { + const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. + pub fn log(self: Tensor) Tensor { + const op = dialects.stablehlo.log(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise square-root of the input Tensor. + pub fn sqrt(self: Tensor) Tensor { + const sqrt_op = dialects.stablehlo.sqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, sqrt_op.result(0)); + } + + /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. + pub fn rsqrt(self: Tensor) Tensor { + const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, rsqrt_op.result(0)); + } + + /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. + pub fn tanh(self: Tensor) Tensor { + const tanh_op = dialects.stablehlo.tanh(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, tanh_op.result(0)); + } + + /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. + pub fn exponentialMinusOne(self: Tensor) Tensor { + const expm1_op = dialects.stablehlo.exponential_minus_one(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, expm1_op.result(0)); + } + + pub const ArangeArgs = struct { + start: i64 = 0, + end: i64, + step: i64 = 1, + }; + /// + /// Returns a Tensor containing evenly spaced values within a given interval. + pub fn arange(args: ArangeArgs, dt: DataType) Tensor { + stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); + stdx.debug.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); + + const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; + const sh = Shape.init(.{n_steps}, dt); + var op = dialects.stablehlo.iota( + mlirCtx(), + 0, + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + var res = _result(sh, op.result(0)); + + if (args.step != 1) { + res = res.scale(args.step); + } + + if (args.start != 0) { + res = res.addConstant(args.start); + } + + return res; + } + + /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. + /// + /// The output dtype will be `.i32`, unless the given axis has a too big dimension, in that case we use `.i64`. + /// In most program this shouldn't matter, because typically this will be used in a comparison, + /// or explicitly converted by the user to do floating point arithmetic. + pub fn iota(sh: Shape, axis_: anytype) Tensor { + const a = sh.axis(axis_); + const dt: DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; + const res_shape = sh.withDtype(dt); + + var op = dialects.stablehlo.iota( + mlirCtx(), + a, + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + pub const LinspaceArgs = struct { + start: f64, + end: f64, + steps: i64, + }; + + /// Returns a Tensor containing 'args.steps' values evenly spaced from 'args.start' to 'args.end', inclusive. + pub fn linspace(args: LinspaceArgs, dt: DataType) Tensor { + stdx.debug.assert(args.start < args.end, "linspace expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); + stdx.debug.assert(args.steps > 0, "linspace expects 'args.steps' to be positive, got {}", .{args.steps}); + stdx.debug.assert(dt.isFloat(), "linspace expects type to be a float, got {} (hint: use arange instead)", .{dt}); + + const sh = Shape.init(.{args.steps}, dt); + var iota_op = dialects.stablehlo.iota( + mlirCtx(), + 0, + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + var res = _result(sh, iota_op.result(0)); + + if (args.steps != 1) { + res = res.scale(args.steps); + } + + if (args.start != 0) { + res = res.addConstant(args.start); + } + + return res; + } + + /// Returns a 0-rank Tensor with the given value. + pub fn scalar(val: anytype, dt: DataType) Tensor { + const data = DataType.Value.init(dt, val); + switch (dt.class()) { + .float => stdx.debug.assert(!std.math.isNan(val), "scalar(NaN) is probably due to compiling a model with an uninitialized field", .{}), + else => {}, + } + return Tensor.constant(.{}, data); + } + + // TODO(Corentin) + //test scalar { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _fwd() [6]Tensor { + // var res: [6]Tensor = undefined; + // const dtypes = .{ .bool, .u8, .i32, .f32, .bf16, .u64 }; + // inline for (0..6) |i| res[i] = scalar(0, dtypes[i]); + // return res; + // } + // }; + + // _ = try zml.testing.compileAndCall(platform, Local._fwd, .{}); + //} + + /// Returns a constant Tensor with the given value. + pub fn constant(val: DataType.Value) Tensor { + const op = dialects.stablehlo.constant( + mlirCtx(), + &.{}, + mlirx.Type.fromDType(mlirCtx(), val.dtype()), + val.constSlice(), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(.init(&.{}, val.dtype()), op.result(0)).autoBroadcast(); + } + + // TODO(Corentin) + pub fn zeroes(sh: Shape) Tensor { + return .constant(sh.dtype().zero()); + } + + // TODO(Corentin) + /// Embeds a buffer with concrete values into an Mlir program. + pub fn constantTensor(val: HostBuffer) Tensor { + const ctx = CompilationContext.current().mlirCtx(); + const loc = ctx.location(@src()); + const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); + const constant_op = dialects.stablehlo.constant(ctx, val.shape().dims(), elem_type, val.bytes(), loc); + return _result(val.shape(), constant_op.result(0)); + } + + /// Returns a Tensor containing the result of the outer product between the input Tensors. + pub fn outer(self: Tensor, other: Tensor) Tensor { + if (self.rank() + other.rank() == 1) { + return self.mul(other); + } + + const res_shape = self.shape().outer(other.shape()); + return self.broad(res_shape).mul(other.broad(res_shape)); + } + + /// Given a tensor and a shape of the same rank, + /// will "broadcast" the given axes, so that `self` has the given shape. + /// This happens by virtually repeating the data several time along each give axes. + /// Note: most of the time the optimizer will make it so that the broadcast doesn't trigger a copy. + /// Note: the tags of the return tensor will be from the `output_shape`. + /// This means if you use and un-tagged broadcast on a tagged tensor, + /// you will lose the tags. + /// To avoid use favorise `.broad(shape)` when working with tagged tensors. + pub fn broadcast(self: Tensor, output_shape: Shape, axes_: []const i64) Tensor { + stdx.debug.assert(axes_.len == self.rank(), "broadcast expects axes_ to map all axes from self to axes of the output shape, got broadcast({f}, {f}, {any})", .{ self, output_shape, axes_ }); + for (0.., axes_) |self_ax, other_ax| { + const d = self.dim(self_ax); + stdx.debug.assert(d == 1 or d == output_shape.dim(other_ax), "broadcast expects shape axes to either be 1-sized or to match the target size. got broadcast({f}, {f}, {any}), error on self axis {d} mapping to other axis {d}", .{ self, output_shape, axes_, self_ax, other_ax }); + } + + const res_shape = output_shape.withDtype(self.dtype()); + if (std.mem.eql(i64, self.dims(), output_shape.dims())) { + // No broadcast needed. We don't emit a new stablehlo value + // but we propagate output_shape tags. + return _result(res_shape, self.value()); + } + const result_type = mlir.rankedTensorType( + res_shape.dims(), + mlirx.Type.fromDType(mlirCtx(), res_shape.dtype()), + ); + const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(res_shape, broadcast_op.result(0)); + } + + /// Broadcasts a Tensor to the given shape, adding axes at the beginning. + pub fn broadcastLeft(self: Tensor, output_shape: Shape) Tensor { + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastLeft expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); + + const a = output_shape.rank() - self.rank(); + if (self.rank() == output_shape.rank() and std.mem.eql(i64, self.dims(), output_shape.dims())) { + return self; + } + + return self.broadcast(output_shape, Shape.range(output_shape.rank(), output_shape.dtype()).dims()[a..]); + } + + /// Broadcasts a Tensor to the given shape, adding axes at the end. + pub fn broadcastRight(self: Tensor, output_shape: Shape) Tensor { + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastRight expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); + + if (self.rank() == output_shape.rank() and self._shape.eql(output_shape)) { + return self; + } + + return self.broadcast(output_shape, Shape.range(self.rank(), output_shape.dtype()).dims()); + } + + /// Broadcasts a Tensor to the given shape, extending dimensions if needed. + pub fn broad(self: Tensor, other: Shape) Tensor { + // TODO: broad is too restrictive because sometime you only want to specify one specific axis + // Note: if you code below, make sure to update Shape.canBroadcastTo. + stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self, other }); + + // Already the right shape + if (std.mem.eql(i64, self.dims(), other.dims())) return self; + + // Non ambiguous broadcasting + // TODO: broad is error prone because of this: + // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } + if (self._shape.rank() == 0 or self._shape.rank() == other.rank()) { + const all_axes = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + return self.broadcast(other, all_axes[0..self.rank()]); + } + + // check that each axis of self maps to an axis of other + var axes_: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (self._shape.tags()) |t| { + axes_.appendAssumeCapacity(@intCast(other.axis(t))); + } + return self.broadcast(other, axes_.constSlice()); + } + + /// Reshapes the input Tensor with the given shape. + pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { + const output_shape = self._shape.reshape(output_shape_); + const tensor_type = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(mlirCtx(), output_shape.dtype())); + const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(output_shape, reshape_value.result(0)); + } + + /// Converts the given 1 element Tensor into a 0-rank Tensor. + pub fn asScalar(self: Tensor) Tensor { + stdx.debug.assert(self.count() == 1, "Tensor.asScalar expects an input with exactly 1-element got {f}", .{self}); + return if (self.rank() == 0) self else self.reshape(.{}); + } + + pub const Pad = struct { + low: i64 = 0, + high: i64 = 0, + interior: i64 = 0, + }; + + /// Pads the input Tensor with the given values. + /// Usage: x.pad(0, .{ .a = .{ .low = 1, .high = 1 }}); + pub fn pad(self: Tensor, padding_value: anytype, paddings: anytype) Tensor { + const _paddings = self.shape().parseAxesOptions(Pad, paddings, .{}); + + const ZEROS = [_]i64{0} ** MAX_RANK; + var low = ZEROS; + var high = ZEROS; + var interior = ZEROS; + + var res_shape = self._shape; + for (_paddings.constSlice(), 0..) |padding, i| { + low[i] = padding.low; + high[i] = padding.high; + interior[i] = padding.interior; + + var d: i64 = self.dim(i); + d += low[i] + (@max(d - 1, 0) * interior[i]) + high[i]; + res_shape._dims.set(i, d); + } + + const rk = self.rank(); + const pad_op = dialects.stablehlo.pad( + mlirCtx(), + self.value(), + Tensor.scalar(padding_value, self.dtype()).value(), + .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(res_shape, pad_op.result(0)); + } + + /// Inserts 1-dim axes at the given position, with the given tags. + /// `.{.a = 5, .b = 4}.insert(.b, .{ .c, .d }) -> .{ .a = 5, .c = 1, .d = 1, .b = 4 }` + pub fn insertAxes(self: Tensor, axis_: anytype, tags: anytype) Tensor { + const tags_ = Shape.parseTags(tags); + const ax = if (@TypeOf(axis_) == @TypeOf(.enum_literal) and axis_ == .last) + self.rank() + else + self.axis(axis_); + + var res_shape = self._shape; + const ones = [_]i64{1} ** MAX_RANK; + res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; + res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + return self.reshape(res_shape); + } + + /// Appends a 1-dim axis, with the given tag. + pub fn appendAxes(self: Tensor, t: anytype) Tensor { + // stdx.debug.assert(self.rank() < Tensor.MAX_RANK - t.len, "appendAxis expects tensor rank to be small enough in order to extend it, got {} and {} (max is {})", .{ self.rank(), t.len, Tensor.MAX_RANK }); + + return self.insertAxes(.last, t); + } + + /// Drops a 1-dim axis at the given index + pub fn squeeze(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); + + const new_shape = self._shape.remove(a); + // log.debug("squeeze({}, {d}={d}) -> ({})", .{ self, axis, a, new_shape }); + + return _result(new_shape, self.reshape(new_shape).value()); + } + + /// Returns a Tensor with the given axes reversed. + pub fn reverse(self: Tensor, axes_: anytype) Tensor { + const actual_axes = self._shape.axes(axes_); + + const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, reverse_op.result(0)); + } + + pub const GatherOpts = struct { indices_are_sorted: bool = false }; + + /// `gather` extracts slices from the given tensor at the specified offsets. + /// example: `values.gather(.{ .a = idx }, .{})` + /// + /// * indices is a named list of integer tensors: eg `.{ .a = idx }`. + /// Each names specify a gathering axis, it must refer to an axis of self, + /// and the corresponding idx Tensor must contains valid indices into axis .a. + /// All indices must have the same shape or broadcast to the same shape. + /// + /// * result is a tensor whose shape is similar to the input shape + /// where the gathered axes have been replaced by axes from 'indices'. + /// + /// Some example input for the base case where we work on one axis: + /// - gather(f:[a], .{ .a = idx:[n]})[n] == f[idx[n]] + /// - gather(f:[a, b], .a, idx:[n])[n, b] == f[idx[n], b] + /// - gather(f:[a,b,c], .{.b = idx:[n,m]})[a, n, m, c] == f[a, idx[n, m], c] + /// + /// If an axis in common between `self` and `indices`, + /// it is treated as a "batching" axis, meaning that semantically + /// the operator is doing a gather one time per dimension of this axis: + /// - gather(f: [a,b,c], .{.b=idx: [a,n]})[a, n] == f[a, idx[a, n]] + /// + /// It's possible to pass several indices: + /// - gather(f: [a,b,c], .{.b=idx_b[n], .c=idx_c[n]})[a, n] == f[a, idx_b[n], idx_c[n]] + /// - gather(f: [a,b,c,d], .{.b=idx_b[a,n], .c=idx_c[a, n]})[a, n, d] == f[a, idx_b[a, n], idx_c[a, n], d] + /// + /// If `self` isn't tagged, you can use `gather_` to specify gathered axis by their position but batching won't be available. + /// + /// For performance it's better to have batching and gathering axes of `self` be the first one, + /// so that gather can + pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { + const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); + var idx_axes: Shape.AxesArray = .{}; + for (idx_tags.slice()) |t| { + idx_axes.appendAssumeCapacity(self.axis(t)); + } + + // TODO: sort indices following self.shape instead of asking the user to do it. + return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); + } + + pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); + for (idx_axes, 0..) |a, i| { + if (i > 0) { + stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); + } + } + var indices_shape = idx_per_axis[0].shape(); + for (idx_per_axis[1..]) |idx| { + if (idx.rank() > indices_shape.rank()) { + indices_shape = idx.shape(); + } + } + for (idx_per_axis) |idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + } + + var idx_batch_axes: Shape.DimsArray = .{}; + + const AxisKind = enum { batching, offset, collapsed, indices }; + var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + if (indices_shape.hasTag(t)) |id_ax| { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { + // we collapsed all gathered axes + self_kind.buffer[self_ax] = .collapsed; + // idx_kind.buffer[id_ax] = .indices; + } else { + self_kind.buffer[self_ax] = .offset; + } + } + + // compute res shape + var res_shape = Shape.init(.{}, self.dtype()); + var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; + for (self_kind.slice(), 0..) |kind, ax_usize| { + const ax: u3 = @intCast(ax_usize); + if (ax == idx_axes[0]) { + // The first val_ax is special cause this is the place where we insert indices axes. + for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + // Batching axis is already in self. + if (is_batching_axis) continue; + + res_shape = res_shape.appendDim(id_inserted_dim, id_axis); + res_kind.appendAssumeCapacity(.indices); + } + } + switch (kind) { + .collapsed => continue, + else => { + res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); + res_kind.appendAssumeCapacity(kind); + }, + } + } + + // This is not a gather, but a dynamicSlice. + // Sometimes the backend recognize this pattern, but not always. + // So let us handle that. + if (indices_shape.count() == 1 and idx_axes.len == 1) { + return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); + } + + var slice_dims: Shape.DimsArray = .{}; + for (self_kind.slice(), self.dims()) |k, d| { + slice_dims.appendAssumeCapacity(switch (k) { + .batching, .collapsed => 1, + .offset => d, + .indices => unreachable, + }); + } + + // TODO: try changing .last by other axis and see the perf impact. + const indices = Tensor.stack(idx_per_axis, .last, .coord); + // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); + const gather_op = dialects.stablehlo.gather( + mlirCtx(), + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), + .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = indices.axis(.coord), + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); + stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + return _result(res_shape, gather_op.result(0)); + } + + // TODO(Corentin) + //test gather { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _idx(idx_shape: anytype) Tensor { + // return Tensor.constant(idx_shape, .{ .i32 = 0 }); + // } + // }; + + // const idx = Local._idx; + + // { + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, + // .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, + // // .{ .{ .a = 10, .b = 20 }, 0, idx(.{ .n = 8 }), .{ .n = 8, .b = 20 } }, + // // Favor val shape, instead of indices shape. + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, + // .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, + // // batching axes are implicits. + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10 }) }, .{ .a = 10 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .b = 20 }) }, .{ .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8 } }, + // // stablehlo.gather is biased toward indices shape (like gatherSlice). + // // This make it awkward to use when you have both batching dimension and new indices dimensions. + // // For now we reject those, and let user explicitly transpose self or indices if needed. + // // .{ .{ .a = 10, .b = 20 }, .{.b = idx(.{ .n = 8, .a = 10 })}, .{ .a = 10, .n = 8 } }, + // // Also handle tuples + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + // }) |testcase| { + // const x_shape, const indices, const res_shape = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const y = gather(x, indices, .{}); + // try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + // try std.testing.expect(y.value().owner().verify()); + // } + + // inline for (.{ + // .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, + // }) |testcase| { + // const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const y = gather_(x, idx_axes, idx_per_axis, .{}); + // try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + // try std.testing.expect(y.value().owner().verify()); + // } + // } + //} + + /// Gathers slices along the given axes with runtime indices. + /// * slice_shape represents the shape of the slices to extract, + /// it must be smaller than original shape. + /// It must use a subset of self axes. + /// If slice_shape is **not** tagged, then it must have the same rank than self. + /// * `indices` represents a set of coordinates. + /// The coordinates are read from the `.coord` axis, or last axis if `.coord` is not found. + /// The coordinate axis must have `slice_shape.rank()` dims. + /// The coordinates represent the "top-left" corner of the slice to extract. + /// * the output tensor starts with axes from `indices`. + /// * if the input tensor has tagged axes, matching `indices` axes, + /// they will be considered "batching" axes. + /// + /// Sample input/output shapes: + /// * gatherSlices([A, B, C, D], .{.b=B', .c=C'}, [N, 2]) -> [N, A, B', C', D] + /// * gatherSlices(x(a,b,c,d), .{.b=B', .c=C'}, g(n,m)) = z(n, a, b', c', d) = x(a, g(n, 0) + b', g(n, 1) + c', d) + /// + /// Note: the axis order of the result is different from gatherValues. + /// This is because gatherSlices, favorizes contiguous copy of the extracted slices, + /// while gatherValues, always copy values one by one, and as such don't have the same issues. + /// In our example the contiguous dimension .d is not sliced + /// and gatherSlices can copy data by group of C'*D elements. + pub fn gatherSlices(self: Tensor, slice_shape_: anytype, indices: Tensor, opts: GatherOpts) Tensor { + const slice_shape = if (@TypeOf(slice_shape_) == Shape) slice_shape_ else Shape.init(slice_shape_, .i32); + // scoped_log.debug("gatherSlice({}, {f}, {})", .{ self, slice_shape, indices }); + + const tagged_api = slice_shape.isFullyTagged(); + if (tagged_api) { + for (slice_shape.tags()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "gatherSlices expects `slices_shape` to only use tags from `self`. But {s} wasn't found in {f}", .{ t, self }); + } + } else { + // For untagged api, we require all slices to be specified. + // Note: we could relax this and right align the slice. + stdx.debug.assert(slice_shape.rank() == self.rank(), "gatherSlices expects `slice_shape.rank()` to match `self.rank()`. Got: gatherSlices({f}, slice={f}). To avoid specifying all axes in `slice_shape`, you can use tags.", .{ self, slice_shape }); + } + + const index_coord_axis = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); + stdx.debug.assert(indices.dim(index_coord_axis) == slice_shape.rank(), "gatherSlices({f}, slice={f}, indices) expects 'indices' to be a tensor [..., {}], got {f}", .{ self, slice_shape, slice_shape.rank(), indices }); + + // Compute result shape + var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); + var slice_dims = self._shape._dims; + var self_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var indices_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var start_index_map: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var self_offset_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); + + if (tagged_api and indices._shape.hasTag(t) != null) { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_batch_axes.appendAssumeCapacity(@intCast(self_ax)); + indices_batch_axes.appendAssumeCapacity(indices._shape.axis(t)); + slice_dims.set(self_ax, 1); + stdx.debug.assert(slice_shape.hasTag(t) == null, "gatherSlices expect axes to be either batches or slices axes. Axis {s} has been found both in `slices={f}` and `indices={f}`", .{ t, slice_shape, indices }); + } else if (maybe_slice_ax) |slice_ax| { + // Specified axes contains the start offset of the slices, + // and are collected in `start_index_map`. + const slice_dim = slice_shape.dim(slice_ax); + stdx.debug.assert(slice_dim <= self._shape.dim(self_ax), "gatherSlices expects `slice_shape` to be smaller than `self.shape()`. On axis {s}, got {f} > {f}.", .{ t, slice_shape, self._shape }); + slice_dims.set(self_ax, slice_dim); + res_shape = res_shape.appendDim(slice_dim, t); + start_index_map.appendAssumeCapacity(@intCast(self_ax)); + self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); + } else { + // non-batching, non-indexed axes + res_shape = res_shape.appendDim(self.dim(self_ax), t); + self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); + } + } + + const gather_op = dialects.stablehlo.gather( + mlirCtx(), + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = self_offset_axes.constSlice(), + .collapsed_slice_dims = &.{}, + .operand_batching_dims = self_batch_axes.constSlice(), + .start_indices_batching_dims = indices_batch_axes.constSlice(), + .start_index_map = start_index_map.constSlice(), + .index_vector_dim = index_coord_axis, + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, gather_op.result(0)); + } + + // TODO(Corentin) + //test gatherSlices { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { + // return self.gatherSlices(slice_shape, indices, opts); + // } + // }; + + // { + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, + // .{ .{ .a = 10 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7 } }, + // .{ .{ .a = 10 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7 } }, + // .{ .{ .a = 10 }, .{ .a = 7 }, .{ .coord = 1, .n = 8 }, .{ .n = 8, .a = 7 } }, + // // tags aren't required. + // .{ .{10}, .{7}, .{ .n = 8, ._ = 1 }, .{ .n = 8, ._ = 7 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, .coord = 1, .m = 9 }, .{ .n = 8, .m = 9, .a = 7, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = 7, .b = 17 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, + // // Note: currently the order of the axes in the slice is not used. + // .{ .{ .a = 10, .b = 20 }, .{ .b = 17, .a = 7 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, + // .{ .{ .a = 10, .b = 20, .c = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17, .c = 20 } }, + // // batching dims + // .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .a = 10, ._ = 1 }, .{ .a = 10, .b = 17 } }, + // .{ .{ .b = 200, .a = 100, .c = 300 }, .{ .c = 300 }, .{ .a = 100, .b = 200, ._ = 1 }, .{ .a = 100, .b = 200, .c = 300 } }, + // }) |testcase| { + // const x_shape, const slice_dims, const idx_shape, const res_shape = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const slice_shape = Shape.init(slice_dims, .u16); + // const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); + // const y = gatherSlices(x, slice_shape, idx, .{}); + // try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + // try std.testing.expect(y.value().owner().verify()); + + // const mod = try zml.compileFn( + // std.testing.allocator, + // Local._gatherSlices, + // .{ x.shape(), slice_shape, idx.shape(), .{ .indices_are_sorted = true } }, + // platform, + // ); + // defer mod.deinit(); + // } + // } + + // // Test with actual values. + // const range = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 2 * 4 * 6 }, .u16); + // defer range.deinit(std.testing.allocator); + // const operand = try range.reshape(.{ .a = 2, .b = 4, .c = 6 }).toDevice(platform); + // defer operand.deinit(); + // const start_indices = (try zml.Buffer.fromArray(platform, [2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })).withTags(.{ .n, ._ }); + // defer start_indices.deinit(); + + // const result = try zml.testing.compileAndCall(platform, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), start_indices, .{} }); + + // const expected = zml.HostBuffer.fromArray(&[2][2][2][3]u16{ + // .{ + // .{ .{ 13, 14, 15 }, .{ 19, 20, 21 } }, + // .{ .{ 37, 38, 39 }, .{ 43, 44, 45 } }, + // }, + // .{ + // .{ .{ 3, 4, 5 }, .{ 9, 10, 11 } }, + // .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, + // }, + // }); + // try zml.testing.expectClose(expected, result, 0); + //} + + pub const ScatterArgs = struct { + input: Tensor, + update: Tensor, + }; + + pub const ScatterOpts = struct { + /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final offset in `self` + /// Result is undefined if the promise is violated. + indices_are_sorted: bool = false, + + /// Promise scatter that slices don't overlap. + /// Result is undefined if the promise is violated. + /// This allows for better code generation, because it means that updates can be applied in parallel. + indices_are_unique: bool = false, + + /// Function used to update previous value in `self` with values from `updates`. + /// If `update_fn` is not associative (ie the order of execution matters), + /// then you should make sure the slices don't overlap, + /// otherwise the result will depend on the runtime scheduling + /// of the operator which is backend specific. + update_fn: *const fn (ScatterArgs) struct { Tensor } = increment, + + pub fn increment(values: ScatterArgs) struct { Tensor } { + return .{values.input.add(values.update)}; + } + + pub fn override(values: ScatterArgs) struct { Tensor } { + return .{values.update}; + } + }; + + /// Generalized version of scatter to many inputs. + /// See `zml.Tensor.scatterSlices` for documentation on scatter. + /// + /// This allows to use the same indices to update several tensors at once, + /// and where the update function is allow to look at elements from the different tensors + /// to compute the final value. + /// + /// This sounds nice but in practice XLA doesn't support this well on GPU, + /// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. + pub fn scatter( + inputs: anytype, + index_tensors: anytype, + updates: anytype, + comptime func: anytype, + context: anytype, + opts: Tensor.ScatterOpts, + ) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const update_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].input = Tensor.init(Shape.init(.{}, inputs[i].dtype())); + args[i].update = Tensor.init(Shape.init(.{}, inputs[i].dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].input.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].update.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const update_block = mlir.Block.init(&block_types, &block_locs); + errdefer update_block.deinit(); + + CompilationContext.current().pushBlock(update_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(update_block); + break :b .{ update_block, result }; + }; + + // Note: I was a bit lazy here, and I only look at tags on the first tensor. + // we probably should check all of them. + const self = meta.first(Tensor, inputs); + const update = meta.first(Tensor, updates); + var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); + + if (indices_per_axis.len == 0) return inputs; + + // validate coord axes: all coord_axes should exist inside self + for (indices_axes.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); + } + + // Handle scalar indices by broadcasting them to the indices with the highest rank. + const indices_shape = blk: { + var higher_rank = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |indices| { + if (indices.rank() > higher_rank.rank()) { + higher_rank = indices.shape(); + } + } + break :blk higher_rank; + }; + for (indices_per_axis.slice()) |*idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); + stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); + idx.* = idx.broad(indices_shape); + } + + // rewrite simple scatters to dynamicUpdateSlice. + if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { + return .{self.dynamicUpdateSlice(index_tensors, update)}; + } + + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. + var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); + const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); + // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; + // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var updates_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); + + const op = dialects.stablehlo.scatter( + mlirCtx(), + &input_values, + &.{indices.value()}, + &updates_values, + update_block, + .{ + .update_window_dims = _collectAxes(AxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(AxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(AxisKind, config.op_kind, .batching).constSlice(), + .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), + .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), + .index_vector_dim = indices.rank() - 1, + .indices_are_sorted = opts.indices_are_sorted, + .unique_indices = opts.indices_are_unique, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), op.result(i)); + } + + return result; + } + + const ScatterConfig = struct { + op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + indices_batch_axes: Shape.DimsArray = .{}, + scatter_to_operand_axes: Shape.DimsArray = .{}, + updates_transpose: Shape.AxesArray = .{}, + }; + + const AxisKind = enum { batching, update_window, inserted_window, window_id }; + + fn scatterConfig( + op: Shape, + update: Shape, + indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: Shape.TagsArray, + ) ScatterConfig { + var op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var indices_batch_axes: Shape.DimsArray = .{}; + var scatter_to_operand_axes: Shape.DimsArray = .{}; + var updates_transpose: Shape.AxesArray = .{}; + + const tagged_api = indices_axes.len > 0; + const indices = indices_per_axis.get(0).shape(); + + if (tagged_api) { + for (indices_axes.constSlice()) |t| { + scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); + } + for (indices.tags()) |t| { + stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); + updates_transpose.appendAssumeCapacity(update.axis(t)); + } + + for (op.tags()) |t| { + if (update.hasTag(t)) |up_ax| { + updates_transpose.appendAssumeCapacity(up_ax); + + if (indices.hasTag(t)) |id_ax| { + if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { + // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim + op_kind.appendAssumeCapacity(.inserted_window); + } else { + // tag is in op, indices and updates -> it's a batching dim + op_kind.appendAssumeCapacity(.batching); + indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); + } + } else { + op_kind.appendAssumeCapacity(.update_window); + } + } else { + op_kind.appendAssumeCapacity(.inserted_window); + } + } + + for (update.tags(), 0..) |t, up_ax| { + // Handle batch axes right away. + if (op.hasTag(t)) |self_ax| { + if (op_kind.get(self_ax) == .batching) { + up_kind.appendAssumeCapacity(.batching); + continue; + } + } + if (indices.hasTag(t) != null) { + up_kind.appendAssumeCapacity(.window_id); + } else if (op.hasTag(t)) |self_ax| { + stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); + up_kind.appendAssumeCapacity(.update_window); + } else { + // TODO: consider accepting untagged update here. + std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); + } + } + } else { + for (0..indices_per_axis.len) |i| { + op_kind.appendAssumeCapacity(.inserted_window); + scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); + up_kind.appendAssumeCapacity(.window_id); + } + for (indices_per_axis.len..op.rank()) |_| { + op_kind.appendAssumeCapacity(.update_window); + } + for (indices_per_axis.len..update.rank()) |_| { + up_kind.appendAssumeCapacity(.update_window); + } + for (0..update.rank()) |i| { + updates_transpose.appendAssumeCapacity(@intCast(i)); + } + } + + return .{ + .op_kind = op_kind, + .up_kind = up_kind, + .indices_batch_axes = indices_batch_axes, + .scatter_to_operand_axes = scatter_to_operand_axes, + .updates_transpose = updates_transpose, + }; + } + + // TODO(Corentin) + //test scatterConfig { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // const Local = struct { + // pub fn _idx(idx_shape: anytype) Tensor { + // return Tensor.constant(idx_shape, .{ .i32 = 0 }); + // } + // }; + + // const idx = Local._idx; + // const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); + + // // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + // { + // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); + // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + // const cfg = scatterConfig(op, update, indices, coords_tags); + // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); + // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); + // } + + // // similar, but use the normalized form where .a is no longer an explicit batching axis. + // { + // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); + // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + // const cfg = scatterConfig(op, update, indices, coords_tags); + // try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); + // try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); + // } + //} + + /// Concatenate all indices tensor in one tensor. + /// + /// Is allowed to reorder stuff to simplify the job of the backend, + /// and to expand the batching dims. + fn scatterPrepareIndices( + cfg: *ScatterConfig, + op: Shape, + update: Shape, + indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: *Shape.TagsArray, + ) Tensor { + var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; + const batching = _collectAxes(AxisKind, cfg.op_kind, .batching); + for (batching.constSlice()) |batch_ax| { + const id_shape = indices_per_axis.get(0).shape(); + // batching requires tagging, so we're sure to have a tag here. + const batch_tag = op.tag(batch_ax); + indices_axes.appendAssumeCapacity(batch_tag); + const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); + indices_per_axis.appendAssumeCapacity(batch_id); + cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; + cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; + old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); + } + cfg.indices_batch_axes = .{}; + + // Reorder the axes so that in indices_per_axis is ordered like in op if possible. + // TODO: transpose updates if needed + var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; + var scatter_to_op_axes: Shape.DimsArray = .{}; + + while (old_scatter_to_op_axes.len > 0) { + const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; + const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); + const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); + + scatter_to_op_axes.appendAssumeCapacity(op_ax); + indices.appendAssumeCapacity(scatter_idx); + } + cfg.scatter_to_operand_axes = scatter_to_op_axes; + + for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { + if (i != sc_ax) { + //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); + break; + } + } + return Tensor.stack(indices.constSlice(), .last, .coord); + } + + /// Update the given tensor, by copying `values` into slice by slice into `self`. + /// The slices are chosen at runtime by interpreting indices as coordinates into `self`. + /// This is a generalized version of `dynamicUpdateSlice` where more than one offset can be specified at a time. + /// + /// ### Arguments + /// + /// - Return a tensor with same shape than `self`, with updated content. + /// - `indices` is a set of Tensor (typically rank 1), representing coordinates into `self`. + /// all indices must have the same shape, but scalars are accepted. + /// - each `indices` entry contains offset along an axes into `self`. + /// Typically axes are identified by their tags, but in the absence of tags on `indices`, + /// The entry in indices will be assigned to axes of `self` from major to minor axis. + /// It is recommended to have indices referencing only major axes of `self` for better performance. + /// - `values` shape is obtained by concatenating the shape of `indices` with the shape of the slices to be extracted. + /// - `opts`: `zml.Tensor.ScatterOpts` des + /// + /// ### Sample input/output shapes with corresponding pseudo-code. + /// + /// Basic `scatterSlices` with the first two axes (.a, .b) being indexed, and full (.c, .d) slice copies: + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N]}, y[N, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],c,d] += y[n, c, d]; + /// }} + /// } + /// return z; + /// } + /// ``` + /// + /// `scatterSlices` with the first three axes (.a, .b, .c) being indexed, and a partial copy of (.c, .d). + /// Note that .c axis is present both in the indices and updates, and `updates.dim(.c) < self.dim(.c)`. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N], .c=off_c[N]}, y[N, C', D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C') |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],off_c[n]+c,d] += y[n, c, d]; + /// }} + /// } + /// return z; + /// } + /// ``` + /// + /// `scatterSlices` with the first axis .a being indexed, and where .b is used as a batching axis. + /// Note that here .b axis is present in `self`, `off_a`, and `updates`, + /// and is not mentionned in the axes of indices. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[B,N]}, y[N, B, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..B) |b| { + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[b,n],b,c,d] += y[n, b, c, d]; + /// }} + /// } + /// } + /// return z; + /// } + /// ``` + /// + /// ### Warnings + /// + /// - if `opts.update_fn` is not associative not all calls to `scatterSlices` are sound. + /// In particular if you scatter overlapping slices, with `zml.Tensor.ScatterOpts.override`, + /// then the result will depend on the execution order that you don't control. + /// - `scatterSlices` is a very expressive operator, and can lead to complicated code generation + /// that requires host<->device synchronization. + /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. + pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { + //scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); + + const UpdateType = @TypeOf(ScatterOpts.increment); + + const Custom = struct { + pub fn inc(values: ScatterArgs, custom: *const UpdateType) struct { Tensor } { + return @call(.auto, custom, .{values}); + } + }; + + return scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; + } + + // TODO(Corentin) + //test scatterSlices { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _scatter(self: Tensor, indices: []const Tensor, updates: Tensor) Tensor { + // return self.scatterSlices( + // indices, + // updates, + // .{ .update_fn = ScatterOpts.increment }, + // ); + // } + + // pub fn _scatterCB(self: Tensor, coords: Tensor, updates: Tensor) Tensor { + // return self.scatterSlices( + // .{ .c = coords.choose1d(.coord, 0), .b = coords.choose1d(.coord, 1) }, + // updates, + // .{ .update_fn = ScatterOpts.increment }, + // ); + // } + + // pub fn _idx(idx_shape: anytype) Tensor { + // return Tensor.constant(idx_shape, .{ .i32 = 0 }); + // } + // }; + + // { + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + // const idx = Local._idx; + + // inline for (.{ + // // This is equivalent to a dynamic update slice, update 3 values at given offset of axis .a: + // .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{ .a = 3 } }, + // // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, + // // Same but with update transposed + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .b = 2, .n = 8 } }, + // // similar, but use the normalized form where a is no longer an explicit batching axis. + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a2 = 10, .n = 8 }), .b = idx(.{ .a2 = 10, .n = 8 }) }, .{ .a2 = 10, .n = 8, .b = 2 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 2 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }), .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 3, .b = 2 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .a = 3, .n = 8, .b = 2 } }, + // }) |testcase| { + // const x_shape, const indices, const updates_shapes = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const updates = Tensor.constant(updates_shapes, .{ .f16 = 0 }); + + // const y = scatterSlices(x, indices, updates, .{}); + // // Shape doesn't change with scatterSlices + // try zml.testing.expectEqualShapes(x.shape(), y.shape()); + // try std.testing.expect(y.value().owner().verify()); + // } + // } + // // Test with actual values, no batching. + // { + // const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); + // const a = (try zml.Buffer.from(platform, a_host.reshape(.{ 3, 3 }), .{})).withTags(.{ .a, .b }); + // defer a.deinit(); + // a_host.deinit(std.testing.allocator); + + // const scatter_indices = try zml.Buffer.fromArray(platform, [2]i32{ 0, 2 }); + // const updates = try zml.Buffer.fromArray(platform, [2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } }); + + // const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; + // const result = try zml.testing.compileAndCall(platform, Local._scatter, .{ + // a, + // &.{scatter_indices.withTags(.{.n})}, + // updates.withTags(.{ .n, .b }), + // }); + // try std.testing.expect(a.shape().eql(result.shape())); + // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + // } + // // Test with setting individual values (no batching) + // { + // const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); + // const a = try zml.Buffer.from(platform, a_host, .{}); + // defer a.deinit(); + // a_host.deinit(std.testing.allocator); + + // const scatter_indices = try zml.Buffer.fromArray(platform, [2]i32{ 2, 7 }); + // const updates = try zml.Buffer.fromArray(platform, [2]i32{ 20, 70 }); + + // const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; + // const result = try zml.testing.compileAndCall(platform, Local._scatter, .{ + // a, + // &.{scatter_indices.withTags(.{.n})}, + // updates.withTags(.{.n}), + // }); + // try std.testing.expect(a.shape().eql(result.shape())); + // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + // } + // { + // // Test with actual values and batching along axis .a + // const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0, .{}); + // defer operand.deinit(); + // const start_indices = (try zml.Buffer.fromArray( + // platform, + // [2][2][3][2]i32{ + // .{ + // .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, + // .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, + // }, + // .{ + // .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, + // .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, + // }, + // }, + // )).withTags(.{ .n, .a, .m, .coord }); + // defer start_indices.deinit(); + + // const values = try zml.Buffer.constant( + // platform, + // Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16), + // 1, + // .{}, + // ); + // defer values.deinit(); + + // const result = try zml.testing.compileAndCall(platform, Local._scatterCB, .{ operand, start_indices, values }); + + // const expected = [2][3][4][2]u16{ + // .{ + // .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, + // .{ .{ 0, 0 }, .{ 0, 0 }, .{ 2, 2 }, .{ 2, 2 } }, + // .{ .{ 0, 0 }, .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 } }, + // }, + // .{ + // .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, + // .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, + // .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, + // }, + // }; + // try std.testing.expect(operand.shape().eql(result.shape())); + // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + // } + //} + + /// Returns a Tensor containing the maximum over a given axis. + pub fn max(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + return reduce( + .{self}, + .{Tensor.constant(self.dtype().minValue())}, + &.{a}, + struct { + pub fn cmp(args: ReduceArgs) struct { Tensor } { + return .{args.left.maximum(args.right)}; + } + }, + .{}, + )[0]; + } + + /// Returns a Tensor containing the minimum over a given axis. + pub fn min(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + return reduce( + .{self}, + .{Tensor.constant(self.dtype().maxValue())}, + &.{a}, + struct { + pub fn cmp(args: ReduceArgs) struct { Tensor } { + return .{args.left.minimum(args.right)}; + } + }, + .{}, + )[0]; + } + + pub const ArgMaxRes = struct { + values: Tensor, + indices: Tensor, + }; + + pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { + const a = x.axis(axis_); + const dt: DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; + + const values, const indices = reduce( + .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, + .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, + &.{a}, + struct { + fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { + const left_gt_right = values.left.cmp(.GT, values.right); + const is_nan = values.left.cmp(.NE, values.left); + const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); + + // If values.left == values.right: keep the smallest idx. + const is_same = values.left.cmp(.EQ, values.right); + const is_first = indices.left.cmp(.LT, indices.right); + const is_same_but_first = is_same.logical(.AND, is_first); + const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + const max_idx = keep_left_idx.select(indices.left, indices.right); + + return .{ max_val, max_idx }; + } + }.cmp, + .{}, + ); + return .{ .values = values, .indices = indices }; + } + + // TODO(Corentin) + //test argMax { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + // const allocator = std.testing.allocator; + // const ArgMaxTest = struct { + // pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { + // return x.argMax(1); + // } + // }; + + // const argmax = try zml.compileFn(allocator, ArgMaxTest._fwd, .{Shape.init(.{ 1, 5 }, .f32)}, platform); + // defer argmax.deinit(); + // // Test with tie + // { + // const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }}); + // const res = argmax.call(.{x}); + // const max_ = res.values.getValue(f32); + // const max_idx = res.indices.getValue(i32); + // try std.testing.expectEqual(max_, 7.9); + // // We should always return the first max found. + // try std.testing.expectEqual(max_idx, 2); + // } + + // // Test with Nan + // { + // const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }}); + // const res = argmax.call(.{x}); + // const max_ = try res.values.getValue(f32); + // const max_idx = try res.indices.getValue(i32); + // try std.testing.expect(std.math.isNan(max_)); + // try std.testing.expectEqual(max_idx, 1); + // } + //} + + pub const ReduceArgs = struct { + left: Tensor, + right: Tensor, + }; + + fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes_)), + }, + .verify = true, + .location = .unknown(mlirCtx()), + }).appendTo(currentBlock()); + + // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. + // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. + // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, + // to find the correct mlir.Value, but we first broadcast before creating the final + // Tensor struct. + var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (0..Tensor.MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { + broadcasting_axes.append(@intCast(i)) catch unreachable; + } + } + + inline for (0..result.len) |i| { + var reduced_shape: Shape = inputs[i].shape(); + for (axes_) |a| { + reduced_shape = reduced_shape.setDim(a, 1); + } + + const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlirCtx(), reduced_shape.dtype())); + const broad_op = dialects.stablehlo.broadcast_in_dim( + mlirCtx(), + reduce_op.result(i), + broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + result[i] = Tensor._result(reduced_shape, broad_op.result(0)); + } + + return result; + } + + pub const SortRes = ArgMaxRes; + + pub const SortArgs = struct { + left: Tensor, + right: Tensor, + }; + + pub const SortOpts = struct { descending: bool = false }; + + /// Returns two Tensors. The first contains the sorted values and the second one contains the sorted indices. + pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { + const a = self.axis(axis_); + const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); + const res = sort_(.{ self, indices }, a, struct { + fn call(direction: dialects.stablehlo.ComparisonDirection.Direction, values: SortArgs, _: SortArgs) Tensor { + return values.left.cmp(direction, values.right); + } + }.call, .{if (opts.descending) .GT else .LT}, true); + return .{ .values = res[0], .indices = res[1] }; + } + + fn sort_(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const sort_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].left = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); + args[i].right = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const sort_block = mlir.Block.init(&block_types, &block_locs); + errdefer sort_block.deinit(); + + CompilationContext.current().pushBlock(sort_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(sort_block); + break :b .{ sort_block, result }; + }; + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + const sort_op = mlir.Operation.make(mlirCtx(), "stablehlo.sort", .{ + .operands = .{ .variadic = &.{&input_values} }, + .result_type_inference = true, + .blocks = &.{sort_block}, + .attributes = &.{ + .named(mlirCtx(), "dimension", mlir.integerAttribute(mlirCtx(), .i64, axis_)), + .named(mlirCtx(), "is_stable", mlir.boolAttribute(mlirCtx(), is_stable)), + }, + .verify = true, + .location = .unknown(mlirCtx()), + }).appendTo(currentBlock()); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); + } + + return result; + } + + pub const ArgSortOpts = SortOpts; + + /// Returns a Tensor containing the indices corresponding to the sorted values over the given axis. + pub fn argsort(self: Tensor, axis_: anytype, opts: ArgSortOpts) Tensor { + return self.sort(axis_, .{ .descending = opts.descending }).indices; + } + + // TODO(Corentin) + //test argsort { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { + // return x.argsort(axis_, opts); + // } + // }; + + // var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + // defer arena_state.deinit(); + // const allocator = arena_state.allocator(); + // // 2D Tensor - dim = 1, ascending + // { + // const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 }); + // const res = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{} }); + // const res_cpu = try res.toHostAlloc(allocator); + // try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); + // } + // // 3D Tensor, dim = 1, descending + // { + // const x = try zml.Buffer.fromSlice(platform, .{ 1, 5, 10 }, &[_]f16{ + // -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, + // 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, + // 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, + // 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, + // 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, + // }); + // const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{ .descending = true } }); + // const res = try res_dev.toHostAlloc(allocator); + // try std.testing.expectEqualSlices(i32, &.{ + // 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, + // 2, 0, 4, 4, 1, 3, 4, 4, 1, 0, + // 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, + // 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, + // 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, + // }, res.items(i32)); + // } + // // 4D Tensor, dim = 3, ascending + // { + // const x = try zml.Buffer.fromSlice(platform, .{ 4, 2, 1, 4 }, &[_]i32{ + // 89, 31, 22, 42, + // 64, 39, 0, 30, + // 64, 71, 46, 31, + // 89, 82, 78, 86, + // 55, 32, 43, 19, + // 93, 24, 45, 72, + // 64, 86, 62, 88, + // 57, 21, 19, 12, + // }); + // const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 3, .{} }); + // const res = try res_dev.toHostAlloc(allocator); + // try std.testing.expectEqualSlices(i32, &.{ + // 2, 1, 3, 0, + // 2, 3, 1, 0, + // 3, 2, 0, 1, + // 2, 1, 3, 0, + // 3, 1, 2, 0, + // 1, 2, 3, 0, + // 2, 0, 1, 3, + // 3, 2, 1, 0, + // }, res.items(i32)); + // } + //} + + pub const TopKOpts = SortOpts; + + /// Returns a Tensor representing the result of Top-K over the given axis. + pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: TopKOpts) SortRes { + stdx.debug.assert(k > 0, "topK expects a k > 0, got 0", .{}); + const err_msg = "topK named axis should be an integer or a named axis, eg `x.topK(.{{ .best_token = .token }}, 16)` or `x.topK(-1, 16)`"; + const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { + .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, + .@"struct" => |info| blk: { + stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); + break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; + }, + else => stdx.debug.compileError(err_msg, .{}), + }; + var result = self.sort(a, opts); + result.values = result.values.slice1d(a, .{ .end = k }); + result.indices = result.indices.slice1d(a, .{ .end = k }); + if (has_name) |new_name| { + result.values._shape._tags.set(a, new_name.ptr); + result.indices._shape._tags.set(a, new_name.ptr); + } + return result; + } + + // TODO(Corentin) + // + //pub const MaxPoolRes = ArgMaxRes; + /// Computes the 1d maxPool operation on the input Tensor. + //pub fn maxPool1d(self: Tensor, opts: struct { + // window_dimensions: i64, + // window_strides: ?i64, + // base_dilations: i64 = 1, + // window_dilations: i64 = 1, + // padding: [2]i64 = .{ 0, 0 }, + //}) MaxPoolRes { + // // TODO migrate to the following syntax. + // // maxPool(.{.a = .{ .stride = 5, .dilation = 2, .padding = .{0, 1} }, + // // .b = .{ .stride = 8, .dilation = 2, .padding = .{0, 1} }), + // // maxPool(.{ + // // .stride = .{ .a = 5, .b = 8 }, + // // .dilation = .{ .a = 2, .b = 2 }, + // // .padding = .{ .a = .{ 0, 2 }, .b = .{0, 2} + // // }) + + // // TODO: support maxPool on non last axis + // const a = self.axis(-1); + // const ones = [_]i64{1} ** Tensor.MAX_RANK; + // var window_dimensions = ones; + // window_dimensions[a] = opts.window_dimensions; + // var window_strides = window_dimensions; + // if (opts.window_strides) |stride| window_strides[a] = stride; + + // var base_dilations = ones; + // base_dilations[a] = opts.base_dilations; + // var window_dilations = ones; + // window_dilations[a] = opts.window_dilations; + + // var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + // padding[a] = opts.padding; + + // return ops.reduceWindow( + // MaxPoolRes.cmp, + // .{ .values = self, .indices = iota(self._shape, a) }, + // .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, + // .{ + // .window_dimensions = window_dimensions[0..self.rank()], + // .window_strides = window_strides[0..self.rank()], + // .base_dilations = base_dilations[0..self.rank()], + // .window_dilations = window_dilations[0..self.rank()], + // .padding = padding[0..self.rank()], + // }, + // ); + //} + + ///// Computes the 2d maxPool operation on the input Tensor. + //pub fn maxPool2d(self: Tensor, opts: struct { + // window_dimensions: [2]i64, + // window_strides: ?[2]i64 = null, + // base_dilations: [2]i64 = .{ 1, 1 }, + // window_dilations: [2]i64 = .{ 1, 1 }, + // padding: [2][2]i64 = .{ .{ 0, 0 }, .{ 0, 0 } }, + //}) MaxPoolRes { + // // TODO: rewrite using modern ZML + // stdx.debug.guard(self.rank() == 3 or self.rank() == 4, @src()); + + // // TODO: support maxPool on non last axis + // // Note: the problem is initPoolArg assuming last axis + // const a = self.axis(-1); + + // const window_dimensions = initPoolArg(self.rank(), &opts.window_dimensions); + // const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), &ws) else window_dimensions; + // const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); + // const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); + + // var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + // padding[a - 1] = opts.padding[0]; + // padding[a] = opts.padding[1]; + + // return ops.reduceWindow( + // MaxPoolRes.cmp, + // .{ .values = self, .indices = iota(self._shape, a) }, + // .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, + // .{ + // .window_dimensions = window_dimensions[0..self.rank()], + // .window_strides = window_strides[0..self.rank()], + // .base_dilations = base_dilation[0..self.rank()], + // .window_dilations = window_dilations[0..self.rank()], + // .padding = padding[0..self.rank()], + // }, + // ); + //} + + /// Chunk a given tensor into exactly n parts of equal shape. + /// `self.dim(axis_)` must be divisible by n_chunks. + pub fn chunkExact(self: Tensor, axis_: anytype, n_chunks: comptime_int) [n_chunks]Tensor { + const a = self.axis(axis_); + const d = self.dim(a); + const chunk_size = @divExact(d, n_chunks); + var chunks: [n_chunks]Tensor = undefined; + for (0..n_chunks) |i| { + const start: i64 = @as(i64, @intCast(i)) * chunk_size; + chunks[i] = self.slice1d(a, .{ .start = start, .end = start + chunk_size }); + } + return chunks; + } + + // TODO(Corentin) + //test chunkExact { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, + // .{ .{ .a = 12, .b = 2 }, .a, 3, .{ .a = 4, .b = 2 } }, + // .{ .{ 12, 2 }, 0, 3, .{ 4, 2 } }, + // }) |testcase| { + // const x_shape, const ax, const n_chunks, const res = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const chunks = x.chunkExact(ax, n_chunks); + + // const res_shape = Shape.init(res, .f16); + // for (chunks) |chk| { + // try zml.testing.expectEqualShapes(res_shape, chk.shape()); + // } + // } + //} + + /// Chunk a given tensor into n parts of equal shape, and one part with the remaining items. + /// When `self.dim(axis_)` is divisible by `n_chunks` it will return exactly `n_chunks`. + pub fn chunkAllowTrailing( + self: Tensor, + axis_: i64, + n_chunks: comptime_int, + ) []Tensor { + const a = self.axis(axis_); + const d = self.dim(a); + const chunk_size: i64 = @divFloor(d, n_chunks); + const tail_chunk_size: i64 = @rem(d, chunk_size); + + const allocator = self.getContext().allocator(); + var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); + for (0..n_chunks) |i| { + const start: i64 = @as(i64, @intCast(i)) * chunk_size; + chunks.appendAssumeCapacity( + self.slice1d(a, .{ .start = start, .end = start + chunk_size }), + ); + } + if (tail_chunk_size != 0) { + const start: i64 = n_chunks * chunk_size; + chunks.appendAssumeCapacity(self.slice1d(a, .{ .start = start })); + } + return chunks.items; + } + + // TODO(Corentin) + //test chunkAllowTrailing { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, + // .{ .{ .a = 10, .b = 2 }, .a, 3, .{ .a = 3, .b = 2 }, .{ .a = 1, .b = 2 } }, + // .{ .{ 10, 2 }, 0, 3, .{ 3, 2 }, .{ 1, 2 } }, + // .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, + // }) |testcase| { + // const x_shape, const ax, const n_chunks, const res, const trailing = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const chunks = x.chunkAllowTrailing(x.axis(ax), n_chunks); + + // const res_shape = Shape.init(res, .f16); + // for (chunks[0..n_chunks]) |chk| { + // try zml.testing.expectEqualShapes(res_shape, chk.shape()); + // } + // const trailing_shape = Shape.init(trailing, .f16); + // if (trailing_shape.rank() > 0) { + // try std.testing.expectEqual(n_chunks + 1, chunks.len); + // try zml.testing.expectEqualShapes(trailing_shape, chunks[n_chunks].shape()); + // } else { + // try std.testing.expectEqual(n_chunks, chunks.len); + // } + // } + //} + + pub fn split(self: Tensor, axis_: anytype, split_sizes: []const i64) []Tensor { + stdx.debug.assert(split_sizes.len > 0, "split expects at least one 'split_sizes', got 0", .{}); + + const a = self.axis(axis_); + const d = self.dim(a); + var split_sum: i64 = 0; + for (split_sizes) |n| split_sum += n; + stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); + + const allocator = self.getContext().allocator(); + const res = allocator.alloc(Tensor, split_sizes.len) catch @panic("OOM"); + errdefer allocator.dealloc(res); + + var start: i64 = 0; + for (split_sizes, 0..) |n, i| { + res[i] = self.slice1d(a, .{ .start = start, .end = start + n }); + start += n; + } + return res; + } + + /// Slices the input Tensor along a specific axis, with a start offset known at runtime. + /// Note: this doesn't support tagging, if you have tags, + /// you should use `dynamicSlice` directly. + pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { + stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {f}", .{slice_.start}); + + const a = self.axis(axis_); + const new_shape = self._shape.set(a, slice_.len); + + var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** MAX_RANK; + start_indices[a] = slice_.start.value(); + + const op = dialects.stablehlo.dynamic_slice( + mlirCtx(), + self.value(), + new_shape.dims(), + start_indices[0..self.rank()], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(new_shape, op.result(0)); + } + + pub const DynSlice = struct { start: Tensor, len: i64 }; + + /// Slices a Tensor across many axes, with runtime known offsets. + /// + /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. + /// When using the tagged API it is allowed to not specify some axes. + /// But with the non-tagged API all slices need to be specified. + /// Examples: + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ .a = .{ .start = a_off, .len = 11}); + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ + /// .a = .{ .start = a_off, .len = 11 }, + /// .b = .{ .start = b_off, .len = 12 }, + /// }); + /// Tensor(.{ 20,30,40}).dynamicSlice(.{.{ .start = scalar(0, .i32), .len = 20 }, .{ .start = b_off, .len = 12 }, .{ .start = scalar(0, .i32), .len = 40 }}); + /// ``` + pub fn dynamicSlice(self: Tensor, slices_: anytype) Tensor { + // TODO: the untagged api is a bit verbose. Should I allow: `Tensor(.{ 20,30,40}).dynamicSlice(.{.{}, .{ .start = b_off, .len = 12 }, .{}});` ?? + // + const slices, const slices_tags = Shape.parseStruct(DynSlice, slices_); + + // TODO use slices and slices_tags for the format. + // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" + // which is kinda ugly. + + const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; + const zero = Tensor.scalar(0, idx_dtype).value(); + var offset_values = [_]*const mlir.Value{zero} ** MAX_RANK; + var res_shape = self._shape; + for (slices.constSlice(), 0..) |slice_, i| { + const offset = slice_.start; + const len = slice_.len; + if (slices_tags.len == 0) { + stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {d} and {d}", .{ self.rank(), slices.len }); + + offset_values[i] = offset.value(); + res_shape._dims.set(i, len); + + stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {d} and {d} for index {d}", .{ len, self.dim(i), i }); + } else { + const t = slices_tags.get(i); + const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {f})", .{ t, self._shape }); + + stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {d} and {d} for axis {s}", .{ len, self.dim(a), t }); + + offset_values[a] = offset.value(); + res_shape._dims.set(a, len); + } + } + const op = dialects.stablehlo.dynamic_slice(mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + // TODO(Corentin) + //test dynamicSlice { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + // const T = f32; + + // { + // const x = try zml.Buffer.fromArray(platform, [10]T{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + // const z = try zml.Buffer.scalar(platform, 4, .i32); + // const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 0, .{ .len = 2, .start = z } }); + + // try std.testing.expectEqual([2]T{ 4, 5 }, try res.getValue([2]T)); + // } + + // { + // // Strided + // const x = try zml.Buffer.fromArray(platform, [2][5]T{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const z = try zml.Buffer.scalar(platform, 3, .i32); + + // const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 1, .{ .len = 2, .start = z } }); + // try std.testing.expectEqual([4]T{ 3, 4, 8, 9 }, res.getValue([4]T)); + // } + //} + + /// Updates a slice of the input Tensor along a specific axis using the given 'update' Tensor, with a start offset known at runtime. + /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. + pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { + const placeholder = Tensor.scalar(0, .i32); + var start_indices = [_]Tensor{placeholder} ** MAX_RANK; + start_indices[self.axis(axis_)] = offset; + return self.dynamicUpdateSlice(start_indices[0..self.rank()], update); + } + + /// Updates a part of the input Tensor using the given 'update' Tensor, with runtime known offsets. + /// + /// The offsets are specified similarly to the dynamicSlice api. + /// It's semantically equivalent to: + /// self.dynamicSlice(offsets_) := update + /// Examples: + /// ``` + /// Tensor(.{ .a = 2, .b = 5 }).dynamicUpdateSlice(.{ .a = scalar(1, .i32) }, Tensor(.{ .b = 5 })); + /// ``` + pub fn dynamicUpdateSlice(self: Tensor, offset_: anytype, update_: Tensor) Tensor { + // TODO: add updateSlice for when the offset isn't dynamic + stdx.debug.assert(self.dtype() == update_.dtype(), "dynamicUpdateSlice expects input and 'update_' tensors to be of the same type, got {} and {}", .{ self.dtype(), update_.dtype() }); + + const offset, const offset_tags = Shape.parseStruct(Tensor, offset_); + // log.debug("offset: {any}, offset_tags: {any}", .{ offset, offset_tags }); + for (offset.constSlice(), 0..) |start_idx, i| { + stdx.debug.assert(start_idx.rank() == 0, "dynamicUpdateSlice expects 'offset_' tensor ranks to be equal to 0, got {} at index {}", .{ start_idx.rank(), i }); + } + + const tagged_api = update_._shape.isFullyTagged() and self._shape.isFullyTagged() and offset_tags.len > 0; + // When using tags, we can safely insert axis with a 1-dim. + // the offset into the inserted axis will need to be specified through indices. + var update = update_; + if (tagged_api) { + // Check that all update tags are known. + for (update._shape._tags.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "dynamicUpdateSlice expects 'update_' tensor tags to be a subset of input tensor tags but {s} is missing (input shape is {f})", .{ t, self._shape }); + } + + var update_shape = self._shape; + var prev_ax: i8 = -1; + for (self._shape.tags(), 0..) |t, self_ax| { + if (update._shape.hasTag(t)) |up_ax| { + stdx.debug.assert(up_ax == prev_ax + 1, "dynamicUpdateSlice expects 'update_' and input tensor axis to have the same order, got {f} and {f}. (hint: you need to explicitly transpose 'update_')", .{ update_, self }); + + update_shape._dims.set(self_ax, update.dim(up_ax)); + prev_ax = up_ax; + } else { + update_shape._dims.set(self_ax, 1); + } + } + update = update.reshape(update_shape); + } + + stdx.debug.assert(self.rank() == update.rank(), "dynamicUpdateSlice expects input and computed update tensors to have the same rank, got {f} and {f}", .{ self, update }); + + for (self.dims(), update.dims(), 0..) |self_d, up_d, ax| { + const t = self._shape.debugTag(ax); + stdx.debug.assert(up_d <= self_d, "dynamicUpdateSlice expects 'update_' dimensions to be less than or equal to their corresponding dimension in input tensor, got {} and {} for axis .{s}", .{ up_d, self_d, t }); + + if (tagged_api and up_d < self_d) { + const axis_has_offset = std.mem.indexOfScalar(Shape.Tag, offset_tags.constSlice(), self._shape._tags.get(ax)) != null; + + stdx.debug.assert(axis_has_offset, "dynamicUpdateSlice expects 'update_' dimensions to be equal to their corresponding dimension in input tensor, got {} and {} for axis .{s} (hint: you need to provide an offset)", .{ up_d, self_d, t }); + } + } + + const idx_dtype = if (offset.len > 0) offset.get(0).dtype() else .i32; + const zero = Tensor.scalar(0, idx_dtype).value(); + var offset_values: [MAX_RANK]*const mlir.Value = undefined; + if (offset_tags.len == 0) { + // Without offset tags we need the same number of offset than rank. + stdx.debug.assert(self.rank() == offset.len, "dynamicUpdateSlice expects input tensor rank and 'offset_' length to be equal, got {} and {}", .{ self.rank(), offset.len }); + + for (offset.constSlice(), 0..) |idx, i| { + offset_values[i] = idx.value(); + } + } else { + // If an axis isn't specified, update the full slice. + // This is only allowed when using tagged sliced. + offset_values = .{zero} ** MAX_RANK; + for (offset.constSlice(), offset_tags.constSlice()) |start, t| { + const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); + offset_values[a] = start.value(); + } + } + + const op = dialects.stablehlo.dynamic_update_slice( + mlirCtx(), + self.value(), + update.value(), + offset_values[0..self.rank()], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + // TODO(Corentin) + //test dynamicUpdateSlice { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // { + // const x = try zml.Buffer.fromArray(platform, [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + // const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); + // const idx = try zml.Buffer.scalar(platform, 4, .i32); + // const res = try zml.testing.compileAndCall( + // platform, + // struct { + // pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(idx_, y_); + // } + // }._fwd, + // .{ x.withTags(.{.a}), .{ .a = idx }, y.withTags(.{.a}) }, + // ); + // try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32)); + // } + + // { + // // Updates 2D, tagged api + // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); + // const idx = try zml.Buffer.scalar(platform, 3, .i32); + + // const res = try zml.testing.compileAndCall( + // platform, + // struct { + // pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); + // } + // }._fwd, + // .{ x.withTags(.{ .a, .b }), idx, y.withTags(.{.a}) }, + // ); + // try std.testing.expectEqualDeep( + // [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, + // try res.getValue([2][5]f32), + // ); + // } + + // { + // // Updates 2D slice, un-tagged api. Note that `y` needs to have a 1 dimension axis. + // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const y = try zml.Buffer.fromArray(platform, [2][1]f32{ .{-1}, .{-1} }); + // const idx = try zml.Buffer.scalar(platform, 3, .i32); + // const res = try zml.testing.compileAndCall( + // platform, + // struct { + // pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); + // } + // }._fwd, + // .{ x, idx, y }, + // ); + // try std.testing.expectEqualDeep( + // [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, + // res.getValue([2][5]f32), + // ); + // } + + // { + // // Updates 2D, partial update + // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const y = try zml.Buffer.fromArray(platform, [1]f32{-1}); + // const idx_a = try zml.Buffer.scalar(platform, 1, .i32); + // const idx_b = try zml.Buffer.scalar(platform, 3, .i32); + // const res = try zml.testing.compileAndCall( + // platform, + // struct { + // pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(idx_, y_); + // } + // }._fwd, + // .{ x.withTags(.{ .a, .b }), .{ .a = idx_a, .b = idx_b }, y.withTags(.{.a}) }, + // ); + // try std.testing.expectEqualDeep( + // [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, + // res.getValue([2][5]f32), + // ); + // } + + // { + // // Updates 2D, partial update, un-tagged api. + // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const y = try zml.Buffer.fromArray(platform, [1][1]f32{.{-1}}); + // const idx_a = try zml.Buffer.scalar(platform, 1, .i32); + // const idx_b = try zml.Buffer.scalar(platform, 3, .i32); + // const A = struct { + // pub fn _fwd(x_: Tensor, idx_: [2]Tensor, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(&idx_, y_); + // } + // }; + // const res = try zml.testing.compileAndCall(platform, A._fwd, .{ x, .{ idx_a, idx_b }, y }); + // try std.testing.expectEqualDeep( + // [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, + // res.getValue([2][5]f32), + // ); + // } + //} + + /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. + pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); + + if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); + if (self.rank() != 0 and other.rank() == 0) return self.cmp(direction, other.broadcast(self._shape, &.{})); + + stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); + + const op = dialects.stablehlo.compare( + mlirCtx(), + self.value(), + other.value(), + dialects.stablehlo.ComparisonDirection.init(self.getContext().mlirCtx(), direction), + getComparisonType(mlirCtx(), self.dtype()), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(self._shape.withDtype(.bool), op.result(0)); + } + + /// For each vector in the input tensor, + /// creates a diagonal-matrix where diagonal values are set to the vector values. + pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]@TypeOf(.enum_literal)) Tensor { + stdx.debug.assert(self.rank() < MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ MAX_RANK - 1, self }); + const a = self.axis(axis_); + const d = self.dim(a); + var res_shape = self._shape; + res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; + res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; + + const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); + const zeros = Tensor.constant(res_shape, self.dtype().zero()); + + const x = Tensor.iota(res_shape, a); + const y = Tensor.iota(res_shape, a + 1); + var res = x.cmp(.EQ, y).select(values, zeros); + res._shape = res_shape; + return res; + } + + // TODO(Corentin) + //test toDiagonal { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _toDiag(input: Tensor) Tensor { + // const res = input.toDiagonal(-1, .{ .x, .y }); + // std.debug.assert(res.dim(.x) == input.dim(-1)); + // std.debug.assert(res.dim(.y) == input.dim(-1)); + // return res; + // } + // }; + + // const x = try zml.Buffer.fromArray(platform, [2][2]u8{ .{ 1, 2 }, .{ 3, 4 } }); + // { + // const res = try zml.testing.compileAndCall(platform, Local._toDiag, .{x}); + // try std.testing.expectEqual( + // [2][2][2]u8{ .{ + // .{ 1, 0 }, + // .{ 0, 2 }, + // }, .{ + // .{ 3, 0 }, + // .{ 0, 4 }, + // } }, + // try res.getValue([2][2][2]u8), + // ); + // } + //} + + /// For each matrix specified by the two axes, returns the lower triangular part of it. + /// The other elements are set to 0. + /// Usage: `.{ .b = 32, .w = 20, .h = 20 }.triangular(.{ .w, .h}, 0);` + /// + /// * if `num_diagonals` is set to 0, the diagonal is not modified. + /// * if set to -1, the diagonal is set to 0 + /// * if set to n, the n "quasi diagonal" above the diagonal are conserved. + /// + /// To get the upper triangular part, swap the order of axes: + /// `.{ .b = 32, .w = 20, .h = 20 }.triangular(.{ .h, .w }, 0);` + pub fn triangular(self: Tensor, axes_: anytype, num_diagonals: i32) Tensor { + stdx.debug.assertComptime(stdx.meta.isTuple(@TypeOf(axes_)) and axes_.len == 2, "triangular expects exactly two axes to work on.", .{}); + const _axes = self.axes(axes_); + + const x = Tensor.iota(self.shape(), _axes.get(0)); + const y = Tensor.iota(self.shape(), _axes.get(1)); + + const zeros = Tensor.constant(self.shape(), self.dtype().zero()); + return x.addConstant(num_diagonals).cmp(.GE, y).select(self, zeros); + } + + // TODO(Corentin) + //test triangular { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _tri(input: Tensor, num_diagonals: i32) Tensor { + // return input.triangular(.{ -2, -1 }, num_diagonals); + // } + // }; + + // const x = try zml.Buffer.fromArray(platform, [3][3]u8{ + // .{ 1, 1, 1 }, + // .{ 1, 1, 1 }, + // .{ 1, 1, 1 }, + // }); + // { + // const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, 0 }); + // try std.testing.expectEqual( + // [3][3]u8{ + // .{ 1, 0, 0 }, + // .{ 1, 1, 0 }, + // .{ 1, 1, 1 }, + // }, + // try res.getValue([3][3]u8), + // ); + // } + // { + // const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, 1 }); + // try std.testing.expectEqual( + // [3][3]u8{ + // .{ 1, 1, 0 }, + // .{ 1, 1, 1 }, + // .{ 1, 1, 1 }, + // }, + // try res.getValue([3][3]u8), + // ); + // } + // { + // const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, -1 }); + // try std.testing.expectEqual( + // [3][3]u8{ + // .{ 0, 0, 0 }, + // .{ 1, 0, 0 }, + // .{ 1, 1, 0 }, + // }, + // try res.getValue([3][3]u8), + // ); + // } + //} + + /// For each element at index `i`, if `bool_tensor[i] == true`, `output[i] = on_true[i]` + /// otherwise, if `bool_tensor[i] == false`, `output[i] = on_false[i]` + pub fn select(bool_tensor: Tensor, on_true: Tensor, on_false: Tensor) Tensor { + stdx.debug.assert(bool_tensor.dtype() == .bool, "select expects input tensor type to be a boolean, got {}", .{bool_tensor.dtype()}); + stdx.debug.assert(on_true.dtype() == on_false.dtype(), "select expects 'on_true' and 'on_false' tensor types to be equal, got {} and {}", .{ on_true.dtype(), on_false.dtype() }); + + if (bool_tensor.rank() != 0 and on_true.rank() == 0) { + return bool_tensor.select(on_true.broad(bool_tensor.shape()), on_false); + } + if (bool_tensor.rank() != 0 and on_false.rank() == 0) { + return bool_tensor.select(on_true, on_false.broad(bool_tensor.shape())); + } + + stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); + stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); + + const op = dialects.stablehlo.select( + mlirCtx(), + bool_tensor.value(), + on_true.value(), + on_false.value(), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(on_true._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. + pub fn not(self: Tensor) Tensor { + const op = dialects.stablehlo.not(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. + pub fn any(self: Tensor, axis_: anytype) Tensor { + const pred = self.cmp(.NE, Tensor.constant(self.dims(), self.dtype().zero())); + return reduce( + .{pred}, + .{Tensor.scalar(false, .bool)}, + &.{self.axis(axis_)}, + struct { + pub fn acc(args: ReduceArgs) Tensor { + return args.left.logical(.OR, args.right); + } + }.acc, + .{}, + )[0]; + } + + /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. + pub fn all(self: Tensor, axis_: anytype) Tensor { + const pred = if (self.dtype() == .bool) self else self.cmp(.NE, Tensor.scalar(0, self.dtype())); + return reduce( + .{pred}, + .{Tensor.scalar(true, .bool)}, + &.{self.axis(axis_)}, + struct { + pub fn acc(args: ReduceArgs) Tensor { + return args.left.logical(.AND, args.right); + } + }.acc, + .{}, + )[0]; + } + + /// Given a set of N vectors of lengths A, B, C, D, + /// returns N tensors of rank N, and shape (A, B, C, D). + /// For any coordinate (a, b, c, d), we have: + /// + /// - res[0][a, b, c, d] == A[a] + /// - res[1][a, b, c, d] == B[b] + /// - res[2][a, b, c, d] == C[c] + /// - res[3][a, b, c, d] == D[d] + /// + /// This is implemented with broadcasting, so typically it won't copy. + /// In Pytorch/Numpy this is know as `meshgrid` with "ij" mode. + /// See `zml.torch.meshgrid` for the "xy" mode. + pub fn cartesianProduct(comptime N: u3, vectors: [N]Tensor) [N]Tensor { + var out: @TypeOf(vectors) = undefined; + _cartesianProduct(&vectors, &out); + return out; + } + + fn _cartesianProduct(vectors: []const Tensor, out: []Tensor) void { + stdx.debug.assert(vectors.len >= 1, "cartesianProduct expects at least one input.", .{}); + stdx.debug.assert(vectors.len < Tensor.MAX_RANK, "cartesianProduct expects at most {} input vectors, received {} !", .{ Tensor.MAX_RANK - 1, vectors.len }); + for (vectors) |x| { + stdx.debug.assert(x.rank() <= 1, "cartesianProduct expects 0 or 1 rank input vectors. Got: {any}", .{vectors}); + stdx.debug.assert(vectors[0].dtype() == x.dtype(), "cartesianProduct expects input vectors to have all the same dtype. Got: {any}", .{vectors}); + } + + var res_shape = Shape.init(.{}, vectors[0].dtype()); + for (vectors) |x| { + if (x.rank() == 0) { + res_shape = res_shape.appendDim(1, null); + } else { + res_shape = res_shape.appendDim(x.dim(0), x.shape().tag(0)); + } + } + + for (out, vectors, 0..) |*o, x, i| { + o.* = x.broadcast(res_shape, &[1]i64{@intCast(i)}); + } + } + + // TODO(Corentin) + //test cartesianProduct { + // const zml = @import("zml.zig"); + // const client = zml.testing.env(); + + // const x = try zml.Buffer.fromSlice(client, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); + // const y = try zml.Buffer.fromSlice(client, .{4}, &[_]i32{ 0, 1, 2, 3 }); + + // const Local = struct { + // pub fn _cartesianProduct2(a: Tensor, b: Tensor) [2]Tensor { + // return cartesianProduct(2, .{ a, b }); + // } + // }; + + // { + // const xs, const ys = try zml.testing.compileAndCall(client, Local._cartesianProduct2, .{ x, y }); + // try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); + // try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); + // try std.testing.expectEqualDeep( + // [6][4]i32{ + // .{ 0, 0, 0, 0 }, + // .{ 1, 1, 1, 1 }, + // .{ 2, 2, 2, 2 }, + // .{ 3, 3, 3, 3 }, + // .{ 4, 4, 4, 4 }, + // .{ 5, 5, 5, 5 }, + // }, + // try xs.getValue([6][4]i32), + // ); + // try std.testing.expectEqualDeep( + // [6][4]i32{ + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // }, + // try ys.getValue([6][4]i32), + // ); + // } + //} + + /// Given a set of N vectors of lengths A, B, C, D, + /// returns 1 tensors of rank N+1, and shape (A, B, C, D, N). + /// For any coordinate (a, b, c, d), we have: + /// + /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) + pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { + var out = stdx.BoundedArray(Tensor, Tensor.MAX_RANK).init(vectors.len) catch unreachable; + _cartesianProduct(vectors, out.slice()); + + return Tensor.stack(out.constSlice(), .last, .coord); + } + + // TODO(Corentin) + //test cartesianProductStacked { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + // const x = try zml.Buffer.fromSlice(platform, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); + // const y = try zml.Buffer.fromSlice(platform, .{4}, &[_]i32{ 0, 1, 2, 3 }); + + // const Local = struct { + // pub fn _fwd(a: Tensor, b: Tensor) Tensor { + // return cartesianProductStacked(&.{ a, b }); + // } + // }; + + // const z = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, y }); + // try std.testing.expectEqualDeep( + // [6][4][2]i32{ + // .{ .{ 0, 0 }, .{ 0, 1 }, .{ 0, 2 }, .{ 0, 3 } }, + // .{ .{ 1, 0 }, .{ 1, 1 }, .{ 1, 2 }, .{ 1, 3 } }, + // .{ .{ 2, 0 }, .{ 2, 1 }, .{ 2, 2 }, .{ 2, 3 } }, + // .{ .{ 3, 0 }, .{ 3, 1 }, .{ 3, 2 }, .{ 3, 3 } }, + // .{ .{ 4, 0 }, .{ 4, 1 }, .{ 4, 2 }, .{ 4, 3 } }, + // .{ .{ 5, 0 }, .{ 5, 1 }, .{ 5, 2 }, .{ 5, 3 } }, + // }, + // try z.getValue([6][4][2]i32), + // ); + //} + + fn binaryOp( + op_name: []const u8, + op_fn: fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation, + ) fn (Tensor, Tensor) Tensor { + return struct { + pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {f} and {f}", .{ op_name, self, other }); + + if (self.rank() == 0 and other.rank() != 0) { + return binaryOpHelper(self.broad(other._shape), other); + } + + if (self.rank() != 0 and other.rank() == 0) { + return binaryOpHelper(self, other.broad(self._shape)); + } + + var other_ = other; + var same_shape = self._shape.eql(other._shape); + if (!same_shape and std.mem.eql(Shape.Tag, self._shape.tags(), other._shape.tags()) and other._shape.canBroadcastTo(self._shape)) { + // Only a restrictive version of broadcasting is allowed here, where all the tags matches already. + // Typical use case: `x.div(x.sum(.a))` + same_shape = true; + other_ = other.broad(self._shape); + } + + stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); + + const ret = @call(.auto, op_fn, .{ mlirCtx(), self.value(), other_.value(), .unknown(mlirCtx()) }).appendTo(currentBlock()); + return _result(self._shape, ret.result(0)); + } + }.binaryOpHelper; + } + + /// Insert code that will print the content of the given buffer at runtime. + /// Only for debug purpose, it inserts device to host synchronization + /// so it will slow down the program execution. + pub fn print(input: Tensor) Tensor { + _ = input; // autofix + // TODO(Corentin) + //return callback.call(callback.Print, .{input}, &.{input.shape()})[0]; + } + + fn mlirCtx() *mlir.Context { + return CompilationContext.current().mlir_ctx; + } + + fn currentBlock() *mlir.Block { + return CompilationContext.current().currentScope().block; + } +}; + +fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { + // TODO use shape + var result = [_]i64{1} ** Tensor.MAX_RANK; + const start = rank - data.len; + @memcpy(result[start .. start + data.len], data); + return result; +} + +fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Tensor.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(Tensor.MAX_RANK, i64), window_dilations: @Vector(Tensor.MAX_RANK, i64), window_strides: @Vector(Tensor.MAX_RANK, i64)) Shape { + // TODO use shape + var input_dims = [_]i64{1} ** Tensor.MAX_RANK; + @memcpy(input_dims[0..in_dims.len], in_dims); + + const input_dims_: @Vector(Tensor.MAX_RANK, i64) = input_dims; + const splat_one: @Vector(Tensor.MAX_RANK, i64) = @splat(1); + const dilated_input_shape: @Vector(Tensor.MAX_RANK, i64) = (input_dims_ - splat_one) * base_dilations + splat_one; + var pad_slice0: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); + var pad_slice1: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); + if (padding.len > 1) { + var idx: usize = 0; + while (idx < in_dims.len * 2) : (idx += 2) { + pad_slice0[idx / 2] = padding[idx]; + pad_slice1[idx / 2] = padding[idx + 1]; + } + } + const padded_input_shape: @Vector(Tensor.MAX_RANK, i64) = pad_slice0 + dilated_input_shape + pad_slice1; + const dilated_window_shape = (window_dimensions - splat_one) * window_dilations + splat_one; + const dims = @divFloor(padded_input_shape - dilated_window_shape, window_strides) + splat_one; + const dims_arr: [Tensor.MAX_RANK]i64 = @bitCast(dims); + return Shape.init(dims_arr[0..in_dims.len], dt); +} + +fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialects.stablehlo.CompareType { + return dialects.stablehlo.CompareType.init(ctx, switch (dtype.class()) { + .bool => .UNSIGNED, + .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, + .float => .FLOAT, + .complex => @panic("Can't compare complex numbers"), + }); +} + +// TODO(Corentin) +//test "Tensor.maxPool1d" { +// const zml = @import("zml.zig"); +// const platform = zml.testing.env(); +// +// const MaxPool = struct { +// pub fn _fwd(x: zml.Tensor) Tensor.ArgMaxRes { +// return x.maxPool1d(.{ +// .window_dimensions = 3, +// .window_strides = 2, +// }); +// } +// }; +// +// var data: [20]f32 = undefined; +// for (&data, 0..) |*v, i| v.* = @floatFromInt(i); +// +// const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5 }, &data); +// const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); +// try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); +// try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); +// try std.testing.expectEqualDeep( +// [2][2][2]f32{ +// .{ .{ 2, 4 }, .{ 7, 9 } }, +// .{ .{ 12, 14 }, .{ 17, 19 } }, +// }, +// result.values.getValue([2][2][2]f32), +// ); +//} +// +//test "Tensor.maxPool2d" { +// const zml = @import("zml.zig"); +// const platform = zml.testing.env(); +// +// const MaxPool = struct { +// pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { +// return x.maxPool2d(.{ +// .window_dimensions = .{ 3, 2 }, +// .window_strides = .{ 2, 1 }, +// }); +// } +// }; +// +// var data: [100]f32 = undefined; +// for (&data, 0..) |*v, i| v.* = @floatFromInt(i); +// const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5, 5 }, &data); +// +// const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); +// try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); +// try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); +// var buffer: [2][2][2][4]f32 = undefined; +// _ = try result.values.toHost(std.mem.asBytes(&buffer)); +// try std.testing.expectEqualDeep( +// [2][2][2][4]f32{ +// .{ +// .{ .{ 11, 12, 13, 14 }, .{ 21, 22, 23, 24 } }, +// .{ .{ 36, 37, 38, 39 }, .{ 46, 47, 48, 49 } }, +// }, +// .{ +// .{ .{ 61, 62, 63, 64 }, .{ 71, 72, 73, 74 } }, +// .{ .{ 86, 87, 88, 89 }, .{ 96, 97, 98, 99 } }, +// }, +// }, +// buffer, +// ); +//} + +fn toI64(values: anytype) []i64 { + var res: [Tensor.MAX_RANK]i64 = undefined; + for (values, 0..) |val, i| res[i] = @intCast(val); + return res[0..values.len]; +} + +fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { + var perm: stdx.BoundedArray(struct { u8, bool }, Tensor.MAX_RANK) = .{}; + // Don't rewrite on invalid inputs. + if (permutation.len > x.rank()) return false; + for (permutation) |ax| { + const squeezable = x.dim(ax) == 1; + perm.appendAssumeCapacity(.{ @intCast(ax), squeezable }); + } + + var effective_ax: u8 = 0; + for (0..perm.len) |i| { + const ax, const squeezable = perm.get(i); + if (squeezable) { + // Effectively squeeze this axis by decrementing axes coming after by 1. + for (i..perm.len) |j| { + if (perm.buffer[j][0] > ax) { + perm.buffer[j][0] -= 1; + } + } + continue; + } + + if (ax != effective_ax) return false; + effective_ax += 1; + } + + return true; +} + +test transposeIsJustAReshape { + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 0, 1, 2 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 1, 0, 2 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 2, 1, 0 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 64, 8, 1, 128 }, .bf16), &.{ 0, 2, 1, 3 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 64, 8, 155, 128 }, .bf16), &.{ 0, 2, 1, 3 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 64, 1, 1, 128 }, .bf16), &.{ 1, 2, 0, 3 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ .b = 1, .h = 10, .q = 155, .hd = 1 }, .f32), &.{ 0, 2, 1, 3 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 2, 3, 1 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 1, 3, 2 })); +} + +test "unused tensor" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _fwd(x: Tensor) Tensor { + const y = x.addConstant(1); + _ = y; + return x; + } + }; + + const mod = try zml.compileFn(std.testing.allocator, Local._fwd, .{Shape.init(.{10}, .f32)}, platform); + defer mod.deinit(); +} From b1709af1814404810cf57b6cf6defac810cd30ac Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 3 Dec 2025 11:15:10 +0100 Subject: [PATCH 4/6] Tackled some TODOs on operations --- zml2/tensor.zig | 298 ++++++++++++++++++++++++++++++------------------ 1 file changed, 187 insertions(+), 111 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index c74955c0..ed9895bc 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1245,22 +1245,16 @@ pub const Tensor = struct { return max_.add(log_sum_exp); } - // TODO(Corentin) /// Returns a Tensor containing the sum of elements over the given axis. /// Output shape is the input shape with the axis_ dim set to 1. - //pub fn sum(self: Tensor, axis_: anytype) Tensor { - // const a = self.axis(axis_); - // return ops.reduce( - // struct { - // pub fn acc(x: Tensor, res: Tensor) Tensor { - // return res.add(x.convert(res.dtype())); - // } - // }.acc, - // self, - // Tensor.scalar(0, self.dtype()), - // &.{a}, - // ); - //} + pub fn sum(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + return reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { + pub fn acc(args: ReduceArgs) Tensor { + return args.right.add(args.left.convert(args.right.dtype())); + } + }.acc, .{})[0]; + } /// Returns a Tensor containing the mean of elements over the given axis. /// Output shape is the input shape with the axis_ dim set to 1. @@ -1908,9 +1902,8 @@ pub const Tensor = struct { return _result(.init(&.{}, val.dtype()), op.result(0)).autoBroadcast(); } - // TODO(Corentin) pub fn zeroes(sh: Shape) Tensor { - return .constant(sh.dtype().zero()); + return .constant(sh.dtype().zero()).broad(sh); } // TODO(Corentin) @@ -3098,6 +3091,23 @@ pub const Tensor = struct { pub const ArgMaxRes = struct { values: Tensor, indices: Tensor, + + fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { + const left_gt_right = values.left.cmp(.GT, values.right); + const is_nan = values.left.cmp(.NE, values.left); + const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); + + // If values.left == values.right: keep the smallest idx. + const is_same = values.left.cmp(.EQ, values.right); + const is_first = indices.left.cmp(.LT, indices.right); + const is_same_but_first = is_same.logical(.AND, is_first); + const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + const max_idx = keep_left_idx.select(indices.left, indices.right); + + return .{ max_val, max_idx }; + } }; pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { @@ -3108,24 +3118,7 @@ pub const Tensor = struct { .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, &.{a}, - struct { - fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { - const left_gt_right = values.left.cmp(.GT, values.right); - const is_nan = values.left.cmp(.NE, values.left); - const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(values.left, values.right); - - // If values.left == values.right: keep the smallest idx. - const is_same = values.left.cmp(.EQ, values.right); - const is_first = indices.left.cmp(.LT, indices.right); - const is_same_but_first = is_same.logical(.AND, is_first); - const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); - const max_idx = keep_left_idx.select(indices.left, indices.right); - - return .{ max_val, max_idx }; - } - }.cmp, + ArgMaxRes.cmp, .{}, ); return .{ .values = values, .indices = indices }; @@ -3447,93 +3440,176 @@ pub const Tensor = struct { return result; } - // TODO(Corentin) - // - //pub const MaxPoolRes = ArgMaxRes; + pub const ReduceWindowOpts = struct { + // TODO replace with Shape + window_dimensions: []const i64, + window_strides: []const i64, + base_dilations: []const i64, + window_dilations: []const i64, + padding: []const [2]i64, + }; + + fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce_window", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlirCtx(), "window_dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_dimensions)), + .named(mlirCtx(), "window_strides", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_strides)), + .named(mlirCtx(), "base_dilations", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.base_dilations)), + .named(mlirCtx(), "window_dilations", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_dilations)), + // Cast the [][2]i64 to []i64 (safe) + .named(mlirCtx(), "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{}, mlir.integerType(mlirCtx(), .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), + }, + .verify = true, + .location = .unknown(mlirCtx()), + }).appendTo(currentBlock()); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), reduce_op.result(i)); + } + + return result; + } + + pub const MaxPoolRes = ArgMaxRes; + /// Computes the 1d maxPool operation on the input Tensor. - //pub fn maxPool1d(self: Tensor, opts: struct { - // window_dimensions: i64, - // window_strides: ?i64, - // base_dilations: i64 = 1, - // window_dilations: i64 = 1, - // padding: [2]i64 = .{ 0, 0 }, - //}) MaxPoolRes { - // // TODO migrate to the following syntax. - // // maxPool(.{.a = .{ .stride = 5, .dilation = 2, .padding = .{0, 1} }, - // // .b = .{ .stride = 8, .dilation = 2, .padding = .{0, 1} }), - // // maxPool(.{ - // // .stride = .{ .a = 5, .b = 8 }, - // // .dilation = .{ .a = 2, .b = 2 }, - // // .padding = .{ .a = .{ 0, 2 }, .b = .{0, 2} - // // }) + pub fn maxPool1d(self: Tensor, opts: struct { + window_dimensions: i64, + window_strides: ?i64, + base_dilations: i64 = 1, + window_dilations: i64 = 1, + padding: [2]i64 = .{ 0, 0 }, + }) MaxPoolRes { + // TODO migrate to the following syntax. + // maxPool(.{.a = .{ .stride = 5, .dilation = 2, .padding = .{0, 1} }, + // .b = .{ .stride = 8, .dilation = 2, .padding = .{0, 1} }), + // maxPool(.{ + // .stride = .{ .a = 5, .b = 8 }, + // .dilation = .{ .a = 2, .b = 2 }, + // .padding = .{ .a = .{ 0, 2 }, .b = .{0, 2} + // }) - // // TODO: support maxPool on non last axis - // const a = self.axis(-1); - // const ones = [_]i64{1} ** Tensor.MAX_RANK; - // var window_dimensions = ones; - // window_dimensions[a] = opts.window_dimensions; - // var window_strides = window_dimensions; - // if (opts.window_strides) |stride| window_strides[a] = stride; + // TODO: support maxPool on non last axis + const a = self.axis(-1); + const ones = [_]i64{1} ** Tensor.MAX_RANK; + var window_dimensions = ones; + window_dimensions[a] = opts.window_dimensions; + var window_strides = window_dimensions; + if (opts.window_strides) |stride| window_strides[a] = stride; - // var base_dilations = ones; - // base_dilations[a] = opts.base_dilations; - // var window_dilations = ones; - // window_dilations[a] = opts.window_dilations; + var base_dilations = ones; + base_dilations[a] = opts.base_dilations; + var window_dilations = ones; + window_dilations[a] = opts.window_dilations; - // var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; - // padding[a] = opts.padding; + var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + padding[a] = opts.padding; - // return ops.reduceWindow( - // MaxPoolRes.cmp, - // .{ .values = self, .indices = iota(self._shape, a) }, - // .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, - // .{ - // .window_dimensions = window_dimensions[0..self.rank()], - // .window_strides = window_strides[0..self.rank()], - // .base_dilations = base_dilations[0..self.rank()], - // .window_dilations = window_dilations[0..self.rank()], - // .padding = padding[0..self.rank()], - // }, - // ); - //} + const result = reduceWindow( + .{ self, iota(self._shape, a) }, + .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, + .{ + .window_dimensions = window_dimensions[0..self.rank()], + .window_strides = window_strides[0..self.rank()], + .base_dilations = base_dilations[0..self.rank()], + .window_dilations = window_dilations[0..self.rank()], + .padding = padding[0..self.rank()], + }, + MaxPoolRes.cmp, + .{}, + ); + return .{ .values = result[0], .indices = result[1] }; + } - ///// Computes the 2d maxPool operation on the input Tensor. - //pub fn maxPool2d(self: Tensor, opts: struct { - // window_dimensions: [2]i64, - // window_strides: ?[2]i64 = null, - // base_dilations: [2]i64 = .{ 1, 1 }, - // window_dilations: [2]i64 = .{ 1, 1 }, - // padding: [2][2]i64 = .{ .{ 0, 0 }, .{ 0, 0 } }, - //}) MaxPoolRes { - // // TODO: rewrite using modern ZML - // stdx.debug.guard(self.rank() == 3 or self.rank() == 4, @src()); + /// Computes the 2d maxPool operation on the input Tensor. + pub fn maxPool2d(self: Tensor, opts: struct { + window_dimensions: [2]i64, + window_strides: ?[2]i64 = null, + base_dilations: [2]i64 = .{ 1, 1 }, + window_dilations: [2]i64 = .{ 1, 1 }, + padding: [2][2]i64 = .{ .{ 0, 0 }, .{ 0, 0 } }, + }) MaxPoolRes { + // TODO: rewrite using modern ZML + stdx.debug.guard(self.rank() == 3 or self.rank() == 4, @src()); - // // TODO: support maxPool on non last axis - // // Note: the problem is initPoolArg assuming last axis - // const a = self.axis(-1); + // TODO: support maxPool on non last axis + // Note: the problem is initPoolArg assuming last axis + const a = self.axis(-1); - // const window_dimensions = initPoolArg(self.rank(), &opts.window_dimensions); - // const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), &ws) else window_dimensions; - // const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); - // const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); + const window_dimensions = initPoolArg(self.rank(), &opts.window_dimensions); + const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), &ws) else window_dimensions; + const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); + const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); - // var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; - // padding[a - 1] = opts.padding[0]; - // padding[a] = opts.padding[1]; + var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + padding[a - 1] = opts.padding[0]; + padding[a] = opts.padding[1]; - // return ops.reduceWindow( - // MaxPoolRes.cmp, - // .{ .values = self, .indices = iota(self._shape, a) }, - // .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, - // .{ - // .window_dimensions = window_dimensions[0..self.rank()], - // .window_strides = window_strides[0..self.rank()], - // .base_dilations = base_dilation[0..self.rank()], - // .window_dilations = window_dilations[0..self.rank()], - // .padding = padding[0..self.rank()], - // }, - // ); - //} + const result = reduceWindow( + .{ self, iota(self._shape, a) }, + .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, + .{ + .window_dimensions = window_dimensions[0..self.rank()], + .window_strides = window_strides[0..self.rank()], + .base_dilations = base_dilation[0..self.rank()], + .window_dilations = window_dilations[0..self.rank()], + .padding = padding[0..self.rank()], + }, + MaxPoolRes.cmp, + .{}, + ); + return .{ .values = result[0], .indices = result[1] }; + } /// Chunk a given tensor into exactly n parts of equal shape. /// `self.dim(axis_)` must be divisible by n_chunks. From 9b6d4965c8dbf2a8e8f9fdb3b8f5bb9aa6dcad8f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 3 Dec 2025 17:27:55 +0100 Subject: [PATCH 5/6] Basic compilation support --- mlir/dialects/func.zig | 101 ++++++--- zml2/BUILD.bazel | 37 +++- zml2/exe.zig | 226 +++++++++++++++++++ zml2/main.zig | 31 +++ zml2/meta.zig | 30 ++- zml2/mlirx.zig | 63 +++--- zml2/module.zig | 487 +++++++++++++++++++++++++++++++++++++++++ zml2/pjrtx.zig | 12 +- zml2/platform.zig | 15 ++ zml2/shape.zig | 2 +- zml2/tensor.zig | 133 +++++++---- zml2/zml.zig | 19 +- 12 files changed, 1015 insertions(+), 141 deletions(-) create mode 100644 zml2/exe.zig create mode 100644 zml2/module.zig diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 868ff3bc..b9f8b7f5 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -3,50 +3,91 @@ const std = @import("std"); const mlir = @import("mlir"); const stdx = @import("stdx"); -pub fn func( - ctx: mlir.Context, - args: struct { - sym_name: []const u8, - args: []const mlir.Type, - arg_attrs: []const mlir.Attribute = &.{}, - results: []const mlir.Type, - res_attrs: []const mlir.Attribute = &.{}, - block: mlir.Block, - location: mlir.Location, - }, -) mlir.Operation { - var attrs_tuple_buffer = stdx.BoundedArray(mlir.AttrTuple, 4){}; - attrs_tuple_buffer.appendAssumeCapacity(.{ "sym_name", .string(ctx, args.sym_name) }); - attrs_tuple_buffer.appendAssumeCapacity(.{ "function_type", .type_(.function(ctx, args.args, args.results)) }); - if (args.arg_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "arg_attrs", .array(ctx, args.arg_attrs) }); - } +const FuncOpArgs = struct { + pub const Visibility = enum { + public, + private, + }; - if (args.res_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "res_attrs", .array(ctx, args.res_attrs) }); + name: []const u8, + args: ?[]const *const mlir.Type = null, + args_attributes: ?[]const *const mlir.Attribute = null, + results: ?[]const *const mlir.Type = null, + results_attributes: ?[]const *const mlir.Attribute = null, + block: *mlir.Block, + location: *const mlir.Location, + no_inline: bool = false, + visibility: Visibility = .public, +}; + +pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .{}; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .{}; + + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .{}; + attr_tuples_buffer.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), + .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), + .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( + ctx, + args.args orelse args: { + for (0..args.block.numArguments()) |i| { + args_buffer.appendAssumeCapacity(args.block.argument(i).type_()); + } + break :args args_buffer.constSlice(); + }, + args.results orelse results: { + const terminator = args.block.terminator() orelse { + @panic("block has no terminator"); + }; + for (0..terminator.numOperands()) |i| { + results_buffer.appendAssumeCapacity(terminator.operand(i).type_()); + } + break :results results_buffer.constSlice(); + }, + ))), + }); + + if (args.args_attributes) |args_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); + } + if (args.results_attributes) |results_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, results_attributes))); + } + if (args.no_inline) { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); } return mlir.Operation.make(ctx, "func.func", .{ .blocks = &.{args.block}, - .attributes = attrs_tuple_buffer.constSlice(), + .attributes = attr_tuples_buffer.constSlice(), .location = args.location, }); } -pub fn call(ctx: mlir.Context, name: [:0]const u8, values: []const mlir.Value, results: []const mlir.Type, loc: mlir.Location) mlir.Operation { +pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Value, results: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.call", .{ - .variadic_operands = &.{values}, - .results = results, - .verify = true, - .attributes = &.{.{ "callee", .symbol(ctx, name) }}, - .location = loc, + .operands = .{ .variadic = &.{values} }, + .results = .{ .flat = results }, + .attributes = &.{ + .named(ctx, "callee", mlir.flatSymbolRefAttribute(ctx, name)), + }, + .location = location, }); } -pub fn return_(ctx: mlir.Context, values: []const mlir.Value, loc: mlir.Location) mlir.Operation { +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.return", .{ - .operands = values, + .operands = .{ .flat = &.{value} }, + .location = location, .verify = false, - .location = loc, + }); +} + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "func.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, }); } diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 90c95102..277e5169 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -1,5 +1,16 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +upb_c_proto_library( + name = "xla_data_upb", + deps = ["@xla//xla:xla_data_proto"], +) + +upb_c_proto_library( + name = "xla_compile_options_upb", + deps = ["@xla//xla/pjrt/proto:compile_options_proto"], +) cc_library( name = "posix", @@ -9,30 +20,36 @@ cc_library( zig_library( name = "zml", srcs = [ - "zml.zig", - "platform.zig", - "pjrtx.zig", "dtype.zig", + "exe.zig", "floats.zig", + "meta.zig", + "mlirx.zig", + "module.zig", + "pjrtx.zig", + "platform.zig", "shape.zig", "tensor.zig", - "mlirx.zig", - "meta.zig", + "zml.zig", ], + main = "zml.zig", + visibility = ["//visibility:public"], zigopts = [ "-lc", "-freference-trace=20", ], - main = "zml.zig", - visibility = ["//visibility:public"], deps = [ - "//runtimes", + ":posix", + ":xla_compile_options_upb", + ":xla_data_upb", + "//async", "//mlir", "//mlir/dialects", "//pjrt", + "//runtimes", "//stdx", + "//upb", "@rules_zig//zig/runfiles", - ":posix", ], ) diff --git a/zml2/exe.zig b/zml2/exe.zig new file mode 100644 index 00000000..23dffafa --- /dev/null +++ b/zml2/exe.zig @@ -0,0 +1,226 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); +const meta = @import("meta.zig"); +const Shape = @import("shape.zig").Shape; + +const Platform = @import("platform.zig").Platform; + +// TODO(Corentin) +const Buffer = struct {}; + +pub const Exe = struct { + platform: Platform, + exe: *pjrt.LoadedExecutable, + + context: ?*pjrt.ExecuteContext = null, + + input_shapes: []const Shape, + output_shapes: []const Shape, + + num_devices: u8, + + arena: std.heap.ArenaAllocator, + + pub fn init( + allocator: std.mem.Allocator, + platform: Platform, + exe: *pjrt.LoadedExecutable, + input_shapes: []const Shape, + output_shapes: []const Shape, + num_devices: u8, + ) !Exe { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); + const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); + + return .{ + .platform = platform, + .exe = exe, + .input_shapes = input_shapes_copy, + .output_shapes = output_shapes_copy, + .num_devices = num_devices, + .arena = arena, + }; + } + + pub fn deinit(self: *const Exe) void { + self.arena.deinit(); + } + + pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { + return Arguments.init(allocator, self.input_shapes, self.num_devices); + } + + pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { + return Results.init(allocator, self.output_shapes, self.num_devices, self.platform); + } + + pub const FlatBuffers = struct { + buffers: []const [*]*pjrt.Buffer, + raw_buffers: []const *pjrt.Buffer, + + num_devices: usize, + + pub fn init(allocator: std.mem.Allocator, count: usize, num_devices: usize) !FlatBuffers { + const raw_buffers = try allocator.alloc(*pjrt.Buffer, num_devices * count); + errdefer allocator.free(raw_buffers); + + const buffers = try allocator.alloc([*]*pjrt.Buffer, num_devices); + errdefer allocator.free(buffers); + + for (0..num_devices) |i| { + buffers[i] = raw_buffers[i * count ..].ptr; + } + + return .{ + .buffers = buffers, + .raw_buffers = raw_buffers, + .num_devices = num_devices, + }; + } + + pub fn deinit(self: *const FlatBuffers, allocator: std.mem.Allocator) void { + allocator.free(self.buffers); + allocator.free(self.raw_buffers); + } + }; + + pub const Arguments = struct { + flat_buffers: FlatBuffers, + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Arguments, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn set(self: *Arguments, v: anytype) void { + return self.setPartial(v, 0); + } + + pub fn setPartial(self: *Arguments, v: anytype, offset: usize) void { + const LocalContext = struct { + self: *Arguments, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = offset }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + } + }; + + pub const Results = struct { + platform: Platform, + flat_buffers: FlatBuffers, + + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: Platform) !Results { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .platform = platform, + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Results, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn get(self: *Results, comptime T: type) T { + var result: T = undefined; + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *Buffer) void { + var shards: Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, &result); + return result; + } + + pub fn fill(self: *Results, v: anytype) void { + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *Buffer) void { + //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + var shards: Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, v); + } + }; + + pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { + var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; + const sharding = self.platform.sharding(); + + self.exe.execute(self.platform.pjrt_api, .{ + .arguments = arguments.flat_buffers.buffers, + .num_args = arguments.expected_shapes.len, + .results = results_.flat_buffers.buffers, + .events = events[0..sharding.num_partitions], + // this allows to tell a specific buffer shouldn't be donated, + // even if it has been marked as "can be donated" during compilation. + // TODO: expose it ? + .non_donatable_input_indices = &.{}, + .context = self.context, + }) catch |err| { + std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); + }; + + for (events[0..sharding.num_partitions]) |e| { + if (e) |ev| { + ev.await_(self.platform.pjrt_api) catch unreachable; + } + } + } +}; diff --git a/zml2/main.zig b/zml2/main.zig index 66d7a6cd..41ca7d1a 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -2,6 +2,26 @@ const std = @import("std"); const async = @import("async"); const zml = @import("zml"); +pub const Model = struct { + weight: zml.Tensor, + bias: zml.Tensor, + + pub fn init() !Model { + return .{ + .weight = .init(zml.Shape.init(.{ .m = 4096, .n = 4096 }, .f32)), + .bias = .init(zml.Shape.init(.{ .m = 4096 }, .f32)), + }; + } + + pub fn deinit(self: *Model) void { + _ = self; + } + + pub fn forward(self: Model, input: zml.Tensor) zml.Tensor { + return self.weight.dot(input, .n).add(self.bias); + } +}; + pub fn main() !void { try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } @@ -10,6 +30,17 @@ pub fn asyncMain() !void { zml.init(); defer zml.deinit(); + var gpa: std.heap.DebugAllocator(.{}) = .init; + defer _ = gpa.deinit(); + + const allocator = gpa.allocator(); + var platform = try zml.Platform.init(.cpu, .{}); defer platform.deinit(); + + var model: Model = try .init(); + defer model.deinit(); + + var exe = try zml.module.compileModel(allocator, Model.forward, model, .{zml.Tensor.init(zml.Shape.init(.{ .n = 4096 }, .f32))}, platform); + defer exe.deinit(); } diff --git a/zml2/meta.zig b/zml2/meta.zig index 891a27ec..9f630742 100644 --- a/zml2/meta.zig +++ b/zml2/meta.zig @@ -427,7 +427,14 @@ test MapRestrict { /// Recursively visit the given struct and calls the callback for each K found. /// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. -pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { +pub fn VisitReturn(comptime cb: anytype) type { + return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| + @Type(std.builtin.Type{ .error_union = .{ .error_set = error_set, .payload = void } }) + else + void; +} + +pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn(cb) { const Callback = @TypeOf(cb); const Ptr = @TypeOf(v); const type_info_v = @typeInfo(Ptr); @@ -436,6 +443,7 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { } const ptr_info = type_info_v.pointer; const Child = ptr_info.child; + const can_error = stdx.meta.FnSignature(cb, null).ReturnErrorSet != null; const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { .pointer => |info| .{ info.child, !info.is_const }, @@ -468,15 +476,15 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { const field_type_info = @typeInfo(field.type); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. switch (field_type_info) { - .pointer => visit(cb, ctx, @field(v, field.name)), - .array, .optional, .@"union", .@"struct" => visit(cb, ctx, &@field(v, field.name)), + .pointer => if (can_error) try visit(cb, ctx, @field(v, field.name)) else visit(cb, ctx, @field(v, field.name)), + .array, .optional, .@"union", .@"struct" => if (can_error) try visit(cb, ctx, &@field(v, field.name)) else visit(cb, ctx, &@field(v, field.name)), else => {}, } }, - .array => |_| for (v) |*elem| visit(cb, ctx, elem), - .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), .@"union" => switch (v.*) { - inline else => |*v_field| visit(cb, ctx, v_field), + inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), }, else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), }, @@ -488,15 +496,15 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { const field_type_info = @typeInfo(field.type); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. if (field_type_info == .pointer) { - visit(cb, ctx, @field(v_elem, field.name)); + if (can_error) try visit(cb, ctx, @field(v_elem, field.name)) else visit(cb, ctx, @field(v_elem, field.name)); } else { - visit(cb, ctx, &@field(v_elem, field.name)); + if (can_error) try visit(cb, ctx, &@field(v_elem, field.name)) else visit(cb, ctx, &@field(v_elem, field.name)); } }, - .array => |_| for (v) |*elem| visit(cb, ctx, elem), - .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), .@"union" => switch (v_elem.*) { - inline else => |*v_field| visit(cb, ctx, v_field), + inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), }, else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), } diff --git a/zml2/mlirx.zig b/zml2/mlirx.zig index 2224b88d..3b13b2d8 100644 --- a/zml2/mlirx.zig +++ b/zml2/mlirx.zig @@ -12,20 +12,25 @@ pub const Type = struct { return switch (dt) { .bool => mlir.integerType(ctx, .i1), .f4e2m1 => mlir.floatType(ctx, .f4e2m1fn), - .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), + .f8e3m4 => mlir.floatType(ctx, .f8e3m4), + .f8e4m3 => mlir.floatType(ctx, .f8e4m3), .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), + .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), .f8e5m2 => mlir.floatType(ctx, .f8e5m2), .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), + .f8e8m0 => @panic("Unsupported type"), .bf16 => mlir.floatType(ctx, .bf16), .f16 => mlir.floatType(ctx, .f16), .f32 => mlir.floatType(ctx, .f32), .f64 => mlir.floatType(ctx, .f64), + .i2 => mlir.integerType(ctx, .i2), .i4 => mlir.integerType(ctx, .i4), .i8 => mlir.integerType(ctx, .i8), .i16 => mlir.integerType(ctx, .i16), .i32 => mlir.integerType(ctx, .i32), .i64 => mlir.integerType(ctx, .i64), + .u2 => mlir.integerType(ctx, .u2), .u4 => mlir.integerType(ctx, .u4), .u8 => mlir.integerType(ctx, .u8), .u16 => mlir.integerType(ctx, .u16), @@ -38,43 +43,49 @@ pub const Type = struct { // TODO(Corentin): Maybe remove that as it now requires a mlir context. // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. - pub fn toDType(mlir_type: *const mlir.Type) dtype.DataType { + pub fn toDType(ctx: *mlir.Context, mlir_type: *const mlir.Type) dtype.DataType { const mapping = .{ - .{ .bool, mlir.IntegerType(.i1) }, + .{ .bool, mlir.integerType(ctx, .i1) }, - .{ .f8e4m3b11fnuz, mlir.FloatType(.f8e4m3b11fnuz) }, - .{ .f8e4m3fn, mlir.FloatType(.f8e4m3fn) }, - .{ .f8e4m3fnuz, mlir.FloatType(.f8e4m3fnuz) }, - .{ .f8e5m2, mlir.FloatType(.f8e5m2) }, - .{ .f8e5m2fnuz, mlir.FloatType(.f8e5m2fnuz) }, - .{ .bf16, mlir.FloatType(.bf16) }, - .{ .f16, mlir.FloatType(.f16) }, - .{ .f32, mlir.FloatType(.f32) }, - .{ .f64, mlir.FloatType(.f64) }, + .{ .f8e4m3b11fnuz, mlir.floatType(ctx, .f8e4m3b11fnuz) }, + .{ .f8e4m3fn, mlir.floatType(ctx, .f8e4m3fn) }, + .{ .f8e4m3fnuz, mlir.floatType(ctx, .f8e4m3fnuz) }, + .{ .f8e5m2, mlir.floatType(ctx, .f8e5m2) }, + .{ .f8e5m2fnuz, mlir.floatType(ctx, .f8e5m2fnuz) }, + .{ .bf16, mlir.floatType(ctx, .bf16) }, + .{ .f16, mlir.floatType(ctx, .f16) }, + .{ .f32, mlir.floatType(ctx, .f32) }, + .{ .f64, mlir.floatType(ctx, .f64) }, - .{ .i4, mlir.IntegerType(.i4) }, - .{ .i8, mlir.IntegerType(.i8) }, - .{ .i16, mlir.IntegerType(.i16) }, - .{ .i32, mlir.IntegerType(.i32) }, - .{ .i64, mlir.IntegerType(.i64) }, + .{ .i4, mlir.integerType(ctx, .i4) }, + .{ .i8, mlir.integerType(ctx, .i8) }, + .{ .i16, mlir.integerType(ctx, .i16) }, + .{ .i32, mlir.integerType(ctx, .i32) }, + .{ .i64, mlir.integerType(ctx, .i64) }, - .{ .u4, mlir.IntegerType(.u4) }, - .{ .u8, mlir.IntegerType(.u8) }, - .{ .u16, mlir.IntegerType(.u16) }, - .{ .u32, mlir.IntegerType(.u32) }, - .{ .u64, mlir.IntegerType(.u64) }, + .{ .u4, mlir.integerType(ctx, .u4) }, + .{ .u8, mlir.integerType(ctx, .u8) }, + .{ .u16, mlir.integerType(ctx, .u16) }, + .{ .u32, mlir.integerType(ctx, .u32) }, + .{ .u64, mlir.integerType(ctx, .u64) }, - .{ .c64, mlir.ComplexType(.c64) }, - .{ .c128, mlir.ComplexType(.c128) }, + .{ .c64, mlir.complexType(ctx, .c64) }, + .{ .c128, mlir.complexType(ctx, .c128) }, }; inline for (mapping) |entry| { const dt, const mlirT = entry; - if (mlirT.isAFn(mlir_type._inner)) { + if (mlirT.eql(mlir_type)) { return dt; } } - std.debug.panic("Could not convert mlir.Type to DataType: {}", .{mlir_type}); + std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); } }; + +pub var once = std.once(struct { + fn call() void { + mlir.registerPasses("Transforms"); + } +}.call); diff --git a/zml2/module.zig b/zml2/module.zig new file mode 100644 index 00000000..f9f68daa --- /dev/null +++ b/zml2/module.zig @@ -0,0 +1,487 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); +const mlir = @import("mlir"); +const c = @import("c"); +const upb = @import("upb"); + +const meta = @import("meta.zig"); +const dialects = @import("mlir/dialects"); +const mlirx = @import("mlirx.zig"); + +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const Exe = @import("exe.zig").Exe; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/module"); + +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); + +pub const CompilationContext = struct { + pub const Scope = struct { + block: *mlir.Block, + id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), + arena: std.heap.ArenaAllocator, + + pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(allocator); + return .{ + .block = block, + .id_to_argument = .empty, + .id_to_donation = .empty, + .arena = arena, + }; + } + + pub fn deinit(self: *Scope) void { + self.arena.deinit(); + } + }; + + allocator: std.mem.Allocator, + + mlir_registry: *mlir.DialectRegistry, + mlir_ctx: *mlir.Context, + mlir_pass_manager: *mlir.PassManager, + //mlir_op_pass_manager: *mlir.OpPassManager, + module: *mlir.Module, + + scopes: stdx.BoundedArray(Scope, 16) = .{}, + + threadlocal var _current: ?*CompilationContext = null; + + pub fn init(allocator: std.mem.Allocator) CompilationContext { + mlir.registerPasses("Transforms"); + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; + inline for (.{ "func", "stablehlo" }) |d| { + mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); + } + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + //const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); + const module = mlir.Module.init(.unknown(mlir_ctx)); + module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); + + const pass_manager = mlir.PassManager.init(mlir_ctx); + { + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + opm.addPipeline(pass) catch unreachable; + } + } + + return .{ + .allocator = allocator, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = pass_manager, + .module = module, + }; + } + + pub fn deinit(self: *CompilationContext) void { + self.mlir_pass_manager.deinit(); + self.module.deinit(); + self.mlir_ctx.deinit(); + self.mlir_registry.deinit(); + } + + pub fn activate(self: *CompilationContext) void { + std.debug.assert(_current == null); + _current = self; + } + + pub fn deactivate(self: *CompilationContext) void { + _ = self; + _current = null; + } + + pub fn current() *CompilationContext { + return _current.?; + } + + pub fn currentScope(self: *CompilationContext) *Scope { + return &self.scopes.slice()[self.scopes.len - 1]; + } + + pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { + const scope = Scope.initFromBlock(self.allocator, block); + self.scopes.appendAssumeCapacity(scope); + } + + pub fn popBlock(self: *CompilationContext) void { + var maybe_popped_scope = self.scopes.pop(); + if (maybe_popped_scope) |*popped| { + popped.deinit(); + } + } + + //pub fn addShardingAttributes( + // self: *const CompilationContext, + // mesh: Mesh, + // arg_attrs: []AttributeList, + // res_attrs: []AttributeList, + // input_shapes: []const Shape, + // input_meshes: []const ?Mesh, + // output_info: OutputInfo, + //) void { + // if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. + + // const default_layout = mlir.NamedAttribute.named(self.mlir_ctx, "mhlo.layout_mode", mlir.stringAttribute(self.mlir_ctx, "default")); + // for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { + // attr.appendAssumeCapacity(default_layout); + // if (input_mesh) |m| { + // // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); + // } else { + // // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + // } + // } + + // for (res_attrs, output_info.shapes) |*attr, shape| { + // attr.appendAssumeCapacity(default_layout); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + // } + //} + + //pub fn getShardingAttr(self: *const CompilationContext, sharding: Sharding) *const mlir.Attribute { + // var sharding_str: stdx.BoundedArray(u8, 128) = .{}; + // sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; + // return mlir.stringAttribute(self.mlir_ctx, sharding_str.constSlice()); + //} +}; + +pub fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), platform: Platform) !Exe { + return compile(allocator, func, .{model} ++ args, platform); +} + +pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { + var compilation_context: CompilationContext = .init(allocator); + defer compilation_context.deinit(); + + const result = emitMlir(&compilation_context, func, args) catch unreachable; + defer result.output_info.deinit(compilation_context.allocator); + defer result.input_info.deinit(compilation_context.allocator); + + _ = result.func.appendTo(compilation_context.module.body()); + const sharding = platform.sharding(); + compilation_context.module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_replicas)); + compilation_context.module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_partitions)); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), platform, compilation_context.module, null) catch unreachable; + + log.warn("******** ZML generated MLIR ********", .{}); + log.warn("{f}", .{compilation_context.module.operation()}); + + const num_devices = sharding.num_partitions * sharding.num_replicas; + const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); + errdefer exe.deinit(); + + return exe; +} + +fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { + const LocalContext = struct { + list: *std.array_list.Managed(Shape), + }; + var list = std.array_list.Managed(Shape).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.shape()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +pub const OutputInfo = struct { + shapes: []Shape, + values: []*const mlir.Value, + donations: []?usize, + + pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + allocator.free(self.values); + allocator.free(self.donations); + } +}; + +fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + value_list: *std.array_list.Managed(*const mlir.Value), + donation_list: *std.array_list.Managed(?usize), + }; + + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); + var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); + errdefer value_list.deinit(); + var donation_list = std.array_list.Managed(?usize).init(allocator); + errdefer donation_list.deinit(); + + var context: LocalContext = .{ .shape_list = &shape_list, .value_list = &value_list, .donation_list = &donation_list }; + + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + try ctx_.value_list.append(tensor.value()); + try ctx_.donation_list.append(tensor.donation()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + .values = try value_list.toOwnedSlice(), + .donations = try donation_list.toOwnedSlice(), + }; +} + +pub const InputInfo = struct { + shapes: []Shape, + + pub fn deinit(self: InputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + } +}; + +fn collectInputInfo(allocator: std.mem.Allocator, v: anytype) !InputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + }; + + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); + + var context: LocalContext = .{ .shape_list = &shape_list }; + + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + }; +} + +const EmitMlirResult = struct { + func: *mlir.Operation, + input_info: InputInfo, + output_info: OutputInfo, +}; + +fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { + const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); + for (res, attributes) |*r, attr| { + r.* = mlir.dictionaryAttribute(mlir_ctx, attr.constSlice()); + } + return res; +} + +fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { + var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); + defer arena.deinit(); + + const module = mlir.Module.init(.unknown(compilation_context.mlir_ctx)); + errdefer module.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + compilation_context.pushBlock(block); + defer compilation_context.popBlock(); + + const LocalContext = struct { + compilation_context: *CompilationContext, + current_argument_id: usize = 0, + }; + var context: LocalContext = .{ + .compilation_context = compilation_context, + }; + meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + std.log.debug("Argument {} - shape: {f}", .{ ctx_.current_argument_id, tensor.shape() }); + const mlir_type = mlir.rankedTensorType( + tensor.dims(), + mlirx.Type.fromDType(ctx_.compilation_context.mlir_ctx, tensor.dtype()), + ); + _ = ctx_.compilation_context.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compilation_context.mlir_ctx)); + ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; + ctx_.current_argument_id += 1; + } + }.cb, &context, &args); + + const input_info = try collectInputInfo(compilation_context.allocator, &args); + errdefer input_info.deinit(compilation_context.allocator); + + const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); + @memset(input_attributes, .{}); + + const output_info = b: { + compilation_context.activate(); + defer compilation_context.deactivate(); + + const result = @call(.auto, func, args); + + const output_info = try collectOutputInfo(compilation_context.allocator, &result); + errdefer output_info.deinit(compilation_context.allocator); + + break :b output_info; + }; + errdefer output_info.deinit(compilation_context.allocator); + + const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); + @memset(output_attributes, .{}); + + for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { + input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); + }; + _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + + //self.add + + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ + .name = "main", + .block = compilation_context.currentScope().block, + .location = .unknown(compilation_context.mlir_ctx), + .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), + }); + + compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { + error.MlirUnexpected => { + std.log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_func}); + // user errors should have triggered a panic before we reach this. + @panic("ZML generated invalid mlir. Please open a bug report"); + }, + }; + + return .{ + .func = mlir_func, + .input_info = input_info, + .output_info = output_info, + }; +} + +fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { + const result = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView(flag) }, + .{ .msg_val = blk: { + const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); + switch (@typeInfo(@TypeOf(value))) { + .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), + .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), + .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), + else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), + } + break :blk @ptrCast(field); + } }, + upb_arena, + ); + + if (result == false) { + return std.mem.Allocator.Error.OutOfMemory; + } +} + +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { + //const tracer = Tracer.init("ai.zml.compilation"); + //const compile_frame = tracer.frameStart("pjrt compilation"); + //defer tracer.frameEnd(compile_frame, "pjrt compilation"); + + const sharding = platform.sharding(); + + var upb_alloc: upb.Allocator = .init(arena); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const options = blk: { + const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); + c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { + const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); + + c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { + const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); + for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { + computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); + } + break :device_assignment_blk device_assignment; + }); + + break :executable_build_options_blk exec_build_options; + }); + + const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); + switch (platform.target) { + .cuda => { + // NVIDIA recommends these settings + // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_libnvptxcompiler", true, upb_arena); + }, + .rocm => { + // Disable Triton GEMM on ROCM. For some reason it's much, much slower when + // enabled on CDNA and it's used on RDNA. Disable it altogether. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + // Use lld from libllvm instead of invoking the ld.lld binary. + // This saves us from having to sandbox it. + try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); + }, + else => {}, + } + + if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { + try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); + if (platform.compilation_options.xla_dump_fusion_visualization) { + try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); + } + if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); + } + } + + break :blk options; + }; + + const loaded_executable = try platform.pjrt_client.compile( + platform.pjrt_api, + arena, + module, + try upb.serialize(options, upb_arena), + ); + errdefer loaded_executable.deinit(); + + return loaded_executable; +} diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index f09ffa1e..207b5841 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -79,13 +79,13 @@ pub const Client = opaque { return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); } - fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer bytecode.deinit(); - module.op().writeBytecodeWithConfig(&bytecode.writer, .{ .desiredEmitedVersion = 1 }) catch |err| { + module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode.writer) catch |err| { log.err("failed to write module bytecode: {}", .{err}); return switch (err) { - std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + error.WriteFailed => error.OutOfMemory, else => |e| e, }; }; @@ -95,9 +95,9 @@ pub const Client = opaque { const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { - break :blk dialects.stablehlo.stablehloGetSmallerVersion(requested_version, dialects.stablehlo.getCurrentVersion()); + break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); } - break :blk dialects.stablehlo.getMinimumVersion(); + break :blk dialects.stablehlo.minimumVersion(); }; dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { @@ -115,7 +115,7 @@ pub const Client = opaque { })); } - pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { return try async.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); } diff --git a/zml2/platform.zig b/zml2/platform.zig index d942290b..e67e7128 100644 --- a/zml2/platform.zig +++ b/zml2/platform.zig @@ -55,6 +55,21 @@ pub const Platform = struct { return all_devices; } + pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; + + pub fn sharding(self: Platform) Sharding { + // replicas run the same function but with different inputs, + // while partitions contribute to one evaluation over a shared input. + // Inside an inference process, we generally don't want replicas, + // as it's best to fully isolate replicas on different processes. + // For now we hardcode num_replicas = 1. + const num_devices: u8 = @intCast(self.getDevices().len); + return if (self.compilation_options.sharding_enabled) + .{ .num_replicas = 1, .num_partitions = num_devices } + else + .{ .num_replicas = 1, .num_partitions = 1 }; + } + pub fn withCompilationOptions(self: Platform, opts: CompilationOptions) Platform { var res = self; res.compilation_options = opts; diff --git a/zml2/shape.zig b/zml2/shape.zig index 564f33b5..e45313c9 100644 --- a/zml2/shape.zig +++ b/zml2/shape.zig @@ -180,7 +180,7 @@ pub const Shape = struct { }; } - fn toTag(v: anytype) Tag { + pub fn toTag(v: anytype) Tag { const T = @TypeOf(v); return switch (T) { EnumLiteral => @tagName(v).ptr, diff --git a/zml2/tensor.zig b/zml2/tensor.zig index ed9895bc..9ed4ace2 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -10,7 +10,7 @@ const mlirx = @import("mlirx.zig"); const dialects = @import("mlir/dialects"); const DataType = @import("dtype.zig").DataType; -const CompilationContext = struct {}; +const CompilationContext = @import("module.zig").CompilationContext; pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; @@ -21,6 +21,10 @@ pub const Tensor = struct { _shape: Shape, _value: ?*const mlir.Value = null, + pub fn init(shape_: Shape) Tensor { + return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; + } + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor({f})", .{self._shape}); } @@ -72,15 +76,15 @@ pub const Tensor = struct { /// Creates a Tensor from a mlir.Value /// /// The shape is derived from the type of the mlir.Value. - pub fn fromMlirValue(val: mlir.Value) Tensor { - const ranked_tensor = val.getType().as(mlir.RankedTensorType).?; - const n = ranked_tensor.getRank(); + pub fn fromMlirValue(val: *const mlir.Value) Tensor { + const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; + const n = ranked_tensor.rank(); stdx.debug.assert(n <= MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, MAX_RANK }); - var sh: Shape = .{ ._dtype = mlirx.Type.toDType(ranked_tensor.getElementType()) }; + var sh: Shape = .{ ._dtype = mlirx.Type.toDType(mlirCtx(), ranked_tensor.elementType()) }; for (0..n) |i| { - sh._dims.appendAssumeCapacity(ranked_tensor.getDimension(i)); + sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } sh._tags.resize(n) catch unreachable; @@ -158,8 +162,8 @@ pub const Tensor = struct { /// /// This will fail if used outside of a compilation context. pub fn value(self: Tensor) *const mlir.Value { - if (self.getContext().currentScope().id_to_argument.get(self.id)) |argument_index| { - return self.getContext().currentScope().block.argument(argument_index); + if (CompilationContext.current().currentScope().id_to_argument.get(self.id)) |argument_index| { + return CompilationContext.current().currentScope().block.argument(argument_index); } else if (self._value) |v| { return v; } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); @@ -219,7 +223,7 @@ pub const Tensor = struct { res_shape = res_shape.withDtype(dt); const op = dialects.stablehlo.bitcast_convert( - self.getContext().mlirCtx(), + mlirCtx(), self.value(), mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), .unknown(mlirCtx()), @@ -567,10 +571,13 @@ pub const Tensor = struct { stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); const a = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.mean)); + _ = a; // autofix const b = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.stddev)); - const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); - const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(sh, op.result(0)); + _ = b; // autofix + unreachable; + //const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); + //const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); + //return _result(sh, op.result(0)); } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. @@ -1117,13 +1124,11 @@ pub const Tensor = struct { res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); } - const mlir_ctx = lhs.getContext().mlir_ctx; - const loc = mlir.Location.unknown(mlirCtx()); const op = dialects.stablehlo.dot_general( - mlir_ctx, + mlirCtx(), lhs.value(), rhs.value(), - mlir.rankedTensorType(res_shape.dims(), mlir.Type.fromDType(mlir_ctx, res_shape.dtype())), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), .{ .lhs_batching_dimensions = lhs_batching_axes.constSlice(), .rhs_batching_dimensions = rhs_batching_axes.constSlice(), @@ -1131,7 +1136,7 @@ pub const Tensor = struct { .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), .dot_precision = .fast, }, - loc, + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } @@ -1374,7 +1379,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), - mlirx.tensorType(self.getContext().mlirCtx(), new_shape), + mlirx.tensorType(mlirCtx(), new_shape), .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); @@ -1908,12 +1913,13 @@ pub const Tensor = struct { // TODO(Corentin) /// Embeds a buffer with concrete values into an Mlir program. - pub fn constantTensor(val: HostBuffer) Tensor { - const ctx = CompilationContext.current().mlirCtx(); - const loc = ctx.location(@src()); - const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); - const constant_op = dialects.stablehlo.constant(ctx, val.shape().dims(), elem_type, val.bytes(), loc); - return _result(val.shape(), constant_op.result(0)); + pub fn constantTensor() Tensor { + @panic("unimplemented"); + //const ctx = CompilationContext.current().mlirCtx(); + //const loc = ctx.location(@src()); + //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); + //const constant_op = dialects.stablehlo.constant(ctx, val.shape().dims(), elem_type, val.bytes(), loc); + //return _result(val.shape(), constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. @@ -2141,6 +2147,8 @@ pub const Tensor = struct { return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); } + const GatherAxisKind = enum { batching, offset, collapsed, indices }; + pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); for (idx_axes, 0..) |a, i| { @@ -2160,8 +2168,7 @@ pub const Tensor = struct { var idx_batch_axes: Shape.DimsArray = .{}; - const AxisKind = enum { batching, offset, collapsed, indices }; - var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + var self_kind: stdx.BoundedArray(GatherAxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); @@ -2182,7 +2189,7 @@ pub const Tensor = struct { // compute res shape var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; + var res_kind: stdx.BoundedArray(GatherAxisKind, MAX_RANK) = .{}; for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); if (ax == idx_axes[0]) { @@ -2230,11 +2237,11 @@ pub const Tensor = struct { indices.value(), slice_dims.constSlice(), .{ - .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), - .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), + .offset_dims = _collectAxes(GatherAxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(GatherAxisKind, self_kind, .batching).constSlice(), .start_indices_batching_dims = idx_batch_axes.constSlice(), - .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .start_index_map = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), .index_vector_dim = indices.axis(.coord), .indices_are_sorted = opts.indices_are_sorted, }, @@ -2618,9 +2625,9 @@ pub const Tensor = struct { &updates_values, update_block, .{ - .update_window_dims = _collectAxes(AxisKind, config.up_kind, .update_window).constSlice(), - .inserted_window_dims = _collectAxes(AxisKind, config.op_kind, .inserted_window).constSlice(), - .input_batching_dims = _collectAxes(AxisKind, config.op_kind, .batching).constSlice(), + .update_window_dims = _collectAxes(ScatterAxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(ScatterAxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(ScatterAxisKind, config.op_kind, .batching).constSlice(), .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), .index_vector_dim = indices.rank() - 1, @@ -2638,14 +2645,14 @@ pub const Tensor = struct { } const ScatterConfig = struct { - op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, - up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, indices_batch_axes: Shape.DimsArray = .{}, scatter_to_operand_axes: Shape.DimsArray = .{}, updates_transpose: Shape.AxesArray = .{}, }; - const AxisKind = enum { batching, update_window, inserted_window, window_id }; + const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; fn scatterConfig( op: Shape, @@ -2653,8 +2660,8 @@ pub const Tensor = struct { indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), indices_axes: Shape.TagsArray, ) ScatterConfig { - var op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; - var up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; var indices_batch_axes: Shape.DimsArray = .{}; var scatter_to_operand_axes: Shape.DimsArray = .{}; var updates_transpose: Shape.AxesArray = .{}; @@ -2788,7 +2795,7 @@ pub const Tensor = struct { indices_axes: *Shape.TagsArray, ) Tensor { var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; - const batching = _collectAxes(AxisKind, cfg.op_kind, .batching); + const batching = _collectAxes(ScatterAxisKind, cfg.op_kind, .batching); for (batching.constSlice()) |batch_ax| { const id_shape = indices_per_axis.get(0).shape(); // batching requires tagging, so we're sure to have a tag here. @@ -3664,8 +3671,10 @@ pub const Tensor = struct { const chunk_size: i64 = @divFloor(d, n_chunks); const tail_chunk_size: i64 = @rem(d, chunk_size); - const allocator = self.getContext().allocator(); - var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + var chunks = std.ArrayList(Tensor).initCapacity(arena.allocator(), n_chunks + 1) catch @panic("OOM"); for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; chunks.appendAssumeCapacity( @@ -3723,9 +3732,10 @@ pub const Tensor = struct { for (split_sizes) |n| split_sum += n; stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - const allocator = self.getContext().allocator(); - const res = allocator.alloc(Tensor, split_sizes.len) catch @panic("OOM"); - errdefer allocator.dealloc(res); + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const res = arena.allocator().alloc(Tensor, split_sizes.len) catch @panic("OOM"); var start: i64 = 0; for (split_sizes, 0..) |n, i| { @@ -4046,7 +4056,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), other.value(), - dialects.stablehlo.ComparisonDirection.init(self.getContext().mlirCtx(), direction), + dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction), getComparisonType(mlirCtx(), self.dtype()), .unknown(mlirCtx()), ).appendTo(currentBlock()); @@ -4365,7 +4375,7 @@ pub const Tensor = struct { fn binaryOp( op_name: []const u8, - op_fn: fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation, + op_fn: fn (*mlir.Context, *const mlir.Value, *const mlir.Value, *const mlir.Location) *mlir.Operation, ) fn (Tensor, Tensor) Tensor { return struct { pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { @@ -4390,7 +4400,7 @@ pub const Tensor = struct { stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); - const ret = @call(.auto, op_fn, .{ mlirCtx(), self.value(), other_.value(), .unknown(mlirCtx()) }).appendTo(currentBlock()); + const ret = @call(.auto, op_fn, .{ mlirCtx(), self.value(), other_.value(), mlir.Location.unknown(mlirCtx()) }).appendTo(currentBlock()); return _result(self._shape, ret.result(0)); } }.binaryOpHelper; @@ -4412,6 +4422,11 @@ pub const Tensor = struct { fn currentBlock() *mlir.Block { return CompilationContext.current().currentScope().block; } + + /// Returns the donation data of the tensor. + pub fn donation(self: Tensor) ?usize { + return CompilationContext.current().currentScope().id_to_donation.get(self.id); + } }; fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { @@ -4522,6 +4537,28 @@ fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialects.stablehlo.Comp // ); //} +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { + var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (bounded_array.constSlice(), 0..) |v, ax| { + if (v == value) { + res.appendAssumeCapacity(@intCast(ax)); + } + } + return res; +} + +fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, Tensor.MAX_RANK) } { + const AxesT = @TypeOf(axes_); + const axes_is_scalar = AxesT == @TypeOf(.enum_literal) or AxesT == comptime_int or @typeInfo(AxesT) == .int; + + const coord_axes = if (axes_is_scalar) + stdx.BoundedArray(u3, Tensor.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable + else + self.axes(axes_); + + return .{ axes_is_scalar, coord_axes }; +} + fn toI64(values: anytype) []i64 { var res: [Tensor.MAX_RANK]i64 = undefined; for (values, 0..) |val, i| res[i] = @intCast(val); diff --git a/zml2/zml.zig b/zml2/zml.zig index b1cc842c..03769e68 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -7,7 +7,6 @@ const std = @import("std"); const c = @import("c"); -const mlir = @import("mlir"); const runfiles = @import("runfiles"); pub const platform = @import("platform.zig"); @@ -15,16 +14,18 @@ pub const Platform = platform.Platform; pub const Target = platform.Target; pub const CompilationOptions = platform.CompilationOptions; pub const floats = @import("floats.zig"); -const dtype = @import("dtype.zig"); +pub const dtype = @import("dtype.zig"); pub const Data = dtype.Data; pub const DataType = dtype.DataType; pub const pjrt = @import("pjrtx.zig"); - -var mlir_once = std.once(struct { - fn call() void { - mlir.registerPasses("Transforms"); - } -}.call); +pub const module = @import("module.zig"); +pub const exe = @import("exe.zig"); +pub const meta = @import("meta.zig"); +pub const mlir = @import("mlirx.zig"); +pub const shape = @import("shape.zig"); +pub const Shape = shape.Shape; +pub const tensor = @import("tensor.zig"); +pub const Tensor = tensor.Tensor; var runfiles_once = std.once(struct { fn call_() !void { @@ -55,7 +56,7 @@ var runfiles_once = std.once(struct { pub fn init() void { runfiles_once.call(); - mlir_once.call(); + mlir.once.call(); } pub fn deinit() void {} From b0e4ece290e5936a2f6e2e5dc37cdfa76f046b7c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 4 Dec 2025 14:20:57 +0100 Subject: [PATCH 6/6] Move some operations from tensor.zig to ops.zig --- zml2/BUILD.bazel | 1 + zml2/ops.zig | 711 +++++++++++++++++++++++++++++++++++++++++++++ zml2/tensor.zig | 735 ++--------------------------------------------- 3 files changed, 736 insertions(+), 711 deletions(-) create mode 100644 zml2/ops.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 277e5169..ff846ad2 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -26,6 +26,7 @@ zig_library( "meta.zig", "mlirx.zig", "module.zig", + "ops.zig", "pjrtx.zig", "platform.zig", "shape.zig", diff --git a/zml2/ops.zig b/zml2/ops.zig new file mode 100644 index 00000000..fc99d38c --- /dev/null +++ b/zml2/ops.zig @@ -0,0 +1,711 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const meta = @import("meta.zig"); +const mlir = @import("mlir"); +const mlirx = @import("mlirx.zig"); +const dialects = @import("mlir/dialects"); + +const Tensor = @import("tensor.zig").Tensor; +const Shape = @import("shape.zig").Shape; +const CompilationContext = @import("module.zig").CompilationContext; + +pub const ReduceArgs = struct { + left: Tensor, + right: Tensor, +}; + +pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlir_ctx, "dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, axes_)), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); + + // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. + // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. + // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, + // to find the correct mlir.Value, but we first broadcast before creating the final + // Tensor struct. + var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (0..Tensor.MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { + broadcasting_axes.append(@intCast(i)) catch unreachable; + } + } + + inline for (0..result.len) |i| { + var reduced_shape: Shape = inputs[i].shape(); + for (axes_) |a| { + reduced_shape = reduced_shape.setDim(a, 1); + } + + const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlir_ctx, reduced_shape.dtype())); + const broad_op = dialects.stablehlo.broadcast_in_dim( + mlir_ctx, + reduce_op.result(i), + broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], + tensor_type, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); + + result[i] = Tensor._result(reduced_shape, broad_op.result(0)); + } + + return result; +} + +pub const ReduceWindowOpts = struct { + // TODO replace with Shape + window_dimensions: []const i64, + window_strides: []const i64, + base_dilations: []const i64, + window_dilations: []const i64, + padding: []const [2]i64, +}; + +pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_window", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlir_ctx, "window_dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dimensions)), + .named(mlir_ctx, "window_strides", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_strides)), + .named(mlir_ctx, "base_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.base_dilations)), + .named(mlir_ctx, "window_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dilations)), + // Cast the [][2]i64 to []i64 (safe) + .named(mlir_ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{}, mlir.integerType(mlir_ctx, .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), reduce_op.result(i)); + } + + return result; +} + +pub const SortArgs = struct { + left: Tensor, + right: Tensor, +}; + +pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const sort_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].left = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); + args[i].right = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const sort_block = mlir.Block.init(&block_types, &block_locs); + errdefer sort_block.deinit(); + + CompilationContext.current().pushBlock(sort_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(sort_block); + break :b .{ sort_block, result }; + }; + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + const sort_op = mlir.Operation.make(mlir_ctx, "stablehlo.sort", .{ + .operands = .{ .variadic = &.{&input_values} }, + .result_type_inference = true, + .blocks = &.{sort_block}, + .attributes = &.{ + .named(mlir_ctx, "dimension", mlir.integerAttribute(mlir_ctx, .i64, axis_)), + .named(mlir_ctx, "is_stable", mlir.boolAttribute(mlir_ctx, is_stable)), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); + } + + return result; +} + +pub const ScatterArgs = struct { + input: Tensor, + update: Tensor, +}; + +/// Generalized version of scatter to many inputs. +/// See `zml.Tensor.scatterSlices` for documentation on scatter. +/// +/// This allows to use the same indices to update several tensors at once, +/// and where the update function is allow to look at elements from the different tensors +/// to compute the final value. +/// +/// This sounds nice but in practice XLA doesn't support this well on GPU, +/// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. +pub fn scatter( + inputs: anytype, + index_tensors: anytype, + updates: anytype, + comptime func: anytype, + context: anytype, + opts: Tensor.ScatterOpts, +) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const update_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].input = Tensor.init(Shape.init(.{}, inputs[i].dtype())); + args[i].update = Tensor.init(Shape.init(.{}, inputs[i].dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].input.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].update.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const update_block = mlir.Block.init(&block_types, &block_locs); + errdefer update_block.deinit(); + + CompilationContext.current().pushBlock(update_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(update_block); + break :b .{ update_block, result }; + }; + + // Note: I was a bit lazy here, and I only look at tags on the first tensor. + // we probably should check all of them. + const self = meta.first(Tensor, inputs); + const update = meta.first(Tensor, updates); + var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); + + if (indices_per_axis.len == 0) return inputs; + + // validate coord axes: all coord_axes should exist inside self + for (indices_axes.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); + } + + // Handle scalar indices by broadcasting them to the indices with the highest rank. + const indices_shape = blk: { + var higher_rank = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |indices| { + if (indices.rank() > higher_rank.rank()) { + higher_rank = indices.shape(); + } + } + break :blk higher_rank; + }; + for (indices_per_axis.slice()) |*idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); + stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); + idx.* = idx.broad(indices_shape); + } + + // rewrite simple scatters to dynamicUpdateSlice. + if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { + return .{self.dynamicUpdateSlice(index_tensors, update)}; + } + + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. + var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); + const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); + // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; + // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var updates_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); + + const op = dialects.stablehlo.scatter( + mlir_ctx, + &input_values, + &.{indices.value()}, + &updates_values, + update_block, + .{ + .update_window_dims = _collectAxes(ScatterAxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(ScatterAxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(ScatterAxisKind, config.op_kind, .batching).constSlice(), + .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), + .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), + .index_vector_dim = indices.rank() - 1, + .indices_are_sorted = opts.indices_are_sorted, + .unique_indices = opts.indices_are_unique, + }, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), op.result(i)); + } + + return result; +} + +const ScatterConfig = struct { + op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, + indices_batch_axes: Shape.DimsArray = .{}, + scatter_to_operand_axes: Shape.DimsArray = .{}, + updates_transpose: Shape.AxesArray = .{}, +}; + +const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; + +fn scatterConfig( + op: Shape, + update: Shape, + indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: Shape.TagsArray, +) ScatterConfig { + var op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; + var indices_batch_axes: Shape.DimsArray = .{}; + var scatter_to_operand_axes: Shape.DimsArray = .{}; + var updates_transpose: Shape.AxesArray = .{}; + + const tagged_api = indices_axes.len > 0; + const indices = indices_per_axis.get(0).shape(); + + if (tagged_api) { + for (indices_axes.constSlice()) |t| { + scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); + } + for (indices.tags()) |t| { + stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); + updates_transpose.appendAssumeCapacity(update.axis(t)); + } + + for (op.tags()) |t| { + if (update.hasTag(t)) |up_ax| { + updates_transpose.appendAssumeCapacity(up_ax); + + if (indices.hasTag(t)) |id_ax| { + if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { + // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim + op_kind.appendAssumeCapacity(.inserted_window); + } else { + // tag is in op, indices and updates -> it's a batching dim + op_kind.appendAssumeCapacity(.batching); + indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); + } + } else { + op_kind.appendAssumeCapacity(.update_window); + } + } else { + op_kind.appendAssumeCapacity(.inserted_window); + } + } + + for (update.tags(), 0..) |t, up_ax| { + // Handle batch axes right away. + if (op.hasTag(t)) |self_ax| { + if (op_kind.get(self_ax) == .batching) { + up_kind.appendAssumeCapacity(.batching); + continue; + } + } + if (indices.hasTag(t) != null) { + up_kind.appendAssumeCapacity(.window_id); + } else if (op.hasTag(t)) |self_ax| { + stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); + up_kind.appendAssumeCapacity(.update_window); + } else { + // TODO: consider accepting untagged update here. + std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); + } + } + } else { + for (0..indices_per_axis.len) |i| { + op_kind.appendAssumeCapacity(.inserted_window); + scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); + up_kind.appendAssumeCapacity(.window_id); + } + for (indices_per_axis.len..op.rank()) |_| { + op_kind.appendAssumeCapacity(.update_window); + } + for (indices_per_axis.len..update.rank()) |_| { + up_kind.appendAssumeCapacity(.update_window); + } + for (0..update.rank()) |i| { + updates_transpose.appendAssumeCapacity(@intCast(i)); + } + } + + return .{ + .op_kind = op_kind, + .up_kind = up_kind, + .indices_batch_axes = indices_batch_axes, + .scatter_to_operand_axes = scatter_to_operand_axes, + .updates_transpose = updates_transpose, + }; +} + +// TODO(Corentin) +//test scatterConfig { +// const zml = @import("zml.zig"); +// const platform = zml.testing.env(); + +// var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); +// defer comp.deinit(); +// comp.activate(); +// defer comp.deactivate(); + +// const Local = struct { +// pub fn _idx(idx_shape: anytype) Tensor { +// return Tensor.constant(idx_shape, .{ .i32 = 0 }); +// } +// }; + +// const idx = Local._idx; +// const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); + +// // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b +// { +// const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); +// const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + +// const cfg = scatterConfig(op, update, indices, coords_tags); +// try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); +// try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); +// } + +// // similar, but use the normalized form where .a is no longer an explicit batching axis. +// { +// const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); +// const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + +// const cfg = scatterConfig(op, update, indices, coords_tags); +// try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); +// try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); +// } +//} + +/// Concatenate all indices tensor in one tensor. +/// +/// Is allowed to reorder stuff to simplify the job of the backend, +/// and to expand the batching dims. +fn scatterPrepareIndices( + cfg: *ScatterConfig, + op: Shape, + update: Shape, + indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: *Shape.TagsArray, +) Tensor { + var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; + const batching = _collectAxes(ScatterAxisKind, cfg.op_kind, .batching); + for (batching.constSlice()) |batch_ax| { + const id_shape = indices_per_axis.get(0).shape(); + // batching requires tagging, so we're sure to have a tag here. + const batch_tag = op.tag(batch_ax); + indices_axes.appendAssumeCapacity(batch_tag); + const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); + indices_per_axis.appendAssumeCapacity(batch_id); + cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; + cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; + old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); + } + cfg.indices_batch_axes = .{}; + + // Reorder the axes so that in indices_per_axis is ordered like in op if possible. + // TODO: transpose updates if needed + var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; + var scatter_to_op_axes: Shape.DimsArray = .{}; + + while (old_scatter_to_op_axes.len > 0) { + const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; + const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); + const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); + + scatter_to_op_axes.appendAssumeCapacity(op_ax); + indices.appendAssumeCapacity(scatter_idx); + } + cfg.scatter_to_operand_axes = scatter_to_op_axes; + + for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { + if (i != sc_ax) { + //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); + break; + } + } + return Tensor.stack(indices.constSlice(), .last, .coord); +} + +pub const GatherAxisKind = enum { batching, offset, collapsed, indices }; + +pub const GatherOpts = struct { indices_are_sorted: bool = false }; + +pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + const mlir_ctx = CompilationContext.current().mlir_ctx; + + stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); + for (idx_axes, 0..) |a, i| { + if (i > 0) { + stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); + } + } + var indices_shape = idx_per_axis[0].shape(); + for (idx_per_axis[1..]) |idx| { + if (idx.rank() > indices_shape.rank()) { + indices_shape = idx.shape(); + } + } + for (idx_per_axis) |idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + } + + var idx_batch_axes: Shape.DimsArray = .{}; + + var self_kind: stdx.BoundedArray(GatherAxisKind, Shape.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + if (indices_shape.hasTag(t)) |id_ax| { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { + // we collapsed all gathered axes + self_kind.buffer[self_ax] = .collapsed; + // idx_kind.buffer[id_ax] = .indices; + } else { + self_kind.buffer[self_ax] = .offset; + } + } + + // compute res shape + var res_shape = Shape.init(.{}, self.dtype()); + var res_kind: stdx.BoundedArray(GatherAxisKind, Shape.MAX_RANK) = .{}; + for (self_kind.slice(), 0..) |kind, ax_usize| { + const ax: u3 = @intCast(ax_usize); + if (ax == idx_axes[0]) { + // The first val_ax is special cause this is the place where we insert indices axes. + for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + // Batching axis is already in self. + if (is_batching_axis) continue; + + res_shape = res_shape.appendDim(id_inserted_dim, id_axis); + res_kind.appendAssumeCapacity(.indices); + } + } + switch (kind) { + .collapsed => continue, + else => { + res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); + res_kind.appendAssumeCapacity(kind); + }, + } + } + + // This is not a gather, but a dynamicSlice. + // Sometimes the backend recognize this pattern, but not always. + // So let us handle that. + if (indices_shape.count() == 1 and idx_axes.len == 1) { + return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); + } + + var slice_dims: Shape.DimsArray = .{}; + for (self_kind.slice(), self.dims()) |k, d| { + slice_dims.appendAssumeCapacity(switch (k) { + .batching, .collapsed => 1, + .offset => d, + .indices => unreachable, + }); + } + + // TODO: try changing .last by other axis and see the perf impact. + const indices = Tensor.stack(idx_per_axis, .last, .coord); + // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); + const gather_op = dialects.stablehlo.gather( + mlir_ctx, + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = _collectAxes(GatherAxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(GatherAxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), + .start_index_map = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = indices.axis(.coord), + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); + + const mlir_shape = Tensor.fromMlirValue(gather_op.result(0)).shape(); + stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + return Tensor._result(res_shape, gather_op.result(0)); +} + +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { + var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (bounded_array.constSlice(), 0..) |v, ax| { + if (v == value) { + res.appendAssumeCapacity(@intCast(ax)); + } + } + return res; +} diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 9ed4ace2..f3c4515e 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -8,6 +8,7 @@ const Shape = @import("shape.zig").Shape; const mlir = @import("mlir"); const mlirx = @import("mlirx.zig"); const dialects = @import("mlir/dialects"); +const ops = @import("ops.zig"); const DataType = @import("dtype.zig").DataType; const CompilationContext = @import("module.zig").CompilationContext; @@ -1254,8 +1255,8 @@ pub const Tensor = struct { /// Output shape is the input shape with the axis_ dim set to 1. pub fn sum(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - return reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { - pub fn acc(args: ReduceArgs) Tensor { + return ops.reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { + pub fn acc(args: ops.ReduceArgs) Tensor { return args.right.add(args.left.convert(args.right.dtype())); } }.acc, .{})[0]; @@ -2105,7 +2106,7 @@ pub const Tensor = struct { return _result(self._shape, reverse_op.result(0)); } - pub const GatherOpts = struct { indices_are_sorted: bool = false }; + pub const GatherOpts = ops.GatherOpts; /// `gather` extracts slices from the given tensor at the specified offsets. /// example: `values.gather(.{ .a = idx }, .{})` @@ -2144,113 +2145,7 @@ pub const Tensor = struct { } // TODO: sort indices following self.shape instead of asking the user to do it. - return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); - } - - const GatherAxisKind = enum { batching, offset, collapsed, indices }; - - pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); - for (idx_axes, 0..) |a, i| { - if (i > 0) { - stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); - } - } - var indices_shape = idx_per_axis[0].shape(); - for (idx_per_axis[1..]) |idx| { - if (idx.rank() > indices_shape.rank()) { - indices_shape = idx.shape(); - } - } - for (idx_per_axis) |idx| { - stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); - } - - var idx_batch_axes: Shape.DimsArray = .{}; - - var self_kind: stdx.BoundedArray(GatherAxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; - - for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); - if (indices_shape.hasTag(t)) |id_ax| { - // tag is both in self and indices -> it's a batching dim - // Note: tags are required for batching. - self_kind.buffer[self_ax] = .batching; - idx_batch_axes.appendAssumeCapacity(id_ax); - stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); - } else if (is_gather_axis) { - // we collapsed all gathered axes - self_kind.buffer[self_ax] = .collapsed; - // idx_kind.buffer[id_ax] = .indices; - } else { - self_kind.buffer[self_ax] = .offset; - } - } - - // compute res shape - var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(GatherAxisKind, MAX_RANK) = .{}; - for (self_kind.slice(), 0..) |kind, ax_usize| { - const ax: u3 = @intCast(ax_usize); - if (ax == idx_axes[0]) { - // The first val_ax is special cause this is the place where we insert indices axes. - for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { - const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); - // Batching axis is already in self. - if (is_batching_axis) continue; - - res_shape = res_shape.appendDim(id_inserted_dim, id_axis); - res_kind.appendAssumeCapacity(.indices); - } - } - switch (kind) { - .collapsed => continue, - else => { - res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); - res_kind.appendAssumeCapacity(kind); - }, - } - } - - // This is not a gather, but a dynamicSlice. - // Sometimes the backend recognize this pattern, but not always. - // So let us handle that. - if (indices_shape.count() == 1 and idx_axes.len == 1) { - return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); - } - - var slice_dims: Shape.DimsArray = .{}; - for (self_kind.slice(), self.dims()) |k, d| { - slice_dims.appendAssumeCapacity(switch (k) { - .batching, .collapsed => 1, - .offset => d, - .indices => unreachable, - }); - } - - // TODO: try changing .last by other axis and see the perf impact. - const indices = Tensor.stack(idx_per_axis, .last, .coord); - // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); - const gather_op = dialects.stablehlo.gather( - mlirCtx(), - self.value(), - indices.value(), - slice_dims.constSlice(), - .{ - .offset_dims = _collectAxes(GatherAxisKind, res_kind, .offset).constSlice(), - .collapsed_slice_dims = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), - .operand_batching_dims = _collectAxes(GatherAxisKind, self_kind, .batching).constSlice(), - .start_indices_batching_dims = idx_batch_axes.constSlice(), - .start_index_map = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), - .index_vector_dim = indices.axis(.coord), - .indices_are_sorted = opts.indices_are_sorted, - }, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); - stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); - return _result(res_shape, gather_op.result(0)); + return ops.gather_(self, idx_axes.slice(), idx_per_axis.slice(), opts); } // TODO(Corentin) @@ -2482,11 +2377,6 @@ pub const Tensor = struct { // try zml.testing.expectClose(expected, result, 0); //} - pub const ScatterArgs = struct { - input: Tensor, - update: Tensor, - }; - pub const ScatterOpts = struct { /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final offset in `self` /// Result is undefined if the promise is violated. @@ -2502,337 +2392,17 @@ pub const Tensor = struct { /// then you should make sure the slices don't overlap, /// otherwise the result will depend on the runtime scheduling /// of the operator which is backend specific. - update_fn: *const fn (ScatterArgs) struct { Tensor } = increment, + update_fn: *const fn (ops.ScatterArgs) struct { Tensor } = increment, - pub fn increment(values: ScatterArgs) struct { Tensor } { + pub fn increment(values: ops.ScatterArgs) struct { Tensor } { return .{values.input.add(values.update)}; } - pub fn override(values: ScatterArgs) struct { Tensor } { + pub fn override(values: ops.ScatterArgs) struct { Tensor } { return .{values.update}; } }; - /// Generalized version of scatter to many inputs. - /// See `zml.Tensor.scatterSlices` for documentation on scatter. - /// - /// This allows to use the same indices to update several tensors at once, - /// and where the update function is allow to look at elements from the different tensors - /// to compute the final value. - /// - /// This sounds nice but in practice XLA doesn't support this well on GPU, - /// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. - pub fn scatter( - inputs: anytype, - index_tensors: anytype, - updates: anytype, - comptime func: anytype, - context: anytype, - opts: Tensor.ScatterOpts, - ) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const update_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); - var args: ArgsType = undefined; - var block_types: [2 * inputs.len]*const mlir.Type = undefined; - - inline for (0..inputs.len) |i| { - args[i].input = Tensor.init(Shape.init(.{}, inputs[i].dtype())); - args[i].update = Tensor.init(Shape.init(.{}, inputs[i].dtype())); - - block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].input.dtype())); - block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].update.dtype())); - } - - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); - const update_block = mlir.Block.init(&block_types, &block_locs); - errdefer update_block.deinit(); - - CompilationContext.current().pushBlock(update_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(update_block); - break :b .{ update_block, result }; - }; - - // Note: I was a bit lazy here, and I only look at tags on the first tensor. - // we probably should check all of them. - const self = meta.first(Tensor, inputs); - const update = meta.first(Tensor, updates); - var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); - - if (indices_per_axis.len == 0) return inputs; - - // validate coord axes: all coord_axes should exist inside self - for (indices_axes.constSlice()) |t| { - stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); - } - - // Handle scalar indices by broadcasting them to the indices with the highest rank. - const indices_shape = blk: { - var higher_rank = indices_per_axis.get(0).shape(); - for (indices_per_axis.constSlice()[1..]) |indices| { - if (indices.rank() > higher_rank.rank()) { - higher_rank = indices.shape(); - } - } - break :blk higher_rank; - }; - for (indices_per_axis.slice()) |*idx| { - stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); - stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); - idx.* = idx.broad(indices_shape); - } - - // rewrite simple scatters to dynamicUpdateSlice. - if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { - return .{self.dynamicUpdateSlice(index_tensors, update)}; - } - - // TODO: ideally we should catch all possible scatter errors and provide nice error messages. - var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); - const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); - // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; - // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); - - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - var updates_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); - - const op = dialects.stablehlo.scatter( - mlirCtx(), - &input_values, - &.{indices.value()}, - &updates_values, - update_block, - .{ - .update_window_dims = _collectAxes(ScatterAxisKind, config.up_kind, .update_window).constSlice(), - .inserted_window_dims = _collectAxes(ScatterAxisKind, config.op_kind, .inserted_window).constSlice(), - .input_batching_dims = _collectAxes(ScatterAxisKind, config.op_kind, .batching).constSlice(), - .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), - .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), - .index_vector_dim = indices.rank() - 1, - .indices_are_sorted = opts.indices_are_sorted, - .unique_indices = opts.indices_are_unique, - }, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - inline for (0..result.len) |i| { - result[i] = Tensor._result(inputs[i].shape(), op.result(i)); - } - - return result; - } - - const ScatterConfig = struct { - op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, - up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, - indices_batch_axes: Shape.DimsArray = .{}, - scatter_to_operand_axes: Shape.DimsArray = .{}, - updates_transpose: Shape.AxesArray = .{}, - }; - - const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; - - fn scatterConfig( - op: Shape, - update: Shape, - indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), - indices_axes: Shape.TagsArray, - ) ScatterConfig { - var op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; - var up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; - var scatter_to_operand_axes: Shape.DimsArray = .{}; - var updates_transpose: Shape.AxesArray = .{}; - - const tagged_api = indices_axes.len > 0; - const indices = indices_per_axis.get(0).shape(); - - if (tagged_api) { - for (indices_axes.constSlice()) |t| { - scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); - } - for (indices.tags()) |t| { - stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); - updates_transpose.appendAssumeCapacity(update.axis(t)); - } - - for (op.tags()) |t| { - if (update.hasTag(t)) |up_ax| { - updates_transpose.appendAssumeCapacity(up_ax); - - if (indices.hasTag(t)) |id_ax| { - if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { - // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim - op_kind.appendAssumeCapacity(.inserted_window); - } else { - // tag is in op, indices and updates -> it's a batching dim - op_kind.appendAssumeCapacity(.batching); - indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); - } - } else { - op_kind.appendAssumeCapacity(.update_window); - } - } else { - op_kind.appendAssumeCapacity(.inserted_window); - } - } - - for (update.tags(), 0..) |t, up_ax| { - // Handle batch axes right away. - if (op.hasTag(t)) |self_ax| { - if (op_kind.get(self_ax) == .batching) { - up_kind.appendAssumeCapacity(.batching); - continue; - } - } - if (indices.hasTag(t) != null) { - up_kind.appendAssumeCapacity(.window_id); - } else if (op.hasTag(t)) |self_ax| { - stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); - up_kind.appendAssumeCapacity(.update_window); - } else { - // TODO: consider accepting untagged update here. - std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); - } - } - } else { - for (0..indices_per_axis.len) |i| { - op_kind.appendAssumeCapacity(.inserted_window); - scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); - up_kind.appendAssumeCapacity(.window_id); - } - for (indices_per_axis.len..op.rank()) |_| { - op_kind.appendAssumeCapacity(.update_window); - } - for (indices_per_axis.len..update.rank()) |_| { - up_kind.appendAssumeCapacity(.update_window); - } - for (0..update.rank()) |i| { - updates_transpose.appendAssumeCapacity(@intCast(i)); - } - } - - return .{ - .op_kind = op_kind, - .up_kind = up_kind, - .indices_batch_axes = indices_batch_axes, - .scatter_to_operand_axes = scatter_to_operand_axes, - .updates_transpose = updates_transpose, - }; - } - - // TODO(Corentin) - //test scatterConfig { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); - - // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - // defer comp.deinit(); - // comp.activate(); - // defer comp.deactivate(); - - // const Local = struct { - // pub fn _idx(idx_shape: anytype) Tensor { - // return Tensor.constant(idx_shape, .{ .i32 = 0 }); - // } - // }; - - // const idx = Local._idx; - // const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); - - // // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b - // { - // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); - // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); - - // const cfg = scatterConfig(op, update, indices, coords_tags); - // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); - // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); - // } - - // // similar, but use the normalized form where .a is no longer an explicit batching axis. - // { - // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); - // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); - - // const cfg = scatterConfig(op, update, indices, coords_tags); - // try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); - // try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); - // } - //} - - /// Concatenate all indices tensor in one tensor. - /// - /// Is allowed to reorder stuff to simplify the job of the backend, - /// and to expand the batching dims. - fn scatterPrepareIndices( - cfg: *ScatterConfig, - op: Shape, - update: Shape, - indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), - indices_axes: *Shape.TagsArray, - ) Tensor { - var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; - const batching = _collectAxes(ScatterAxisKind, cfg.op_kind, .batching); - for (batching.constSlice()) |batch_ax| { - const id_shape = indices_per_axis.get(0).shape(); - // batching requires tagging, so we're sure to have a tag here. - const batch_tag = op.tag(batch_ax); - indices_axes.appendAssumeCapacity(batch_tag); - const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); - indices_per_axis.appendAssumeCapacity(batch_id); - cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; - cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; - old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); - } - cfg.indices_batch_axes = .{}; - - // Reorder the axes so that in indices_per_axis is ordered like in op if possible. - // TODO: transpose updates if needed - var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; - var scatter_to_op_axes: Shape.DimsArray = .{}; - - while (old_scatter_to_op_axes.len > 0) { - const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; - const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); - const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); - - scatter_to_op_axes.appendAssumeCapacity(op_ax); - indices.appendAssumeCapacity(scatter_idx); - } - cfg.scatter_to_operand_axes = scatter_to_op_axes; - - for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { - if (i != sc_ax) { - //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); - break; - } - } - return Tensor.stack(indices.constSlice(), .last, .coord); - } - /// Update the given tensor, by copying `values` into slice by slice into `self`. /// The slices are chosen at runtime by interpreting indices as coordinates into `self`. /// This is a generalized version of `dynamicUpdateSlice` where more than one offset can be specified at a time. @@ -2912,12 +2482,12 @@ pub const Tensor = struct { const UpdateType = @TypeOf(ScatterOpts.increment); const Custom = struct { - pub fn inc(values: ScatterArgs, custom: *const UpdateType) struct { Tensor } { + pub fn inc(values: ops.ScatterArgs, custom: *const UpdateType) struct { Tensor } { return @call(.auto, custom, .{values}); } }; - return scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; + return ops.scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; } // TODO(Corentin) @@ -3066,12 +2636,12 @@ pub const Tensor = struct { /// Returns a Tensor containing the maximum over a given axis. pub fn max(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - return reduce( + return ops.reduce( .{self}, .{Tensor.constant(self.dtype().minValue())}, &.{a}, struct { - pub fn cmp(args: ReduceArgs) struct { Tensor } { + pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { return .{args.left.maximum(args.right)}; } }, @@ -3082,12 +2652,12 @@ pub const Tensor = struct { /// Returns a Tensor containing the minimum over a given axis. pub fn min(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - return reduce( + return ops.reduce( .{self}, .{Tensor.constant(self.dtype().maxValue())}, &.{a}, struct { - pub fn cmp(args: ReduceArgs) struct { Tensor } { + pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { return .{args.left.minimum(args.right)}; } }, @@ -3099,7 +2669,7 @@ pub const Tensor = struct { values: Tensor, indices: Tensor, - fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { + fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) struct { Tensor, Tensor } { const left_gt_right = values.left.cmp(.GT, values.right); const is_nan = values.left.cmp(.NE, values.left); const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); @@ -3121,7 +2691,7 @@ pub const Tensor = struct { const a = x.axis(axis_); const dt: DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; - const values, const indices = reduce( + const values, const indices = ops.reduce( .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, &.{a}, @@ -3166,189 +2736,22 @@ pub const Tensor = struct { // } //} - pub const ReduceArgs = struct { - left: Tensor, - right: Tensor, - }; - - fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); - var args: ArgsType = undefined; - var block_types: [2 * inits.len]*const mlir.Type = undefined; - - inline for (0..inits.len) |i| { - args[i].left = Tensor.init(inits[i].shape()); - args[i].right = Tensor.init(inits[i].shape()); - - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); - } - - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); - const reduce_block = mlir.Block.init(&block_types, &block_locs); - errdefer reduce_block.deinit(); - - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inits.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inits.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); - break :b .{ reduce_block, result }; - }; - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - var init_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - - const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes_)), - }, - .verify = true, - .location = .unknown(mlirCtx()), - }).appendTo(currentBlock()); - - // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. - // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. - // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, - // to find the correct mlir.Value, but we first broadcast before creating the final - // Tensor struct. - var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; - for (0..Tensor.MAX_RANK) |i| { - if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { - broadcasting_axes.append(@intCast(i)) catch unreachable; - } - } - - inline for (0..result.len) |i| { - var reduced_shape: Shape = inputs[i].shape(); - for (axes_) |a| { - reduced_shape = reduced_shape.setDim(a, 1); - } - - const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlirCtx(), reduced_shape.dtype())); - const broad_op = dialects.stablehlo.broadcast_in_dim( - mlirCtx(), - reduce_op.result(i), - broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], - tensor_type, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - result[i] = Tensor._result(reduced_shape, broad_op.result(0)); - } - - return result; - } - pub const SortRes = ArgMaxRes; - pub const SortArgs = struct { - left: Tensor, - right: Tensor, - }; - pub const SortOpts = struct { descending: bool = false }; /// Returns two Tensors. The first contains the sorted values and the second one contains the sorted indices. pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { const a = self.axis(axis_); const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); - const res = sort_(.{ self, indices }, a, struct { - fn call(direction: dialects.stablehlo.ComparisonDirection.Direction, values: SortArgs, _: SortArgs) Tensor { + const res = ops.sort(.{ self, indices }, a, struct { + fn call(direction: dialects.stablehlo.ComparisonDirection.Direction, values: ops.SortArgs, _: ops.SortArgs) Tensor { return values.left.cmp(direction, values.right); } }.call, .{if (opts.descending) .GT else .LT}, true); return .{ .values = res[0], .indices = res[1] }; } - fn sort_(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const sort_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); - var args: ArgsType = undefined; - var block_types: [2 * inputs.len]*const mlir.Type = undefined; - - inline for (0..inputs.len) |i| { - args[i].left = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); - args[i].right = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); - - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); - block_types[i + inputs.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); - } - - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); - const sort_block = mlir.Block.init(&block_types, &block_locs); - errdefer sort_block.deinit(); - - CompilationContext.current().pushBlock(sort_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inputs.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(sort_block); - break :b .{ sort_block, result }; - }; - - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - const sort_op = mlir.Operation.make(mlirCtx(), "stablehlo.sort", .{ - .operands = .{ .variadic = &.{&input_values} }, - .result_type_inference = true, - .blocks = &.{sort_block}, - .attributes = &.{ - .named(mlirCtx(), "dimension", mlir.integerAttribute(mlirCtx(), .i64, axis_)), - .named(mlirCtx(), "is_stable", mlir.boolAttribute(mlirCtx(), is_stable)), - }, - .verify = true, - .location = .unknown(mlirCtx()), - }).appendTo(currentBlock()); - - inline for (0..result.len) |i| { - result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); - } - - return result; - } - pub const ArgSortOpts = SortOpts; /// Returns a Tensor containing the indices corresponding to the sorted values over the given axis. @@ -3447,86 +2850,6 @@ pub const Tensor = struct { return result; } - pub const ReduceWindowOpts = struct { - // TODO replace with Shape - window_dimensions: []const i64, - window_strides: []const i64, - base_dilations: []const i64, - window_dilations: []const i64, - padding: []const [2]i64, - }; - - fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); - var args: ArgsType = undefined; - var block_types: [2 * inits.len]*const mlir.Type = undefined; - - inline for (0..inits.len) |i| { - args[i].left = Tensor.init(inits[i].shape()); - args[i].right = Tensor.init(inits[i].shape()); - - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); - } - - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); - const reduce_block = mlir.Block.init(&block_types, &block_locs); - errdefer reduce_block.deinit(); - - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inits.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inits.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); - break :b .{ reduce_block, result }; - }; - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - var init_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - - const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce_window", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlirCtx(), "window_dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_dimensions)), - .named(mlirCtx(), "window_strides", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_strides)), - .named(mlirCtx(), "base_dilations", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.base_dilations)), - .named(mlirCtx(), "window_dilations", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_dilations)), - // Cast the [][2]i64 to []i64 (safe) - .named(mlirCtx(), "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{}, mlir.integerType(mlirCtx(), .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), - }, - .verify = true, - .location = .unknown(mlirCtx()), - }).appendTo(currentBlock()); - - inline for (0..result.len) |i| { - result[i] = Tensor._result(inputs[i].shape(), reduce_op.result(i)); - } - - return result; - } - pub const MaxPoolRes = ArgMaxRes; /// Computes the 1d maxPool operation on the input Tensor. @@ -3562,7 +2885,7 @@ pub const Tensor = struct { var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; padding[a] = opts.padding; - const result = reduceWindow( + const result = ops.reduceWindow( .{ self, iota(self._shape, a) }, .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, .{ @@ -3602,7 +2925,7 @@ pub const Tensor = struct { padding[a - 1] = opts.padding[0]; padding[a] = opts.padding[1]; - const result = reduceWindow( + const result = ops.reduceWindow( .{ self, iota(self._shape, a) }, .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, .{ @@ -4222,12 +3545,12 @@ pub const Tensor = struct { /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. pub fn any(self: Tensor, axis_: anytype) Tensor { const pred = self.cmp(.NE, Tensor.constant(self.dims(), self.dtype().zero())); - return reduce( + return ops.reduce( .{pred}, .{Tensor.scalar(false, .bool)}, &.{self.axis(axis_)}, struct { - pub fn acc(args: ReduceArgs) Tensor { + pub fn acc(args: ops.ReduceArgs) Tensor { return args.left.logical(.OR, args.right); } }.acc, @@ -4238,12 +3561,12 @@ pub const Tensor = struct { /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. pub fn all(self: Tensor, axis_: anytype) Tensor { const pred = if (self.dtype() == .bool) self else self.cmp(.NE, Tensor.scalar(0, self.dtype())); - return reduce( + return ops.reduce( .{pred}, .{Tensor.scalar(true, .bool)}, &.{self.axis(axis_)}, struct { - pub fn acc(args: ReduceArgs) Tensor { + pub fn acc(args: ops.ReduceArgs) Tensor { return args.left.logical(.AND, args.right); } }.acc, @@ -4537,16 +3860,6 @@ fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialects.stablehlo.Comp // ); //} -pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { - var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; - for (bounded_array.constSlice(), 0..) |v, ax| { - if (v == value) { - res.appendAssumeCapacity(@intCast(ax)); - } - } - return res; -} - fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, Tensor.MAX_RANK) } { const AxesT = @TypeOf(axes_); const axes_is_scalar = AxesT == @TypeOf(.enum_literal) or AxesT == comptime_int or @typeInfo(AxesT) == .int;