From 6036d23ebfea8cc904a00b7b4919df0e30f61dab Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 25 Aug 2025 15:54:43 +0200 Subject: [PATCH 01/47] WIP --- BUILD.bazel | 4 + mlir/BUILD.bazel | 28 +- mlir/context.zig | 0 mlir/main.zig | 73 +++ mlir/registry.zig | 1115 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 1219 insertions(+), 1 deletion(-) create mode 100644 mlir/context.zig create mode 100644 mlir/main.zig create mode 100644 mlir/registry.zig diff --git a/BUILD.bazel b/BUILD.bazel index aa1656d4..4c17c21a 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -14,5 +14,9 @@ zls_completion( "//examples/llama", "//stdx", "//zml", + "//mlir:main", + # "//async", + # "//stdx", + # "//zml", ], ) diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 87982a7e..0dccf25d 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") cc_library( name = "c", @@ -28,3 +28,29 @@ zig_test( name = "test", deps = [":mlir"], ) + +zig_library( + name = "mlir2", + main = "registry.zig", + deps = [":c"], +) + +zig_binary( + name = "main", + main = "main.zig", + visibility = ["//visibility:public"], + deps = [ + ":mlir2", + # ":c", + # "//stdx", + ], +) + +zig_test( + name = "test2", + deps = [ + ":mlir2", + # ":c", + # "//stdx", + ], +) diff --git a/mlir/context.zig b/mlir/context.zig new file mode 100644 index 00000000..e69de29b diff --git a/mlir/main.zig b/mlir/main.zig new file mode 100644 index 00000000..8095de85 --- /dev/null +++ b/mlir/main.zig @@ -0,0 +1,73 @@ +const std = @import("std"); + +const mlir = @import("mlir2"); + +pub fn main() !void { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + // const handle = mlir.DialectHandle.fromString("func"); + + inline for (.{ "arith", "func" }) |d| { + registry.registerDialect(d); + } + + const ctx = try mlir.Context.init(.{ + .registry = registry, + .threading = true, + }); + defer ctx.deinit(); + + std.debug.print(">>> {f}\n", .{ctx}); + ctx.loadAllAvailableDialects(); + std.debug.print(">>> {f}\n", .{ctx}); + + const unk = mlir.Location.unknown(ctx); + _ = unk; // autofix + + const attr = mlir.integerAttribute(ctx, @as(i32, 5)); + _ = attr; // autofix + + const one: i32 = 1; + const two: i32 = 2; + + const dict = mlir.DictionaryAttribute.init(ctx, &.{ + .named(ctx, "pouet", mlir.integerAttribute(ctx, one)), + .named(ctx, "pouet2", mlir.integerAttribute(ctx, two)), + }); + _ = dict; // autofix + + const op = try mlir.Operation.parse(ctx, "func.func private @pouet()", "pouet"); + + const mod = mlir.Module.init(mlir.Location.unknown(ctx)); + + const body = mod.body(); + _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); + _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); + _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); + + body.appendOwnedOperation(op); + + // op.deinit(); + // body.detach(); + // body.deinit(); + + // std.debug.print(">>>>>> {any}\n", .{op.block().?.parentOperation()}); + + // std.debug.print(">>>>>> {any}\n", .{op.block().?.parentOperation()}); + + std.debug.print(">>> {f}\n", .{body}); + // const attr = mlir.integerAttribute(ctx, @as(i32, 5)); + // const attr2: *const mlir.Attribute = @ptrCast(attr); + + // std.debug.print("coucou 2 {any} {f}\n", .{ + // ctx.isRegisteredOperation("func.func"), + // attr2, + // }); + + // var buffer: [256]u8 = undefined; + // const vv = "coucou\n"; + // var writer = std.fs.File.stdout().writer(&buffer); + // try writer.interface.writeAll(vv); + // try writer.interface.flush(); +} diff --git a/mlir/registry.zig b/mlir/registry.zig new file mode 100644 index 00000000..69e44975 --- /dev/null +++ b/mlir/registry.zig @@ -0,0 +1,1115 @@ +const std = @import("std"); + +const c = @import("c"); + +const log = std.log.scoped(.mlir); + +test { + std.testing.refAllDeclsRecursive(@This()); +} + +const Error = error{ + /// Invalid Mlir was created. + InvalidMlir, + /// Another Mlir error. Check the log for more context. + MlirUnexpected, + /// A resource/executor was not found. + NotFound, +} || std.mem.Allocator.Error; + +pub inline fn stringRef(str: []const u8) c.MlirStringRef { + return .{ .data = str.ptr, .length = str.len }; +} + +pub inline fn string(str: c.MlirStringRef) []const u8 { + // Note: mlir.StringRef need not to be null terminated. + return str.data[0..str.length]; +} + +pub const StringCallbackCtx = struct { + writer: *std.Io.Writer, + err: ?std.Io.Writer.Error = null, +}; + +pub fn stringCallback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { + const ctx: *StringCallbackCtx = @ptrCast(@alignCast(ctx_)); + ctx.writer.writeAll(string(str)) catch |err| { + ctx.err = err; + }; +} + +pub fn registerPasses(comptime passes: []const u8) void { + @field(c, "mlirRegister" ++ passes ++ "Passes")(); +} + +fn isPtrConst(comptime T: type) bool { + return @typeInfo(T).pointer.is_const; +} + +fn CastedPtr(comptime T1: type, comptime T2: type) type { + return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; +} + +pub fn StdMethods(comptime T: type, comptime NativeT: type) type { + return struct { + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; + pub const Ptr = if (IsPtrConst) *const T else *T; + + pub fn ptr(self: anytype) NativeT { + return .{ .ptr = @ptrCast(@constCast(self)) }; + } + + fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return struct { + fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var ctx: StringCallbackCtx = .{ .writer = writer }; + f(self.ptr(), stringCallback, &ctx); + if (ctx.err) |err| { + return err; + } + } + }.format; + } + + pub fn eql(f: anytype) fn (self: *const T, other: *const T) bool { + return struct { + fn eql(self: *const T, other: *const T) bool { + return f(self.ptr(), other.ptr()); + } + }.eql; + } + + pub fn isA(self: *const T, OtherType: type) ?CastedPtr(@TypeOf(self), OtherType) { + return if (OtherType.isAFn(self.ptr())) @ptrCast(self) else null; + } + }; +} + +pub const DialectRegistry = opaque { + const M = StdMethods(DialectRegistry, c.MlirDialectRegistry); + + pub const ptr = M.ptr; + + pub fn init() std.mem.Allocator.Error!*DialectRegistry { + return @ptrCast(c.mlirDialectRegistryCreate().ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *DialectRegistry) void { + c.mlirDialectRegistryDestroy(self.ptr()); + } + + pub fn registerDialect(self: *DialectRegistry, comptime name: []const u8) void { + DialectHandle.fromString(name).insertDialect(self); + } +}; + +pub const Context = opaque { + const M = StdMethods(Context, c.MlirContext); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirContextEqual); + + pub const InitOptions = struct { + registry: ?*const DialectRegistry = null, + threading: bool = false, + }; + + pub fn init(opts: InitOptions) std.mem.Allocator.Error!*Context { + const ctx = if (opts.registry) |r| c.mlirContextCreateWithRegistry( + r.ptr(), + opts.threading, + ) else c.mlirContextCreateWithThreading(opts.threading); + return @ptrCast(ctx.ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *Context) void { + c.mlirContextDestroy(self.ptr()); + } + + pub fn setMultiThreading(self: *Context, enabled: bool) void { + c.mlirContextEnableMultithreading(self.ptr(), enabled); + } + + pub fn appendDialectRegistry(self: *Context, registry: *const DialectRegistry) void { + c.mlirContextAppendDialectRegistry(self.ptr(), registry.ptr()); + } + + pub fn loadAllAvailableDialects(self: *Context) void { + c.mlirContextLoadAllAvailableDialects(self.ptr()); + } + + pub fn numRegisteredDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumRegisteredDialects(self.ptr())); + } + + pub fn numLoadedDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumLoadedDialects(self.ptr())); + } + + pub fn numThreads(self: *const Context) usize { + return @intCast(c.mlirContextGetNumThreads(self.ptr())); + } + + pub fn isRegisteredOperation(self: *const Context, op: []const u8) bool { + return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); + } + + pub fn format(self: *const Context, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ + @typeName(Context), + self.numThreads(), + self.numRegisteredDialects(), + self.numLoadedDialects(), + }); + } +}; + +pub const Location = opaque { + const M = StdMethods(Location, c.MlirLocation); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirLocationEqual); + pub const format = M.format(c.mlirLocationPrint); + + pub fn fromSrc(ctx: *Context, src: std.builtin.SourceLocation) *const Location { + return fileLineCol( + ctx, + src.file, + src.line, + src.column, + ); + } + + pub fn fileLineCol(ctx: *Context, file: []const u8, line: usize, column: usize) *const Location { + return @ptrCast(c.mlirLocationFileLineColGet( + ctx.ptr(), + stringRef(file), + @intCast(line), + @intCast(column), + ).ptr); + } + + pub fn callSite(callee: *const Location, caller: *const Location) *const Location { + return @ptrCast(c.mlirLocationCallSiteGet(callee.ptr(), caller.ptr()).ptr); + } + + pub fn fused(ctx: *Context, locs: []const *const Location, metadata: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFusedGet( + ctx.ptr(), + @intCast(locs.len), + @ptrCast(locs), + metadata.ptr(), + ).ptr); + } + + pub fn named(self: *const Location, ctx: *Context, loc_name: []const u8) *const Location { + return @ptrCast(c.mlirLocationNameGet(ctx.ptr(), stringRef(loc_name), self.ptr()).ptr); + } + + pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { + var buf: [256]u8 = undefined; + return loc.named(loc, ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { + buf[buf.len - 3 ..].* = "...".*; + break :blk &buf; + }); + } + + pub fn fromAttribute(attribute: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFromAttribute(attribute.ptr()).ptr); + } + + pub fn unknown(ctx: *Context) *const Location { + return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); + } +}; + +pub const Type = opaque { + const M = StdMethods(Type, c.MlirType); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { + return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); + } +}; + +pub const IndexType = opaque { + const M = StdMethods(IndexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAIndex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context) *const IndexType { + return @ptrCast(c.mlirIndexTypeGet(ctx.ptr()).ptr); + } +}; + +pub const Signedness = enum { + signless, + signed, + unsigned, +}; + +pub const IntegerType = opaque { + const M = StdMethods(IntegerType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + fn get(ctx: *Context, bitwidth: usize, sign: Signedness) *const IntegerType { + return @ptrCast(switch (sign) { + .signless => c.mlirIntegerTypeGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .signed => c.mlirIntegerTypeSignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .unsigned => c.mlirIntegerTypeUnsignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + }); + } +}; + +pub fn integerType(ctx: *Context, bitwidth: usize, sign: Signedness) *const Type { + return @ptrCast(IntegerType.get(ctx, bitwidth, sign)); +} + +pub const FloatTypes = enum { + f4e2m1fn, + f6e2m3fn, + f6e3m2fn, + f8e3m4, + f8e4m3, + f8e4m3b11fnuz, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + f8e8m0fnu, + bf16, + f16, + f32, + f64, + // f80, + // f128, + tf32, +}; + +pub fn FloatType(comptime ft: FloatTypes) type { + const Config = switch (ft) { + .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, + .f6e2m3fn => .{ c.mlirTypeIsAFloat6E2M3FN, c.mlirFloat6E2M3FNTypeGet }, + .f6e3m2fn => .{ c.mlirTypeIsAFloat6E3M2FN, c.mlirFloat6E3M2FNTypeGet }, + .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, + .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, + .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, + .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, + .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, + .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, + .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, + .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, + .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, + .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, + .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, + .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, + .tf32 => .{ c.mlirTypeIsATF32, c.mlirTF32TypeGet }, + }; + + return opaque { + const Self = @This(); + const M = StdMethods(Self, c.MlirType); + + pub const isAFn = Config[0]; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context) *const Self { + const getter = Config[1]; + return @ptrCast(getter(ctx.ptr()).ptr); + } + }; +} + +pub fn floatType(comptime ft: FloatTypes, ctx: *Context) *const Type { + return @ptrCast(FloatType(ft).get(ctx)); +} + +pub const Attribute = opaque { + const M = StdMethods(Attribute, c.MlirAttribute); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, attr: []const u8) Error!*const Attribute { + return @ptrCast(c.mlirAttributeParseGet(ctx.ptr(), stringRef(attr)).ptr orelse return Error.InvalidMlir); + } +}; + +pub const StringAttribute = opaque { + const M = StdMethods(StringAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAString; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, str: []const u8) *const StringAttribute { + return @ptrCast(c.mlirStringAttrGet(ctx.ptr(), stringRef(str)).ptr); + } + + pub fn value(self: *const StringAttribute) []const u8 { + return string(c.mlirStringAttrGetValue(self.ptr())); + } +}; + +pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { + return @ptrCast(StringAttribute.init(ctx, str)); +} + +pub const IntegerAttribute = opaque { + const M = StdMethods(IntegerAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, value_: anytype) *const IntegerAttribute { + const ti = @typeInfo(@TypeOf(value_)).int; + const sign: Signedness = switch (ti.signedness) { + .signed => .signless, + .unsigned => .unsigned, + }; + return @ptrCast(c.mlirIntegerAttrGet( + IntegerType.get(ctx, @intCast(ti.bits), sign).ptr(), + @intCast(value_), + ).ptr); + } + + pub fn value(self: *const IntegerAttribute, comptime T: type) T { + const getValue = switch (@typeInfo(T).int.signedness) { + .signed => c.mlirIntegerAttrGetValueInt, + .unsigned => c.mlirIntegerAttrGetValueUInt, + }; + return @intCast(getValue(self.ptr())); + } +}; + +pub fn integerAttribute(ctx: *Context, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, value_)); +} + +pub const BoolAttribute = opaque { + const M = StdMethods(BoolAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsABool; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, value_: bool) *const BoolAttribute { + return @ptrCast(c.mlirBoolAttrGet(ctx.ptr(), @intFromBool(value_)).ptr); + } + + pub fn value(self: *const BoolAttribute) bool { + return c.mlirBoolAttrGetValue(self.ptr()); + } +}; + +pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { + return @ptrCast(BoolAttribute.init(ctx, value)); +} + +pub const TypeAttribute = opaque { + const M = StdMethods(TypeAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(type_: *const Type) *const TypeAttribute { + return @ptrCast(c.mlirTypeAttrGet(type_.ptr()).ptr); + } + + pub fn value(self: *const TypeAttribute) *const Type { + return @ptrCast(c.mlirTypeAttrGetValue(self.ptr()).ptr); + } +}; + +pub fn typeAttribute(type_: *const Type) *const TypeAttribute { + return @ptrCast(TypeAttribute.init(type_)); +} + +pub const Dialect = struct { + const M = StdMethods(Dialect, c.MlirDialect); + + pub const ptr = M.ptr; + + pub fn context(self: *const Dialect) *Context { + return @ptrCast(c.mlirDialectGetContext(self.ptr()).ptr); + } + + pub fn namespace(self: *const Dialect) []const u8 { + return string(c.mlirDialectGetNamespace(self.ptr())); + } + + pub fn format(self: *const Dialect, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(Dialect), self.namespace() }); + } +}; + +pub const DialectHandle = opaque { + const M = StdMethods(DialectHandle, c.MlirDialectHandle); + + pub const ptr = M.ptr; + + pub fn format(self: *const DialectHandle, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(DialectHandle), self.namespace() }); + } + + pub fn namespace(self: *const DialectHandle) []const u8 { + return string(c.mlirDialectHandleGetNamespace(self.ptr())); + } + + pub fn insertDialect(self: *const DialectHandle, registry: *DialectRegistry) void { + c.mlirDialectHandleInsertDialect(self.ptr(), registry.ptr()); + } + + pub fn registerDialect(self: *const DialectHandle, ctx: *Context) void { + c.mlirDialectHandleRegisterDialect(self.ptr(), ctx.ptr()); + } + + pub fn loadDialect(self: *const DialectHandle, ctx: *Context) *Dialect { + return @ptrCast(c.mlirDialectHandleLoadDialect(self.ptr(), ctx.ptr()).ptr); + } + + pub fn fromString(comptime namespace_: []const u8) *const DialectHandle { + return @ptrCast(@field(c, "mlirGetDialectHandle__" ++ namespace_ ++ "__")().ptr); + } +}; + +pub const Identifier = opaque { + const M = StdMethods(Identifier, c.MlirIdentifier); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirIdentifierEqual); + + pub fn context(self: *const Identifier) *Context { + return @ptrCast(c.mlirIdentifierGetContext(self.ptr()).ptr); + } + + pub fn get(ctx: *Context, str_: []const u8) *const Identifier { + return @ptrCast(c.mlirIdentifierGet(ctx.ptr(), stringRef(str_)).ptr); + } + + pub fn str(self: *const Identifier) []const u8 { + return string(c.mlirIdentifierStr(self.ptr())); + } + + pub fn format(self: *const Identifier, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .str = \"{s}\" }}", .{ @typeName(Identifier), self.str() }); + } +}; + +pub const Module = opaque { + const M = StdMethods(Module, c.MlirModule); + + const ptr = M.ptr; + + pub fn init(loc: *const Location) *const Module { + return @ptrCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr); + } + + pub fn deinit(self: *Module) void { + c.mlirModuleDestroy(self.ptr()); + } + + pub fn fromOperation(op: *Operation) *const Module { + return @ptrCast(c.mlirModuleFromOperation(op.ptr()).ptr); + } + + pub fn operation(self: *const Module) *Operation { + return @ptrCast(c.mlirModuleGetOperation(self.ptr()).ptr); + } + + pub fn parse(ctx: *Context, source: []const u8) Error!*const Module { + return @ptrCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir); + } + + pub fn context(self: *const Module) *Context { + return @ptrCast(c.mlirModuleGetContext(self.ptr()).ptr); + } + + pub fn body(self: *const Module) *Block { + return @ptrCast(c.mlirModuleGetBody(self.ptr()).ptr); + } +}; + +pub const Block = opaque { + const M = StdMethods(Block, c.MlirBlock); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirBlockEqual); + pub const format = M.format(c.mlirBlockPrint); + + pub fn init(args: []const *const Type, locs: []const *const Location) *Block { + return @ptrCast(c.mlirBlockCreate(@intCast(args.len), @ptrCast(args), @ptrCast(locs)).ptr); + } + + pub fn deinit(self: *Block) void { + std.debug.assert(self.parentOperation() == null); + c.mlirBlockDestroy(self.ptr()); + } + + pub fn argument(self: *const Block, index: usize) *const Value { + return @ptrCast(c.mlirBlockGetArgument(self.ptr(), @intCast(index)).ptr); + } + + pub fn numArguments(self: *const Block) usize { + return @intCast(c.mlirBlockGetNumArguments(self.ptr())); + } + + pub fn addArgument(self: *Block, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockAddArgument(self.ptr(), type_.ptr(), loc.ptr()).ptr); + } + + pub fn insertArgument(self: *Block, index: usize, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockInsertArgument(self.ptr(), @intCast(index), type_.ptr(), loc.ptr()).ptr); + } + + pub fn appendOwnedOperation(self: *Block, op: *Operation) void { + c.mlirBlockAppendOwnedOperation(self.ptr(), op.ptr()); + } + + pub fn appendOwnedOperations(self: *Block, ops: []const *Operation) void { + for (ops) |op| { + self.appendOwnedOperation(op); + } + } + + pub fn parentOperation(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); + } + + pub fn detach(self: *Block) void { + c.mlirBlockDetach(self.ptr()); + } +}; + +pub const NamedAttribute = struct { + inner: c.MlirNamedAttribute, + + pub fn ptr(self: NamedAttribute) c.MlirNamedAttribute { + return self.inner; + } + + pub fn name(self: NamedAttribute) *const Identifier { + return @ptrCast(self.inner.name.ptr); + } + + pub fn attribute(self: NamedAttribute) *const Attribute { + return @ptrCast(self.inner.attribute.ptr); + } + + pub fn named(ctx: *Context, name_: []const u8, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = Identifier.get(ctx, name_).ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn init(name_: *const Identifier, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = name_.ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn format(self: NamedAttribute, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s}{{ .{s} = \"{f}\" }}", .{ + @typeName(NamedAttribute), + self.name().str(), + self.attribute(), + }); + } +}; + +pub const DictionaryAttribute = opaque { + const M = StdMethods(DictionaryAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADictionary; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attributes: []const NamedAttribute) *const DictionaryAttribute { + return @ptrCast(c.mlirDictionaryAttrGet( + ctx.ptr(), + @intCast(attributes.len), + @ptrCast(attributes.ptr), + ).ptr); + } + + pub fn size(self: *const DictionaryAttribute) usize { + return @intCast(c.mlirDictionaryAttrGetNumElements(self.ptr())); + } + + pub fn get(self: *const DictionaryAttribute, pos: usize) NamedAttribute { + return .{ .inner = c.mlirDictionaryAttrGetElement(self.ptr(), @bitCast(pos)) }; + } + + pub fn getByName(self: *const DictionaryAttribute, name: []const u8) ?*const Attribute { + return @ptrCast(c.mlirDictionaryAttrGetElementByName(self.ptr(), stringRef(name)).ptr); + } +}; + +pub const Value = opaque { + const M = StdMethods(Value, c.MlirValue); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + pub const isA = M.isA; + + pub fn type_(val: *const Value) *const Type { + return @ptrCast(c.mlirValueGetType(val.ptr()).ptr); + } + + pub fn setType(self: *Value, type__: *const Type) void { + c.mlirValueSetType(self.ptr(), type__.ptr()); + } + + pub fn firstUse(self: *const Value) *const OpOperand { + return @ptrCast(c.mlirValueGetFirstUse(self.ptr()).ptr); + } + + pub fn replaceAllUsesWith(self: *const Value, with: *const Value) void { + c.mlirValueReplaceAllUsesOfWith(self.ptr(), with.ptr()); + } +}; + +pub const OpOperand = opaque { + const M = StdMethods(OpOperand, c.MlirOpOperand); + + pub const ptr = M.ptr; + + pub fn owner(self: *const OpOperand) *Operation { + return @ptrCast(c.mlirOpOperandGetOwner(self.ptr()).ptr); + } + + pub fn number(self: *const OpOperand) usize { + return @intCast(c.mlirOpOperandGetOperandNumber(self.ptr())); + } + + pub fn nextUse(self: *const OpOperand) ?*const OpOperand { + return @ptrCast(c.mlirOpOperandGetNextUse(self.ptr()).ptr); + } +}; + +pub const OpResult = opaque { + const M = StdMethods(OpResult, c.MlirValue); + + pub const isAFn = c.mlirValueIsAOpResult; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn owner(self: *const OpResult) *Operation { + return @ptrCast(c.mlirOpResultGetOwner(self.ptr()).ptr); + } + + pub fn resultNumber(self: *const OpResult) usize { + return @intCast(c.mlirOpResultGetResultNumber(self.ptr())); + } +}; + +pub const BlockArgument = opaque { + const M = StdMethods(BlockArgument, c.MlirValue); + + pub const isAFn = c.mlirValueIsABlockArgument; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn block(arg: *const BlockArgument) *Block { + return @ptrCast(c.mlirBlockArgumentGetOwner(arg.ptr()).ptr); + } + + pub fn number(self: *const BlockArgument) usize { + return @intCast(c.mlirBlockArgumentGetArgNumber(self.ptr())); + } + + pub fn setType(self: *BlockArgument, type_: *const Type) void { + c.mlirBlockArgumentSetType(self.ptr(), type_.ptr()); + } +}; + +pub const OperationState = struct { + const M = StdMethods(OperationState, c.MlirOperationState); + + name: []const u8, + location: *const Location, + results: []const *const Type = &.{}, + operands: []const *const Value = &.{}, + regions: []*const Region = &.{}, + successors: []*const Block = &.{}, + attributes: []NamedAttribute = &.{}, + result_type_inference: bool = false, + + pub const ptr = M.ptr; + + pub fn inner(self: OperationState) c.MlirOperationState { + return .{ + .name = stringRef(self.name), + .nResults = @intCast(self.results.len), + .results = @constCast(@ptrCast(self.results)), + .nOperands = @intCast(self.operands.len), + .operands = @constCast(@ptrCast(self.operands)), + .nRegions = @intCast(self.regions.len), + .regions = @constCast(@ptrCast(self.regions)), + .nSuccessors = @intCast(self.successors.len), + .successors = @constCast(@ptrCast(self.successors)), + .nAttributes = @intCast(self.attributes.len), + .attributes = @constCast(@ptrCast(self.attributes)), + .enableResultTypeInference = self.result_type_inference, + }; + } +}; + +pub const Operation = opaque { + const M = StdMethods(Operation, c.MlirOperation); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirOperationEqual); + pub const format = M.format(c.mlirOperationPrint); + + pub fn init(state: *OperationState) !*Operation { + var os = state.inner(); + return @ptrCast(c.mlirOperationCreate(&os).ptr orelse return Error.InvalidMlir); + } + + pub fn deinit(self: *Operation) void { + std.debug.assert(self.block() == null); + c.mlirOperationDestroy(self.ptr()); + } + + pub fn parse(ctx: *Context, source: []const u8, name: []const u8) Error!*Operation { + return @ptrCast(c.mlirOperationCreateParse(ctx.ptr(), stringRef(source), stringRef(name)).ptr orelse return Error.InvalidMlir); + } + + pub fn clone(self: *const Operation) *Operation { + return @ptrCast(c.mlirOperationClone(self.ptr()).ptr); + } + + pub fn block(self: *const Operation) ?*Block { + return @ptrCast(c.mlirOperationGetBlock(self.ptr()).ptr); + } + + pub fn parent(self: *const Operation) ?*Operation { + return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); + } + + pub const PrintFlags = struct { + elide_large_elements_attrs: ?usize = null, + debug_info: bool = false, + debug_info_pretty_form: bool = true, + print_generic_op_form: bool = false, + use_local_scope: bool = false, + assume_verified: bool = false, + + const FmtCtx = struct { + self: *const Operation, + flags: PrintFlags, + }; + }; + + fn formatWithFlags(fctx: PrintFlags.FmtCtx, writer: *std.io.Writer) std.Io.Writer.Error!void { + const flags = c.mlirOpPrintingFlagsCreate(); + defer c.mlirOpPrintingFlagsDestroy(flags); + + if (fctx.flags.elide_large_elements_attrs) |v| { + c.mlirOpPrintingFlagsElideLargeElementsAttrs(flags, @intCast(v)); + } + c.mlirOpPrintingFlagsEnableDebugInfo( + flags, + fctx.flags.debug_info, + fctx.flags.debug_info_pretty_form, + ); + if (fctx.flags.print_generic_op_form) { + c.mlirOpPrintingFlagsPrintGenericOpForm(flags); + } + if (fctx.flags.use_local_scope) { + c.mlirOpPrintingFlagsUseLocalScope(flags); + } + if (fctx.flags.assume_verified) { + c.mlirOpPrintingFlagsAssumeVerified(flags); + } + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationPrintWithFlags(fctx.self.ptr(), flags, stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } + } + + pub fn fmt(self: *const Operation, flags: PrintFlags) std.fmt.Alt(PrintFlags.FmtCtx, formatWithFlags) { + return .{ .data = .{ + .self = self, + .flags = flags, + } }; + } + + pub const MakeArgs = struct { + pub const AttrTuple = struct { []const u8, *const Attribute }; + + operands: []const *const Value, + operands_variadic: ?[]const usize = null, + results: []const *const Type, + results_variadic: ?[]const usize = null, + result_type_inference: bool = false, + n_regions: usize = 0, + attributes: ?[]const AttrTuple = null, + blocks: ?[]const *const Block = null, + verify: bool = true, + location: *const Location, + }; + + pub fn try_make(ctx: *Context, op_name: []const u8, args: MakeArgs) !*Operation { + _ = ctx; // autofix + var state: OperationState = .{ + .name = op_name, + .location = args.location, + .result_type_inference = args.result_type_inference, + }; + // std.debug.assert(!(args.operands != null and args.variadic_operands != null)); + state.addOperands(operands); + + // if (args.operands) |operands| { + // state.addOperands(operands); + // // } else if (args.variadic_operands) |operands_segments| { + // // const MAX_SEGMENTS = 32; + // // var segments: std.BoundedArray(i32, MAX_SEGMENTS) = .{}; + + // // for (operands_segments) |operands| { + // // state.addOperands(operands); + // // segments.appendAssumeCapacity(@intCast(operands.len)); + // // } + // // state.addAttribute(ctx, "operandSegmentSizes", .denseElements(ctx, &.{@intCast(segments.len)}, .i32, segments.constSlice())); + // // } else if (args.tt_variadic_operands) |operands_segments| { + // // // stablehlo and triton seems to disagree on the expected type of operandSegmentSizes, let's fix that. + // // const MAX_SEGMENTS = 32; + // // var segments: std.BoundedArray(i32, MAX_SEGMENTS) = .{}; + + // // for (operands_segments) |operands| { + // // state.addOperands(operands); + // // segments.appendAssumeCapacity(@intCast(operands.len)); + // // } + // // state.addAttribute(ctx, "operandSegmentSizes", .dense(ctx, .i32, segments.constSlice())); + // } + // std.debug.assert(!(args.results != null and args.variadic_results != null)); + // if (args.results) |results| { + // state.addResults(results); + // // } else if (args.variadic_results) |result_segments| { + // // for (result_segments) |results| { + // // state.addResults(results); + // // } + // } + // for (0..args.n_regions) |_| { + // var region_ = Region.init() catch { + // @panic("Failed to create MLIR region"); + // }; + // state.addRegion(®ion_); + // } + // if (args.attributes) |attrs| { + // for (attrs) |attr| { + // state.addAttributeRaw( + // Identifier.get(ctx, attr[0]), + // attr[1], + // ); + // } + // } + // if (args.blocks) |blocks_| { + // for (blocks_) |block_| { + // var region_ = Region.init() catch { + // @panic("Failed to create MLIR region"); + // }; + // region_.appendBlock(block_); + // state.addRegion(®ion_); + // } + // } + + const new_op = try Operation.init(&state); + if (args.verify and new_op.verify() == false) { + log.err("Failed to verify MLIR operation:\n{f}", .{ + new_op.fmt(.{ + .print_generic_op_form = true, + .debug_info = true, + }), + }); + return Error.InvalidMlir; + } + + return new_op; + } + + pub fn make(ctx: *Context, op_name: []const u8, args: MakeArgs) *Operation { + return try_make(ctx, op_name, args) catch { + @panic("Failed to create MLIR operation"); + }; + } + + pub fn verify(self: *const Operation) bool { + return c.mlirOperationVerify(self.ptr()); + } + + pub fn location(self: *const Operation) *const Location { + return @ptrCast(c.mlirOperationGetLocation(self.ptr()).ptr); + } + + pub const WalkOrder = enum(c.MlirWalkOrder) { + pre_order = c.MlirWalkPreOrder, + post_order = c.MlirWalkPostOrder, + }; + + pub const WalkResult = enum(c.MlirWalkResult) { + advance = c.MlirWalkResultAdvance, + interrupt = c.MlirWalkResultInterrupt, + skip = c.MlirWalkResultSkip, + }; + + pub fn walk(self: *Operation, order: WalkOrder, ctx: anytype, walkfn: fn (ctx_: @TypeOf(ctx), op: *Operation) WalkResult) void { + var inner_ctx = .{ .ctx = ctx }; + const ContextType = @TypeOf(inner_ctx); + + c.mlirOperationWalk( + self._inner, + (struct { + pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { + const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); + return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); + } + }).callback, + &inner_ctx, + @intFromEnum(order), + ); + } +}; + +pub const Region = opaque { + const M = StdMethods(Region, c.MlirRegion); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirRegionEqual); + + pub fn init() Error!*Region { + return @ptrCast(c.mlirRegionCreate().ptr orelse return Error.InvalidMlir); + } + + pub fn deinit(self: *Region) void { + c.mlirRegionDestroy(self.ptr()); + } + + pub fn appendOwnedBlock(self: *Region, block: *Block) void { + c.mlirRegionAppendOwnedBlock(self.ptr(), block.ptr()); + } + + pub fn insertOwnedBlock(self: *Region, index: usize, block: *Block) void { + c.mlirRegionInsertOwnedBlock(self.ptr(), @intCast(index), block.ptr()); + } + + pub fn insertOwnedBlockBefore(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockBefore(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn insertOwnedBlockAfter(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockAfter(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn firstBlock(self: *const Region) ?*Block { + return @ptrCast(c.mlirRegionGetFirstBlock(self.ptr()).ptr); + } + + pub fn takeBody(self: *Region, from: *Region) void { + c.mlirRegionTakeBody(self.ptr(), from.ptr()); + } + + pub fn nextInOperation(self: *const Region) *Region { + return @ptrCast(c.mlirRegionGetNextInOperation(self.ptr()).ptr); + } +}; + +// pub const Block = opaque { +// const M = StdMethods(Block, c.MlirBlock); +// // pub const wrapOr = helpers.wrapOr(Block, c.mlirBlockIsNull); +// // pub const deinit = helpers.deinit(Block, c.mlirBlockDestroy); + +// // pub const eql = helpers.eql(Block, c.mlirBlockEqual); + +// pub fn init(args: []*const Type, locs: []const *const Location) !Block { +// return @ptrCast(c.mlirBlockCreate(@intCast(args.len), @ptrCast(args.ptr), @ptrCast(locs.ptr)) orelse return Error.InvalidMlir); +// } + +// pub fn argument(self: Block, index: usize) Value { +// return .{ ._inner = c.mlirBlockGetArgument(self._inner, @intCast(index)) }; +// } + +// pub fn numArguments(self: Block) usize { +// return @intCast(c.mlirBlockGetNumArguments(self._inner)); +// } + +// pub fn addArgument(self: *Block, typ: Type, loc: Location) Value { +// return .{ ._inner = c.mlirBlockAddArgument(self._inner, typ._inner, loc._inner) }; +// } + +// pub fn insertArgument(self: *Block, index: usize, typ: Type, loc: Location) Value { +// return .{ ._inner = c.mlirBlockInsertArgument(self._inner, @intCast(index), typ._inner, loc._inner) }; +// } + +// pub fn equals(self: Block, other: Block) bool { +// return c.mlirBlockEqual(self._inner, other._inner); +// } + +// pub fn appendOperation(self: Block, op: Operation) void { +// c.mlirBlockAppendOwnedOperation(self._inner, op._inner); +// } + +// pub fn appendOperations(self: *Block, ops: []const Operation) void { +// for (ops) |op| { +// c.mlirBlockAppendOwnedOperation(self._inner, op._inner); +// } +// } + +// pub const RecursiveOpts = enum { open, hermetic }; + +// pub fn appendValueRecursive(self: Block, value: Value, opt: RecursiveOpts) void { +// switch (value.kind()) { +// .op_result => |parent_op| self.appendOperationRecursive(parent_op, opt), +// .block_argument => |arg| { +// // Hermetic blocks are not allowed to use arguments from other blocks. +// stdx.debug.assert(opt == .open or self.eql(arg.block()), "Can't add {} from {?x} block to {?x} block", .{ arg, arg.block()._inner.ptr, self._inner.ptr }); +// }, +// .null => @panic("InvalidMlir"), +// } +// } + +// pub fn appendOperationRecursive(self: Block, op: Operation, opt: RecursiveOpts) void { +// if (op.block()) |prev_block| { +// // Hermetic blocks are not allowed to reference values from other blocks. +// stdx.debug.assert(opt == .open or self.equals(prev_block), "Can't add {} from {?x} block to {?x} block", .{ op, prev_block._inner.ptr, self._inner.ptr }); +// return; +// } +// for (0..op.numOperands()) |i| { +// self.appendValueRecursive(op.operand(i), opt); +// } +// self.appendOperation(op); +// } +// }; From 16ddc3d6a18a36272c9f11d8a75344845ed81804 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 27 Aug 2025 19:21:18 +0200 Subject: [PATCH 02/47] WIP --- BUILD.bazel | 8 +- mlir/BUILD.bazel | 7 +- mlir/dialects/BUILD.bazel | 12 +- mlir/dialects/dialects.zig | 11 +- mlir/dialects/func.zig | 57 ++-- mlir/main.zig | 55 +++- mlir/registry.zig | 601 +++++++++++++++++++++++++------------ 7 files changed, 500 insertions(+), 251 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 4c17c21a..f891cd2d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -10,10 +10,10 @@ zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ - "//async", - "//examples/llama", - "//stdx", - "//zml", + # "//async", + # "//examples/llama", + # "//stdx", + # "//zml", "//mlir:main", # "//async", # "//stdx", diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 0dccf25d..f0ba476f 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -32,7 +32,11 @@ zig_test( zig_library( name = "mlir2", main = "registry.zig", - deps = [":c"], + visibility = ["//visibility:public"], + deps = [ + ":c", + "//stdx", + ], ) zig_binary( @@ -41,6 +45,7 @@ zig_binary( visibility = ["//visibility:public"], deps = [ ":mlir2", + "//mlir/dialects", # ":c", # "//stdx", ], diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 101f67ff..4fbc61bd 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -3,18 +3,18 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "dialects", srcs = [ - "arith.zig", + # "arith.zig", "func.zig", - "math.zig", - "scf.zig", - "tensor.zig", + # "math.zig", + # "scf.zig", + # "tensor.zig", ], import_name = "mlir/dialects", main = "dialects.zig", visibility = ["//visibility:public"], deps = [ - "//mlir", - "//mlir/dialects/stablehlo", + "//mlir:mlir2", + # "//mlir/dialects/stablehlo", "//stdx", ], ) diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index 47f74d8e..b930086b 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,11 +1,12 @@ const std = @import("std"); -pub const arith = @import("arith.zig"); pub const func = @import("func.zig"); -pub const math = @import("math.zig"); -pub const scf = @import("scf.zig"); -pub const tensor = @import("tensor.zig"); -pub const stablehlo = @import("mlir/dialects/stablehlo"); + +// pub const arith = @import("arith.zig"); +// pub const math = @import("math.zig"); +// pub const scf = @import("scf.zig"); +// pub const tensor = @import("tensor.zig"); +// pub const stablehlo = @import("mlir/dialects/stablehlo"); test { std.testing.refAllDecls(@This()); diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 868ff3bc..50eccf96 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -1,29 +1,27 @@ const std = @import("std"); -const mlir = @import("mlir"); +const mlir = @import("mlir2"); const stdx = @import("stdx"); -pub fn func( - ctx: mlir.Context, - args: struct { - sym_name: []const u8, - args: []const mlir.Type, - arg_attrs: []const mlir.Attribute = &.{}, - results: []const mlir.Type, - res_attrs: []const mlir.Attribute = &.{}, - block: mlir.Block, - location: mlir.Location, - }, -) mlir.Operation { - var attrs_tuple_buffer = stdx.BoundedArray(mlir.AttrTuple, 4){}; - attrs_tuple_buffer.appendAssumeCapacity(.{ "sym_name", .string(ctx, args.sym_name) }); - attrs_tuple_buffer.appendAssumeCapacity(.{ "function_type", .type_(.function(ctx, args.args, args.results)) }); - if (args.arg_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "arg_attrs", .array(ctx, args.arg_attrs) }); - } +const FuncOpArgs = struct { + name: []const u8, + args: []const *const mlir.Type, + args_attributes: ?[]const *const mlir.Attribute = null, + results: []const *const mlir.Type, + results_attributes: ?[]const *const mlir.Attribute = null, + block: *mlir.Block, + location: *const mlir.Location, +}; - if (args.res_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "res_attrs", .array(ctx, args.res_attrs) }); +pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { + var attrs_tuple_buffer = stdx.BoundedArray(mlir.Operation.MakeArgs.AttrTuple, 4){}; + attrs_tuple_buffer.appendAssumeCapacity(.{ "sym_name", mlir.stringAttribute(ctx, args.name) }); + attrs_tuple_buffer.appendAssumeCapacity(.{ "function_type", mlir.typeAttribute(mlir.functionType(ctx, args.args, args.results)) }); + if (args.args_attributes) |args_attributes| { + attrs_tuple_buffer.appendAssumeCapacity(.{ "arg_attrs", mlir.arrayAttribute(ctx, args_attributes) }); + } + if (args.results_attributes) |results_attributes| { + attrs_tuple_buffer.appendAssumeCapacity(.{ "res_attrs", mlir.arrayAttribute(ctx, results_attributes) }); } return mlir.Operation.make(ctx, "func.func", .{ @@ -33,20 +31,21 @@ pub fn func( }); } -pub fn call(ctx: mlir.Context, name: [:0]const u8, values: []const mlir.Value, results: []const mlir.Type, loc: mlir.Location) mlir.Operation { +pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Value, results: []const *const mlir.Type, loc: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.call", .{ - .variadic_operands = &.{values}, - .results = results, - .verify = true, - .attributes = &.{.{ "callee", .symbol(ctx, name) }}, + .operands = .{ .variadic = &.{values} }, + .results = .{ .flat = results }, + .attributes = &.{ + .{ "callee", mlir.flatSymbolRefAttribute(ctx, name) }, + }, .location = loc, }); } -pub fn return_(ctx: mlir.Context, values: []const mlir.Value, loc: mlir.Location) mlir.Operation { +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, loc: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.return", .{ - .operands = values, - .verify = false, + .operands = .{ .flat = values }, .location = loc, + .verify = false, }); } diff --git a/mlir/main.zig b/mlir/main.zig index 8095de85..177d2a5a 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -1,5 +1,6 @@ const std = @import("std"); +const dialects = @import("mlir/dialects"); const mlir = @import("mlir2"); pub fn main() !void { @@ -25,28 +26,50 @@ pub fn main() !void { const unk = mlir.Location.unknown(ctx); _ = unk; // autofix - const attr = mlir.integerAttribute(ctx, @as(i32, 5)); - _ = attr; // autofix + // const attr = mlir.integerAttribute(ctx, @as(i32, 5)); + // _ = attr; // autofix - const one: i32 = 1; - const two: i32 = 2; + // const one: i32 = 1; + // const two: i32 = 2; - const dict = mlir.DictionaryAttribute.init(ctx, &.{ - .named(ctx, "pouet", mlir.integerAttribute(ctx, one)), - .named(ctx, "pouet2", mlir.integerAttribute(ctx, two)), + // const dict = mlir.DictionaryAttribute.init(ctx, &.{ + // .named(ctx, "pouet", mlir.integerAttribute(ctx, one)), + // .named(ctx, "pouet2", mlir.integerAttribute(ctx, two)), + // }); + // _ = dict; // autofix + + const int64 = mlir.integerType(ctx, 64, .signless); + + const block = mlir.Block.init(&.{ int64, int64 }, &.{ .unknown(ctx), .unknown(ctx) }); + + block.appendOwnedOperation(dialects.func.return_(ctx, &.{block.argument(0)}, .unknown(ctx))); + + const op = dialects.func.func(ctx, .{ + .name = "pouet", + .args = &.{ int64, int64 }, + .results = &.{int64}, + .block = block, + .location = .unknown(ctx), }); - _ = dict; // autofix - const op = try mlir.Operation.parse(ctx, "func.func private @pouet()", "pouet"); + // const op = try mlir.Operation.parse(ctx, "func.func private @pouet()", "pouet"); - const mod = mlir.Module.init(mlir.Location.unknown(ctx)); + // // // const mod = mlir.Module.init(mlir.Location.unknown(ctx)); - const body = mod.body(); - _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); - _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); - _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); + // const op = mlir.Operation.make(ctx, "arith.addi", .{ + // .operands = .{ .flat = &.{ block.argument(0), block.argument(1) } }, + // .results = .{ .variadic = &.{ &.{ int64, int64 }, &.{int64} } }, + // }); + // defer op.deinit(); - body.appendOwnedOperation(op); + std.debug.print(">>>>>> {f}\n", .{op.fmt(.{ .print_generic_op_form = true })}); + + // const body = mod.body(); + // _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); + // _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); + // _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); + + // body.appendOwnedOperation(op); // op.deinit(); // body.detach(); @@ -56,7 +79,7 @@ pub fn main() !void { // std.debug.print(">>>>>> {any}\n", .{op.block().?.parentOperation()}); - std.debug.print(">>> {f}\n", .{body}); + // std.debug.print(">>> {f}\n", .{body}); // const attr = mlir.integerAttribute(ctx, @as(i32, 5)); // const attr2: *const mlir.Attribute = @ptrCast(attr); diff --git a/mlir/registry.zig b/mlir/registry.zig index 69e44975..b4b17d44 100644 --- a/mlir/registry.zig +++ b/mlir/registry.zig @@ -1,6 +1,7 @@ const std = @import("std"); const c = @import("c"); +const stdx = @import("stdx"); const log = std.log.scoped(.mlir); @@ -50,7 +51,7 @@ fn CastedPtr(comptime T1: type, comptime T2: type) type { return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; } -pub fn StdMethods(comptime T: type, comptime NativeT: type) type { +pub fn Methods(comptime T: type, comptime NativeT: type) type { return struct { pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; pub const Ptr = if (IsPtrConst) *const T else *T; @@ -86,7 +87,7 @@ pub fn StdMethods(comptime T: type, comptime NativeT: type) type { } pub const DialectRegistry = opaque { - const M = StdMethods(DialectRegistry, c.MlirDialectRegistry); + const M = Methods(DialectRegistry, c.MlirDialectRegistry); pub const ptr = M.ptr; @@ -104,7 +105,7 @@ pub const DialectRegistry = opaque { }; pub const Context = opaque { - const M = StdMethods(Context, c.MlirContext); + const M = Methods(Context, c.MlirContext); pub const ptr = M.ptr; pub const eql = M.eql(c.mlirContextEqual); @@ -165,7 +166,7 @@ pub const Context = opaque { }; pub const Location = opaque { - const M = StdMethods(Location, c.MlirLocation); + const M = Methods(Location, c.MlirLocation); pub const ptr = M.ptr; pub const eql = M.eql(c.mlirLocationEqual); @@ -224,7 +225,7 @@ pub const Location = opaque { }; pub const Type = opaque { - const M = StdMethods(Type, c.MlirType); + const M = Methods(Type, c.MlirType); pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); @@ -237,7 +238,7 @@ pub const Type = opaque { }; pub const IndexType = opaque { - const M = StdMethods(IndexType, c.MlirType); + const M = Methods(IndexType, c.MlirType); pub const isAFn = c.mlirTypeIsAIndex; pub const ptr = M.ptr; @@ -256,7 +257,7 @@ pub const Signedness = enum { }; pub const IntegerType = opaque { - const M = StdMethods(IntegerType, c.MlirType); + const M = Methods(IntegerType, c.MlirType); pub const isAFn = c.mlirTypeIsAInteger; pub const ptr = M.ptr; @@ -298,7 +299,7 @@ pub const FloatTypes = enum { }; pub fn FloatType(comptime ft: FloatTypes) type { - const Config = switch (ft) { + const _isAFn, const getter = switch (ft) { .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, .f6e2m3fn => .{ c.mlirTypeIsAFloat6E2M3FN, c.mlirFloat6E2M3FNTypeGet }, .f6e3m2fn => .{ c.mlirTypeIsAFloat6E3M2FN, c.mlirFloat6E3M2FNTypeGet }, @@ -319,15 +320,14 @@ pub fn FloatType(comptime ft: FloatTypes) type { return opaque { const Self = @This(); - const M = StdMethods(Self, c.MlirType); + const M = Methods(Self, c.MlirType); - pub const isAFn = Config[0]; + pub const isAFn = _isAFn; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); pub fn get(ctx: *Context) *const Self { - const getter = Config[1]; return @ptrCast(getter(ctx.ptr()).ptr); } }; @@ -338,7 +338,7 @@ pub fn floatType(comptime ft: FloatTypes, ctx: *Context) *const Type { } pub const Attribute = opaque { - const M = StdMethods(Attribute, c.MlirAttribute); + const M = Methods(Attribute, c.MlirAttribute); pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); @@ -348,10 +348,18 @@ pub const Attribute = opaque { pub fn parse(ctx: *Context, attr: []const u8) Error!*const Attribute { return @ptrCast(c.mlirAttributeParseGet(ctx.ptr(), stringRef(attr)).ptr orelse return Error.InvalidMlir); } + + pub fn context(self: *const Context) *Context { + return @ptrCast(c.mlirAttributeGetContext(self.ptr()).ptr); + } + + pub fn dialect(self: *const Context) *const Dialect { + return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); + } }; pub const StringAttribute = opaque { - const M = StdMethods(StringAttribute, c.MlirAttribute); + const M = Methods(StringAttribute, c.MlirAttribute); pub const isAFn = c.mlirAttributeIsAString; pub const ptr = M.ptr; @@ -372,7 +380,7 @@ pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { } pub const IntegerAttribute = opaque { - const M = StdMethods(IntegerAttribute, c.MlirAttribute); + const M = Methods(IntegerAttribute, c.MlirAttribute); pub const isAFn = c.mlirAttributeIsAInteger; pub const ptr = M.ptr; @@ -405,7 +413,7 @@ pub fn integerAttribute(ctx: *Context, value_: anytype) *const Attribute { } pub const BoolAttribute = opaque { - const M = StdMethods(BoolAttribute, c.MlirAttribute); + const M = Methods(BoolAttribute, c.MlirAttribute); pub const isAFn = c.mlirAttributeIsABool; pub const ptr = M.ptr; @@ -426,7 +434,7 @@ pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { } pub const TypeAttribute = opaque { - const M = StdMethods(TypeAttribute, c.MlirAttribute); + const M = Methods(TypeAttribute, c.MlirAttribute); pub const isAFn = c.mlirAttributeIsAType; pub const ptr = M.ptr; @@ -442,12 +450,12 @@ pub const TypeAttribute = opaque { } }; -pub fn typeAttribute(type_: *const Type) *const TypeAttribute { +pub fn typeAttribute(type_: *const Type) *const Attribute { return @ptrCast(TypeAttribute.init(type_)); } pub const Dialect = struct { - const M = StdMethods(Dialect, c.MlirDialect); + const M = Methods(Dialect, c.MlirDialect); pub const ptr = M.ptr; @@ -465,7 +473,7 @@ pub const Dialect = struct { }; pub const DialectHandle = opaque { - const M = StdMethods(DialectHandle, c.MlirDialectHandle); + const M = Methods(DialectHandle, c.MlirDialectHandle); pub const ptr = M.ptr; @@ -495,7 +503,7 @@ pub const DialectHandle = opaque { }; pub const Identifier = opaque { - const M = StdMethods(Identifier, c.MlirIdentifier); + const M = Methods(Identifier, c.MlirIdentifier); pub const ptr = M.ptr; pub const eql = M.eql(c.mlirIdentifierEqual); @@ -518,7 +526,7 @@ pub const Identifier = opaque { }; pub const Module = opaque { - const M = StdMethods(Module, c.MlirModule); + const M = Methods(Module, c.MlirModule); const ptr = M.ptr; @@ -552,7 +560,7 @@ pub const Module = opaque { }; pub const Block = opaque { - const M = StdMethods(Block, c.MlirBlock); + const M = Methods(Block, c.MlirBlock); pub const ptr = M.ptr; pub const eql = M.eql(c.mlirBlockEqual); @@ -645,7 +653,7 @@ pub const NamedAttribute = struct { }; pub const DictionaryAttribute = opaque { - const M = StdMethods(DictionaryAttribute, c.MlirAttribute); + const M = Methods(DictionaryAttribute, c.MlirAttribute); pub const isAFn = c.mlirAttributeIsADictionary; pub const ptr = M.ptr; @@ -656,7 +664,7 @@ pub const DictionaryAttribute = opaque { return @ptrCast(c.mlirDictionaryAttrGet( ctx.ptr(), @intCast(attributes.len), - @ptrCast(attributes.ptr), + @ptrCast(attributes), ).ptr); } @@ -665,7 +673,7 @@ pub const DictionaryAttribute = opaque { } pub fn get(self: *const DictionaryAttribute, pos: usize) NamedAttribute { - return .{ .inner = c.mlirDictionaryAttrGetElement(self.ptr(), @bitCast(pos)) }; + return .{ .inner = c.mlirDictionaryAttrGetElement(self.ptr(), @intCast(pos)) }; } pub fn getByName(self: *const DictionaryAttribute, name: []const u8) ?*const Attribute { @@ -674,7 +682,7 @@ pub const DictionaryAttribute = opaque { }; pub const Value = opaque { - const M = StdMethods(Value, c.MlirValue); + const M = Methods(Value, c.MlirValue); pub const ptr = M.ptr; pub const eql = M.eql(c.mlirValueEqual); @@ -699,7 +707,7 @@ pub const Value = opaque { }; pub const OpOperand = opaque { - const M = StdMethods(OpOperand, c.MlirOpOperand); + const M = Methods(OpOperand, c.MlirOpOperand); pub const ptr = M.ptr; @@ -717,7 +725,7 @@ pub const OpOperand = opaque { }; pub const OpResult = opaque { - const M = StdMethods(OpResult, c.MlirValue); + const M = Methods(OpResult, c.MlirValue); pub const isAFn = c.mlirValueIsAOpResult; pub const ptr = M.ptr; @@ -734,7 +742,7 @@ pub const OpResult = opaque { }; pub const BlockArgument = opaque { - const M = StdMethods(BlockArgument, c.MlirValue); + const M = Methods(BlockArgument, c.MlirValue); pub const isAFn = c.mlirValueIsABlockArgument; pub const ptr = M.ptr; @@ -755,47 +763,49 @@ pub const BlockArgument = opaque { }; pub const OperationState = struct { - const M = StdMethods(OperationState, c.MlirOperationState); + inner: c.MlirOperationState, - name: []const u8, - location: *const Location, - results: []const *const Type = &.{}, - operands: []const *const Value = &.{}, - regions: []*const Region = &.{}, - successors: []*const Block = &.{}, - attributes: []NamedAttribute = &.{}, - result_type_inference: bool = false, + pub fn init(name: []const u8, location: *const Location) OperationState { + return .{ .inner = c.mlirOperationStateGet(stringRef(name), location.ptr()) }; + } - pub const ptr = M.ptr; + pub fn addResults(self: *OperationState, results: []const *const Type) void { + c.mlirOperationStateAddResults(&self.inner, @intCast(results.len), @ptrCast(results)); + } - pub fn inner(self: OperationState) c.MlirOperationState { - return .{ - .name = stringRef(self.name), - .nResults = @intCast(self.results.len), - .results = @constCast(@ptrCast(self.results)), - .nOperands = @intCast(self.operands.len), - .operands = @constCast(@ptrCast(self.operands)), - .nRegions = @intCast(self.regions.len), - .regions = @constCast(@ptrCast(self.regions)), - .nSuccessors = @intCast(self.successors.len), - .successors = @constCast(@ptrCast(self.successors)), - .nAttributes = @intCast(self.attributes.len), - .attributes = @constCast(@ptrCast(self.attributes)), - .enableResultTypeInference = self.result_type_inference, - }; + pub fn addOperands(self: *OperationState, operands: []const *const Value) void { + c.mlirOperationStateAddOperands(&self.inner, @intCast(operands.len), @ptrCast(operands)); + } + + pub fn addOwnedRegions(self: *OperationState, regions: []const *Region) void { + c.mlirOperationStateAddOwnedRegions(&self.inner, @intCast(regions.len), @ptrCast(regions)); + } + + pub fn addSuccessors(self: *OperationState, successors: []*const Block) void { + c.mlirOperationStateAddSuccessors(&self.inner, @intCast(successors.len), @ptrCast(successors)); + } + + pub fn addAttributes(self: *OperationState, attributes: []const NamedAttribute) void { + c.mlirOperationStateAddAttributes(&self.inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); + } + + pub fn enableResultTypeInference(self: *OperationState) void { + c.mlirOperationStateEnableResultTypeInference(&self.inner); } }; pub const Operation = opaque { - const M = StdMethods(Operation, c.MlirOperation); + const M = Methods(Operation, c.MlirOperation); + + const MAX_SEGMENTS = 32; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirOperationEqual); pub const format = M.format(c.mlirOperationPrint); pub fn init(state: *OperationState) !*Operation { - var os = state.inner(); - return @ptrCast(c.mlirOperationCreate(&os).ptr orelse return Error.InvalidMlir); + defer state.* = undefined; + return @ptrCast(c.mlirOperationCreate(&state.inner).ptr orelse return Error.InvalidMlir); } pub fn deinit(self: *Operation) void { @@ -803,8 +813,12 @@ pub const Operation = opaque { c.mlirOperationDestroy(self.ptr()); } - pub fn parse(ctx: *Context, source: []const u8, name: []const u8) Error!*Operation { - return @ptrCast(c.mlirOperationCreateParse(ctx.ptr(), stringRef(source), stringRef(name)).ptr orelse return Error.InvalidMlir); + pub fn parse(ctx: *Context, source: []const u8, name_: []const u8) Error!*Operation { + return @ptrCast(c.mlirOperationCreateParse(ctx.ptr(), stringRef(source), stringRef(name_)).ptr orelse return Error.InvalidMlir); + } + + pub fn name(self: *const Operation) []const u8 { + return string(c.mlirOperationGetName(self.ptr())); } pub fn clone(self: *const Operation) *Operation { @@ -819,6 +833,38 @@ pub const Operation = opaque { return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); } + pub fn removeFromParent(self: *Operation) void { + c.mlirOperationRemoveFromParent(self.ptr()); + } + + pub fn numOperands(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumOperands(self.ptr())); + } + + pub fn operand(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetOperand(self.ptr(), @intCast(index)).ptr); + } + + pub fn setOperand(self: *Operation, index: usize, value: *const Value) void { + c.mlirOperationSetOperand(self.ptr(), @intCast(index), value.ptr()); + } + + pub fn numResults(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumResults(self.ptr())); + } + + pub fn result(self: *const Operation, index: usize) Value { + return @ptrCast(c.mlirOperationGetResult(self.ptr(), @intCast(index)).ptr); + } + + pub fn nextInBlock(self: *const Operation) *const Operation { + return @ptrCast(c.mlirOperationGetNextInBlock(self.ptr()).ptr); + } + + pub fn context(self: *const Operation) *Context { + return @ptrCast(c.mlirOperationGetContext(self.ptr()).ptr); + } + pub const PrintFlags = struct { elide_large_elements_attrs: ?usize = null, debug_info: bool = false, @@ -827,13 +873,13 @@ pub const Operation = opaque { use_local_scope: bool = false, assume_verified: bool = false, - const FmtCtx = struct { + const Ctx = struct { self: *const Operation, flags: PrintFlags, }; }; - fn formatWithFlags(fctx: PrintFlags.FmtCtx, writer: *std.io.Writer) std.Io.Writer.Error!void { + fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { const flags = c.mlirOpPrintingFlagsCreate(); defer c.mlirOpPrintingFlagsDestroy(flags); @@ -861,7 +907,7 @@ pub const Operation = opaque { } } - pub fn fmt(self: *const Operation, flags: PrintFlags) std.fmt.Alt(PrintFlags.FmtCtx, formatWithFlags) { + pub fn fmt(self: *const Operation, flags: PrintFlags) std.fmt.Alt(PrintFlags.Ctx, formatWithFlags) { return .{ .data = .{ .self = self, .flags = flags, @@ -871,82 +917,70 @@ pub const Operation = opaque { pub const MakeArgs = struct { pub const AttrTuple = struct { []const u8, *const Attribute }; - operands: []const *const Value, - operands_variadic: ?[]const usize = null, - results: []const *const Type, - results_variadic: ?[]const usize = null, + pub const Operands = union(enum) { + flat: []const *const Value, + variadic: []const []const *const Value, + }; + + pub const ResultTypes = union(enum) { + flat: []const *const Type, + variadic: []const []const *const Type, + }; + + operands: ?Operands = null, + results: ?ResultTypes = null, result_type_inference: bool = false, - n_regions: usize = 0, attributes: ?[]const AttrTuple = null, - blocks: ?[]const *const Block = null, + blocks: ?[]const *Block = null, verify: bool = true, - location: *const Location, + location: ?*const Location = null, }; - pub fn try_make(ctx: *Context, op_name: []const u8, args: MakeArgs) !*Operation { - _ = ctx; // autofix - var state: OperationState = .{ - .name = op_name, - .location = args.location, - .result_type_inference = args.result_type_inference, + pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { + var state: OperationState = .init(name_, args.location orelse .unknown(ctx)); + if (args.result_type_inference) { + state.enableResultTypeInference(); + } + if (args.operands) |operands| switch (operands) { + .flat => |v| state.addOperands(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_operands| { + state.addOperands(segment_operands); + sizes.appendAssumeCapacity(@intCast(segment_operands.len)); + } + state.addAttributes(&.{ + .named(ctx, "operandSegmentSizes", denseArrayAttribute(ctx, .i32, sizes.constSlice())), + }); + }, }; - // std.debug.assert(!(args.operands != null and args.variadic_operands != null)); - state.addOperands(operands); - - // if (args.operands) |operands| { - // state.addOperands(operands); - // // } else if (args.variadic_operands) |operands_segments| { - // // const MAX_SEGMENTS = 32; - // // var segments: std.BoundedArray(i32, MAX_SEGMENTS) = .{}; - - // // for (operands_segments) |operands| { - // // state.addOperands(operands); - // // segments.appendAssumeCapacity(@intCast(operands.len)); - // // } - // // state.addAttribute(ctx, "operandSegmentSizes", .denseElements(ctx, &.{@intCast(segments.len)}, .i32, segments.constSlice())); - // // } else if (args.tt_variadic_operands) |operands_segments| { - // // // stablehlo and triton seems to disagree on the expected type of operandSegmentSizes, let's fix that. - // // const MAX_SEGMENTS = 32; - // // var segments: std.BoundedArray(i32, MAX_SEGMENTS) = .{}; - - // // for (operands_segments) |operands| { - // // state.addOperands(operands); - // // segments.appendAssumeCapacity(@intCast(operands.len)); - // // } - // // state.addAttribute(ctx, "operandSegmentSizes", .dense(ctx, .i32, segments.constSlice())); - // } - // std.debug.assert(!(args.results != null and args.variadic_results != null)); - // if (args.results) |results| { - // state.addResults(results); - // // } else if (args.variadic_results) |result_segments| { - // // for (result_segments) |results| { - // // state.addResults(results); - // // } - // } - // for (0..args.n_regions) |_| { - // var region_ = Region.init() catch { - // @panic("Failed to create MLIR region"); - // }; - // state.addRegion(®ion_); - // } - // if (args.attributes) |attrs| { - // for (attrs) |attr| { - // state.addAttributeRaw( - // Identifier.get(ctx, attr[0]), - // attr[1], - // ); - // } - // } - // if (args.blocks) |blocks_| { - // for (blocks_) |block_| { - // var region_ = Region.init() catch { - // @panic("Failed to create MLIR region"); - // }; - // region_.appendBlock(block_); - // state.addRegion(®ion_); - // } - // } - + if (args.results) |results| switch (results) { + .flat => |v| state.addResults(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_results| { + state.addResults(segment_results); + sizes.appendAssumeCapacity(@intCast(segment_results.len)); + } + state.addAttributes(&.{ + .named(ctx, "resultSegmentSizes", denseArrayAttribute(ctx, .i32, sizes.constSlice())), + }); + }, + }; + if (args.attributes) |attrs| { + for (attrs) |attr_tuple| { + const attr_name, const attr = attr_tuple; + const named_attr = NamedAttribute.named(ctx, attr_name, attr); + state.addAttributes(&.{named_attr}); + } + } + if (args.blocks) |blocks| { + const region = Region.init(); + for (blocks) |block_| { + region.appendOwnedBlock(block_); + } + state.addOwnedRegions(&.{region}); + } const new_op = try Operation.init(&state); if (args.verify and new_op.verify() == false) { log.err("Failed to verify MLIR operation:\n{f}", .{ @@ -957,7 +991,6 @@ pub const Operation = opaque { }); return Error.InvalidMlir; } - return new_op; } @@ -1002,16 +1035,58 @@ pub const Operation = opaque { @intFromEnum(order), ); } + + pub const BytecodeWriterConfig = struct { + desired_emit_version: ?usize = null, + + const Ctx = struct { + self: *const Operation, + config: ?BytecodeWriterConfig, + }; + }; + + pub fn writeBytecode(self: *const Operation, config: ?BytecodeWriterConfig, writer: *std.Io.Writer) !void { + if (config) |config_| { + const bc_config = c.mlirBytecodeWriterConfigCreate(); + defer c.mlirBytecodeWriterConfigDestroy(bc_config); + if (config_.desired_emit_version) |version| { + c.mlirBytecodeWriterConfigDesiredEmitVersion(bc_config, @intCast(version)); + } + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationWriteBytecodeWithConfig(self.ptr(), bc_config, stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } + return; + } + + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationWriteBytecode(self.ptr(), stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } + } + + fn fmtBytecodeImpl(ctx: BytecodeWriterConfig.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + try ctx.self.writeBytecode(ctx.config, writer); + } + + pub fn fmtBytecode(self: *const Operation, config: ?BytecodeWriterConfig) std.fmt.Alt(BytecodeWriterConfig.Ctx, fmtBytecodeImpl) { + return .{ .data = .{ + .self = self, + .config = config, + } }; + } }; pub const Region = opaque { - const M = StdMethods(Region, c.MlirRegion); + const M = Methods(Region, c.MlirRegion); pub const ptr = M.ptr; pub const eql = M.eql(c.mlirRegionEqual); - pub fn init() Error!*Region { - return @ptrCast(c.mlirRegionCreate().ptr orelse return Error.InvalidMlir); + pub fn init() *Region { + return @ptrCast(c.mlirRegionCreate().ptr); } pub fn deinit(self: *Region) void { @@ -1047,69 +1122,215 @@ pub const Region = opaque { } }; -// pub const Block = opaque { -// const M = StdMethods(Block, c.MlirBlock); -// // pub const wrapOr = helpers.wrapOr(Block, c.mlirBlockIsNull); -// // pub const deinit = helpers.deinit(Block, c.mlirBlockDestroy); +pub const ShapedType = opaque { + const M = Methods(ShapedType, c.MlirType); -// // pub const eql = helpers.eql(Block, c.mlirBlockEqual); + pub const isAFn = c.mlirTypeIsAShaped; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; -// pub fn init(args: []*const Type, locs: []const *const Location) !Block { -// return @ptrCast(c.mlirBlockCreate(@intCast(args.len), @ptrCast(args.ptr), @ptrCast(locs.ptr)) orelse return Error.InvalidMlir); -// } + pub fn elementType(self: *const ShapedType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } -// pub fn argument(self: Block, index: usize) Value { -// return .{ ._inner = c.mlirBlockGetArgument(self._inner, @intCast(index)) }; -// } + pub fn rank(self: *const ShapedType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } -// pub fn numArguments(self: Block) usize { -// return @intCast(c.mlirBlockGetNumArguments(self._inner)); -// } + pub fn dimension(self: *const ShapedType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } +}; -// pub fn addArgument(self: *Block, typ: Type, loc: Location) Value { -// return .{ ._inner = c.mlirBlockAddArgument(self._inner, typ._inner, loc._inner) }; -// } +pub const RankedTensorType = opaque { + const M = Methods(RankedTensorType, c.MlirType); -// pub fn insertArgument(self: *Block, index: usize, typ: Type, loc: Location) Value { -// return .{ ._inner = c.mlirBlockInsertArgument(self._inner, @intCast(index), typ._inner, loc._inner) }; -// } + pub const isAFn = c.mlirTypeIsARankedTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); -// pub fn equals(self: Block, other: Block) bool { -// return c.mlirBlockEqual(self._inner, other._inner); -// } + pub fn init(dimensions: []const i64, elemType: *const Type) *const RankedTensorType { + return @ptrCast(c.mlirRankedTensorTypeGet( + @intCast(dimensions.len), + @ptrCast(dimensions), + elemType.ptr(), + c.mlirAttributeGetNull(), + ).ptr); + } -// pub fn appendOperation(self: Block, op: Operation) void { -// c.mlirBlockAppendOwnedOperation(self._inner, op._inner); -// } + pub fn elementType(self: *const RankedTensorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } -// pub fn appendOperations(self: *Block, ops: []const Operation) void { -// for (ops) |op| { -// c.mlirBlockAppendOwnedOperation(self._inner, op._inner); -// } -// } + pub fn rank(self: *const RankedTensorType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } -// pub const RecursiveOpts = enum { open, hermetic }; + pub fn dimension(self: *const RankedTensorType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } -// pub fn appendValueRecursive(self: Block, value: Value, opt: RecursiveOpts) void { -// switch (value.kind()) { -// .op_result => |parent_op| self.appendOperationRecursive(parent_op, opt), -// .block_argument => |arg| { -// // Hermetic blocks are not allowed to use arguments from other blocks. -// stdx.debug.assert(opt == .open or self.eql(arg.block()), "Can't add {} from {?x} block to {?x} block", .{ arg, arg.block()._inner.ptr, self._inner.ptr }); -// }, -// .null => @panic("InvalidMlir"), -// } -// } + pub fn shaped(self: *const RankedTensorType) *const ShapedType { + return @ptrCast(self); + } +}; -// pub fn appendOperationRecursive(self: Block, op: Operation, opt: RecursiveOpts) void { -// if (op.block()) |prev_block| { -// // Hermetic blocks are not allowed to reference values from other blocks. -// stdx.debug.assert(opt == .open or self.equals(prev_block), "Can't add {} from {?x} block to {?x} block", .{ op, prev_block._inner.ptr, self._inner.ptr }); -// return; -// } -// for (0..op.numOperands()) |i| { -// self.appendValueRecursive(op.operand(i), opt); -// } -// self.appendOperation(op); -// } -// }; +pub fn rankedTensorType(dimensions: []const i64, elemType: *const Type) *const Type { + return @ptrCast(RankedTensorType.init(dimensions, elemType)); +} + +pub const DenseElementsAttribute = opaque { + const M = Methods(DenseElementsAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADenseElements; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(shaped_type: *const ShapedType, values: []const u8) *const DenseElementsAttribute { + return @ptrCast(c.mlirDenseElementsAttrRawBufferGet( + shaped_type.ptr(), + @intCast(values.len), + @ptrCast(values), + ).ptr orelse unreachable); + } +}; + +pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: []const u8) *const Attribute { + return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); +} + +pub const DenseArrayTypes = enum { + bool, + i8, + i16, + i32, + i64, + f32, + f64, + + fn ZigType(comptime ElementType: @This()) type { + return switch (ElementType) { + .bool => i32, + .i8 => i8, + .i16 => i16, + .i32 => i32, + .i64 => i64, + .f32 => f32, + .f64 => f64, + }; + } +}; + +pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { + const ElementTypeZig = ElementType.ZigType(); + + const _isAFn, const getFn, const getElementFn = switch (ElementType) { + .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, + .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, + .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, + .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, + .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, + .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, + .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, + }; + + return struct { + const Self = @This(); + + const M = Methods(@This(), c.MlirAttribute); + + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, values: []const ElementTypeZig) *const Self { + return @ptrCast(getFn(ctx.ptr(), @intCast(values.len), @ptrCast(values)).ptr); + } + + pub fn get(self: *const Self, pos: usize) ElementTypeZig { + return getElementFn(self.ptr(), @intCast(pos)); + } + + pub fn len(self: *const Self) usize { + return @intCast(c.mlirDenseArrayGetNumElements(self.ptr())); + } + }; +} + +pub fn denseArrayAttribute(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { + return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); +} + +pub const ArrayAttribute = opaque { + const M = Methods(ArrayAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAArray; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attrs: []const *const Attribute) *const ArrayAttribute { + return @ptrCast(c.mlirArrayAttrGet(ctx.ptr(), @intCast(attrs.len), @ptrCast(attrs)).ptr); + } + + pub fn size(self: *const ArrayAttribute) usize { + return @intCast(c.mlirArrayAttrGetNumElements(self.ptr())); + } + + pub fn get(self: *const ArrayAttribute, index: usize) Attribute { + return @ptrCast(c.mlirArrayAttrGetElement(self.ptr(), @intCast(index)).ptr); + } +}; + +pub fn arrayAttribute(ctx: *Context, attrs: []const *const Attribute) *const Attribute { + return @ptrCast(ArrayAttribute.init(ctx, attrs)); +} + +pub const FunctionType = opaque { + const M = Methods(FunctionType, c.MlirType); + pub const isAFn = c.mlirTypeIsAFunction; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(ctx: *Context, args: []const *const Type, results: []const *const Type) *const FunctionType { + return @ptrCast(c.mlirFunctionTypeGet( + ctx.ptr(), + @intCast(args.len), + @ptrCast(args.ptr), + @intCast(results.len), + @ptrCast(results.ptr), + ).ptr); + } +}; + +pub fn functionType(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { + return @ptrCast(FunctionType.init(ctx, args, results)); +} + +pub const FlatSymbolRefAttribute = opaque { + const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAFlatSymbolRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, symbol_: []const u8) *const FlatSymbolRefAttribute { + return @ptrCast(c.mlirFlatSymbolRefAttrGet(ctx.ptr(), stringRef(symbol_)).ptr); + } + + pub fn symbol(self: *const FlatSymbolRefAttribute) []const u8 { + return string(c.mlirFlatSymbolRefAttrGetValue(self.ptr())); + } +}; + +pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { + return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); +} From a49efee290c259a6ae79ab09b82abfd4f31b6bca Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Aug 2025 01:25:22 +0200 Subject: [PATCH 03/47] WIP --- mlir/dialects/BUILD.bazel | 2 +- mlir/dialects/arith.zig | 78 +- mlir/dialects/dialects.zig | 4 +- mlir/dialects/func.zig | 69 +- mlir/dialects/math.zig | 36 - mlir/dialects/scf.zig | 61 - mlir/dialects/stablehlo/BUILD.bazel | 2 +- mlir/dialects/stablehlo/stablehlo.zig | 2585 ++++++++++++------------- mlir/main.zig | 53 +- mlir/registry.zig | 138 +- 10 files changed, 1493 insertions(+), 1535 deletions(-) delete mode 100644 mlir/dialects/math.zig delete mode 100644 mlir/dialects/scf.zig diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 4fbc61bd..34400ca9 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -14,7 +14,7 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//mlir:mlir2", - # "//mlir/dialects/stablehlo", + "//mlir/dialects/stablehlo", "//stdx", ], ) diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig index 39cbc2a1..4e8e0ed9 100644 --- a/mlir/dialects/arith.zig +++ b/mlir/dialects/arith.zig @@ -1,59 +1,61 @@ const std = @import("std"); -const mlir = @import("mlir"); +const mlir = @import("mlir2"); -pub fn constant(ctx: mlir.Context, value: mlir.Attribute, location: mlir.Location) mlir.Operation { +pub fn constant(ctx: *mlir.Context, value: *const mlir.Attribute, location: *const mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, "arith.constant", .{ - .attributes = &.{.{ "value", value }}, + .attributes = &.{ + .named(ctx, "value", value), + }, .result_type_inference = true, .location = location, }); } -fn binary_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation { +fn binaryOp(comptime op_name: []const u8) type { return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { + pub fn call(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, + .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .location = location, }); } - }.call; + }; } -fn cast_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Type, mlir.Location) mlir.Operation { +fn castOp(comptime op_name: []const u8) type { return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, new_type: mlir.Type, location: mlir.Location) mlir.Operation { + pub fn call(ctx: mlir.Context, value: *const mlir.Value, new_type: *const mlir.Type, location: *const mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .results = &.{new_type}, + .operands = .{ .float = &.{value} }, + .results = .{ .flat = &.{new_type} }, .location = location, }); } - }.call; + }; } -pub const addi = binary_fn("arith.addi"); -pub const addf = binary_fn("arith.addf"); -pub const subi = binary_fn("arith.subi"); -pub const subf = binary_fn("arith.subf"); -pub const muli = binary_fn("arith.muli"); -pub const mulf = binary_fn("arith.mulf"); -pub const divsi = binary_fn("arith.divsi"); -pub const divui = binary_fn("arith.divui"); -pub const divf = binary_fn("arith.divf"); -pub const maxnumf = binary_fn("arith.maxnumf"); -pub const maxnumi = binary_fn("arith.maxnumi"); -pub const extsi = cast_fn("arith.extsi"); -pub const extui = cast_fn("arith.extui"); -pub const extf = cast_fn("arith.extf"); -pub const trunci = cast_fn("arith.trunci"); -pub const truncf = cast_fn("arith.truncf"); -pub const fptosi = cast_fn("arith.fptosi"); -pub const fptoui = cast_fn("arith.fptoui"); -pub const sitofp = cast_fn("arith.sitofp"); -pub const uitofp = cast_fn("arith.uitofp"); +pub const addi = binaryOp("arith.addi").call; +pub const addf = binaryOp("arith.addf").call; +pub const subi = binaryOp("arith.subi").call; +pub const subf = binaryOp("arith.subf").call; +pub const muli = binaryOp("arith.muli").call; +pub const mulf = binaryOp("arith.mulf").call; +pub const divsi = binaryOp("arith.divsi").call; +pub const divui = binaryOp("arith.divui").call; +pub const divf = binaryOp("arith.divf").call; +pub const maxnumf = binaryOp("arith.maxnumf").call; +pub const maxnumi = binaryOp("arith.maxnumi").call; +pub const extsi = castOp("arith.extsi").call; +pub const extui = castOp("arith.extui").call; +pub const extf = castOp("arith.extf").call; +pub const trunci = castOp("arith.trunci").call; +pub const truncf = castOp("arith.truncf").call; +pub const fptosi = castOp("arith.fptosi").call; +pub const fptoui = castOp("arith.fptoui").call; +pub const sitofp = castOp("arith.sitofp").call; +pub const uitofp = castOp("arith.uitofp").call; pub const CmpIPredicate = enum { eq, @@ -68,12 +70,12 @@ pub const CmpIPredicate = enum { uge, }; -pub fn cmpi(ctx: mlir.Context, predicate: CmpIPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn cmpi(ctx: mlir.Context, predicate: CmpIPredicate, lhs: *const mlir.Value, rhs: *const mlir.Value, location: mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, "arith.cmpi", .{ - .operands = &.{ lhs, rhs }, + .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .{ "predicate", .int(ctx, .i64, @intFromEnum(predicate)) }, + .{ "predicate", mlir.integerAttribute(ctx, .i64, @intFromEnum(predicate)) }, }, .location = location, }); @@ -98,12 +100,12 @@ pub const CmpFPredicate = enum { true, }; -pub fn cmpf(ctx: mlir.Context, predicate: CmpFPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn cmpf(ctx: mlir.Context, predicate: CmpFPredicate, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, "arith.cmpf", .{ - .operands = &.{ lhs, rhs }, + .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .{ "predicate", .int(ctx, .i64, @intFromEnum(predicate)) }, + .named(ctx, "predicate", mlir.integerAttribute(ctx, .i64, @intFromEnum(predicate))), }, .location = location, }); diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index b930086b..3649d443 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,13 +1,13 @@ const std = @import("std"); +pub const stablehlo = @import("mlir/dialects/stablehlo"); + pub const func = @import("func.zig"); // pub const arith = @import("arith.zig"); // pub const math = @import("math.zig"); // pub const scf = @import("scf.zig"); // pub const tensor = @import("tensor.zig"); -// pub const stablehlo = @import("mlir/dialects/stablehlo"); - test { std.testing.refAllDecls(@This()); } diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 50eccf96..178aab75 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -4,48 +4,89 @@ const mlir = @import("mlir2"); const stdx = @import("stdx"); const FuncOpArgs = struct { + pub const Visibility = enum { + public, + private, + }; + name: []const u8, - args: []const *const mlir.Type, + args: ?[]const *const mlir.Type = null, args_attributes: ?[]const *const mlir.Attribute = null, - results: []const *const mlir.Type, + results: ?[]const *const mlir.Type = null, results_attributes: ?[]const *const mlir.Attribute = null, block: *mlir.Block, location: *const mlir.Location, + no_inline: bool = true, + visibility: Visibility = .public, }; pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { - var attrs_tuple_buffer = stdx.BoundedArray(mlir.Operation.MakeArgs.AttrTuple, 4){}; - attrs_tuple_buffer.appendAssumeCapacity(.{ "sym_name", mlir.stringAttribute(ctx, args.name) }); - attrs_tuple_buffer.appendAssumeCapacity(.{ "function_type", mlir.typeAttribute(mlir.functionType(ctx, args.args, args.results)) }); + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .{}; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .{}; + + var attr_tuples_buffer = stdx.BoundedArray(mlir.NamedAttribute, 16).fromSlice(&.{ + .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), + .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), + .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( + ctx, + args.args orelse args: { + for (0..args.block.numArguments()) |i| { + args_buffer.appendAssumeCapacity(args.block.argument(i).type_()); + } + break :args args_buffer.constSlice(); + }, + args.results orelse results: { + const terminator = args.block.terminator() orelse { + @panic("block has no terminator"); + }; + for (0..terminator.numOperands()) |i| { + results_buffer.appendAssumeCapacity(terminator.operand(i).type_()); + } + break :results results_buffer.constSlice(); + }, + ))), + }) catch unreachable; + if (args.args_attributes) |args_attributes| { - attrs_tuple_buffer.appendAssumeCapacity(.{ "arg_attrs", mlir.arrayAttribute(ctx, args_attributes) }); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); } if (args.results_attributes) |results_attributes| { - attrs_tuple_buffer.appendAssumeCapacity(.{ "res_attrs", mlir.arrayAttribute(ctx, results_attributes) }); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, results_attributes))); + } + if (args.no_inline) { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); } return mlir.Operation.make(ctx, "func.func", .{ .blocks = &.{args.block}, - .attributes = attrs_tuple_buffer.constSlice(), + .attributes = attr_tuples_buffer.constSlice(), .location = args.location, }); } -pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Value, results: []const *const mlir.Type, loc: *const mlir.Location) *mlir.Operation { +pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Value, results: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.call", .{ .operands = .{ .variadic = &.{values} }, .results = .{ .flat = results }, .attributes = &.{ - .{ "callee", mlir.flatSymbolRefAttribute(ctx, name) }, + .named(ctx, "callee", mlir.flatSymbolRefAttribute(ctx, name)), }, - .location = loc, + .location = location, }); } -pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, loc: *const mlir.Location) *mlir.Operation { +pub fn return_(ctx: *mlir.Context, values: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.return", .{ - .operands = .{ .flat = values }, - .location = loc, + .operands = .{ .flat = &.{values} }, + .location = location, .verify = false, }); } + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "func.return", .{ + .operands = .{ .variadic = &.{values} }, + .verify = false, + .location = location, + }); +} diff --git a/mlir/dialects/math.zig b/mlir/dialects/math.zig deleted file mode 100644 index 62ef622c..00000000 --- a/mlir/dialects/math.zig +++ /dev/null @@ -1,36 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -const namespace = "math"; - -fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{value}, - .results = &.{value.getType()}, - .location = location, - }); - } - }; -} - -fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{ lhs, rhs }, - .results = &.{}, - .location = location, - }); - } - }; -} - -pub const ipowi = binary_fn("ipowi").call; -pub const fpowi = binary_fn("fpowi").call; -pub const tanh = unary_fn("tanh").call; -pub const sqrt = unary_fn("sqrt").call; -pub const exp = unary_fn("exp").call; -pub const exp2 = unary_fn("exp2").call; -pub const log = unary_fn("log").call; diff --git a/mlir/dialects/scf.zig b/mlir/dialects/scf.zig deleted file mode 100644 index 61fcb189..00000000 --- a/mlir/dialects/scf.zig +++ /dev/null @@ -1,61 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -pub fn ForBody(ExtraArgs: type) type { - return fn (mlir.Context, mlir.Block, ExtraArgs) mlir.Operation; -} - -pub const ForRange = struct { - start: mlir.Value, - end: mlir.Value, - step: mlir.Value, -}; - -pub fn @"for"( - ExtraArgs: type, - ctx: mlir.Context, - range: ForRange, - init_values: []const mlir.Value, - body: ForBody(ExtraArgs), - extra_args: ExtraArgs, - loc: mlir.Location, -) mlir.Operation { - const n_args = init_values.len; - var init_types_buf: [32]mlir.Type = undefined; - var locs_buf: [32]mlir.Location = undefined; - - // The first block argument is the for loop induction variable, - // followed then by all the loop-carried variables. - const init_types = init_types_buf[0 .. n_args + 1]; - const locs = locs_buf[0 .. n_args + 1]; - init_types[0] = range.start.getType(); - locs[0] = loc; - for (1.., init_values) |i, val| { - init_types[i] = val.getType(); - locs[i] = loc; - } - - const block = mlir.Block.init(init_types, locs) catch unreachable; - const yield_op = @call(.auto, body, .{ ctx, block, extra_args }); - std.debug.assert(std.mem.eql(u8, "scf.yield", yield_op.name().str())); - block.appendOperationRecursive(yield_op, .open); - - const for_op = mlir.Operation.make(ctx, "scf.for", .{ - .variadic_operands = &.{ &.{ range.start, range.end, range.step }, init_values }, - .results = init_types[1..], - .blocks = &.{block}, - .location = loc, - .verify = false, - }); - return for_op; -} - -pub fn yield(ctx: mlir.Context, res: []const mlir.Value, loc: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "scf.yield", .{ - .variadic_operands = &.{res}, - .results = &.{}, - .location = loc, - .verify = false, - }); -} diff --git a/mlir/dialects/stablehlo/BUILD.bazel b/mlir/dialects/stablehlo/BUILD.bazel index 45e69239..442e8c7b 100644 --- a/mlir/dialects/stablehlo/BUILD.bazel +++ b/mlir/dialects/stablehlo/BUILD.bazel @@ -6,8 +6,8 @@ zig_library( main = "stablehlo.zig", visibility = ["//mlir/dialects:__subpackages__"], deps = [ - "//mlir", "//mlir:c", + "//mlir:mlir2", "//stdx", "@stablehlo//:stablehlo_dialect_capi", ], diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 0c60fd01..83ea5669 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -1,1325 +1,1248 @@ const std = @import("std"); const c = @import("c"); -const mlir = @import("mlir"); +const mlir = @import("mlir2"); +const stringRef = mlir.stringRef; const stdx = @import("stdx"); -pub const abs = functors.unary_fn("stablehlo.abs").call; -pub const cosine = functors.unary_fn("stablehlo.cosine").call; -pub const sine = functors.unary_fn("stablehlo.sine").call; -pub const exponential = functors.unary_fn("stablehlo.exponential").call; -pub const exponential_minus_one = functors.unary_fn("stablehlo.exponential_minus_one").call; -pub const floor = functors.unary_fn("stablehlo.floor").call; -pub const log = functors.unary_fn("stablehlo.log").call; -pub const log_plus_one = functors.unary_fn("stablehlo.log_plus_one").call; -pub const not = functors.unary_fn("stablehlo.not").call; -pub const negate = functors.unary_fn("stablehlo.negate").call; -pub const sqrt = functors.unary_fn("stablehlo.sqrt").call; -pub const tanh = functors.unary_fn("stablehlo.tanh").call; -pub const cbrt = functors.unary_fn("stablehlo.cbrt").call; -pub const ceil = functors.unary_fn("stablehlo.ceil").call; -pub const rsqrt = functors.unary_fn("stablehlo.rsqrt").call; -pub const count_leading_zeros = functors.unary_fn("stablehlo.count_leading_zeros").call; -pub const is_finite = functors.unary_fn("stablehlo.is_finite").call; -pub const logistic = functors.unary_fn("stablehlo.logistic").call; -pub const popcnt = functors.unary_fn("stablehlo.popcnt").call; -pub const sign = functors.unary_fn("stablehlo.sign").call; -pub const real = functors.unary_fn("stablehlo.real").call; -pub const imag = functors.unary_fn("stablehlo.imag").call; - -pub const add = functors.binary_fn("stablehlo.add").call; -pub const multiply = functors.binary_fn("stablehlo.multiply").call; -pub const divide = functors.binary_fn("stablehlo.divide").call; -pub const subtract = functors.binary_fn("stablehlo.subtract").call; -pub const or_ = functors.binary_fn("stablehlo.or").call; -pub const xor = functors.binary_fn("stablehlo.xor").call; -pub const and_ = functors.binary_fn("stablehlo.and").call; -pub const atan2 = functors.binary_fn("stablehlo.atan2").call; -pub const maximum = functors.binary_fn("stablehlo.maximum").call; -pub const minimum = functors.binary_fn("stablehlo.minimum").call; -pub const power = functors.binary_fn("stablehlo.power").call; -pub const remainder = functors.binary_fn("stablehlo.remainder").call; -pub const shift_left = functors.binary_fn("stablehlo.shift_left").call; -pub const shift_right_arithmetic = functors.binary_fn("stablehlo.shift_right_arithmetic").call; -pub const shift_right_logical = functors.binary_fn("stablehlo.shift_right_logical").call; -pub const complex = functors.binary_fn("stablehlo.complex").call; - -const functors = struct { - fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); - } - }; - } - - pub fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .location = location, - }); - } - }; - } -}; - -pub fn return_(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{&.{value}}, - .verify = false, - .location = location, - }); -} - -pub fn returns_(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{values}, - .verify = false, - .location = location, - }); -} - -pub fn bitcast_convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.bitcast_convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn cholesky(ctx: mlir.Context, value: mlir.Value, lower: bool, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.cholesky", .{ - .operands = &.{value}, - .result_type_inference = true, - .attributes = &.{ - .{ "lower", .i1FromBool(ctx, lower) }, - }, - .location = location, - }); -} - -pub fn clamp(ctx: mlir.Context, min: mlir.Value, value: mlir.Value, max: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.clamp", .{ - .operands = &.{ min, value, max }, - .result_type_inference = true, - .location = location, - }); -} - -pub const DotPrecision = union(enum) { - fast, - high, - highest, - algorithm: DotAlgorithm, - - pub fn precisionAttr(self: DotPrecision, ctx: mlir.Context) mlir.Attribute { - const precision = PrecisionAttribute.init(ctx, switch (self) { - .fast => .DEFAULT, - .high => .HIGH, - .highest => .HIGHEST, - // When we specify the dot algorithm, we should not specify the precision. - .algorithm => .DEFAULT, - }); - return precision.asAttr(); - } - - pub fn algorithmAttr(self: DotPrecision, ctx: mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { - return switch (self) { - .algorithm => |algo| algo.asAttr(ctx, operand_type), - else => null, - }; - } -}; - -pub const DotAlgorithm = struct { - accumulation: mlir.FloatTypes, - // Note stablehlo distinguish between left/right component_count - // but all the supported algorithm have the same component_count on both side. - component_count: u8 = 1, - num_primitive_operations: u8 = 1, - allow_imprecise_accumulation: bool = false, - - // bf16_6x: each input is decomposed to 3 bf16 components, then 6 dot operations are done on those components, and the result is accumulated in f32. - // not sure where this is available. - pub const bf16_6x: DotAlgorithm = .{ - .operand = .bf16, - .accumulation = .f32, - .component_count = 1, - .num_primitive_operations = 6, - .allow_imprecise_accumulation = false, - }; - - pub fn asAttr(self: DotAlgorithm, ctx: mlir.Context, tensor_type: mlir.RankedTensorType) mlir.Attribute { - const elem_type = tensor_type.getElementType(); - - return mlir.Attribute.wrap(c.stablehloDotAlgorithmGet( - ctx._inner, - elem_type._inner, - elem_type._inner, - self.accumulation.asType(ctx)._inner, - self.component_count, - self.component_count, - self.num_primitive_operations, - self.allow_imprecise_accumulation, - )); - } -}; - -/// General matrix multiplication "a la Einstein sum" -/// Note: stablehlo doesn't do type inference for dot_general -pub fn dot_general( - ctx: mlir.Context, - lhs: mlir.Value, - rhs: mlir.Value, - result_type: mlir.Type, - location: mlir.Location, - opts: struct { - lhs_batching_dimensions: []const i64, - rhs_batching_dimensions: []const i64, - lhs_contracting_dimensions: []const i64, - rhs_contracting_dimensions: []const i64, - precision: DotPrecision, - }, -) mlir.Operation { - const precisions: [2]mlir.Attribute = @splat(opts.precision.precisionAttr(ctx)); - const attributes = [3]mlir.AttrTuple{ - .{ - "dot_dimension_numbers", DotDimensionNumbersAttribute.init(ctx, .{ - .lhs_batching_dimensions = opts.lhs_batching_dimensions, - .rhs_batching_dimensions = opts.rhs_batching_dimensions, - .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, - .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, - }).asAttr(), - }, - .{ "precision_config", .array(ctx, &precisions) }, - // keep algorithm as the last attribute so we can omit it when it's not set. - .{ "algorithm", opts.precision.algorithmAttr(ctx, lhs.getType().as(mlir.RankedTensorType).?) orelse undefined }, - }; - const n_attributes = if (opts.precision == .algorithm) attributes.len else attributes.len - 1; - return mlir.Operation.make(ctx, "stablehlo.dot_general", .{ - .operands = &.{ lhs, rhs }, - .results = &.{result_type}, - .attributes = attributes[0..n_attributes], - .location = location, - }); -} - -pub fn constant( - ctx: mlir.Context, - dims: []const i64, - elem_type: mlir.DenseElementsAttributeTypes, - raw_bytes: []const u8, - location: mlir.Location, -) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.constant", .{ - .operands = &.{}, - .results = &.{.tensor(dims, elem_type.mlirType(ctx))}, - .attributes = &.{.{ "value", .denseElementsFromBytes(ctx, dims, elem_type, raw_bytes) }}, - .location = location, - }); -} - -pub fn convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn broadcast_in_dim(ctx: mlir.Context, operand: mlir.Value, dims: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ - .operands = &.{operand}, - .results = &.{result_type}, - .attributes = &.{ - .{ "broadcast_dimensions", .dense(ctx, .i64, dims) }, - }, - .location = location, - }); -} - -pub fn transpose(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location, opts: struct { permutation: []const i64 }) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.transpose", .{ - .operands = &.{value}, - .results = &.{result_type}, - .attributes = &.{ - .{ "permutation", .dense(ctx, .i64, opts.permutation) }, - }, - .location = location, - }); -} - -pub fn slice(ctx: mlir.Context, operand: mlir.Value, start_indices: []const i64, limit_indices: []const i64, strides: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.slice", .{ - .operands = &.{operand}, - .results = &.{result_type}, - .attributes = &.{ - .{ "start_indices", .dense(ctx, .i64, start_indices) }, - .{ "limit_indices", .dense(ctx, .i64, limit_indices) }, - .{ "strides", .dense(ctx, .i64, strides) }, - }, - .location = location, - }); -} - -pub fn concatenate(ctx: mlir.Context, inputs: []const mlir.Value, dimension: i64, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ - .operands = inputs, - .result_type_inference = true, - .attributes = &.{ - .{ "dimension", .int(ctx, .i64, dimension) }, - }, - .location = location, - }); -} - -pub fn reshape(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.reshape", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn select(ctx: mlir.Context, condition: mlir.Value, then: mlir.Value, else_: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.select", .{ - .operands = &.{ condition, then, else_ }, - .results = &.{then.getType()}, - .location = location, - }); -} - -pub fn gather( - ctx: mlir.Context, - value: mlir.Value, - indices: mlir.Value, - slice_sizes: []const i64, - location: mlir.Location, - args: struct { - offset_dims: []const i64, - collapsed_slice_dims: []const i64, - operand_batching_dims: []const i64, - start_indices_batching_dims: []const i64, - start_index_map: []const i64, - index_vector_dim: i64, - indices_are_sorted: bool = false, - }, -) mlir.Operation { - return mlir.Operation.make( - ctx, - "stablehlo.gather", - .{ - .operands = &.{ value, indices }, - .result_type_inference = true, - .attributes = &.{ - .{ "dimension_numbers", GatherDimensionNumbersAttribute.init( - ctx, - args.offset_dims, - args.collapsed_slice_dims, - args.operand_batching_dims, - args.start_indices_batching_dims, - args.start_index_map, - args.index_vector_dim, - ).asAttr() }, - .{ "slice_sizes", .dense(ctx, .i64, slice_sizes) }, - .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, - }, - .location = location, - }, - ); -} - -fn elementTypeOrSelf(typ: mlir.Type) mlir.Type { - return if (typ.as(mlir.ShapedType)) |shaped| { - return shaped.elementType(); - } else typ; -} - -pub const ScatterArgs = struct { - update_window_dims: []const i64, - inserted_window_dims: []const i64, - input_batching_dims: []const i64, - scatter_indices_batching_dims: []const i64, - scatter_dims_to_operand_dims: []const i64, - index_vector_dim: i64, - indices_are_sorted: bool = false, - unique_indices: bool = false, - - pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: mlir.Context) mlir.Attribute { - return .{ ._inner = c.stablehloScatterDimensionNumbersGet( - ctx._inner, - @intCast(self.update_window_dims.len), - self.update_window_dims.ptr, - @intCast(self.inserted_window_dims.len), - self.inserted_window_dims.ptr, - @intCast(self.input_batching_dims.len), - self.input_batching_dims.ptr, - @intCast(self.scatter_indices_batching_dims.len), - self.scatter_indices_batching_dims.ptr, - @intCast(self.scatter_dims_to_operand_dims.len), - self.scatter_dims_to_operand_dims.ptr, - self.index_vector_dim, - ) }; - } -}; - -pub fn scatter( - ctx: mlir.Context, - inputs: []const mlir.Value, - scatter_indices: []const mlir.Value, - updates: []const mlir.Value, - update_block: mlir.Block, - args: ScatterArgs, - location: mlir.Location, -) mlir.Operation { - return mlir.Operation.make( - ctx, - "stablehlo.scatter", - .{ - .variadic_operands = &.{ inputs, scatter_indices, updates }, - .blocks = &.{update_block}, - .attributes = &.{ - .{ "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx) }, - .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, - .{ "unique_indices", .boolean(ctx, args.unique_indices) }, - }, - .result_type_inference = true, - .location = location, - }, - ); -} - -pub fn iota(ctx: mlir.Context, dimension: i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.iota", .{ - .operands = &.{}, - .results = &.{result_type}, - .attributes = &.{ - .{ "iota_dimension", .int(ctx, .i64, dimension) }, - }, - .location = location, - }); -} - -pub fn reverse(ctx: mlir.Context, operand: mlir.Value, dimensions: []const i64, location: mlir.Location) mlir.Operation { - const result_type = operand.getType(); - return mlir.Operation.make(ctx, "stablehlo.reverse", .{ - .operands = &.{operand}, - .results = &.{result_type}, - .attributes = &.{ - .{ "dimensions", .dense(ctx, .i64, dimensions) }, - }, - .location = location, - }); -} - -pub fn compare(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, comparison_direction: ComparisonDirection, compare_type: CompareType, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.compare", .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .attributes = &.{ - .{ "comparison_direction", comparison_direction.asAttr() }, - .{ "compare_type", compare_type.asAttr() }, - }, - .location = location, - }); -} - -pub fn reduce( - ctx: mlir.Context, - inputs: []const mlir.Value, - init_values: []const mlir.Value, - dimensions: []const i64, - blkctx: anytype, - blkfn: fn (anytype, mlir.Context, []const mlir.Value, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; - - const block_n_args = inputs.len + init_values.len; - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0..block_n_args]; - var reduce_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type: mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); - reduce_elem_types[i] = arg_type; - reduce_elem_types[inputs.len + i] = arg_type; - } - var block = mlir.Block.open(reduce_elem_types[0..block_n_args], locations) catch unreachable; - { - defer block.close(); - - var block_inputs: [MaxBlockArguments / 2]mlir.Value = undefined; - var block_accs: [MaxBlockArguments / 2]mlir.Value = undefined; - for (0..inputs.len) |i| { - block_inputs[i] = block.argument(i); - block_accs[i] = block.argument(inputs.len + i); - } - _ = blkfn(blkctx, ctx, block_inputs[0..inputs.len], block_accs[0..init_values.len]); - } - - return mlir.Operation.make(ctx, "stablehlo.reduce", .{ - .variadic_operands = &.{ inputs, init_values }, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimensions", .dense(ctx, .i64, dimensions) }, - }, - .location = location, - }); -} - -pub fn sort( - ctx: mlir.Context, - inputs: []const mlir.Value, - dimension: i64, - is_stable: bool, - blkctx: anytype, - compfn: fn (anytype, mlir.Context, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; - - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0 .. inputs.len * 2]; - var sort_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type: mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); - sort_elem_types[i * 2] = arg_type; - sort_elem_types[i * 2 + 1] = arg_type; - } - var block = mlir.Block.init(sort_elem_types[0 .. inputs.len * 2], locations) catch unreachable; - - var block_inputs: [MaxBlockArguments]mlir.Value = undefined; - for (0..inputs.len * 2) |i| { - block_inputs[i] = block.argument(i); - } - _ = compfn(blkctx, ctx, block_inputs[0 .. inputs.len * 2]); - - return mlir.Operation.make(ctx, "stablehlo.sort", .{ - .variadic_operands = &.{inputs}, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimension", .int(ctx, .i64, dimension) }, - .{ "is_stable", .boolean(ctx, is_stable) }, - }, - .location = location, - }); -} - -pub fn dynamic_slice(ctx: mlir.Context, operand: mlir.Value, new_dims: []const i64, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.dynamic_slice", .{ - .variadic_operands = &.{ &.{operand}, start_indices }, - .result_type_inference = true, - .attributes = &.{ - .{ "slice_sizes", .dense(ctx, .i64, new_dims) }, - }, - .location = location, - }); -} - -pub fn round_nearest_afz(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_afz", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - -pub fn round_nearest_even(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_even", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - -pub const PadOpts = struct { - low: []const i64, - high: []const i64, - interior: []const i64, -}; - -pub fn pad(ctx: mlir.Context, value: mlir.Value, padding_value: mlir.Value, opts: PadOpts, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.pad", .{ - .operands = &.{ value, padding_value }, - .result_type_inference = true, - .attributes = &.{ - .{ "edge_padding_low", .dense(ctx, .i64, opts.low) }, - .{ "edge_padding_high", .dense(ctx, .i64, opts.high) }, - .{ "interior_padding", .dense(ctx, .i64, opts.interior) }, - }, - .location = location, - }); -} - -pub const TriangularSolveOpts = struct { - left_side: bool, - lower: bool, - unit_diagonal: bool, - transpose_a: Transpose.Type, -}; - -pub fn triangular_solve(ctx: mlir.Context, value: mlir.Value, other: mlir.Value, location: mlir.Location, opts: TriangularSolveOpts) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ - .operands = &.{ value, other }, - .result_type_inference = true, - .attributes = &.{ - .{ "left_side", .i1FromBool(ctx, opts.left_side) }, - .{ "lower", .i1FromBool(ctx, opts.lower) }, - .{ "unit_diagonal", .i1FromBool(ctx, opts.unit_diagonal) }, - .{ "transpose_a", Transpose.init(ctx, opts.transpose_a).asAttr() }, - }, - .location = location, - }); -} - -pub const FftOpts = struct { - kind: FftType.Type, - length: []const i64, -}; - -pub fn fft(ctx: mlir.Context, value: mlir.Value, location: mlir.Location, opts: FftOpts) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.fft", .{ - .operands = &.{value}, - .result_type_inference = true, - .attributes = &.{ - .{ "fft_type", FftType.init(ctx, opts.kind).asAttr() }, - .{ "fft_length", .dense(ctx, .i64, opts.length) }, - }, - .location = location, - }); -} - -pub fn rng(ctx: mlir.Context, a: mlir.Value, b: mlir.Value, shape: mlir.Value, rng_distribution: RngDistribution.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.rng", .{ - .operands = &.{ a, b, shape }, - .result_type_inference = true, - .attributes = &.{ - .{ "rng_distribution", RngDistribution.init(ctx, rng_distribution).asAttr() }, - }, - .location = location, - }); -} - -pub fn rng_bit_generator(ctx: mlir.Context, rng_algorithm: RngAlgorithm.Type, initial_state: mlir.Value, res_state_type: mlir.Type, res_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ - .operands = &.{initial_state}, - .results = &.{ res_state_type, res_type }, - .attributes = &.{ - .{ "rng_algorithm", RngAlgorithm.init(ctx, rng_algorithm).asAttr() }, - }, - .location = location, - }); -} - -pub fn reduce_precision(ctx: mlir.Context, value: mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ - .operands = &.{value}, - .result_type_inference = true, - .attributes = &.{ - .{ "exponent_bits", .int(ctx, .i32, exponent_bits) }, - .{ "mantissa_bits", .int(ctx, .i32, mantissa_bits) }, - }, - .location = location, - }); -} - -pub fn dynamic_update_slice(ctx: mlir.Context, operand: mlir.Value, update: mlir.Value, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ - .variadic_operands = &.{ &.{operand}, &.{update}, start_indices }, - .result_type_inference = true, - .location = location, - }); -} - -pub fn tuple(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.tuple", .{ - .operands = values, - .result_type_inference = true, - .location = location, - }); -} - -pub fn get_tuple_element(ctx: mlir.Context, tuple_value: mlir.Value, index: i64, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ - .operands = &.{tuple_value}, - .result_type_inference = true, - .attributes = &.{ - .{ "index", .int(ctx, .i32, index) }, - }, - .location = location, - }); -} - -pub const ConvolutionOpts = struct { - window_strides: []const i64, - pad_value: []const i64, - pad_shape: []const i64 = &.{}, - lhs_dilation: []const i64, - rhs_dilation: []const i64, - window_reversal: []const bool, - input_batch_dimension: i64, - input_feature_dimension: i64, - input_spatial_dimensions: []const i64, - kernel_input_feature_dimension: i64, - kernel_output_feature_dimension: i64, - kernel_spatial_dimensions: []const i64, - output_batch_dimension: i64, - output_feature_dimension: i64, - output_spatial_dimensions: []const i64, - feature_group_count: i64, - batch_group_count: i64, - precision_config: []const PrecisionAttribute.Precision = &.{}, -}; - -pub fn convolution( - ctx: mlir.Context, - lhs: mlir.Value, - rhs: mlir.Value, - opts: ConvolutionOpts, - res_type: mlir.Type, - location: mlir.Location, -) mlir.Operation { - var max_precisions: [2]mlir.Attribute = undefined; - for (opts.precision_config, 0..) |p, i| { - max_precisions[i] = PrecisionAttribute.init(ctx, p).asAttr(); - } - var window_reversal: [3]i32 = undefined; - for (opts.window_reversal, 0..) |w, i| { - window_reversal[i] = @intCast(@intFromBool(w)); - } - return mlir.Operation.make(ctx, "stablehlo.convolution", .{ - .operands = &.{ lhs, rhs }, - .results = &.{res_type}, - .attributes = &.{ - .{ "window_strides", .dense(ctx, .i64, opts.window_strides) }, - .{ "padding", .denseElements(ctx, opts.pad_shape, .i64, opts.pad_value) }, - .{ "lhs_dilation", .dense(ctx, .i64, opts.lhs_dilation) }, - .{ "rhs_dilation", .dense(ctx, .i64, opts.rhs_dilation) }, - .{ "window_reversal", .dense(ctx, .bool, window_reversal[0..opts.window_reversal.len]) }, - .{ - "dimension_numbers", ConvDimensionNumbersAttribute.init(ctx, .{ - .input_batch_dimension = opts.input_batch_dimension, - .input_feature_dimension = opts.input_feature_dimension, - .input_spatial_dimensions = opts.input_spatial_dimensions, - .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, - .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, - .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, - .output_batch_dimension = opts.output_batch_dimension, - .output_feature_dimension = opts.output_feature_dimension, - .output_spatial_dimensions = opts.output_spatial_dimensions, - }).asAttr(), - }, - .{ "feature_group_count", .int(ctx, .i64, opts.feature_group_count) }, - .{ "batch_group_count", .int(ctx, .i64, opts.batch_group_count) }, - .{ "precision_config", .array(ctx, &max_precisions) }, - }, - .location = location, - }); -} - -pub const CustomCallOpts = struct { - pub const ApiVersion = enum(i32) { - original = 1, - status_returning = 2, - status_returning_unified = 3, - typed_ffi = 4, - }; - - call_target_name: [:0]const u8, - has_side_effect: bool, - backend_config: ?mlir.Attribute, - operand_layouts: ?[]const []const usize = null, - result_layouts: ?[]const []const usize = null, - output_operand_aliases: []const i64 = &.{}, - additional_attributes: []const mlir.AttrTuple = &.{}, - api_version: ApiVersion, -}; - -pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { - const MAX_OPERANDS = 64; - const MAX_RESULTS = 16; - - const backend_config = opts.backend_config orelse mlir.Attribute.string(ctx, ""); - if (@intFromEnum(opts.api_version) < @intFromEnum(CustomCallOpts.ApiVersion.typed_ffi)) { - stdx.debug.assert( - backend_config.isA(mlir.StringAttribute), - "API version < 4 requires a string as backend_config, got {}", - .{backend_config}, - ); - } else { - stdx.debug.assert( - backend_config.isA(mlir.DictionaryAttribute), - "API version >= 4 requires a dictionary as backend_config, got {}", - .{backend_config}, - ); - } - - var attrs: stdx.BoundedArray(mlir.AttrTuple, 32) = .{}; - attrs.appendSliceAssumeCapacity(&[_]mlir.AttrTuple{ - .{ "api_version", .int(ctx, .i32, @intFromEnum(opts.api_version)) }, - .{ "call_target_name", .string(ctx, opts.call_target_name) }, - .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, - .{ "backend_config", backend_config }, - }); - - { - var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - for (opts.output_operand_aliases) |alias| { - output_operand_aliases.appendAssumeCapacity( - OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), - ); - } - attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); - } - - const MINOR_TO_MAJOR = blk: { - const MAX_RANK = 8; - var ret: [MAX_RANK]usize = undefined; - for (0..MAX_RANK) |i| { - ret[i] = @intCast(MAX_RANK - i - 1); - } - break :blk ret; - }; - - if (opts.operand_layouts) |layouts| { - var operand_layouts: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; - for (layouts) |ol| { - operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); - } - attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); - } else { - const operand_layouts = blk: { - var ret: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; - for (inputs) |input| { - const ranked_type = input.getType().as(mlir.RankedTensorType).?; - const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; - ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); - } - break :blk ret; - }; - attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); - } - - if (opts.result_layouts) |layouts| { - var result_layouts: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - for (layouts) |rl| { - result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); - } - attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); - } else { - const result_layouts = blk: { - var ret: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - for (res_types) |t| { - const ranked_t = t.as(mlir.RankedTensorType).?; - const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; - ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); - } - break :blk ret; - }; - attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); - } - - attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); - - return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ - .operands = inputs, - .results = res_types, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub const DotDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsADotDimensionNumbers; - const Self = DotDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub fn init(ctx: mlir.Context, args: struct { - lhs_batching_dimensions: []const i64, - rhs_batching_dimensions: []const i64, - lhs_contracting_dimensions: []const i64, - rhs_contracting_dimensions: []const i64, - }) Self { - return .{ - ._inner = c.stablehloDotDimensionNumbersGet( - ctx._inner, - @intCast(args.lhs_batching_dimensions.len), - args.lhs_batching_dimensions.ptr, - @intCast(args.rhs_batching_dimensions.len), - args.rhs_batching_dimensions.ptr, - @intCast(args.lhs_contracting_dimensions.len), - args.lhs_contracting_dimensions.ptr, - @intCast(args.rhs_contracting_dimensions.len), - args.rhs_contracting_dimensions.ptr, - ), - }; - } - - pub fn getLhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self._inner)); - } - - pub fn getLhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self._inner, @intCast(pos)); - } - - pub fn getRhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self._inner)); - } - - pub fn getRhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self._inner, @intCast(pos)); - } - - pub fn getLhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self._inner)); - } - - pub fn getLhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self._inner, @intCast(pos)); - } - - pub fn getRhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self._inner)); - } - - pub fn getRhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self._inner, @intCast(pos)); - } -}; - -pub const GatherDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsAGatherDimensionNumbers; - const Self = GatherDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub fn init( - ctx: mlir.Context, - offset_dims: []const i64, - collapsed_slice_dims: []const i64, - operand_batching_dims: []const i64, - start_indices_batching_dims: []const i64, - start_index_map: []const i64, - index_vector_dim: i64, - ) Self { - return .{ - ._inner = c.stablehloGatherDimensionNumbersGet( - ctx._inner, - @intCast(offset_dims.len), - offset_dims.ptr, - @intCast(collapsed_slice_dims.len), - collapsed_slice_dims.ptr, - @intCast(operand_batching_dims.len), - operand_batching_dims.ptr, - @intCast(start_indices_batching_dims.len), - start_indices_batching_dims.ptr, - @intCast(start_index_map.len), - start_index_map.ptr, - index_vector_dim, - ), - }; - } - - pub fn getOffsetDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self._inner)); - } - - pub fn getOffsetDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self._inner, @intCast(pos)); - } - - pub fn getCollapsedSliceDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self._inner)); - } - - pub fn getCollapsedSliceDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self._inner, @intCast(pos)); - } - - pub fn getStartIndexMapSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self._inner)); - } - - pub fn getOperandBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self._inner)); - } - - pub fn getOperandBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self._inner, @intCast(pos)); - } - - pub fn getStartIndicesBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self._inner)); - } - - pub fn getStartIndicesBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self._inner, @intCast(pos)); - } - - pub fn getStartIndexMapElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self._inner, @intCast(pos)); - } - - pub fn getIndexVectorDim(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self._inner)); - } -}; - -pub const ConvDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsAConvDimensionNumbers; - const Self = ConvDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub fn init(ctx: mlir.Context, args: struct { - input_batch_dimension: i64, - input_feature_dimension: i64, - input_spatial_dimensions: []const i64, - kernel_input_feature_dimension: i64, - kernel_output_feature_dimension: i64, - kernel_spatial_dimensions: []const i64, - output_batch_dimension: i64, - output_feature_dimension: i64, - output_spatial_dimensions: []const i64, - }) Self { - return .{ - ._inner = c.stablehloConvDimensionNumbersGet( - ctx._inner, - args.input_batch_dimension, - args.input_feature_dimension, - @intCast(args.input_spatial_dimensions.len), - args.input_spatial_dimensions.ptr, - args.kernel_input_feature_dimension, - args.kernel_output_feature_dimension, - @intCast(args.kernel_spatial_dimensions.len), - args.kernel_spatial_dimensions.ptr, - args.output_batch_dimension, - args.output_feature_dimension, - @intCast(args.output_spatial_dimensions.len), - args.output_spatial_dimensions.ptr, - ), - }; - } - - pub fn getInputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputBatchDimension(self._inner); - } - - pub fn getInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self._inner); - } - - pub fn getInputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self._inner)); - } - - pub fn getInputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self._inner, @intCast(pos)); - } - - pub fn getKernelInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self._inner); - } - - pub fn getKernelOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self._inner); - } - - pub fn getKernelSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self._inner)); - } - - pub fn getKernelSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self._inner, @intCast(pos)); - } - - pub fn getOutputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self._inner); - } - - pub fn getOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self._inner); - } - - pub fn getOutputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self._inner)); - } - - pub fn getOutputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self._inner, @intCast(pos)); - } -}; - -pub const OutputOperandAliasAttribute = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsAOutputOperandAlias; - pub const asAttr = mlir.Attribute.fromAny(OutputOperandAliasAttribute); - pub const eql = mlir.Attribute.eqlAny(OutputOperandAliasAttribute); - - pub fn init( - ctx: mlir.Context, - output_tuple_indices: []const i64, - operand_index: i64, - operand_tuple_indices: []const i64, - ) OutputOperandAliasAttribute { - return .{ ._inner = c.stablehloOutputOperandAliasGet( - ctx._inner, - @intCast(output_tuple_indices.len), - output_tuple_indices.ptr, - @intCast(operand_index), - @intCast(operand_tuple_indices.len), - operand_tuple_indices.ptr, - ) }; - } -}; - -pub const PrecisionAttribute = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsAPrecisionAttr; - const Self = PrecisionAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Precision = enum { - DEFAULT, - HIGH, - HIGHEST, - }; - - pub fn init(ctx: mlir.Context, value: Precision) Self { - return .{ ._inner = c.stablehloPrecisionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Precision { - const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self._inner)); - return std.meta.stringToEnum(Precision, value) orelse unreachable; - } -}; - -pub const ComparisonDirection = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsAComparisonDirectionAttr; - const Self = ComparisonDirection; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Direction = enum { - EQ, - NE, - GE, - GT, - LE, - LT, - }; - - pub fn init(ctx: mlir.Context, value: Direction) Self { - return .{ ._inner = c.stablehloComparisonDirectionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Direction { - const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self._inner)); - return std.meta.stringToEnum(Direction, value) orelse unreachable; - } -}; - -pub const CompareType = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsAComparisonTypeAttr; - const Self = CompareType; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - SIGNED, - UNSIGNED, - FLOAT, - TOTALORDER, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloComparisonTypeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const Transpose = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsATransposeAttr; - const Self = Transpose; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - NO_TRANSPOSE, - TRANSPOSE, - ADJOINT, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloTransposeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const FftType = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsAFftTypeAttr; - const Self = FftType; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - FFT, - IFFT, - RFFT, - IRFFT, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloFftTypeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const RngDistribution = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsARngDistributionAttr; - const Self = RngDistribution; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - UNIFORM, - NORMAL, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloRngDistributionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngDistributionAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const RngAlgorithm = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsARngAlgorithmAttr; - const Self = RngAlgorithm; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - DEFAULT, - THREE_FRY, - PHILOX, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloRngAlgorithmAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngAlgorithmAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub fn stablehloVersionFromCompatibilityRequirement(requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { - const state = struct { - var buf: [32]u8 = undefined; - - fn call(req: c.MlirStablehloCompatibilityRequirement) []u8 { - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - c.stablehloVersionFromCompatibilityRequirement(req, (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); - - return buf[0..stream.pos]; +pub const abs = unaryOp("stablehlo.abs").call; +pub const cosine = unaryOp("stablehlo.cosine").call; +pub const sine = unaryOp("stablehlo.sine").call; +pub const exponential = unaryOp("stablehlo.exponential").call; +pub const exponential_minus_one = unaryOp("stablehlo.exponential_minus_one").call; +pub const floor = unaryOp("stablehlo.floor").call; +pub const log = unaryOp("stablehlo.log").call; +pub const log_plus_one = unaryOp("stablehlo.log_plus_one").call; +pub const not = unaryOp("stablehlo.not").call; +pub const negate = unaryOp("stablehlo.negate").call; +pub const sqrt = unaryOp("stablehlo.sqrt").call; +pub const tanh = unaryOp("stablehlo.tanh").call; +pub const cbrt = unaryOp("stablehlo.cbrt").call; +pub const ceil = unaryOp("stablehlo.ceil").call; +pub const rsqrt = unaryOp("stablehlo.rsqrt").call; +pub const count_leading_zeros = unaryOp("stablehlo.count_leading_zeros").call; +pub const is_finite = unaryOp("stablehlo.is_finite").call; +pub const logistic = unaryOp("stablehlo.logistic").call; +pub const popcnt = unaryOp("stablehlo.popcnt").call; +pub const sign = unaryOp("stablehlo.sign").call; +pub const real = unaryOp("stablehlo.real").call; +pub const imag = unaryOp("stablehlo.imag").call; +pub const round_nearest_afz = unaryOp("stablehlo.round_nearest_afz").call; +pub const round_nearest_even = unaryOp("stablehlo.round_nearest_even").call; +pub const tuple = unaryOp("stablehlo.tuple").call; + +pub const add = binaryOp("stablehlo.add").call; +pub const multiply = binaryOp("stablehlo.multiply").call; +pub const divide = binaryOp("stablehlo.divide").call; +pub const subtract = binaryOp("stablehlo.subtract").call; +pub const or_ = binaryOp("stablehlo.or").call; +pub const xor = binaryOp("stablehlo.xor").call; +pub const and_ = binaryOp("stablehlo.and").call; +pub const atan2 = binaryOp("stablehlo.atan2").call; +pub const maximum = binaryOp("stablehlo.maximum").call; +pub const minimum = binaryOp("stablehlo.minimum").call; +pub const power = binaryOp("stablehlo.power").call; +pub const remainder = binaryOp("stablehlo.remainder").call; +pub const shift_left = binaryOp("stablehlo.shift_left").call; +pub const shift_right_arithmetic = binaryOp("stablehlo.shift_right_arithmetic").call; +pub const shift_right_logical = binaryOp("stablehlo.shift_right_logical").call; +pub const complex = binaryOp("stablehlo.complex").call; + +pub const bitcast_convert = castOp("stablehlo.bitcast_convert").call; +pub const convert = castOp("stablehlo.convert").call; +pub const reshape = castOp("stablehlo.reshape").call; + +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); } }; - - return state.call(requirement); } -pub fn stablehloGetSmallerVersion(version1: []const u8, version2: []const u8) []const u8 { +fn unaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +pub fn binaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +// pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.cholesky", .{ +// .operands = .{ .flat = &.{value} }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower)) }, +// }, +// .location = location, +// }); +// } + +// pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Value, max: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.clamp", .{ +// .operands = .{ .flat = &.{ min, value, max } }, +// .result_type_inference = true, +// .location = location, +// }); +// } + +// pub const DotPrecision = union(enum) { +// fast, +// high, +// highest, +// algorithm: DotAlgorithm, + +// pub fn precisionAttr(self: DotPrecision, ctx: *mlir.Context) *const mlir.Attribute { +// const precision = PrecisionAttribute.init(ctx, switch (self) { +// .fast => .DEFAULT, +// .high => .HIGH, +// .highest => .HIGHEST, +// // When we specify the dot algorithm, we should not specify the precision. +// .algorithm => .DEFAULT, +// }); +// return precision.asAttr(); +// } + +// pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { +// return switch (self) { +// .algorithm => |algo| algo.asAttr(ctx, operand_type), +// else => null, +// }; +// } +// }; + +// pub const DotAlgorithm = struct { +// accumulation: mlir.FloatTypes, +// // Note stablehlo distinguish between left/right component_count +// // but all the supported algorithm have the same component_count on both side. +// component_count: u8 = 1, +// num_primitive_operations: u8 = 1, +// allow_imprecise_accumulation: bool = false, + +// // bf16_6x: each input is decomposed to 3 bf16 components, then 6 dot operations are done on those components, and the result is accumulated in f32. +// // not sure where this is available. +// pub const bf16_6x: DotAlgorithm = .{ +// .operand = .bf16, +// .accumulation = .f32, +// .component_count = 1, +// .num_primitive_operations = 6, +// .allow_imprecise_accumulation = false, +// }; + +// pub fn asAttr(self: DotAlgorithm, ctx: *mlir.Context, tensor_type: mlir.RankedTensorType) *const mlir.Attribute { +// const elem_type = tensor_type.getElementType(); + +// return mlir.Attribute.wrap(c.stablehloDotAlgorithmGet( +// ctx._inner, +// elem_type._inner, +// elem_type._inner, +// self.accumulation.asType(ctx)._inner, +// self.component_count, +// self.component_count, +// self.num_primitive_operations, +// self.allow_imprecise_accumulation, +// )); +// } +// }; + +// /// General matrix multiplication "a la Einstein sum" +// /// Note: stablehlo doesn't do type inference for dot_general +// pub fn dot_general( +// ctx: *mlir.Context, +// lhs: *const mlir.Value, +// rhs: *const mlir.Value, +// result_type: *const mlir.Type, +// location: *const mlir.Location, +// opts: struct { +// lhs_batching_dimensions: []const i64, +// rhs_batching_dimensions: []const i64, +// lhs_contracting_dimensions: []const i64, +// rhs_contracting_dimensions: []const i64, +// precision: DotPrecision, +// }, +// ) *mlir.Operation { +// const precisions: [2]mlir.Attribute = @splat(opts.precision.precisionAttr(ctx)); +// const attributes = [3]mlir.AttrTuple{ +// .{ +// "dot_dimension_numbers", DotDimensionNumbersAttribute.init(ctx, .{ +// .lhs_batching_dimensions = opts.lhs_batching_dimensions, +// .rhs_batching_dimensions = opts.rhs_batching_dimensions, +// .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, +// .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, +// }).asAttr(), +// }, +// .{ "precision_config", .array(ctx, &precisions) }, +// // keep algorithm as the last attribute so we can omit it when it's not set. +// .{ "algorithm", opts.precision.algorithmAttr(ctx, lhs.getType().as(mlir.RankedTensorType).?) orelse undefined }, +// }; +// const n_attributes = if (opts.precision == .algorithm) attributes.len else attributes.len - 1; +// return mlir.Operation.make(ctx, "stablehlo.dot_general", .{ +// .operands = .{ .flat = &.{ lhs, rhs } }, +// .results = .{ .flat = &.{result_type} }, +// .attributes = attributes[0..n_attributes], +// .location = location, +// }); +// } + +// pub fn constant( +// ctx: *mlir.Context, +// dims: []const i64, +// elem_type: mlir.DenseElementsAttributeTypes, +// raw_bytes: []const u8, +// location: *const mlir.Location, +// ) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.constant", .{ +// .operands = .{ .flat = &.{} }, +// .results = &.{.tensor(dims, elem_type.mlirType(ctx))}, +// .attributes = &.{ +// .{ "value", .denseElementsFromBytes(ctx, dims, elem_type, raw_bytes) }, +// }, +// .location = location, +// }); +// } + +// pub fn broadcast_in_dim(ctx: *mlir.Context, operand: *const mlir.Value, dims: []const i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ +// .operands = .{ .flat = &.{operand} }, +// .results = .{ .flat = &.{result_type} }, +// .attributes = &.{ +// .{ "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims) }, +// }, +// .location = location, +// }); +// } + +// pub fn transpose(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location, opts: struct { permutation: []const i64 }) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.transpose", .{ +// .operands = .{ .flat = &.{value} }, +// .results = .{ .flat = &.{result_type} }, +// .attributes = &.{ +// .{ "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation) }, +// }, +// .location = location, +// }); +// } + +// pub fn slice(ctx: *mlir.Context, operand: *const mlir.Value, start_indices: []const i64, limit_indices: []const i64, strides: []const i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.slice", .{ +// .operands = .{ .flat = &.{operand} }, +// .results = .{ .flat = &.{result_type} }, +// .attributes = &.{ +// .{ "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices) }, +// .{ "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices) }, +// .{ "strides", mlir.denseArrayAttribute(ctx, .i64, strides) }, +// }, +// .location = location, +// }); +// } + +// pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimension: i64, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ +// .operands = inputs, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "dimension", mlir.integerAttribute(ctx, .i64, dimension) }, +// }, +// .location = location, +// }); +// } + +// pub fn select(ctx: *mlir.Context, condition: *const mlir.Value, then: *const mlir.Value, else_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.select", .{ +// .operands = .{ .flat = &.{ condition, then, else_ } }, +// .results = &.{then.getType()}, +// .location = location, +// }); +// } + +// pub fn gather( +// ctx: *mlir.Context, +// value: *const mlir.Value, +// indices: *const mlir.Value, +// slice_sizes: []const i64, +// location: *const mlir.Location, +// args: struct { +// offset_dims: []const i64, +// collapsed_slice_dims: []const i64, +// operand_batching_dims: []const i64, +// start_indices_batching_dims: []const i64, +// start_index_map: []const i64, +// index_vector_dim: i64, +// indices_are_sorted: bool = false, +// }, +// ) *mlir.Operation { +// return mlir.Operation.make( +// ctx, +// "stablehlo.gather", +// .{ +// .operands = .{ .flat = &.{ value, indices } }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "dimension_numbers", GatherDimensionNumbersAttribute.init( +// ctx, +// args.offset_dims, +// args.collapsed_slice_dims, +// args.operand_batching_dims, +// args.start_indices_batching_dims, +// args.start_index_map, +// args.index_vector_dim, +// ).asAttr() }, +// .{ "slice_sizes", .dense(ctx, .i64, slice_sizes) }, +// .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, +// }, +// .location = location, +// }, +// ); +// } + +// fn elementTypeOrSelf(typ: *const mlir.Type) *const mlir.Type { +// return if (typ.as(mlir.ShapedType)) |shaped| { +// return shaped.elementType(); +// } else typ; +// } + +// pub const ScatterArgs = struct { +// update_window_dims: []const i64, +// inserted_window_dims: []const i64, +// input_batching_dims: []const i64, +// scatter_indices_batching_dims: []const i64, +// scatter_dims_to_operand_dims: []const i64, +// index_vector_dim: i64, +// indices_are_sorted: bool = false, +// unique_indices: bool = false, + +// pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { +// return .{ ._inner = c.stablehloScatterDimensionNumbersGet( +// ctx._inner, +// @intCast(self.update_window_dims.len), +// self.update_window_dims.ptr, +// @intCast(self.inserted_window_dims.len), +// self.inserted_window_dims.ptr, +// @intCast(self.input_batching_dims.len), +// self.input_batching_dims.ptr, +// @intCast(self.scatter_indices_batching_dims.len), +// self.scatter_indices_batching_dims.ptr, +// @intCast(self.scatter_dims_to_operand_dims.len), +// self.scatter_dims_to_operand_dims.ptr, +// self.index_vector_dim, +// ) }; +// } +// }; + +// pub fn scatter( +// ctx: *mlir.Context, +// inputs: []const *const mlir.Value, +// scatter_indices: []const *const mlir.Value, +// updates: []const *const mlir.Value, +// update_block: mlir.Block, +// args: ScatterArgs, +// location: *const mlir.Location, +// ) *mlir.Operation { +// return mlir.Operation.make( +// ctx, +// "stablehlo.scatter", +// .{ +// .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, +// .blocks = &.{update_block}, +// .attributes = &.{ +// .{ "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx) }, +// .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, +// .{ "unique_indices", .boolean(ctx, args.unique_indices) }, +// }, +// .result_type_inference = true, +// .location = location, +// }, +// ); +// } + +// pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.iota", .{ +// .operands = .{ .flat = &.{} }, +// .results = .{ .flat = &.{result_type} }, +// .attributes = &.{ +// .{ "iota_dimension", mlir.integerType(ctx, .i64, dimension) }, +// }, +// .location = location, +// }); +// } + +// pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []const i64, location: *const mlir.Location) *mlir.Operation { +// const result_type = operand.getType(); +// return mlir.Operation.make(ctx, "stablehlo.reverse", .{ +// .operands = .{ .flat = &.{operand} }, +// .results = .{ .flat = &.{result_type} }, +// .attributes = &.{ +// .{ "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions) }, +// }, +// .location = location, +// }); +// } + +// pub fn compare(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, comparison_direction: ComparisonDirection, compare_type: CompareType, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.compare", .{ +// .operands = .{ .flat = &.{ lhs, rhs } }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "comparison_direction", comparison_direction.asAttr() }, +// .{ "compare_type", compare_type.asAttr() }, +// }, +// .location = location, +// }); +// } + +// pub fn reduce( +// ctx: *mlir.Context, +// inputs: []const *const mlir.Value, +// init_values: []const *const mlir.Value, +// dimensions: []const i64, +// blkctx: anytype, +// blkfn: fn (anytype, mlir.Context, []const *const mlir.Value, []const *const mlir.Value) mlir.Operation, +// location: *const mlir.Location, +// ) *mlir.Operation { +// const MaxBlockArguments = 32; + +// const block_n_args = inputs.len + init_values.len; +// const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0..block_n_args]; +// var reduce_elem_types: [MaxBlockArguments]*const mlir.Type = undefined; +// for (inputs, 0..) |input, i| { +// const arg_type: *const mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); +// reduce_elem_types[i] = arg_type; +// reduce_elem_types[inputs.len + i] = arg_type; +// } +// var block = mlir.Block.open(reduce_elem_types[0..block_n_args], locations) catch unreachable; +// { +// defer block.close(); + +// var block_inputs: [MaxBlockArguments / 2]*const mlir.Value = undefined; +// var block_accs: [MaxBlockArguments / 2]*const mlir.Value = undefined; +// for (0..inputs.len) |i| { +// block_inputs[i] = block.argument(i); +// block_accs[i] = block.argument(inputs.len + i); +// } +// _ = blkfn(blkctx, ctx, block_inputs[0..inputs.len], block_accs[0..init_values.len]); +// } + +// return mlir.Operation.make(ctx, "stablehlo.reduce", .{ +// .operands = .{ .variadic = &.{ inputs, init_values } }, +// .result_type_inference = true, +// .block = block, +// .attributes = &.{ +// .{ "dimensions", .dense(ctx, .i64, dimensions) }, +// }, +// .location = location, +// }); +// } + +// pub fn sort( +// ctx: *mlir.Context, +// inputs: []const *const mlir.Value, +// dimension: i64, +// is_stable: bool, +// blkctx: anytype, +// compfn: fn (anytype, mlir.Context, []const *const mlir.Value) mlir.Operation, +// location: *const mlir.Location, +// ) *mlir.Operation { +// const MaxBlockArguments = 32; + +// const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0 .. inputs.len * 2]; +// var sort_elem_types: [MaxBlockArguments]*const mlir.Type = undefined; +// for (inputs, 0..) |input, i| { +// const arg_type: *const mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); +// sort_elem_types[i * 2] = arg_type; +// sort_elem_types[i * 2 + 1] = arg_type; +// } +// var block = mlir.Block.init(sort_elem_types[0 .. inputs.len * 2], locations) catch unreachable; + +// var block_inputs: [MaxBlockArguments]*const mlir.Value = undefined; +// for (0..inputs.len * 2) |i| { +// block_inputs[i] = block.argument(i); +// } +// _ = compfn(blkctx, ctx, block_inputs[0 .. inputs.len * 2]); + +// return mlir.Operation.make(ctx, "stablehlo.sort", .{ +// .operands = .{ .variadic = &.{inputs} }, +// .result_type_inference = true, +// .block = block, +// .attributes = &.{ +// .{ "dimension", .int(ctx, .i64, dimension) }, +// .{ "is_stable", .boolean(ctx, is_stable) }, +// }, +// .location = location, +// }); +// } + +// pub fn dynamic_slice(ctx: *mlir.Context, operand: *const mlir.Value, new_dims: []const i64, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.dynamic_slice", .{ +// .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "slice_sizes", .dense(ctx, .i64, new_dims) }, +// }, +// .location = location, +// }); +// } + +// pub const PadOpts = struct { +// low: []const i64, +// high: []const i64, +// interior: []const i64, +// }; + +// pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.pad", .{ +// .operands = .{ .flat = &.{ value, padding_value } }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "edge_padding_low", .dense(ctx, .i64, opts.low) }, +// .{ "edge_padding_high", .dense(ctx, .i64, opts.high) }, +// .{ "interior_padding", .dense(ctx, .i64, opts.interior) }, +// }, +// .location = location, +// }); +// } + +// pub const TriangularSolveOpts = struct { +// left_side: bool, +// lower: bool, +// unit_diagonal: bool, +// transpose_a: Transpose.Type, +// }; + +// pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, location: *const mlir.Location, opts: TriangularSolveOpts) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ +// .operands = .{ .flat = &.{ value, other } }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "left_side", .i1FromBool(ctx, opts.left_side) }, +// .{ "lower", .i1FromBool(ctx, opts.lower) }, +// .{ "unit_diagonal", .i1FromBool(ctx, opts.unit_diagonal) }, +// .{ "transpose_a", Transpose.init(ctx, opts.transpose_a).asAttr() }, +// }, +// .location = location, +// }); +// } + +// pub const FftOpts = struct { +// kind: FftType.Type, +// length: []const i64, +// }; + +// pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location, opts: FftOpts) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.fft", .{ +// .operands = .{ .flat = &.{value} }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "fft_type", FftType.init(ctx, opts.kind).asAttr() }, +// .{ "fft_length", .dense(ctx, .i64, opts.length) }, +// }, +// .location = location, +// }); +// } + +// pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.rng", .{ +// .operands = .{ .flat = &.{ a, b, shape } }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "rng_distribution", RngDistribution.init(ctx, rng_distribution).asAttr() }, +// }, +// .location = location, +// }); +// } + +// pub fn rng_bit_generator(ctx: *mlir.Context, rng_algorithm: RngAlgorithm.Type, initial_state: *const mlir.Value, res_state_type: *const mlir.Type, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ +// .operands = .{ .flat = &.{initial_state} }, +// .results = &.{ res_state_type, res_type }, +// .attributes = &.{ +// .{ "rng_algorithm", RngAlgorithm.init(ctx, rng_algorithm).asAttr() }, +// }, +// .location = location, +// }); +// } + +// pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ +// .operands = .{ .flat = &.{value} }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "exponent_bits", .int(ctx, .i32, exponent_bits) }, +// .{ "mantissa_bits", .int(ctx, .i32, mantissa_bits) }, +// }, +// .location = location, +// }); +// } + +// pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, update: *const mlir.Value, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ +// .operands = .{ .variadic = &.{ &.{operand}, &.{update}, start_indices } }, +// .result_type_inference = true, +// .location = location, +// }); +// } + +// pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { +// return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ +// .operands = .{ .flat = &.{tuple_value} }, +// .result_type_inference = true, +// .attributes = &.{ +// .{ "index", .int(ctx, .i32, index) }, +// }, +// .location = location, +// }); +// } + +// pub const ConvolutionOpts = struct { +// window_strides: []const i64, +// pad_value: []const i64, +// pad_shape: []const i64 = &.{}, +// lhs_dilation: []const i64, +// rhs_dilation: []const i64, +// window_reversal: []const bool, +// input_batch_dimension: i64, +// input_feature_dimension: i64, +// input_spatial_dimensions: []const i64, +// kernel_input_feature_dimension: i64, +// kernel_output_feature_dimension: i64, +// kernel_spatial_dimensions: []const i64, +// output_batch_dimension: i64, +// output_feature_dimension: i64, +// output_spatial_dimensions: []const i64, +// feature_group_count: i64, +// batch_group_count: i64, +// precision_config: []const PrecisionAttribute.Precision = &.{}, +// }; + +// pub fn convolution( +// ctx: *mlir.Context, +// lhs: *const mlir.Value, +// rhs: *const mlir.Value, +// opts: ConvolutionOpts, +// res_type: *const mlir.Type, +// location: *const mlir.Location, +// ) *mlir.Operation { +// var max_precisions: [2]mlir.Attribute = undefined; +// for (opts.precision_config, 0..) |p, i| { +// max_precisions[i] = PrecisionAttribute.init(ctx, p).asAttr(); +// } +// var window_reversal: [3]i32 = undefined; +// for (opts.window_reversal, 0..) |w, i| { +// window_reversal[i] = @intCast(@intFromBool(w)); +// } +// return mlir.Operation.make(ctx, "stablehlo.convolution", .{ +// .operands = .{ .flat = &.{ lhs, rhs } }, +// .results = &.{res_type}, +// .attributes = &.{ +// .{ "window_strides", .dense(ctx, .i64, opts.window_strides) }, +// .{ "padding", .denseElements(ctx, opts.pad_shape, .i64, opts.pad_value) }, +// .{ "lhs_dilation", .dense(ctx, .i64, opts.lhs_dilation) }, +// .{ "rhs_dilation", .dense(ctx, .i64, opts.rhs_dilation) }, +// .{ "window_reversal", .dense(ctx, .bool, window_reversal[0..opts.window_reversal.len]) }, +// .{ +// "dimension_numbers", ConvDimensionNumbersAttribute.init(ctx, .{ +// .input_batch_dimension = opts.input_batch_dimension, +// .input_feature_dimension = opts.input_feature_dimension, +// .input_spatial_dimensions = opts.input_spatial_dimensions, +// .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, +// .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, +// .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, +// .output_batch_dimension = opts.output_batch_dimension, +// .output_feature_dimension = opts.output_feature_dimension, +// .output_spatial_dimensions = opts.output_spatial_dimensions, +// }).asAttr(), +// }, +// .{ "feature_group_count", .int(ctx, .i64, opts.feature_group_count) }, +// .{ "batch_group_count", .int(ctx, .i64, opts.batch_group_count) }, +// .{ "precision_config", .array(ctx, &max_precisions) }, +// }, +// .location = location, +// }); +// } + +// pub const CustomCallOpts = struct { +// pub const ApiVersion = enum(i32) { +// original = 1, +// status_returning = 2, +// status_returning_unified = 3, +// typed_ffi = 4, +// }; + +// call_target_name: []const u8, +// has_side_effect: bool = false, +// backend_config: ?*const mlir.Attribute, +// operand_layouts: ?[]const []const usize = null, +// result_layouts: ?[]const []const usize = null, +// output_operand_aliases: ?[]const i64 = null, +// additional_attributes: ?[]const mlir.AttrTuple = &.{}, +// api_version: ApiVersion, +// }; + +// pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, opts: CustomCallOpts, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +// const MAX_OPERANDS = 64; +// const MAX_RESULTS = 16; + +// const backend_config = opts.backend_config orelse mlir.Attribute.string(ctx, ""); +// if (@intFromEnum(opts.api_version) < @intFromEnum(CustomCallOpts.ApiVersion.typed_ffi)) { +// stdx.debug.assert( +// backend_config.isA(mlir.StringAttribute), +// "API version < 4 requires a string as backend_config, got {}", +// .{backend_config}, +// ); +// } else { +// stdx.debug.assert( +// backend_config.isA(mlir.DictionaryAttribute), +// "API version >= 4 requires a dictionary as backend_config, got {}", +// .{backend_config}, +// ); +// } + +// var attrs: stdx.BoundedArray(mlir.AttrTuple, 32) = .{}; +// attrs.appendSliceAssumeCapacity(&[_]mlir.AttrTuple{ +// .{ "api_version", .int(ctx, .i32, @intFromEnum(opts.api_version)) }, +// .{ "call_target_name", .string(ctx, opts.call_target_name) }, +// .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, +// .{ "backend_config", backend_config }, +// }); + +// { +// var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; +// for (opts.output_operand_aliases) |alias| { +// output_operand_aliases.appendAssumeCapacity( +// OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), +// ); +// } +// attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); +// } + +// const MINOR_TO_MAJOR = blk: { +// const MAX_RANK = 8; +// var ret: [MAX_RANK]usize = undefined; +// for (0..MAX_RANK) |i| { +// ret[i] = @intCast(MAX_RANK - i - 1); +// } +// break :blk ret; +// }; + +// if (opts.operand_layouts) |layouts| { +// var operand_layouts: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; +// for (layouts) |ol| { +// operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); +// } +// attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); +// } else { +// const operand_layouts = blk: { +// var ret: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; +// for (inputs) |input| { +// const ranked_type = input.getType().as(mlir.RankedTensorType).?; +// const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; +// ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); +// } +// break :blk ret; +// }; +// attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); +// } + +// if (opts.result_layouts) |layouts| { +// var result_layouts: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; +// for (layouts) |rl| { +// result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); +// } +// attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); +// } else { +// const result_layouts = blk: { +// var ret: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; +// for (res_types) |t| { +// const ranked_t = t.as(mlir.RankedTensorType).?; +// const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; +// ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); +// } +// break :blk ret; +// }; +// attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); +// } + +// attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); + +// return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ +// .operands = inputs, +// .results = res_types, +// .attributes = attrs.constSlice(), +// .location = location, +// }); +// } + +// pub const DotDimensionNumbersAttribute = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; +// const Self = DotDimensionNumbersAttribute; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub fn init(ctx: *mlir.Context, args: struct { +// lhs_batching_dimensions: []const i64, +// rhs_batching_dimensions: []const i64, +// lhs_contracting_dimensions: []const i64, +// rhs_contracting_dimensions: []const i64, +// }) Self { +// return .{ +// ._inner = c.stablehloDotDimensionNumbersGet( +// ctx._inner, +// @intCast(args.lhs_batching_dimensions.len), +// args.lhs_batching_dimensions.ptr, +// @intCast(args.rhs_batching_dimensions.len), +// args.rhs_batching_dimensions.ptr, +// @intCast(args.lhs_contracting_dimensions.len), +// args.lhs_contracting_dimensions.ptr, +// @intCast(args.rhs_contracting_dimensions.len), +// args.rhs_contracting_dimensions.ptr, +// ), +// }; +// } + +// pub fn getLhsBatchingDimensionsSize(self: Self) usize { +// return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self._inner)); +// } + +// pub fn getLhsBatchingDimensionsElem(self: Self, pos: usize) i64 { +// return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self._inner, @intCast(pos)); +// } + +// pub fn getRhsBatchingDimensionsSize(self: Self) usize { +// return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self._inner)); +// } + +// pub fn getRhsBatchingDimensionsElem(self: Self, pos: usize) i64 { +// return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self._inner, @intCast(pos)); +// } + +// pub fn getLhsContractingDimensionsSize(self: Self) usize { +// return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self._inner)); +// } + +// pub fn getLhsContractingDimensionsElem(self: Self, pos: usize) i64 { +// return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self._inner, @intCast(pos)); +// } + +// pub fn getRhsContractingDimensionsSize(self: Self) usize { +// return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self._inner)); +// } + +// pub fn getRhsContractingDimensionsElem(self: Self, pos: usize) i64 { +// return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self._inner, @intCast(pos)); +// } +// }; + +// pub const GatherDimensionNumbersAttribute = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; +// const Self = GatherDimensionNumbersAttribute; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub fn init( +// ctx: *mlir.Context, +// offset_dims: []const i64, +// collapsed_slice_dims: []const i64, +// operand_batching_dims: []const i64, +// start_indices_batching_dims: []const i64, +// start_index_map: []const i64, +// index_vector_dim: i64, +// ) Self { +// return .{ +// ._inner = c.stablehloGatherDimensionNumbersGet( +// ctx._inner, +// @intCast(offset_dims.len), +// offset_dims.ptr, +// @intCast(collapsed_slice_dims.len), +// collapsed_slice_dims.ptr, +// @intCast(operand_batching_dims.len), +// operand_batching_dims.ptr, +// @intCast(start_indices_batching_dims.len), +// start_indices_batching_dims.ptr, +// @intCast(start_index_map.len), +// start_index_map.ptr, +// index_vector_dim, +// ), +// }; +// } + +// pub fn getOffsetDimsSize(self: Self) usize { +// return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self._inner)); +// } + +// pub fn getOffsetDimsElem(self: Self, pos: usize) i64 { +// return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self._inner, @intCast(pos)); +// } + +// pub fn getCollapsedSliceDimsSize(self: Self) usize { +// return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self._inner)); +// } + +// pub fn getCollapsedSliceDimsElem(self: Self, pos: usize) i64 { +// return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self._inner, @intCast(pos)); +// } + +// pub fn getStartIndexMapSize(self: Self) usize { +// return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self._inner)); +// } + +// pub fn getOperandBatchingDimsSize(self: Self) usize { +// return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self._inner)); +// } + +// pub fn getOperandBatchingDimsElem(self: Self, pos: usize) i64 { +// return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self._inner, @intCast(pos)); +// } + +// pub fn getStartIndicesBatchingDimsSize(self: Self) usize { +// return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self._inner)); +// } + +// pub fn getStartIndicesBatchingDimsElem(self: Self, pos: usize) i64 { +// return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self._inner, @intCast(pos)); +// } + +// pub fn getStartIndexMapElem(self: Self, pos: usize) i64 { +// return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self._inner, @intCast(pos)); +// } + +// pub fn getIndexVectorDim(self: Self) usize { +// return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self._inner)); +// } +// }; + +// pub const ConvDimensionNumbersAttribute = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; +// const Self = ConvDimensionNumbersAttribute; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub fn init(ctx: *mlir.Context, args: struct { +// input_batch_dimension: i64, +// input_feature_dimension: i64, +// input_spatial_dimensions: []const i64, +// kernel_input_feature_dimension: i64, +// kernel_output_feature_dimension: i64, +// kernel_spatial_dimensions: []const i64, +// output_batch_dimension: i64, +// output_feature_dimension: i64, +// output_spatial_dimensions: []const i64, +// }) Self { +// return .{ +// ._inner = c.stablehloConvDimensionNumbersGet( +// ctx._inner, +// args.input_batch_dimension, +// args.input_feature_dimension, +// @intCast(args.input_spatial_dimensions.len), +// args.input_spatial_dimensions.ptr, +// args.kernel_input_feature_dimension, +// args.kernel_output_feature_dimension, +// @intCast(args.kernel_spatial_dimensions.len), +// args.kernel_spatial_dimensions.ptr, +// args.output_batch_dimension, +// args.output_feature_dimension, +// @intCast(args.output_spatial_dimensions.len), +// args.output_spatial_dimensions.ptr, +// ), +// }; +// } + +// pub fn getInputBatchDimension(self: Self) i64 { +// return c.stablehloConvDimensionNumbersGetInputBatchDimension(self._inner); +// } + +// pub fn getInputFeatureDimension(self: Self) i64 { +// return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self._inner); +// } + +// pub fn getInputSpatialDimensionsSize(self: Self) usize { +// return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self._inner)); +// } + +// pub fn getInputSpatialDimensionsElem(self: Self, pos: usize) i64 { +// return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self._inner, @intCast(pos)); +// } + +// pub fn getKernelInputFeatureDimension(self: Self) i64 { +// return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self._inner); +// } + +// pub fn getKernelOutputFeatureDimension(self: Self) i64 { +// return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self._inner); +// } + +// pub fn getKernelSpatialDimensionsSize(self: Self) usize { +// return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self._inner)); +// } + +// pub fn getKernelSpatialDimensionsElem(self: Self, pos: usize) i64 { +// return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self._inner, @intCast(pos)); +// } + +// pub fn getOutputBatchDimension(self: Self) i64 { +// return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self._inner); +// } + +// pub fn getOutputFeatureDimension(self: Self) i64 { +// return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self._inner); +// } + +// pub fn getOutputSpatialDimensionsSize(self: Self) usize { +// return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self._inner)); +// } + +// pub fn getOutputSpatialDimensionsElem(self: Self, pos: usize) i64 { +// return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self._inner, @intCast(pos)); +// } +// }; + +// pub const OutputOperandAliasAttribute = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; +// pub const asAttr = mlir.Attribute.fromAny(OutputOperandAliasAttribute); +// pub const eql = mlir.Attribute.eqlAny(OutputOperandAliasAttribute); + +// pub fn init( +// ctx: *mlir.Context, +// output_tuple_indices: []const i64, +// operand_index: i64, +// operand_tuple_indices: []const i64, +// ) OutputOperandAliasAttribute { +// return .{ ._inner = c.stablehloOutputOperandAliasGet( +// ctx._inner, +// @intCast(output_tuple_indices.len), +// output_tuple_indices.ptr, +// @intCast(operand_index), +// @intCast(operand_tuple_indices.len), +// operand_tuple_indices.ptr, +// ) }; +// } +// }; + +// pub const PrecisionAttribute = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; +// const Self = PrecisionAttribute; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub const Precision = enum { +// DEFAULT, +// HIGH, +// HIGHEST, +// }; + +// pub fn init(ctx: *mlir.Context, value: Precision) Self { +// return .{ ._inner = c.stablehloPrecisionAttrGet(ctx._inner, stringRef(@tagName(value))) }; +// } + +// pub fn getValue(self: Self) Precision { +// const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self._inner)); +// return std.meta.stringToEnum(Precision, value) orelse unreachable; +// } +// }; + +// pub const ComparisonDirection = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; +// const Self = ComparisonDirection; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub const Direction = enum { +// EQ, +// NE, +// GE, +// GT, +// LE, +// LT, +// }; + +// pub fn init(ctx: *mlir.Context, value: Direction) Self { +// return .{ ._inner = c.stablehloComparisonDirectionAttrGet(ctx._inner, stringRef(@tagName(value))) }; +// } + +// pub fn getValue(self: Self) Direction { +// const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self._inner)); +// return std.meta.stringToEnum(Direction, value) orelse unreachable; +// } +// }; + +// pub const CompareType = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; +// const Self = CompareType; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub const Type = enum { +// SIGNED, +// UNSIGNED, +// FLOAT, +// TOTALORDER, +// }; + +// pub fn init(ctx: *mlir.Context, value: Type) Self { +// return .{ ._inner = c.stablehloComparisonTypeAttrGet(ctx._inner, stringRef(@tagName(value))) }; +// } + +// pub fn getValue(self: Self) Type { +// const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self._inner)); +// return std.meta.stringToEnum(Type, value) orelse unreachable; +// } +// }; + +// pub const Transpose = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsATransposeAttr; +// const Self = Transpose; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub const Type = enum { +// NO_TRANSPOSE, +// TRANSPOSE, +// ADJOINT, +// }; + +// pub fn init(ctx: *mlir.Context, value: Type) Self { +// return .{ ._inner = c.stablehloTransposeAttrGet(ctx._inner, stringRef(@tagName(value))) }; +// } + +// pub fn getValue(self: Self) Type { +// const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self._inner)); +// return std.meta.stringToEnum(Type, value) orelse unreachable; +// } +// }; + +// pub const FftType = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; +// const Self = FftType; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub const Type = enum { +// FFT, +// IFFT, +// RFFT, +// IRFFT, +// }; + +// pub fn init(ctx: *mlir.Context, value: Type) Self { +// return .{ ._inner = c.stablehloFftTypeAttrGet(ctx._inner, stringRef(@tagName(value))) }; +// } + +// pub fn getValue(self: Self) Type { +// const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self._inner)); +// return std.meta.stringToEnum(Type, value) orelse unreachable; +// } +// }; + +// pub const RngDistribution = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; +// const Self = RngDistribution; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub const Type = enum { +// UNIFORM, +// NORMAL, +// }; + +// pub fn init(ctx: *mlir.Context, value: Type) Self { +// return .{ ._inner = c.stablehloRngDistributionAttrGet(ctx._inner, stringRef(@tagName(value))) }; +// } + +// pub fn getValue(self: Self) Type { +// const value = mlir.fromStringRef(c.stablehloRngDistributionAttrGetValue(self._inner)); +// return std.meta.stringToEnum(Type, value) orelse unreachable; +// } +// }; + +// pub const RngAlgorithm = struct { +// _inner: c.MlirAttribute, + +// pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; +// const Self = RngAlgorithm; +// pub const asAttr = mlir.Attribute.fromAny(Self); +// pub const eql = mlir.Attribute.eqlAny(Self); + +// pub const Type = enum { +// DEFAULT, +// THREE_FRY, +// PHILOX, +// }; + +// pub fn init(ctx: *mlir.Context, value: Type) Self { +// return .{ ._inner = c.stablehloRngAlgorithmAttrGet(ctx._inner, stringRef(@tagName(value))) }; +// } + +// pub fn getValue(self: Self) Type { +// const value = mlir.fromStringRef(c.stablehloRngAlgorithmAttrGetValue(self._inner)); +// return std.meta.stringToEnum(Type, value) orelse unreachable; +// } +// }; + +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: mlir.StringCallbackCtx = .{ .writer = &writer }; + const new_args = args ++ .{ mlir.stringCallback, &sctx }; + @call(.auto, streamFn, new_args); + return writer.buffered(); +} + +pub fn versionFromCompatibilityRequirement(buffer: []u8, requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { + return stringFromStream(buffer, c.stablehloVersionFromCompatibilityRequirement, .{requirement}); +} + +pub fn smallerVersion(version1: []const u8, version2: []const u8) []const u8 { var buf: [32]u8 = undefined; - - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - _ = c.stablehloGetSmallerVersion(mlir.stringRef(version1), mlir.stringRef(version2), (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); - - return if (std.mem.eql(u8, buf[0..stream.pos], version1)) version1 else version2; + const result = stringFromStream(&buf, c.stablehloGetSmallerVersion, .{ stringRef(version1), stringRef(version2) }); + return if (std.mem.eql(u8, result, version1)) version1 else version2; } -pub fn getCurrentVersion() []const u8 { +pub fn currentVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; var str: []const u8 = undefined; var once = std.once(call); fn call() void { - var stream = std.io.fixedBufferStream(&buf); - var writer_ = stream.writer(); - const ContextWriter = @TypeOf(writer_); - - c.stablehloGetCurrentVersion((struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const writer: *ContextWriter = @ptrCast(@alignCast(userdata)); - _ = writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &writer_); - - str = buf[0..stream.pos]; + str = stringFromStream(&buf, c.stablehloGetCurrentVersion, .{}); } }; @@ -1327,25 +1250,14 @@ pub fn getCurrentVersion() []const u8 { return state.str; } -pub fn getMinimumVersion() []const u8 { +pub fn minimumVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; var str: []const u8 = undefined; var once = std.once(call); fn call() void { - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - c.stablehloGetMinimumVersion((struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); - - str = buf[0..stream.pos]; + str = stringFromStream(&buf, c.stablehloGetMinimumVersion, .{}); } }; @@ -1353,14 +1265,35 @@ pub fn getMinimumVersion() []const u8 { return state.str; } -pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: anytype) !void { - var context = .{ .writer = writer }; - const WriterContext = @TypeOf(context); +pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromStringRef( + stringRef(bytecode), + stringRef(target_version), + mlir.stringCallback, + &sctx, + ); + if (sctx.err) |err| { + return err; + } + if (result.ptr == null) { + return error.InvalidMlirBytecodeVersion; + } +} - try mlir.successOr(c.stablehloSerializePortableArtifactFromStringRef(mlir.stringRef(bytecode), mlir.stringRef(target_version), (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context), error.InvalidMlirBytecodeVersion); +pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromModule( + module.ptr(), + stringRef(target_version), + mlir.stringCallback, + &sctx, + true, + ); + if (sctx.err) |err| { + return err; + } + if (c.mlirLogicalResultIsFailure(result)) { + return error.InvalidMlirBytecodeVersion; + } } diff --git a/mlir/main.zig b/mlir/main.zig index 177d2a5a..3963a9ae 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -4,15 +4,18 @@ const dialects = @import("mlir/dialects"); const mlir = @import("mlir2"); pub fn main() !void { + const allocator = std.heap.c_allocator; const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); // const handle = mlir.DialectHandle.fromString("func"); - inline for (.{ "arith", "func" }) |d| { + inline for (.{ "func", "stablehlo" }) |d| { registry.registerDialect(d); } + // mlir.registerPasses("AllStablehlo"); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = true, @@ -26,6 +29,16 @@ pub fn main() !void { const unk = mlir.Location.unknown(ctx); _ = unk; // autofix + var serialized: std.Io.Writer.Allocating = .init(allocator); + defer serialized.deinit(); + + // const stablehlo_version = blk: { + // if (dialects.stablehlo.currentVersion()) |requested_version| { + // break :blk dialects.stablehlo.stablehloGetSmallerVersion(requested_version, dialects.stablehlo.getCurrentVersion()); + // } + // break :blk dialects.stablehlo.minimumVersion(); + // }; + // const attr = mlir.integerAttribute(ctx, @as(i32, 5)); // _ = attr; // autofix @@ -38,31 +51,31 @@ pub fn main() !void { // }); // _ = dict; // autofix - const int64 = mlir.integerType(ctx, 64, .signless); + const module = mlir.Module.init(.unknown(ctx)); + defer module.deinit(); - const block = mlir.Block.init(&.{ int64, int64 }, &.{ .unknown(ctx), .unknown(ctx) }); + const tensor_i32 = mlir.rankedTensorType(&.{ 4096, 4096 }, mlir.integerType(ctx, .i32)); - block.appendOwnedOperation(dialects.func.return_(ctx, &.{block.argument(0)}, .unknown(ctx))); - - const op = dialects.func.func(ctx, .{ + module.body().appendOwnedOperation(dialects.func.func(ctx, .{ .name = "pouet", - .args = &.{ int64, int64 }, - .results = &.{int64}, - .block = block, + .visibility = .private, + .block = blk: { + const body = mlir.Block.init(&.{ tensor_i32, tensor_i32 }, &.{ .unknown(ctx), .unknown(ctx) }); + const add_op = dialects.stablehlo.add(ctx, body.argument(0), body.argument(1), .unknown(ctx)).appendTo(body); + const mul_op = dialects.stablehlo.multiply(ctx, add_op.result(0), body.argument(1), .unknown(ctx)).appendTo(body); + _ = dialects.func.return_(ctx, mul_op.result(0), .unknown(ctx)).appendTo(body); + break :blk body; + }, .location = .unknown(ctx), - }); + })); - // const op = try mlir.Operation.parse(ctx, "func.func private @pouet()", "pouet"); + std.debug.print(">>>>>>\n{f}\n", .{module.operation().fmt(.{ .print_generic_op_form = false })}); + std.debug.print(">>>>>>\n{f}\n", .{module.operation().fmt(.{ .print_generic_op_form = true })}); - // // // const mod = mlir.Module.init(mlir.Location.unknown(ctx)); - - // const op = mlir.Operation.make(ctx, "arith.addi", .{ - // .operands = .{ .flat = &.{ block.argument(0), block.argument(1) } }, - // .results = .{ .variadic = &.{ &.{ int64, int64 }, &.{int64} } }, - // }); - // defer op.deinit(); - - std.debug.print(">>>>>> {f}\n", .{op.fmt(.{ .print_generic_op_form = true })}); + // dialects.stablehlo.serializePortableArtifact2(module, dialects.stablehlo.minimumVersion(), &serialized.writer) catch |err| { + // std.debug.print("failed to serialize to portable artifact: {}\n", .{err}); + // return err; + // }; // const body = mod.body(); // _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); diff --git a/mlir/registry.zig b/mlir/registry.zig index b4b17d44..6c999b83 100644 --- a/mlir/registry.zig +++ b/mlir/registry.zig @@ -250,13 +250,57 @@ pub const IndexType = opaque { } }; -pub const Signedness = enum { - signless, - signed, - unsigned, +pub const IntegerTypes = enum { + i1, + i4, + i8, + i16, + i32, + i64, + i128, + u1, + u4, + u8, + u16, + u32, + u64, + u128, + si1, + si4, + si8, + si16, + si32, + si64, + si128, + + pub fn bitwidth(self: IntegerTypes) usize { + return switch (self) { + .i1, .u1, .si1 => 1, + .i4, .u4, .si4 => 4, + .i8, .u8, .si8 => 8, + .i16, .u16, .si16 => 16, + .i32, .u32, .si32 => 32, + .i64, .u64, .si64 => 64, + .i128, .u128, .si128 => 128, + }; + } + + pub fn signedness(self: IntegerTypes) IntegerType.Signedness { + return switch (self) { + .i1, .i4, .i8, .i16, .i32, .i64, .i128 => .signless, + .si1, .si4, .si8, .si16, .si32, .si64, .si128 => .signed, + .u1, .u4, .u8, .u16, .u32, .u64, .u128 => .unsigned, + }; + } }; pub const IntegerType = opaque { + pub const Signedness = enum { + signless, + signed, + unsigned, + }; + const M = Methods(IntegerType, c.MlirType); pub const isAFn = c.mlirTypeIsAInteger; @@ -264,7 +308,11 @@ pub const IntegerType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); - fn get(ctx: *Context, bitwidth: usize, sign: Signedness) *const IntegerType { + fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { + return exact(ctx, it.bitwidth(), it.signedness()); + } + + fn exact(ctx: *Context, bitwidth: usize, sign: Signedness) *const IntegerType { return @ptrCast(switch (sign) { .signless => c.mlirIntegerTypeGet(ctx.ptr(), @intCast(bitwidth)).ptr, .signed => c.mlirIntegerTypeSignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, @@ -273,8 +321,8 @@ pub const IntegerType = opaque { } }; -pub fn integerType(ctx: *Context, bitwidth: usize, sign: Signedness) *const Type { - return @ptrCast(IntegerType.get(ctx, bitwidth, sign)); +pub fn integerType(ctx: *Context, it: IntegerTypes) *const Type { + return @ptrCast(IntegerType.get(ctx, it)); } pub const FloatTypes = enum { @@ -387,14 +435,9 @@ pub const IntegerAttribute = opaque { pub const eql = M.eql(c.mlirAttributeEqual); pub const format = M.format(c.mlirAttributePrint); - pub fn init(ctx: *Context, value_: anytype) *const IntegerAttribute { - const ti = @typeInfo(@TypeOf(value_)).int; - const sign: Signedness = switch (ti.signedness) { - .signed => .signless, - .unsigned => .unsigned, - }; + pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { return @ptrCast(c.mlirIntegerAttrGet( - IntegerType.get(ctx, @intCast(ti.bits), sign).ptr(), + IntegerType.get(ctx, it).ptr(), @intCast(value_), ).ptr); } @@ -408,8 +451,8 @@ pub const IntegerAttribute = opaque { } }; -pub fn integerAttribute(ctx: *Context, value_: anytype) *const Attribute { - return @ptrCast(IntegerAttribute.init(ctx, value_)); +pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, it, value_)); } pub const BoolAttribute = opaque { @@ -528,26 +571,26 @@ pub const Identifier = opaque { pub const Module = opaque { const M = Methods(Module, c.MlirModule); - const ptr = M.ptr; + pub const ptr = M.ptr; - pub fn init(loc: *const Location) *const Module { - return @ptrCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr); + pub fn init(loc: *const Location) *Module { + return @constCast(@ptrCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr)); } pub fn deinit(self: *Module) void { c.mlirModuleDestroy(self.ptr()); } - pub fn fromOperation(op: *Operation) *const Module { - return @ptrCast(c.mlirModuleFromOperation(op.ptr()).ptr); + pub fn fromOperation(op: *Operation) *Module { + return @constCast(@ptrCast(c.mlirModuleFromOperation(op.ptr()).ptr)); } pub fn operation(self: *const Module) *Operation { return @ptrCast(c.mlirModuleGetOperation(self.ptr()).ptr); } - pub fn parse(ctx: *Context, source: []const u8) Error!*const Module { - return @ptrCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir); + pub fn parse(ctx: *Context, source: []const u8) Error!*Module { + return @constCast(@ptrCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir)); } pub fn context(self: *const Module) *Context { @@ -608,6 +651,10 @@ pub const Block = opaque { pub fn detach(self: *Block) void { c.mlirBlockDetach(self.ptr()); } + + pub fn terminator(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetTerminator(self.ptr()).ptr); + } }; pub const NamedAttribute = struct { @@ -829,6 +876,11 @@ pub const Operation = opaque { return @ptrCast(c.mlirOperationGetBlock(self.ptr()).ptr); } + pub fn appendTo(self: *Operation, block_: *Block) *Operation { + block_.appendOwnedOperation(self); + return self; + } + pub fn parent(self: *const Operation) ?*Operation { return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); } @@ -853,7 +905,7 @@ pub const Operation = opaque { return @intCast(c.mlirOperationGetNumResults(self.ptr())); } - pub fn result(self: *const Operation, index: usize) Value { + pub fn result(self: *const Operation, index: usize) *const Value { return @ptrCast(c.mlirOperationGetResult(self.ptr(), @intCast(index)).ptr); } @@ -930,7 +982,7 @@ pub const Operation = opaque { operands: ?Operands = null, results: ?ResultTypes = null, result_type_inference: bool = false, - attributes: ?[]const AttrTuple = null, + attributes: ?[]const NamedAttribute = null, blocks: ?[]const *Block = null, verify: bool = true, location: ?*const Location = null, @@ -968,11 +1020,7 @@ pub const Operation = opaque { }, }; if (args.attributes) |attrs| { - for (attrs) |attr_tuple| { - const attr_name, const attr = attr_tuple; - const named_attr = NamedAttribute.named(ctx, attr_name, attr); - state.addAttributes(&.{named_attr}); - } + state.addAttributes(attrs); } if (args.blocks) |blocks| { const region = Region.init(); @@ -982,6 +1030,7 @@ pub const Operation = opaque { state.addOwnedRegions(&.{region}); } const new_op = try Operation.init(&state); + errdefer new_op.deinit(); if (args.verify and new_op.verify() == false) { log.err("Failed to verify MLIR operation:\n{f}", .{ new_op.fmt(.{ @@ -1252,11 +1301,11 @@ pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { return @ptrCast(getFn(ctx.ptr(), @intCast(values.len), @ptrCast(values)).ptr); } - pub fn get(self: *const Self, pos: usize) ElementTypeZig { + pub fn element(self: *const Self, pos: usize) ElementTypeZig { return getElementFn(self.ptr(), @intCast(pos)); } - pub fn len(self: *const Self) usize { + pub fn numElements(self: *const Self) usize { return @intCast(c.mlirDenseArrayGetNumElements(self.ptr())); } }; @@ -1278,12 +1327,12 @@ pub const ArrayAttribute = opaque { return @ptrCast(c.mlirArrayAttrGet(ctx.ptr(), @intCast(attrs.len), @ptrCast(attrs)).ptr); } - pub fn size(self: *const ArrayAttribute) usize { - return @intCast(c.mlirArrayAttrGetNumElements(self.ptr())); + pub fn element(self: *const ArrayAttribute, index: usize) Attribute { + return @ptrCast(c.mlirArrayAttrGetElement(self.ptr(), @intCast(index)).ptr); } - pub fn get(self: *const ArrayAttribute, index: usize) Attribute { - return @ptrCast(c.mlirArrayAttrGetElement(self.ptr(), @intCast(index)).ptr); + pub fn numElements(self: *const ArrayAttribute) usize { + return @intCast(c.mlirArrayAttrGetNumElements(self.ptr())); } }; @@ -1334,3 +1383,20 @@ pub const FlatSymbolRefAttribute = opaque { pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); } + +pub const UnitAttribute = opaque { + const M = Methods(UnitAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAUnit; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn get(ctx: *Context) *const UnitAttribute { + return @ptrCast(c.mlirUnitAttrGet(ctx.ptr()).ptr); + } +}; + +pub fn unitAttribute(ctx: *Context) *const Attribute { + return @ptrCast(UnitAttribute.get(ctx)); +} From 9b67afc4583bf283c1523bb1a308c3f1cc79b0a7 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Aug 2025 02:07:41 +0200 Subject: [PATCH 04/47] WIP --- mlir/BUILD.bazel | 3 +- mlir/dialects/BUILD.bazel | 1 + mlir/dialects/func.zig | 9 +- mlir/dialects/stablehlo/stablehlo.zig | 248 +++++++++++++++----------- mlir/main.zig | 6 +- mlir/mlir.h | 3 +- 6 files changed, 158 insertions(+), 112 deletions(-) diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index f0ba476f..73f1d12d 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -8,8 +8,7 @@ cc_library( deps = [ "@llvm-project//mlir:CAPIArith", "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:CAPIMath", - "@llvm-project//mlir:CAPISCF", + "@llvm-project//mlir:CAPIMemRef", "@llvm-project//mlir:CAPITransforms", ], ) diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 34400ca9..9c4bd6b1 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -5,6 +5,7 @@ zig_library( srcs = [ # "arith.zig", "func.zig", + "memref.zig", # "math.zig", # "scf.zig", # "tensor.zig", diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 178aab75..173b8381 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -16,7 +16,7 @@ const FuncOpArgs = struct { results_attributes: ?[]const *const mlir.Attribute = null, block: *mlir.Block, location: *const mlir.Location, - no_inline: bool = true, + no_inline: bool = false, visibility: Visibility = .public, }; @@ -24,7 +24,8 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .{}; var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .{}; - var attr_tuples_buffer = stdx.BoundedArray(mlir.NamedAttribute, 16).fromSlice(&.{ + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .{}; + attr_tuples_buffer.appendSliceAssumeCapacity(&.{ .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( @@ -45,7 +46,7 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { break :results results_buffer.constSlice(); }, ))), - }) catch unreachable; + }); if (args.args_attributes) |args_attributes| { attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); @@ -57,7 +58,7 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); } - return mlir.Operation.make(ctx, "func.func", .{ + return .make(ctx, "func.func", .{ .blocks = &.{args.block}, .attributes = attr_tuples_buffer.constSlice(), .location = args.location, diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 83ea5669..feb9d49f 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -680,117 +680,163 @@ pub fn binaryOp(comptime op_name: []const u8) type { // }); // } -// pub const CustomCallOpts = struct { -// pub const ApiVersion = enum(i32) { -// original = 1, -// status_returning = 2, -// status_returning_unified = 3, -// typed_ffi = 4, -// }; +pub const CustomCallOpts = struct { + pub const ApiVersion = enum(i32) { + original = 1, + status_returning = 2, + status_returning_unified = 3, + typed_ffi = 4, + }; -// call_target_name: []const u8, -// has_side_effect: bool = false, -// backend_config: ?*const mlir.Attribute, -// operand_layouts: ?[]const []const usize = null, -// result_layouts: ?[]const []const usize = null, -// output_operand_aliases: ?[]const i64 = null, -// additional_attributes: ?[]const mlir.AttrTuple = &.{}, -// api_version: ApiVersion, -// }; + pub const BackendConfig = union(ApiVersion) { + original: *const mlir.StringAttribute, + status_returning: *const mlir.StringAttribute, + status_returning_unified: *const mlir.StringAttribute, + typed_ffi: *const mlir.DictionaryAttribute, + }; -// pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, opts: CustomCallOpts, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { -// const MAX_OPERANDS = 64; -// const MAX_RESULTS = 16; + call_target_name: []const u8, + has_side_effect: bool = false, + api_version: ?ApiVersion, + backend_config: ?BackendConfig = null, + operand_layouts: ?[]const []const usize = null, + result_layouts: ?[]const []const usize = null, + output_operand_aliases: ?[]const i64 = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, +}; -// const backend_config = opts.backend_config orelse mlir.Attribute.string(ctx, ""); -// if (@intFromEnum(opts.api_version) < @intFromEnum(CustomCallOpts.ApiVersion.typed_ffi)) { -// stdx.debug.assert( -// backend_config.isA(mlir.StringAttribute), -// "API version < 4 requires a string as backend_config, got {}", -// .{backend_config}, -// ); -// } else { -// stdx.debug.assert( -// backend_config.isA(mlir.DictionaryAttribute), -// "API version >= 4 requires a dictionary as backend_config, got {}", -// .{backend_config}, -// ); -// } +pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, opts: CustomCallOpts, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + const MAX_OPERANDS = 64; + _ = MAX_OPERANDS; // autofix + const MAX_RESULTS = 16; -// var attrs: stdx.BoundedArray(mlir.AttrTuple, 32) = .{}; -// attrs.appendSliceAssumeCapacity(&[_]mlir.AttrTuple{ -// .{ "api_version", .int(ctx, .i32, @intFromEnum(opts.api_version)) }, -// .{ "call_target_name", .string(ctx, opts.call_target_name) }, -// .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, -// .{ "backend_config", backend_config }, -// }); + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .{}; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "call_target_name", mlir.stringAttribute(ctx, opts.call_target_name)), + .named(ctx, "has_side_effect", mlir.boolAttribute(ctx, opts.has_side_effect)), + }); + attrs.appendSliceAssumeCapacity(opts.additional_attributes); -// { -// var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; -// for (opts.output_operand_aliases) |alias| { -// output_operand_aliases.appendAssumeCapacity( -// OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), -// ); -// } -// attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); -// } + if (opts.api_version orelse opts.backend_config) |v| { + attrs.appendAssumeCapacity(.named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v)))); + } + if (opts.backend_config) |bc| { + switch (bc) { + else => |v| attrs.appendAssumeCapacity(.named(ctx, "backend_config", @ptrCast(v))), + } + } -// const MINOR_TO_MAJOR = blk: { -// const MAX_RANK = 8; -// var ret: [MAX_RANK]usize = undefined; -// for (0..MAX_RANK) |i| { -// ret[i] = @intCast(MAX_RANK - i - 1); -// } -// break :blk ret; -// }; + { + var output_operand_aliases: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; + for (opts.output_operand_aliases) |alias| { + _ = alias; // autofix + // output_operand_aliases.appendAssumeCapacity( + // OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), + // ); + } + attrs.appendAssumeCapacity(.named(ctx, "output_operand_aliases", mlir.arrayAttribute(ctx, output_operand_aliases.constSlice()))); + } -// if (opts.operand_layouts) |layouts| { -// var operand_layouts: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; -// for (layouts) |ol| { -// operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); -// } -// attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); -// } else { -// const operand_layouts = blk: { -// var ret: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; -// for (inputs) |input| { -// const ranked_type = input.getType().as(mlir.RankedTensorType).?; -// const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; -// ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); -// } -// break :blk ret; -// }; -// attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); -// } + // var attrs: stdx.BoundedArray(mlir.AttrTuple, 32) = .{}; + // attrs.appendSliceAssumeCapacity(&[_]mlir.AttrTuple{ + // .{ "api_version", .int(ctx, .i32, @intFromEnum(opts.api_version)) }, + // .{ "call_target_name", .string(ctx, opts.call_target_name) }, + // .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, + // .{ "backend_config", backend_config }, + // }); -// if (opts.result_layouts) |layouts| { -// var result_layouts: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; -// for (layouts) |rl| { -// result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); -// } -// attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); -// } else { -// const result_layouts = blk: { -// var ret: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; -// for (res_types) |t| { -// const ranked_t = t.as(mlir.RankedTensorType).?; -// const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; -// ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); -// } -// break :blk ret; -// }; -// attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); -// } + // { + // var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; + // for (opts.output_operand_aliases) |alias| { + // output_operand_aliases.appendAssumeCapacity( + // OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), + // ); + // } + // attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); + // } -// attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); + // const MINOR_TO_MAJOR = blk: { + // const MAX_RANK = 8; + // var ret: [MAX_RANK]usize = undefined; + // for (0..MAX_RANK) |i| { + // ret[i] = @intCast(MAX_RANK - i - 1); + // } + // break :blk ret; + // }; -// return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ -// .operands = inputs, -// .results = res_types, -// .attributes = attrs.constSlice(), -// .location = location, -// }); -// } + // if (opts.operand_layouts) |layouts| { + // var operand_layouts: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; + // for (layouts) |ol| { + // operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); + // } + // attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); + // } else { + // const operand_layouts = blk: { + // var ret: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; + // for (inputs) |input| { + // const ranked_type = input.getType().as(mlir.RankedTensorType).?; + // const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; + // ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); + // } + // break :blk ret; + // }; + // attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); + // } + + // if (opts.result_layouts) |layouts| { + // var result_layouts: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; + // for (layouts) |rl| { + // result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); + // } + // attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); + // } else { + // const result_layouts = blk: { + // var ret: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; + // for (res_types) |t| { + // const ranked_t = t.as(mlir.RankedTensorType).?; + // const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; + // ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); + // } + // break :blk ret; + // }; + // attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); + // } + + // attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); + + return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + const result_type = mlir.integerType(ctx, .i64); + return custom_call(ctx, &.{value}, .{ + .call_target_name = "CreateBuffer", + .api_version = .typed_ffi, + }, &.{result_type}, location); +} + +pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + const result_type = mlir.integerType(ctx, .i64); + const shape = value.type_().as(mlir.ShapedType).?; + _ = shape; // autofix + return custom_call(ctx, &.{value}, .{ + .call_target_name = "Pin", + .api_version = .typed_ffi, + }, &.{result_type}, location); +} + +pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + const result_type = mlir.integerType(ctx, .i64); + return custom_call(ctx, &.{value}, .{ + .call_target_name = "Unpin", + .api_version = .typed_ffi, + }, &.{result_type}, location); +} // pub const DotDimensionNumbersAttribute = struct { // _inner: c.MlirAttribute, diff --git a/mlir/main.zig b/mlir/main.zig index 3963a9ae..4bf94117 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -10,7 +10,7 @@ pub fn main() !void { // const handle = mlir.DialectHandle.fromString("func"); - inline for (.{ "func", "stablehlo" }) |d| { + inline for (.{ "memref", "func", "stablehlo" }) |d| { registry.registerDialect(d); } @@ -56,7 +56,7 @@ pub fn main() !void { const tensor_i32 = mlir.rankedTensorType(&.{ 4096, 4096 }, mlir.integerType(ctx, .i32)); - module.body().appendOwnedOperation(dialects.func.func(ctx, .{ + _ = dialects.func.func(ctx, .{ .name = "pouet", .visibility = .private, .block = blk: { @@ -67,7 +67,7 @@ pub fn main() !void { break :blk body; }, .location = .unknown(ctx), - })); + }).appendTo(module.body()); std.debug.print(">>>>>>\n{f}\n", .{module.operation().fmt(.{ .print_generic_op_form = false })}); std.debug.print(">>>>>>\n{f}\n", .{module.operation().fmt(.{ .print_generic_op_form = true })}); diff --git a/mlir/mlir.h b/mlir/mlir.h index 4ad1523e..633e11f6 100644 --- a/mlir/mlir.h +++ b/mlir/mlir.h @@ -2,8 +2,7 @@ #include #include #include -#include -#include +#include #include #include #include From 66bc168de2919c20a596417956726fcefb160894 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Aug 2025 02:22:01 +0200 Subject: [PATCH 05/47] WIP --- mlir/dialects/stablehlo/stablehlo.zig | 97 ++++++++++++--------------- mlir/registry.zig | 4 ++ 2 files changed, 48 insertions(+), 53 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index feb9d49f..41b99220 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -718,43 +718,30 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, opts: attrs.appendSliceAssumeCapacity(opts.additional_attributes); if (opts.api_version orelse opts.backend_config) |v| { - attrs.appendAssumeCapacity(.named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v)))); + attrs.appendAssumeCapacity( + .named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v))), + ); } - if (opts.backend_config) |bc| { - switch (bc) { - else => |v| attrs.appendAssumeCapacity(.named(ctx, "backend_config", @ptrCast(v))), + if (opts.backend_config) |backend_config| { + switch (backend_config) { + inline else => |v| attrs.appendAssumeCapacity( + .named(ctx, "backend_config", @ptrCast(v)), + ), } } - { - var output_operand_aliases: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; - for (opts.output_operand_aliases) |alias| { - _ = alias; // autofix - // output_operand_aliases.appendAssumeCapacity( - // OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), - // ); + if (opts.output_operand_aliases) |output_operand_aliases| { + var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; + for (output_operand_aliases) |alias| { + buffer.appendAssumeCapacity( + outputOperandAliasAttribute.init(ctx, .{ .operand_index = alias }), + ); } - attrs.appendAssumeCapacity(.named(ctx, "output_operand_aliases", mlir.arrayAttribute(ctx, output_operand_aliases.constSlice()))); + attrs.appendAssumeCapacity( + .named(ctx, "output_operand_aliases", mlir.arrayAttribute(ctx, buffer.constSlice())), + ); } - // var attrs: stdx.BoundedArray(mlir.AttrTuple, 32) = .{}; - // attrs.appendSliceAssumeCapacity(&[_]mlir.AttrTuple{ - // .{ "api_version", .int(ctx, .i32, @intFromEnum(opts.api_version)) }, - // .{ "call_target_name", .string(ctx, opts.call_target_name) }, - // .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, - // .{ "backend_config", backend_config }, - // }); - - // { - // var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - // for (opts.output_operand_aliases) |alias| { - // output_operand_aliases.appendAssumeCapacity( - // OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), - // ); - // } - // attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); - // } - // const MINOR_TO_MAJOR = blk: { // const MAX_RANK = 8; // var ret: [MAX_RANK]usize = undefined; @@ -802,8 +789,6 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, opts: // attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); // } - // attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); - return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ .operands = .{ .flat = inputs }, .results = .{ .flat = result_types }, @@ -1067,29 +1052,35 @@ pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir // } // }; -// pub const OutputOperandAliasAttribute = struct { -// _inner: c.MlirAttribute, +pub const OutputOperandAliasAttribute = opaque { + const M = mlir.Methods(OutputOperandAliasAttribute, c.MlirAttribute); -// pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; -// pub const asAttr = mlir.Attribute.fromAny(OutputOperandAliasAttribute); -// pub const eql = mlir.Attribute.eqlAny(OutputOperandAliasAttribute); + pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); -// pub fn init( -// ctx: *mlir.Context, -// output_tuple_indices: []const i64, -// operand_index: i64, -// operand_tuple_indices: []const i64, -// ) OutputOperandAliasAttribute { -// return .{ ._inner = c.stablehloOutputOperandAliasGet( -// ctx._inner, -// @intCast(output_tuple_indices.len), -// output_tuple_indices.ptr, -// @intCast(operand_index), -// @intCast(operand_tuple_indices.len), -// operand_tuple_indices.ptr, -// ) }; -// } -// }; + pub const InitArgs = struct { + output_tuple_indices: []const i64 = &.{}, + operand_index: i64, + operand_tuple_indices: []const i64 = &.{}, + }; + + pub fn init(ctx: *mlir.Context, args: InitArgs) *OutputOperandAliasAttribute { + return @ptrCast(c.stablehloOutputOperandAliasGet( + ctx.ptr(), + @intCast(args.output_tuple_indices.len), + @ptrCast(args.output_tuple_indices), + @intCast(args.operand_index), + @intCast(args.operand_tuple_indices.len), + @ptrCast(args.operand_tuple_indices), + ).ptr); + } +}; + +pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasAttribute.InitArgs) *mlir.Attribute { + return @ptrCast(OutputOperandAliasAttribute.init(ctx, args)); +} // pub const PrecisionAttribute = struct { // _inner: c.MlirAttribute, diff --git a/mlir/registry.zig b/mlir/registry.zig index 6c999b83..c08a2754 100644 --- a/mlir/registry.zig +++ b/mlir/registry.zig @@ -728,6 +728,10 @@ pub const DictionaryAttribute = opaque { } }; +pub fn dictionaryAttribute(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { + return @ptrCast(DictionaryAttribute.init(ctx, attributes)); +} + pub const Value = opaque { const M = Methods(Value, c.MlirValue); From 96760c0bc3a78bac89a6638dd0108eaf0607c3ac Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Aug 2025 09:29:37 +0200 Subject: [PATCH 06/47] WIP --- mlir/dialects/BUILD.bazel | 1 - mlir/dialects/func.zig | 2 +- mlir/dialects/stablehlo/stablehlo.zig | 7 ++-- mlir/registry.zig | 47 +++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 9c4bd6b1..34400ca9 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -5,7 +5,6 @@ zig_library( srcs = [ # "arith.zig", "func.zig", - "memref.zig", # "math.zig", # "scf.zig", # "tensor.zig", diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 173b8381..d316f160 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -58,7 +58,7 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); } - return .make(ctx, "func.func", .{ + return mlir.Operation.make(ctx, "func.func", .{ .blocks = &.{args.block}, .attributes = attr_tuples_buffer.constSlice(), .location = args.location, diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 41b99220..11b209b2 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -797,9 +797,8 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, opts: }); } -pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - const result_type = mlir.integerType(ctx, .i64); - return custom_call(ctx, &.{value}, .{ +pub fn createBuffer(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, .{ .call_target_name = "CreateBuffer", .api_version = .typed_ffi, }, &.{result_type}, location); @@ -812,7 +811,7 @@ pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.L return custom_call(ctx, &.{value}, .{ .call_target_name = "Pin", .api_version = .typed_ffi, - }, &.{result_type}, location); + }, &.{ result_type}, location); } pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { diff --git a/mlir/registry.zig b/mlir/registry.zig index c08a2754..cd12b6b0 100644 --- a/mlir/registry.zig +++ b/mlir/registry.zig @@ -1178,6 +1178,8 @@ pub const Region = opaque { pub const ShapedType = opaque { const M = Methods(ShapedType, c.MlirType); + pub const MAX_RANK = 16; + pub const isAFn = c.mlirTypeIsAShaped; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); @@ -1214,6 +1216,17 @@ pub const RankedTensorType = opaque { ).ptr); } + pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return init( + other.elementType(), + dims.constSlice(), + ); + } + pub fn elementType(self: *const RankedTensorType) *const Type { return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); } @@ -1404,3 +1417,37 @@ pub const UnitAttribute = opaque { pub fn unitAttribute(ctx: *Context) *const Attribute { return @ptrCast(UnitAttribute.get(ctx)); } + +pub const MemRefType = opaque { + const M = Methods(MemRefType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*Attribute) *const MemRefType { + return @ptrCast(c.mlirMemRefTypeGet(element_type, shape.len, @ptrCast(shape), layout, memory_space).ptr); + } + + pub fn fromShaped(other: *const ShapedType) *const MemRefType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return init( + other.elementType(), + dims.constSlice(), + null, + null, + ); + } + + pub fn shaped(self: *const MemRefType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { + return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); +} From 82ae988c665fddf1ebdc58977912b2be4c117219 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 09:54:08 +0200 Subject: [PATCH 07/47] stablehlo: bound cholesky --- mlir/dialects/stablehlo/stablehlo.zig | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 11b209b2..f4009c08 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -88,16 +88,16 @@ pub fn binaryOp(comptime op_name: []const u8) type { }; } -// pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.cholesky", .{ -// .operands = .{ .flat = &.{value} }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower)) }, -// }, -// .location = location, -// }); -// } +pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.cholesky", .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .attributes = &.{ + mlir.NamedAttribute.named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower))), + }, + .location = location, + }); +} // pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Value, max: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { // return mlir.Operation.make(ctx, "stablehlo.clamp", .{ From e43d42c50d3b84c412b8d628e00d76cb404ebfff Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 09:56:14 +0200 Subject: [PATCH 08/47] stablehlo: bound clamp --- mlir/dialects/stablehlo/stablehlo.zig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index f4009c08..383c36ff 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -99,13 +99,13 @@ pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, locat }); } -// pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Value, max: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.clamp", .{ -// .operands = .{ .flat = &.{ min, value, max } }, -// .result_type_inference = true, -// .location = location, -// }); -// } +pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Value, max: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.clamp", .{ + .operands = .{ .flat = &.{ min, value, max } }, + .result_type_inference = true, + .location = location, + }); +} // pub const DotPrecision = union(enum) { // fast, From d83e68a6cfb65bc802b1375b851b4d3999810e09 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 10:25:27 +0200 Subject: [PATCH 09/47] stablehlo: bound constant --- mlir/dialects/stablehlo/stablehlo.zig | 32 +++++++++++++-------------- mlir/registry.zig | 4 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 383c36ff..35fc701f 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -205,22 +205,22 @@ pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Valu // }); // } -// pub fn constant( -// ctx: *mlir.Context, -// dims: []const i64, -// elem_type: mlir.DenseElementsAttributeTypes, -// raw_bytes: []const u8, -// location: *const mlir.Location, -// ) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.constant", .{ -// .operands = .{ .flat = &.{} }, -// .results = &.{.tensor(dims, elem_type.mlirType(ctx))}, -// .attributes = &.{ -// .{ "value", .denseElementsFromBytes(ctx, dims, elem_type, raw_bytes) }, -// }, -// .location = location, -// }); -// } +pub fn constant( + ctx: *mlir.Context, + dims: []const i64, + elem_type: *const mlir.Type, + raw_bytes: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.constant", .{ + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, + .attributes = &.{ + mlir.NamedAttribute.named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.init(dims, elem_type).shaped(), raw_bytes)), + }, + .location = location, + }); +} // pub fn broadcast_in_dim(ctx: *mlir.Context, operand: *const mlir.Value, dims: []const i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { // return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ diff --git a/mlir/registry.zig b/mlir/registry.zig index cd12b6b0..78ec4698 100644 --- a/mlir/registry.zig +++ b/mlir/registry.zig @@ -1244,8 +1244,8 @@ pub const RankedTensorType = opaque { } }; -pub fn rankedTensorType(dimensions: []const i64, elemType: *const Type) *const Type { - return @ptrCast(RankedTensorType.init(dimensions, elemType)); +pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { + return @ptrCast(RankedTensorType.init(dimensions, elem_type)); } pub const DenseElementsAttribute = opaque { From bcb37bec363f4e70fa8a0b2198815d9c3248e5bc Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 10:37:59 +0200 Subject: [PATCH 10/47] stablehlo: bound broadcast_in_dim --- mlir/dialects/stablehlo/stablehlo.zig | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 35fc701f..d74b06f5 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -222,16 +222,22 @@ pub fn constant( }); } -// pub fn broadcast_in_dim(ctx: *mlir.Context, operand: *const mlir.Value, dims: []const i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ -// .operands = .{ .flat = &.{operand} }, -// .results = .{ .flat = &.{result_type} }, -// .attributes = &.{ -// .{ "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims) }, -// }, -// .location = location, -// }); -// } +pub fn broadcast_in_dim( + ctx: *mlir.Context, + operand: *const mlir.Value, + dims: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + mlir.NamedAttribute.named(ctx, "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims)), + }, + .location = location, + }); +} // pub fn transpose(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location, opts: struct { permutation: []const i64 }) *mlir.Operation { // return mlir.Operation.make(ctx, "stablehlo.transpose", .{ From b01d43c411de743c0f8a562e55fad8ffaa6a76f0 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 10:40:49 +0200 Subject: [PATCH 11/47] stablehlo: bound transpose --- mlir/dialects/stablehlo/stablehlo.zig | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index d74b06f5..3e259b40 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -239,16 +239,22 @@ pub fn broadcast_in_dim( }); } -// pub fn transpose(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location, opts: struct { permutation: []const i64 }) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.transpose", .{ -// .operands = .{ .flat = &.{value} }, -// .results = .{ .flat = &.{result_type} }, -// .attributes = &.{ -// .{ "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation) }, -// }, -// .location = location, -// }); -// } +pub fn transpose( + ctx: *mlir.Context, + value: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + opts: struct { permutation: []const i64 }, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.transpose", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + mlir.NamedAttribute.named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation)), + }, + .location = location, + }); +} // pub fn slice(ctx: *mlir.Context, operand: *const mlir.Value, start_indices: []const i64, limit_indices: []const i64, strides: []const i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { // return mlir.Operation.make(ctx, "stablehlo.slice", .{ From 0ea50b2880d5d9ff37cec125f8c6403e00497d63 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 10:44:49 +0200 Subject: [PATCH 12/47] stablehlo: bound slice --- mlir/dialects/stablehlo/stablehlo.zig | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 3e259b40..2f25edcf 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -256,18 +256,26 @@ pub fn transpose( }); } -// pub fn slice(ctx: *mlir.Context, operand: *const mlir.Value, start_indices: []const i64, limit_indices: []const i64, strides: []const i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.slice", .{ -// .operands = .{ .flat = &.{operand} }, -// .results = .{ .flat = &.{result_type} }, -// .attributes = &.{ -// .{ "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices) }, -// .{ "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices) }, -// .{ "strides", mlir.denseArrayAttribute(ctx, .i64, strides) }, -// }, -// .location = location, -// }); -// } +pub fn slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + start_indices: []const i64, + limit_indices: []const i64, + strides: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.slice", .{ + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + mlir.NamedAttribute.named(ctx, "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices)), + mlir.NamedAttribute.named(ctx, "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices)), + mlir.NamedAttribute.named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), + }, + .location = location, + }); +} // pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimension: i64, location: *const mlir.Location) *mlir.Operation { // return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ From fbb7e94740364d054a31350671093381e1bceb31 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 10:50:13 +0200 Subject: [PATCH 13/47] stablehlo: bound concatenate --- mlir/dialects/stablehlo/stablehlo.zig | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 2f25edcf..3713faaa 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -277,16 +277,16 @@ pub fn slice( }); } -// pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimension: i64, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ -// .operands = inputs, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "dimension", mlir.integerAttribute(ctx, .i64, dimension) }, -// }, -// .location = location, -// }); -// } +pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimension: i64, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ + .operands = .{ .flat = inputs }, + .result_type_inference = true, + .attributes = &.{ + mlir.NamedAttribute.named(ctx, "dimension", mlir.integerAttribute(ctx, .i64, dimension)), + }, + .location = location, + }); +} // pub fn select(ctx: *mlir.Context, condition: *const mlir.Value, then: *const mlir.Value, else_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { // return mlir.Operation.make(ctx, "stablehlo.select", .{ From 7dc6055e0583744461d8416a2b906f9cb96446dd Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 11:19:25 +0200 Subject: [PATCH 14/47] stablehlo: bound select --- mlir/dialects/stablehlo/stablehlo.zig | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 3713faaa..f26727b9 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -288,13 +288,19 @@ pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimens }); } -// pub fn select(ctx: *mlir.Context, condition: *const mlir.Value, then: *const mlir.Value, else_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.select", .{ -// .operands = .{ .flat = &.{ condition, then, else_ } }, -// .results = &.{then.getType()}, -// .location = location, -// }); -// } +pub fn select( + ctx: *mlir.Context, + condition: *const mlir.Value, + then: *const mlir.Value, + else_: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.select", .{ + .operands = .{ .flat = &.{ condition, then, else_ } }, + .results = .{ .flat = &.{then.type_()} }, + .location = location, + }); +} // pub fn gather( // ctx: *mlir.Context, From 750a1639e269fe4fdf9a6024e9cd880ec71936e0 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 11:22:39 +0200 Subject: [PATCH 15/47] stablehlo: bound iota --- mlir/dialects/stablehlo/stablehlo.zig | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index f26727b9..79b8c2fa 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -402,16 +402,16 @@ pub fn select( // ); // } -// pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.iota", .{ -// .operands = .{ .flat = &.{} }, -// .results = .{ .flat = &.{result_type} }, -// .attributes = &.{ -// .{ "iota_dimension", mlir.integerType(ctx, .i64, dimension) }, -// }, -// .location = location, -// }); -// } +pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.iota", .{ + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + mlir.NamedAttribute.named(ctx, "iota_dimension", mlir.integerAttribute(ctx, .i64, dimension)), + }, + .location = location, + }); +} // pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []const i64, location: *const mlir.Location) *mlir.Operation { // const result_type = operand.getType(); From 41678aee1fb1283207e614a8791ccf87ff231027 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 11:22:49 +0200 Subject: [PATCH 16/47] stablehlo: format typo --- mlir/dialects/stablehlo/stablehlo.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 79b8c2fa..e66d3a94 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -837,7 +837,7 @@ pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.L return custom_call(ctx, &.{value}, .{ .call_target_name = "Pin", .api_version = .typed_ffi, - }, &.{ result_type}, location); + }, &.{result_type}, location); } pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { From 45f99a2a6d43d5cce2c044694b18d8f59f8e9bef Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 11:28:11 +0200 Subject: [PATCH 17/47] stablehlo: bound reverse --- mlir/dialects/stablehlo/stablehlo.zig | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index e66d3a94..15797beb 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -413,17 +413,16 @@ pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, l }); } -// pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []const i64, location: *const mlir.Location) *mlir.Operation { -// const result_type = operand.getType(); -// return mlir.Operation.make(ctx, "stablehlo.reverse", .{ -// .operands = .{ .flat = &.{operand} }, -// .results = .{ .flat = &.{result_type} }, -// .attributes = &.{ -// .{ "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions) }, -// }, -// .location = location, -// }); -// } +pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []const i64, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.reverse", .{ + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{operand.type_()} }, + .attributes = &.{ + mlir.NamedAttribute.named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions)), + }, + .location = location, + }); +} // pub fn compare(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, comparison_direction: ComparisonDirection, compare_type: CompareType, location: *const mlir.Location) *mlir.Operation { // return mlir.Operation.make(ctx, "stablehlo.compare", .{ From 7bec82009f03efb9aa2f1ba72b350661afd175d8 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 11:42:58 +0200 Subject: [PATCH 18/47] stablehlo: simplify NamedAttribute usage --- mlir/dialects/stablehlo/stablehlo.zig | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 15797beb..63eb15c4 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -93,7 +93,7 @@ pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, locat .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - mlir.NamedAttribute.named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower))), + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower))), }, .location = location, }); @@ -216,7 +216,7 @@ pub fn constant( .operands = .{ .flat = &.{} }, .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, .attributes = &.{ - mlir.NamedAttribute.named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.init(dims, elem_type).shaped(), raw_bytes)), + .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.init(dims, elem_type).shaped(), raw_bytes)), }, .location = location, }); @@ -233,7 +233,7 @@ pub fn broadcast_in_dim( .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - mlir.NamedAttribute.named(ctx, "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims)), + .named(ctx, "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims)), }, .location = location, }); @@ -250,7 +250,7 @@ pub fn transpose( .operands = .{ .flat = &.{value} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - mlir.NamedAttribute.named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation)), + .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation)), }, .location = location, }); @@ -269,9 +269,9 @@ pub fn slice( .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - mlir.NamedAttribute.named(ctx, "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices)), - mlir.NamedAttribute.named(ctx, "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices)), - mlir.NamedAttribute.named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), + .named(ctx, "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices)), + .named(ctx, "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices)), + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), }, .location = location, }); @@ -282,7 +282,7 @@ pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimens .operands = .{ .flat = inputs }, .result_type_inference = true, .attributes = &.{ - mlir.NamedAttribute.named(ctx, "dimension", mlir.integerAttribute(ctx, .i64, dimension)), + .named(ctx, "dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); @@ -407,7 +407,7 @@ pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, l .operands = .{ .flat = &.{} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - mlir.NamedAttribute.named(ctx, "iota_dimension", mlir.integerAttribute(ctx, .i64, dimension)), + .named(ctx, "iota_dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); @@ -418,7 +418,7 @@ pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []con .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{operand.type_()} }, .attributes = &.{ - mlir.NamedAttribute.named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions)), + .named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions)), }, .location = location, }); From 7429a7e4c627538c650fdead7b233750dca25a7c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 11:43:18 +0200 Subject: [PATCH 19/47] stablehlo: bound compare --- mlir/dialects/stablehlo/stablehlo.zig | 119 ++++++++++++++------------ 1 file changed, 66 insertions(+), 53 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 63eb15c4..46c40661 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -424,17 +424,24 @@ pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []con }); } -// pub fn compare(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, comparison_direction: ComparisonDirection, compare_type: CompareType, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.compare", .{ -// .operands = .{ .flat = &.{ lhs, rhs } }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "comparison_direction", comparison_direction.asAttr() }, -// .{ "compare_type", compare_type.asAttr() }, -// }, -// .location = location, -// }); -// } +pub fn compare( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + comparison_direction: ComparisonDirection.Direction, + compare_type: CompareType.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.compare", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "comparison_direction", comparisonDirection(ctx, comparison_direction)), + .named(ctx, "compare_type", compareType(ctx, compare_type)), + }, + .location = location, + }); +} // pub fn reduce( // ctx: *mlir.Context, @@ -1130,57 +1137,63 @@ pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasA // } // }; -// pub const ComparisonDirection = struct { -// _inner: c.MlirAttribute, +pub const ComparisonDirection = opaque { + const M = mlir.Methods(ComparisonDirection, c.MlirAttribute); -// pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; -// const Self = ComparisonDirection; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); + pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); -// pub const Direction = enum { -// EQ, -// NE, -// GE, -// GT, -// LE, -// LT, -// }; + pub const Direction = enum { + EQ, + NE, + GE, + GT, + LE, + LT, + }; -// pub fn init(ctx: *mlir.Context, value: Direction) Self { -// return .{ ._inner = c.stablehloComparisonDirectionAttrGet(ctx._inner, stringRef(@tagName(value))) }; -// } + pub fn init(ctx: *mlir.Context, value: Direction) *const ComparisonDirection { + return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); + } -// pub fn getValue(self: Self) Direction { -// const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self._inner)); -// return std.meta.stringToEnum(Direction, value) orelse unreachable; -// } -// }; + pub fn getValue(self: *const ComparisonDirection) Direction { + const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Direction, value) orelse unreachable; + } +}; -// pub const CompareType = struct { -// _inner: c.MlirAttribute, +pub fn comparisonDirection(ctx: *mlir.Context, value: ComparisonDirection.Direction) *const mlir.Attribute { + return @ptrCast(ComparisonDirection.init(ctx, value)); +} -// pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; -// const Self = CompareType; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); +pub const CompareType = opaque { + const M = mlir.Methods(CompareType, c.MlirAttribute); -// pub const Type = enum { -// SIGNED, -// UNSIGNED, -// FLOAT, -// TOTALORDER, -// }; + pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); -// pub fn init(ctx: *mlir.Context, value: Type) Self { -// return .{ ._inner = c.stablehloComparisonTypeAttrGet(ctx._inner, stringRef(@tagName(value))) }; -// } + pub const Type = enum { + SIGNED, + UNSIGNED, + FLOAT, + TOTALORDER, + }; -// pub fn getValue(self: Self) Type { -// const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self._inner)); -// return std.meta.stringToEnum(Type, value) orelse unreachable; -// } -// }; + pub fn init(ctx: *mlir.Context, value: Type) *const CompareType { + return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); + } + + pub fn getValue(self: *const CompareType) Type { + const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, value) orelse unreachable; + } +}; + +pub fn compareType(ctx: *mlir.Context, value: CompareType.Type) *const mlir.Attribute { + return @ptrCast(CompareType.init(ctx, value)); +} // pub const Transpose = struct { // _inner: c.MlirAttribute, From 1a81df4a60400cf65f48cd3945f73cdc6785824e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 11:50:21 +0200 Subject: [PATCH 20/47] stablehlo: bound dynamic_slice --- mlir/dialects/stablehlo/stablehlo.zig | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 46c40661..119ebf53 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -524,16 +524,22 @@ pub fn compare( // }); // } -// pub fn dynamic_slice(ctx: *mlir.Context, operand: *const mlir.Value, new_dims: []const i64, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.dynamic_slice", .{ -// .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "slice_sizes", .dense(ctx, .i64, new_dims) }, -// }, -// .location = location, -// }); -// } +pub fn dynamic_slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + new_dims: []const i64, + start_indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.dynamic_slice", .{ + .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, new_dims)), + }, + .location = location, + }); +} // pub const PadOpts = struct { // low: []const i64, From 86c502825f6f1007349852576c065d526be6d693 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 12:00:13 +0200 Subject: [PATCH 21/47] stablehlo: bound pad --- mlir/dialects/stablehlo/stablehlo.zig | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 119ebf53..3e9ce913 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -541,24 +541,24 @@ pub fn dynamic_slice( }); } -// pub const PadOpts = struct { -// low: []const i64, -// high: []const i64, -// interior: []const i64, -// }; +pub const PadOpts = struct { + low: []const i64, + high: []const i64, + interior: []const i64, +}; -// pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.pad", .{ -// .operands = .{ .flat = &.{ value, padding_value } }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "edge_padding_low", .dense(ctx, .i64, opts.low) }, -// .{ "edge_padding_high", .dense(ctx, .i64, opts.high) }, -// .{ "interior_padding", .dense(ctx, .i64, opts.interior) }, -// }, -// .location = location, -// }); -// } +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.pad", .{ + .operands = .{ .flat = &.{ value, padding_value } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "edge_padding_low", mlir.denseArrayAttribute(ctx, .i64, opts.low)), + .named(ctx, "edge_padding_high", mlir.denseArrayAttribute(ctx, .i64, opts.high)), + .named(ctx, "interior_padding", mlir.denseArrayAttribute(ctx, .i64, opts.interior)), + }, + .location = location, + }); +} // pub const TriangularSolveOpts = struct { // left_side: bool, From cad23da7c087c0c0aed042c641a90740cbff2f03 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 12:07:32 +0200 Subject: [PATCH 22/47] stablehlo: bound triangular_solve --- mlir/dialects/stablehlo/stablehlo.zig | 79 ++++++++++++++------------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 3e9ce913..0b5338b8 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -560,26 +560,26 @@ pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const m }); } -// pub const TriangularSolveOpts = struct { -// left_side: bool, -// lower: bool, -// unit_diagonal: bool, -// transpose_a: Transpose.Type, -// }; +pub const TriangularSolveOpts = struct { + left_side: bool, + lower: bool, + unit_diagonal: bool, + transpose_a: TransposeAttribute.Type, +}; -// pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, location: *const mlir.Location, opts: TriangularSolveOpts) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ -// .operands = .{ .flat = &.{ value, other } }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "left_side", .i1FromBool(ctx, opts.left_side) }, -// .{ "lower", .i1FromBool(ctx, opts.lower) }, -// .{ "unit_diagonal", .i1FromBool(ctx, opts.unit_diagonal) }, -// .{ "transpose_a", Transpose.init(ctx, opts.transpose_a).asAttr() }, -// }, -// .location = location, -// }); -// } +pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, location: *const mlir.Location, opts: TriangularSolveOpts) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ + .operands = .{ .flat = &.{ value, other } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "left_side", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.left_side))), + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.lower))), + .named(ctx, "unit_diagonal", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.unit_diagonal))), + .named(ctx, "transpose_a", transposeAttribute(ctx, opts.transpose_a)), + }, + .location = location, + }); +} // pub const FftOpts = struct { // kind: FftType.Type, @@ -1201,29 +1201,32 @@ pub fn compareType(ctx: *mlir.Context, value: CompareType.Type) *const mlir.Attr return @ptrCast(CompareType.init(ctx, value)); } -// pub const Transpose = struct { -// _inner: c.MlirAttribute, +pub const TransposeAttribute = opaque { + const M = mlir.Methods(TransposeAttribute, mlir.Attribute); -// pub const isAFn = c.stablehloAttributeIsATransposeAttr; -// const Self = Transpose; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); + pub const isAFn = c.stablehloAttributeIsATransposeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); -// pub const Type = enum { -// NO_TRANSPOSE, -// TRANSPOSE, -// ADJOINT, -// }; + pub const Type = enum { + NO_TRANSPOSE, + TRANSPOSE, + ADJOINT, + }; -// pub fn init(ctx: *mlir.Context, value: Type) Self { -// return .{ ._inner = c.stablehloTransposeAttrGet(ctx._inner, stringRef(@tagName(value))) }; -// } + pub fn init(ctx: *mlir.Context, value: Type) *const TransposeAttribute { + return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); + } -// pub fn getValue(self: Self) Type { -// const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self._inner)); -// return std.meta.stringToEnum(Type, value) orelse unreachable; -// } -// }; + pub fn getValue(self: *const TransposeAttribute) Type { + const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, value) orelse unreachable; + } +}; + +pub fn transposeAttribute(ctx: *mlir.Context, value: TransposeAttribute.Type) *const mlir.Attribute { + return @ptrCast(TransposeAttribute.init(ctx, value)); +} // pub const FftType = struct { // _inner: c.MlirAttribute, From a82bc3fe8dc29baf6913f86d2d7983bc6391cfcd Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 12:55:58 +0200 Subject: [PATCH 23/47] stablehlo: bound fft --- mlir/dialects/stablehlo/stablehlo.zig | 73 ++++++++++++++------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 0b5338b8..417c66a4 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -581,22 +581,22 @@ pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *co }); } -// pub const FftOpts = struct { -// kind: FftType.Type, -// length: []const i64, -// }; +pub const FftOpts = struct { + kind: FftType.Type, + length: []const i64, +}; -// pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location, opts: FftOpts) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.fft", .{ -// .operands = .{ .flat = &.{value} }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "fft_type", FftType.init(ctx, opts.kind).asAttr() }, -// .{ "fft_length", .dense(ctx, .i64, opts.length) }, -// }, -// .location = location, -// }); -// } +pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location, opts: FftOpts) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.fft", .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "fft_type", fftType(ctx, opts.kind)), + .named(ctx, "fft_length", mlir.denseArrayAttribute(ctx, .i64, opts.length)), + }, + .location = location, + }); +} // pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { // return mlir.Operation.make(ctx, "stablehlo.rng", .{ @@ -1228,30 +1228,33 @@ pub fn transposeAttribute(ctx: *mlir.Context, value: TransposeAttribute.Type) *c return @ptrCast(TransposeAttribute.init(ctx, value)); } -// pub const FftType = struct { -// _inner: c.MlirAttribute, +pub const FftType = opaque { + const M = mlir.Methods(FftType, mlir.Attribute); -// pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; -// const Self = FftType; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); + pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); -// pub const Type = enum { -// FFT, -// IFFT, -// RFFT, -// IRFFT, -// }; + pub const Type = enum { + FFT, + IFFT, + RFFT, + IRFFT, + }; -// pub fn init(ctx: *mlir.Context, value: Type) Self { -// return .{ ._inner = c.stablehloFftTypeAttrGet(ctx._inner, stringRef(@tagName(value))) }; -// } + pub fn init(ctx: *mlir.Context, value: Type) *const FftType { + return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); + } -// pub fn getValue(self: Self) Type { -// const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self._inner)); -// return std.meta.stringToEnum(Type, value) orelse unreachable; -// } -// }; + pub fn getValue(self: *const FftType) Type { + const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, value) orelse unreachable; + } +}; + +pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { + return @ptrCast(FftType.init(ctx, value)); +} // pub const RngDistribution = struct { // _inner: c.MlirAttribute, From 6433c8eb4ac799e31d3b87125250431f5ccf4b50 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Aug 2025 13:07:48 +0200 Subject: [PATCH 24/47] stablehlo: fix custom_call --- mlir/dialects/stablehlo/stablehlo.zig | 177 +++++++++++++++----------- mlir/registry.zig | 35 +++-- 2 files changed, 124 insertions(+), 88 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 417c66a4..0e241aff 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -727,15 +727,15 @@ pub const CustomCallOpts = struct { }; pub const BackendConfig = union(ApiVersion) { - original: *const mlir.StringAttribute, - status_returning: *const mlir.StringAttribute, - status_returning_unified: *const mlir.StringAttribute, + original: []const u8, + status_returning: []const u8, + status_returning_unified: []const u8, typed_ffi: *const mlir.DictionaryAttribute, }; call_target_name: []const u8, - has_side_effect: bool = false, - api_version: ?ApiVersion, + has_side_effect: ?bool = null, + api_version: ?ApiVersion = null, backend_config: ?BackendConfig = null, operand_layouts: ?[]const []const usize = null, result_layouts: ?[]const []const usize = null, @@ -743,36 +743,49 @@ pub const CustomCallOpts = struct { additional_attributes: []const mlir.NamedAttribute = &.{}, }; -pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, opts: CustomCallOpts, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, opts: CustomCallOpts, location: *const mlir.Location) *mlir.Operation { const MAX_OPERANDS = 64; - _ = MAX_OPERANDS; // autofix const MAX_RESULTS = 16; + const MINOR_TO_MAJOR = comptime blk: { + var ret: [mlir.ShapedType.MAX_RANK]usize = undefined; + for (0..ret.len) |i| { + ret[i] = @intCast(mlir.ShapedType.MAX_RANK - i - 1); + } + break :blk ret; + }; var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .{}; attrs.appendSliceAssumeCapacity(&.{ .named(ctx, "call_target_name", mlir.stringAttribute(ctx, opts.call_target_name)), - .named(ctx, "has_side_effect", mlir.boolAttribute(ctx, opts.has_side_effect)), }); attrs.appendSliceAssumeCapacity(opts.additional_attributes); - if (opts.api_version orelse opts.backend_config) |v| { + if (opts.has_side_effect) |has_side_effects| { + attrs.appendAssumeCapacity( + .named(ctx, "has_side_effect", mlir.boolAttribute(ctx, has_side_effects)), + ); + } + + const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; + if (opts.api_version orelse backend_config_api_version) |v| { attrs.appendAssumeCapacity( .named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v))), ); } if (opts.backend_config) |backend_config| { - switch (backend_config) { - inline else => |v| attrs.appendAssumeCapacity( - .named(ctx, "backend_config", @ptrCast(v)), - ), - } + attrs.appendAssumeCapacity( + .named(ctx, "backend_config", switch (backend_config) { + .typed_ffi => |v| @ptrCast(v), + inline else => |v| mlir.stringAttribute(ctx, v), + }), + ); } if (opts.output_operand_aliases) |output_operand_aliases| { var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; for (output_operand_aliases) |alias| { buffer.appendAssumeCapacity( - outputOperandAliasAttribute.init(ctx, .{ .operand_index = alias }), + outputOperandAliasAttribute(ctx, .{ .operand_index = alias }), ); } attrs.appendAssumeCapacity( @@ -780,52 +793,69 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, opts: ); } - // const MINOR_TO_MAJOR = blk: { - // const MAX_RANK = 8; - // var ret: [MAX_RANK]usize = undefined; - // for (0..MAX_RANK) |i| { - // ret[i] = @intCast(MAX_RANK - i - 1); - // } - // break :blk ret; - // }; + var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .{}; + if (opts.operand_layouts) |layouts| { + for (layouts) |ol| { + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + ).shaped(), + ol, + ), + ); + } + } else { + for (inputs) |input| { + const shaped_type = input.type_().isA(mlir.ShapedType).?; + const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + ).shaped(), + ol, + ), + ); + } + } + attrs.appendAssumeCapacity( + .named(ctx, "operand_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + ); - // if (opts.operand_layouts) |layouts| { - // var operand_layouts: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; - // for (layouts) |ol| { - // operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); - // } - // attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); - // } else { - // const operand_layouts = blk: { - // var ret: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; - // for (inputs) |input| { - // const ranked_type = input.getType().as(mlir.RankedTensorType).?; - // const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; - // ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); - // } - // break :blk ret; - // }; - // attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); - // } - - // if (opts.result_layouts) |layouts| { - // var result_layouts: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - // for (layouts) |rl| { - // result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); - // } - // attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); - // } else { - // const result_layouts = blk: { - // var ret: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - // for (res_types) |t| { - // const ranked_t = t.as(mlir.RankedTensorType).?; - // const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; - // ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); - // } - // break :blk ret; - // }; - // attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); - // } + layouts_buffer.clear(); + if (opts.result_layouts) |layouts| { + for (layouts) |rl| { + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + ).shaped(), + rl, + ), + ); + } + } else { + for (result_types) |result_type| { + const shaped_type = result_type.isA(mlir.ShapedType).?; + const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + ).shaped(), + rl, + ), + ); + } + } + attrs.appendAssumeCapacity( + .named(ctx, "result_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + ); return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ .operands = .{ .flat = inputs }, @@ -835,29 +865,25 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, opts: }); } -pub fn createBuffer(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return custom_call(ctx, &.{}, .{ +pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{value}, .{ .call_target_name = "CreateBuffer", .api_version = .typed_ffi, - }, &.{result_type}, location); + }, location); } pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - const result_type = mlir.integerType(ctx, .i64); - const shape = value.type_().as(mlir.ShapedType).?; - _ = shape; // autofix - return custom_call(ctx, &.{value}, .{ + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ .call_target_name = "Pin", .api_version = .typed_ffi, - }, &.{result_type}, location); + }, location); } pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - const result_type = mlir.integerType(ctx, .i64); - return custom_call(ctx, &.{value}, .{ + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ .call_target_name = "Unpin", .api_version = .typed_ffi, - }, &.{result_type}, location); + }, location); } // pub const DotDimensionNumbersAttribute = struct { @@ -1103,7 +1129,7 @@ pub const OutputOperandAliasAttribute = opaque { operand_tuple_indices: []const i64 = &.{}, }; - pub fn init(ctx: *mlir.Context, args: InitArgs) *OutputOperandAliasAttribute { + pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttribute { return @ptrCast(c.stablehloOutputOperandAliasGet( ctx.ptr(), @intCast(args.output_tuple_indices.len), @@ -1115,8 +1141,8 @@ pub const OutputOperandAliasAttribute = opaque { } }; -pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasAttribute.InitArgs) *mlir.Attribute { - return @ptrCast(OutputOperandAliasAttribute.init(ctx, args)); +pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(OutputOperandAliasAttribute.get(ctx, args)); } // pub const PrecisionAttribute = struct { @@ -1306,8 +1332,7 @@ pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { var writer = std.Io.Writer.fixed(buf); var sctx: mlir.StringCallbackCtx = .{ .writer = &writer }; - const new_args = args ++ .{ mlir.stringCallback, &sctx }; - @call(.auto, streamFn, new_args); + @call(.auto, streamFn, args ++ .{ mlir.stringCallback, &sctx }); return writer.buffered(); } diff --git a/mlir/registry.zig b/mlir/registry.zig index 78ec4698..1367b660 100644 --- a/mlir/registry.zig +++ b/mlir/registry.zig @@ -250,6 +250,10 @@ pub const IndexType = opaque { } }; +pub fn indexType(ctx: *Context) *const Type { + return @ptrCast(IndexType.get(ctx)); +} + pub const IntegerTypes = enum { i1, i4, @@ -1207,7 +1211,7 @@ pub const RankedTensorType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); - pub fn init(dimensions: []const i64, elemType: *const Type) *const RankedTensorType { + pub fn get(dimensions: []const i64, elemType: *const Type) *const RankedTensorType { return @ptrCast(c.mlirRankedTensorTypeGet( @intCast(dimensions.len), @ptrCast(dimensions), @@ -1218,12 +1222,12 @@ pub const RankedTensorType = opaque { pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; - for (other.rank()) |i| { + for (0..other.rank()) |i| { dims.appendAssumeCapacity(other.dimension(i)); } - return init( - other.elementType(), + return get( dims.constSlice(), + other.elementType(), ); } @@ -1245,7 +1249,7 @@ pub const RankedTensorType = opaque { }; pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { - return @ptrCast(RankedTensorType.init(dimensions, elem_type)); + return @ptrCast(RankedTensorType.get(dimensions, elem_type)); } pub const DenseElementsAttribute = opaque { @@ -1256,16 +1260,17 @@ pub const DenseElementsAttribute = opaque { pub const eql = M.eql(c.mlirAttributeEqual); pub const format = M.format(c.mlirAttributePrint); - pub fn init(shaped_type: *const ShapedType, values: []const u8) *const DenseElementsAttribute { + pub fn init(shaped_type: *const ShapedType, values: anytype) *const DenseElementsAttribute { + const bytes = std.mem.sliceAsBytes(values); return @ptrCast(c.mlirDenseElementsAttrRawBufferGet( shaped_type.ptr(), - @intCast(values.len), - @ptrCast(values), + @intCast(bytes.len), + @ptrCast(bytes), ).ptr orelse unreachable); } }; -pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: []const u8) *const Attribute { +pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: anytype) *const Attribute { return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); } @@ -1426,13 +1431,19 @@ pub const MemRefType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); - pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*Attribute) *const MemRefType { - return @ptrCast(c.mlirMemRefTypeGet(element_type, shape.len, @ptrCast(shape), layout, memory_space).ptr); + pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { + return @ptrCast(c.mlirMemRefTypeGet( + element_type.ptr(), + @intCast(shape.len), + @ptrCast(shape), + if (layout) |l| l.ptr() else .{}, + if (memory_space) |m| m.ptr() else .{}, + ).ptr); } pub fn fromShaped(other: *const ShapedType) *const MemRefType { var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; - for (other.rank()) |i| { + for (0..other.rank()) |i| { dims.appendAssumeCapacity(other.dimension(i)); } return init( From 277f5be00e2f1accb22d0599876705d35e8e8569 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Aug 2025 13:14:12 +0200 Subject: [PATCH 25/47] stablehlo: bound get_tuple_element --- mlir/dialects/stablehlo/stablehlo.zig | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 0e241aff..93612e38 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -640,16 +640,16 @@ pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.L // }); // } -// pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ -// .operands = .{ .flat = &.{tuple_value} }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "index", .int(ctx, .i32, index) }, -// }, -// .location = location, -// }); -// } +pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ + .operands = .{ .flat = &.{tuple_value} }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "index", mlir.integerAttribute(ctx, .i32, index)), + }, + .location = location, + }); +} // pub const ConvolutionOpts = struct { // window_strides: []const i64, From 73d1f85a16d1ecc9bb506afa9e93a8c21ace93bb Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Aug 2025 13:18:02 +0200 Subject: [PATCH 26/47] stablehlo: bound dynamic_update_slice --- mlir/dialects/stablehlo/stablehlo.zig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 93612e38..82e92cea 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -632,13 +632,13 @@ pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.L // }); // } -// pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, update: *const mlir.Value, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ -// .operands = .{ .variadic = &.{ &.{operand}, &.{update}, start_indices } }, -// .result_type_inference = true, -// .location = location, -// }); -// } +pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, update: *const mlir.Value, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ + .operands = .{ .variadic = &.{ &.{operand}, &.{update}, start_indices } }, + .result_type_inference = true, + .location = location, + }); +} pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ From a2df29defd4c12a6211dd841a4b6b351331ba832 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Aug 2025 13:18:54 +0200 Subject: [PATCH 27/47] stablehlo: bound reduce_precision --- mlir/dialects/stablehlo/stablehlo.zig | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 82e92cea..abf23fce 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -620,17 +620,17 @@ pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.L // }); // } -// pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ -// .operands = .{ .flat = &.{value} }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "exponent_bits", .int(ctx, .i32, exponent_bits) }, -// .{ "mantissa_bits", .int(ctx, .i32, mantissa_bits) }, -// }, -// .location = location, -// }); -// } +pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "exponent_bits", mlir.integerAttribute(ctx, .i32, exponent_bits)), + .named(ctx, "mantissa_bits", mlir.integerAttribute(ctx, .i32, mantissa_bits)), + }, + .location = location, + }); +} pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, update: *const mlir.Value, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ From c62059baacd0fbf9ed2507f0b7f9da20f8614b84 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 28 Aug 2025 13:25:13 +0200 Subject: [PATCH 28/47] stablehlo: bound rng and rng_bit_generator --- mlir/dialects/stablehlo/stablehlo.zig | 146 ++++++++++++++------------ 1 file changed, 79 insertions(+), 67 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index abf23fce..e0cfbdd9 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -598,27 +598,86 @@ pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.L }); } -// pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.rng", .{ -// .operands = .{ .flat = &.{ a, b, shape } }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "rng_distribution", RngDistribution.init(ctx, rng_distribution).asAttr() }, -// }, -// .location = location, -// }); -// } +pub const RngDistribution = opaque { + const M = mlir.Methods(RngDistribution, c.MlirAttribute); -// pub fn rng_bit_generator(ctx: *mlir.Context, rng_algorithm: RngAlgorithm.Type, initial_state: *const mlir.Value, res_state_type: *const mlir.Type, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { -// return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ -// .operands = .{ .flat = &.{initial_state} }, -// .results = &.{ res_state_type, res_type }, -// .attributes = &.{ -// .{ "rng_algorithm", RngAlgorithm.init(ctx, rng_algorithm).asAttr() }, -// }, -// .location = location, -// }); -// } + pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + UNIFORM, + NORMAL, + }; + + pub fn get(ctx: *mlir.Context, value_: Type) *const RngDistribution { + return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + } + + pub fn value(self: *const RngDistribution) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngDistributionAttrGetValue(self.ptr())), + ) orelse unreachable; + } +}; + +pub fn rngDistribution(ctx: *mlir.Context, value: RngDistribution.Type) *const mlir.Attribute { + return @ptrCast(RngDistribution.get(ctx, value)); +} + +pub const RngAlgorithm = opaque { + const M = mlir.Methods(RngAlgorithm, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + DEFAULT, + THREE_FRY, + PHILOX, + }; + + pub fn get(ctx: *mlir.Context, value_: Type) *const RngAlgorithm { + return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + } + + pub fn value(self: *const RngAlgorithm) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngAlgorithmAttrGetValue(self.ptr())), + ) orelse unreachable; + } +}; + +pub fn rngAlgorithm(ctx: *mlir.Context, value: RngAlgorithm.Type) *const mlir.Attribute { + return @ptrCast(RngAlgorithm.get(ctx, value)); +} + +pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.rng", .{ + .operands = .{ .flat = &.{ a, b, shape } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "rng_distribution", rngDistribution(ctx, rng_distribution)), + }, + .location = location, + }); +} + +pub fn rng_bit_generator(ctx: *mlir.Context, rng_algorithm: RngAlgorithm.Type, initial_state: *const mlir.Value, res_state_type: *const mlir.Type, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ + .operands = .{ .flat = &.{initial_state} }, + .results = &.{ res_state_type, res_type }, + .attributes = &.{ + .named(ctx, "rng_algorithm", rngAlgorithm(ctx, rng_algorithm)), + }, + .location = location, + }); +} pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ @@ -1282,53 +1341,6 @@ pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { return @ptrCast(FftType.init(ctx, value)); } -// pub const RngDistribution = struct { -// _inner: c.MlirAttribute, - -// pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; -// const Self = RngDistribution; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); - -// pub const Type = enum { -// UNIFORM, -// NORMAL, -// }; - -// pub fn init(ctx: *mlir.Context, value: Type) Self { -// return .{ ._inner = c.stablehloRngDistributionAttrGet(ctx._inner, stringRef(@tagName(value))) }; -// } - -// pub fn getValue(self: Self) Type { -// const value = mlir.fromStringRef(c.stablehloRngDistributionAttrGetValue(self._inner)); -// return std.meta.stringToEnum(Type, value) orelse unreachable; -// } -// }; - -// pub const RngAlgorithm = struct { -// _inner: c.MlirAttribute, - -// pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; -// const Self = RngAlgorithm; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); - -// pub const Type = enum { -// DEFAULT, -// THREE_FRY, -// PHILOX, -// }; - -// pub fn init(ctx: *mlir.Context, value: Type) Self { -// return .{ ._inner = c.stablehloRngAlgorithmAttrGet(ctx._inner, stringRef(@tagName(value))) }; -// } - -// pub fn getValue(self: Self) Type { -// const value = mlir.fromStringRef(c.stablehloRngAlgorithmAttrGetValue(self._inner)); -// return std.meta.stringToEnum(Type, value) orelse unreachable; -// } -// }; - fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { var writer = std.Io.Writer.fixed(buf); var sctx: mlir.StringCallbackCtx = .{ .writer = &writer }; From a0025304bee14cf816de0b6b49a280ac3427e94c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 13:44:35 +0200 Subject: [PATCH 29/47] mlir: made mlir.floatType ft parameter allowed at runtime --- mlir/registry.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mlir/registry.zig b/mlir/registry.zig index 1367b660..7536e701 100644 --- a/mlir/registry.zig +++ b/mlir/registry.zig @@ -385,8 +385,10 @@ pub fn FloatType(comptime ft: FloatTypes) type { }; } -pub fn floatType(comptime ft: FloatTypes, ctx: *Context) *const Type { - return @ptrCast(FloatType(ft).get(ctx)); +pub fn floatType(ft: FloatTypes, ctx: *Context) *const Type { + return switch (ft) { + inline else => |v| @ptrCast(FloatType(v).get(ctx)), + }; } pub const Attribute = opaque { From 333c352fc9abcbe0cfaf7be737053c0324ccdf81 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 13:45:22 +0200 Subject: [PATCH 30/47] stablehlo: bound dot_general --- mlir/dialects/stablehlo/stablehlo.zig | 331 +++++++++++++------------- 1 file changed, 170 insertions(+), 161 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index e0cfbdd9..33dcbc7b 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -107,103 +107,104 @@ pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Valu }); } -// pub const DotPrecision = union(enum) { -// fast, -// high, -// highest, -// algorithm: DotAlgorithm, +pub const DotPrecision = union(enum) { + fast, + high, + highest, + algorithm: DotAlgorithm, -// pub fn precisionAttr(self: DotPrecision, ctx: *mlir.Context) *const mlir.Attribute { -// const precision = PrecisionAttribute.init(ctx, switch (self) { -// .fast => .DEFAULT, -// .high => .HIGH, -// .highest => .HIGHEST, -// // When we specify the dot algorithm, we should not specify the precision. -// .algorithm => .DEFAULT, -// }); -// return precision.asAttr(); -// } + pub fn precision(self: DotPrecision) PrecisionAttribute.Precision { + return switch (self) { + .fast => .DEFAULT, + .high => .HIGH, + .highest => .HIGHEST, + // When we specify the dot algorithm, we should not specify the precision. + .algorithm => .DEFAULT, + }; + } -// pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { -// return switch (self) { -// .algorithm => |algo| algo.asAttr(ctx, operand_type), -// else => null, -// }; -// } -// }; + pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { + return switch (self) { + .algorithm => |algo| algo.asAttr(ctx, operand_type), + else => null, + }; + } +}; -// pub const DotAlgorithm = struct { -// accumulation: mlir.FloatTypes, -// // Note stablehlo distinguish between left/right component_count -// // but all the supported algorithm have the same component_count on both side. -// component_count: u8 = 1, -// num_primitive_operations: u8 = 1, -// allow_imprecise_accumulation: bool = false, +pub const DotAlgorithm = struct { + accumulation: mlir.FloatTypes, + // Note stablehlo distinguish between left/right component_count + // but all the supported algorithm have the same component_count on both side. + component_count: u8 = 1, + num_primitive_operations: u8 = 1, + allow_imprecise_accumulation: bool = false, -// // bf16_6x: each input is decomposed to 3 bf16 components, then 6 dot operations are done on those components, and the result is accumulated in f32. -// // not sure where this is available. -// pub const bf16_6x: DotAlgorithm = .{ -// .operand = .bf16, -// .accumulation = .f32, -// .component_count = 1, -// .num_primitive_operations = 6, -// .allow_imprecise_accumulation = false, -// }; + // bf16_6x: each input is decomposed to 3 bf16 components, then 6 dot operations are done on those components, and the result is accumulated in f32. + // not sure where this is available. + pub const bf16_6x: DotAlgorithm = .{ + .accumulation = .f32, + .component_count = 1, + .num_primitive_operations = 6, + .allow_imprecise_accumulation = false, + }; +}; -// pub fn asAttr(self: DotAlgorithm, ctx: *mlir.Context, tensor_type: mlir.RankedTensorType) *const mlir.Attribute { -// const elem_type = tensor_type.getElementType(); +pub fn dotAlgorithmAttribute(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { + return @ptrCast(c.stablehloDotAlgorithmGet( + ctx.ptr(), + element_type.ptr(), + element_type.ptr(), + mlir.floatType(dot_algorithm.accumulation, ctx).ptr(), + dot_algorithm.component_count, + dot_algorithm.component_count, + dot_algorithm.num_primitive_operations, + dot_algorithm.allow_imprecise_accumulation, + ).ptr); +} -// return mlir.Attribute.wrap(c.stablehloDotAlgorithmGet( -// ctx._inner, -// elem_type._inner, -// elem_type._inner, -// self.accumulation.asType(ctx)._inner, -// self.component_count, -// self.component_count, -// self.num_primitive_operations, -// self.allow_imprecise_accumulation, -// )); -// } -// }; - -// /// General matrix multiplication "a la Einstein sum" -// /// Note: stablehlo doesn't do type inference for dot_general -// pub fn dot_general( -// ctx: *mlir.Context, -// lhs: *const mlir.Value, -// rhs: *const mlir.Value, -// result_type: *const mlir.Type, -// location: *const mlir.Location, -// opts: struct { -// lhs_batching_dimensions: []const i64, -// rhs_batching_dimensions: []const i64, -// lhs_contracting_dimensions: []const i64, -// rhs_contracting_dimensions: []const i64, -// precision: DotPrecision, -// }, -// ) *mlir.Operation { -// const precisions: [2]mlir.Attribute = @splat(opts.precision.precisionAttr(ctx)); -// const attributes = [3]mlir.AttrTuple{ -// .{ -// "dot_dimension_numbers", DotDimensionNumbersAttribute.init(ctx, .{ -// .lhs_batching_dimensions = opts.lhs_batching_dimensions, -// .rhs_batching_dimensions = opts.rhs_batching_dimensions, -// .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, -// .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, -// }).asAttr(), -// }, -// .{ "precision_config", .array(ctx, &precisions) }, -// // keep algorithm as the last attribute so we can omit it when it's not set. -// .{ "algorithm", opts.precision.algorithmAttr(ctx, lhs.getType().as(mlir.RankedTensorType).?) orelse undefined }, -// }; -// const n_attributes = if (opts.precision == .algorithm) attributes.len else attributes.len - 1; -// return mlir.Operation.make(ctx, "stablehlo.dot_general", .{ -// .operands = .{ .flat = &.{ lhs, rhs } }, -// .results = .{ .flat = &.{result_type} }, -// .attributes = attributes[0..n_attributes], -// .location = location, -// }); -// } +/// General matrix multiplication "a la Einstein sum" +/// Note: stablehlo doesn't do type inference for dot_general +pub fn dot_general( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + opts: struct { + lhs_batching_dimensions: []const i64, + rhs_batching_dimensions: []const i64, + lhs_contracting_dimensions: []const i64, + rhs_contracting_dimensions: []const i64, + dot_precision: DotPrecision, + }, +) *mlir.Operation { + const precisions: [2]*const mlir.Attribute = @splat(precisionAttribute(ctx, opts.dot_precision.precision())); + const attributes = [3]mlir.NamedAttribute{ + .named( + ctx, + "dot_dimension_numbers", + dotDimensionNumbersAttribute(ctx, .{ + .lhs_batching_dimensions = opts.lhs_batching_dimensions, + .rhs_batching_dimensions = opts.rhs_batching_dimensions, + .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, + .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, + }), + ), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &precisions)), + // keep algorithm as the last attribute so we can omit it when it's not set. + .named(ctx, "algorithm", switch (opts.dot_precision) { + .algorithm => |v| dotAlgorithmAttribute(ctx, v, lhs.type_()), + else => undefined, + }), + }; + const n_attributes = if (opts.dot_precision == .algorithm) attributes.len else attributes.len - 1; + return mlir.Operation.make(ctx, "stablehlo.dot_general", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes[0..n_attributes], + .location = location, + }); +} pub fn constant( ctx: *mlir.Context, @@ -945,67 +946,71 @@ pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir }, location); } -// pub const DotDimensionNumbersAttribute = struct { -// _inner: c.MlirAttribute, +pub const DotDimensionNumbersAttribute = opaque { + const M = mlir.Methods(DotDimensionNumbersAttribute, mlir.Attribute); -// pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; -// const Self = DotDimensionNumbersAttribute; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); + pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); -// pub fn init(ctx: *mlir.Context, args: struct { -// lhs_batching_dimensions: []const i64, -// rhs_batching_dimensions: []const i64, -// lhs_contracting_dimensions: []const i64, -// rhs_contracting_dimensions: []const i64, -// }) Self { -// return .{ -// ._inner = c.stablehloDotDimensionNumbersGet( -// ctx._inner, -// @intCast(args.lhs_batching_dimensions.len), -// args.lhs_batching_dimensions.ptr, -// @intCast(args.rhs_batching_dimensions.len), -// args.rhs_batching_dimensions.ptr, -// @intCast(args.lhs_contracting_dimensions.len), -// args.lhs_contracting_dimensions.ptr, -// @intCast(args.rhs_contracting_dimensions.len), -// args.rhs_contracting_dimensions.ptr, -// ), -// }; -// } + pub const InitArgs = struct { + lhs_batching_dimensions: []const i64, + rhs_batching_dimensions: []const i64, + lhs_contracting_dimensions: []const i64, + rhs_contracting_dimensions: []const i64, + }; -// pub fn getLhsBatchingDimensionsSize(self: Self) usize { -// return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self._inner)); -// } + pub fn init(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttribute { + return @ptrCast(c.stablehloDotDimensionNumbersGet( + ctx.ptr(), + @intCast(args.lhs_batching_dimensions.len), + args.lhs_batching_dimensions.ptr, + @intCast(args.rhs_batching_dimensions.len), + args.rhs_batching_dimensions.ptr, + @intCast(args.lhs_contracting_dimensions.len), + args.lhs_contracting_dimensions.ptr, + @intCast(args.rhs_contracting_dimensions.len), + args.rhs_contracting_dimensions.ptr, + ).ptr); + } -// pub fn getLhsBatchingDimensionsElem(self: Self, pos: usize) i64 { -// return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self._inner, @intCast(pos)); -// } + pub fn getLhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self.ptr())); + } -// pub fn getRhsBatchingDimensionsSize(self: Self) usize { -// return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self._inner)); -// } + pub fn getLhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); + } -// pub fn getRhsBatchingDimensionsElem(self: Self, pos: usize) i64 { -// return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self._inner, @intCast(pos)); -// } + pub fn getRhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self.ptr())); + } -// pub fn getLhsContractingDimensionsSize(self: Self) usize { -// return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self._inner)); -// } + pub fn getRhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); + } -// pub fn getLhsContractingDimensionsElem(self: Self, pos: usize) i64 { -// return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self._inner, @intCast(pos)); -// } + pub fn getLhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self.ptr())); + } -// pub fn getRhsContractingDimensionsSize(self: Self) usize { -// return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self._inner)); -// } + pub fn getLhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self.ptr(), @intCast(pos)); + } -// pub fn getRhsContractingDimensionsElem(self: Self, pos: usize) i64 { -// return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self._inner, @intCast(pos)); -// } -// }; + pub fn getRhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self.ptr())); + } + + pub fn getRhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self.ptr(), @intCast(pos)); + } +}; + +pub fn dotDimensionNumbersAttribute(ctx: *mlir.Context, args: DotDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(DotDimensionNumbersAttribute.init(ctx, args)); +} // pub const GatherDimensionNumbersAttribute = struct { // _inner: c.MlirAttribute, @@ -1204,29 +1209,33 @@ pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasA return @ptrCast(OutputOperandAliasAttribute.get(ctx, args)); } -// pub const PrecisionAttribute = struct { -// _inner: c.MlirAttribute, +pub const PrecisionAttribute = opaque { + const M = mlir.Methods(PrecisionAttribute, mlir.Attribute); -// pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; -// const Self = PrecisionAttribute; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); + pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); -// pub const Precision = enum { -// DEFAULT, -// HIGH, -// HIGHEST, -// }; + pub const Precision = enum { + DEFAULT, + HIGH, + HIGHEST, + }; -// pub fn init(ctx: *mlir.Context, value: Precision) Self { -// return .{ ._inner = c.stablehloPrecisionAttrGet(ctx._inner, stringRef(@tagName(value))) }; -// } + pub fn init(ctx: *mlir.Context, value: Precision) *const PrecisionAttribute { + return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); + } -// pub fn getValue(self: Self) Precision { -// const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self._inner)); -// return std.meta.stringToEnum(Precision, value) orelse unreachable; -// } -// }; + pub fn getValue(self: *const PrecisionAttribute) Precision { + const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Precision, value) orelse unreachable; + } +}; + +pub fn precisionAttribute(ctx: *mlir.Context, precision: PrecisionAttribute.Precision) *const mlir.Attribute { + return @ptrCast(PrecisionAttribute.init(ctx, precision)); +} pub const ComparisonDirection = opaque { const M = mlir.Methods(ComparisonDirection, c.MlirAttribute); From 00760331d324958197d3730846aa8a0ade115b37 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 14:25:54 +0200 Subject: [PATCH 31/47] stablehlo: bound gather --- mlir/dialects/stablehlo/stablehlo.zig | 228 ++++++++++++++------------ 1 file changed, 123 insertions(+), 105 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 33dcbc7b..a63bec55 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -303,45 +303,45 @@ pub fn select( }); } -// pub fn gather( -// ctx: *mlir.Context, -// value: *const mlir.Value, -// indices: *const mlir.Value, -// slice_sizes: []const i64, -// location: *const mlir.Location, -// args: struct { -// offset_dims: []const i64, -// collapsed_slice_dims: []const i64, -// operand_batching_dims: []const i64, -// start_indices_batching_dims: []const i64, -// start_index_map: []const i64, -// index_vector_dim: i64, -// indices_are_sorted: bool = false, -// }, -// ) *mlir.Operation { -// return mlir.Operation.make( -// ctx, -// "stablehlo.gather", -// .{ -// .operands = .{ .flat = &.{ value, indices } }, -// .result_type_inference = true, -// .attributes = &.{ -// .{ "dimension_numbers", GatherDimensionNumbersAttribute.init( -// ctx, -// args.offset_dims, -// args.collapsed_slice_dims, -// args.operand_batching_dims, -// args.start_indices_batching_dims, -// args.start_index_map, -// args.index_vector_dim, -// ).asAttr() }, -// .{ "slice_sizes", .dense(ctx, .i64, slice_sizes) }, -// .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, -// }, -// .location = location, -// }, -// ); -// } +pub fn gather( + ctx: *mlir.Context, + value: *const mlir.Value, + indices: *const mlir.Value, + slice_sizes: []const i64, + location: *const mlir.Location, + args: struct { + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, + indices_are_sorted: bool = false, + }, +) *mlir.Operation { + return mlir.Operation.make( + ctx, + "stablehlo.gather", + .{ + .operands = .{ .flat = &.{ value, indices } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "dimension_numbers", gatherDimensionNumbersAttribute( + ctx, + args.offset_dims, + args.collapsed_slice_dims, + args.operand_batching_dims, + args.start_indices_batching_dims, + args.start_index_map, + args.index_vector_dim, + )), + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, slice_sizes)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), + }, + .location = location, + }, + ); +} // fn elementTypeOrSelf(typ: *const mlir.Type) *const mlir.Type { // return if (typ.as(mlir.ShapedType)) |shaped| { @@ -1012,85 +1012,103 @@ pub fn dotDimensionNumbersAttribute(ctx: *mlir.Context, args: DotDimensionNumber return @ptrCast(DotDimensionNumbersAttribute.init(ctx, args)); } -// pub const GatherDimensionNumbersAttribute = struct { -// _inner: c.MlirAttribute, +pub const GatherDimensionNumbersAttribute = opaque { + const M = mlir.Methods(GatherDimensionNumbersAttribute, mlir.Attribute); -// pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; -// const Self = GatherDimensionNumbersAttribute; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); + pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.eql(c.mlirAttributePrint); -// pub fn init( -// ctx: *mlir.Context, -// offset_dims: []const i64, -// collapsed_slice_dims: []const i64, -// operand_batching_dims: []const i64, -// start_indices_batching_dims: []const i64, -// start_index_map: []const i64, -// index_vector_dim: i64, -// ) Self { -// return .{ -// ._inner = c.stablehloGatherDimensionNumbersGet( -// ctx._inner, -// @intCast(offset_dims.len), -// offset_dims.ptr, -// @intCast(collapsed_slice_dims.len), -// collapsed_slice_dims.ptr, -// @intCast(operand_batching_dims.len), -// operand_batching_dims.ptr, -// @intCast(start_indices_batching_dims.len), -// start_indices_batching_dims.ptr, -// @intCast(start_index_map.len), -// start_index_map.ptr, -// index_vector_dim, -// ), -// }; -// } + pub fn init( + ctx: *mlir.Context, + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, + ) *const GatherDimensionNumbersAttribute { + return @ptrCast(c.stablehloGatherDimensionNumbersGet( + ctx.ptr(), + @intCast(offset_dims.len), + offset_dims.ptr, + @intCast(collapsed_slice_dims.len), + collapsed_slice_dims.ptr, + @intCast(operand_batching_dims.len), + operand_batching_dims.ptr, + @intCast(start_indices_batching_dims.len), + start_indices_batching_dims.ptr, + @intCast(start_index_map.len), + start_index_map.ptr, + index_vector_dim, + ).ptr); + } -// pub fn getOffsetDimsSize(self: Self) usize { -// return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self._inner)); -// } + pub fn getOffsetDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self.ptr())); + } -// pub fn getOffsetDimsElem(self: Self, pos: usize) i64 { -// return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self._inner, @intCast(pos)); -// } + pub fn getOffsetDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self.ptr(), @intCast(pos)); + } -// pub fn getCollapsedSliceDimsSize(self: Self) usize { -// return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self._inner)); -// } + pub fn getCollapsedSliceDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self.ptr())); + } -// pub fn getCollapsedSliceDimsElem(self: Self, pos: usize) i64 { -// return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self._inner, @intCast(pos)); -// } + pub fn getCollapsedSliceDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self.ptr(), @intCast(pos)); + } -// pub fn getStartIndexMapSize(self: Self) usize { -// return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self._inner)); -// } + pub fn getStartIndexMapSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self.ptr())); + } -// pub fn getOperandBatchingDimsSize(self: Self) usize { -// return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self._inner)); -// } + pub fn getOperandBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self.ptr())); + } -// pub fn getOperandBatchingDimsElem(self: Self, pos: usize) i64 { -// return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self._inner, @intCast(pos)); -// } + pub fn getOperandBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self.ptr(), @intCast(pos)); + } -// pub fn getStartIndicesBatchingDimsSize(self: Self) usize { -// return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self._inner)); -// } + pub fn getStartIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self.ptr())); + } -// pub fn getStartIndicesBatchingDimsElem(self: Self, pos: usize) i64 { -// return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self._inner, @intCast(pos)); -// } + pub fn getStartIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self.ptr(), @intCast(pos)); + } -// pub fn getStartIndexMapElem(self: Self, pos: usize) i64 { -// return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self._inner, @intCast(pos)); -// } + pub fn getStartIndexMapElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self.ptr(), @intCast(pos)); + } -// pub fn getIndexVectorDim(self: Self) usize { -// return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self._inner)); -// } -// }; + pub fn getIndexVectorDim(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self.ptr())); + } +}; + +pub fn gatherDimensionNumbersAttribute( + ctx: *mlir.Context, + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, +) *const mlir.Attribute { + return @ptrCast(GatherDimensionNumbersAttribute.init( + ctx, + offset_dims, + collapsed_slice_dims, + operand_batching_dims, + start_indices_batching_dims, + start_index_map, + index_vector_dim, + )); +} // pub const ConvDimensionNumbersAttribute = struct { // _inner: c.MlirAttribute, From 1caa45638c43a707df3eebd57933eb6894a4487b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 14:31:10 +0200 Subject: [PATCH 32/47] stablehlo: bound scatter --- mlir/dialects/stablehlo/stablehlo.zig | 102 +++++++++++++------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index a63bec55..a702fc4f 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -349,59 +349,59 @@ pub fn gather( // } else typ; // } -// pub const ScatterArgs = struct { -// update_window_dims: []const i64, -// inserted_window_dims: []const i64, -// input_batching_dims: []const i64, -// scatter_indices_batching_dims: []const i64, -// scatter_dims_to_operand_dims: []const i64, -// index_vector_dim: i64, -// indices_are_sorted: bool = false, -// unique_indices: bool = false, +pub const ScatterArgs = struct { + update_window_dims: []const i64, + inserted_window_dims: []const i64, + input_batching_dims: []const i64, + scatter_indices_batching_dims: []const i64, + scatter_dims_to_operand_dims: []const i64, + index_vector_dim: i64, + indices_are_sorted: bool = false, + unique_indices: bool = false, -// pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { -// return .{ ._inner = c.stablehloScatterDimensionNumbersGet( -// ctx._inner, -// @intCast(self.update_window_dims.len), -// self.update_window_dims.ptr, -// @intCast(self.inserted_window_dims.len), -// self.inserted_window_dims.ptr, -// @intCast(self.input_batching_dims.len), -// self.input_batching_dims.ptr, -// @intCast(self.scatter_indices_batching_dims.len), -// self.scatter_indices_batching_dims.ptr, -// @intCast(self.scatter_dims_to_operand_dims.len), -// self.scatter_dims_to_operand_dims.ptr, -// self.index_vector_dim, -// ) }; -// } -// }; + pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { + return @ptrCast(c.stablehloScatterDimensionNumbersGet( + ctx.ptr(), + @intCast(self.update_window_dims.len), + self.update_window_dims.ptr, + @intCast(self.inserted_window_dims.len), + self.inserted_window_dims.ptr, + @intCast(self.input_batching_dims.len), + self.input_batching_dims.ptr, + @intCast(self.scatter_indices_batching_dims.len), + self.scatter_indices_batching_dims.ptr, + @intCast(self.scatter_dims_to_operand_dims.len), + self.scatter_dims_to_operand_dims.ptr, + self.index_vector_dim, + ).ptr); + } +}; -// pub fn scatter( -// ctx: *mlir.Context, -// inputs: []const *const mlir.Value, -// scatter_indices: []const *const mlir.Value, -// updates: []const *const mlir.Value, -// update_block: mlir.Block, -// args: ScatterArgs, -// location: *const mlir.Location, -// ) *mlir.Operation { -// return mlir.Operation.make( -// ctx, -// "stablehlo.scatter", -// .{ -// .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, -// .blocks = &.{update_block}, -// .attributes = &.{ -// .{ "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx) }, -// .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, -// .{ "unique_indices", .boolean(ctx, args.unique_indices) }, -// }, -// .result_type_inference = true, -// .location = location, -// }, -// ); -// } +pub fn scatter( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + scatter_indices: []const *const mlir.Value, + updates: []const *const mlir.Value, + update_block: *mlir.Block, + args: ScatterArgs, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make( + ctx, + "stablehlo.scatter", + .{ + .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, + .blocks = &.{update_block}, + .attributes = &.{ + .named(ctx, "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), + .named(ctx, "unique_indices", mlir.boolAttribute(ctx, args.unique_indices)), + }, + .result_type_inference = true, + .location = location, + }, + ); +} pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.iota", .{ From 1bbb2242030bf0f2a19987c2630dfc0aeccba2e2 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 14:55:21 +0200 Subject: [PATCH 33/47] stablehlo: bound convolution --- mlir/dialects/stablehlo/stablehlo.zig | 276 +++++++++++++------------- 1 file changed, 139 insertions(+), 137 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index a702fc4f..4bcfc1b7 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -711,72 +711,70 @@ pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, ind }); } -// pub const ConvolutionOpts = struct { -// window_strides: []const i64, -// pad_value: []const i64, -// pad_shape: []const i64 = &.{}, -// lhs_dilation: []const i64, -// rhs_dilation: []const i64, -// window_reversal: []const bool, -// input_batch_dimension: i64, -// input_feature_dimension: i64, -// input_spatial_dimensions: []const i64, -// kernel_input_feature_dimension: i64, -// kernel_output_feature_dimension: i64, -// kernel_spatial_dimensions: []const i64, -// output_batch_dimension: i64, -// output_feature_dimension: i64, -// output_spatial_dimensions: []const i64, -// feature_group_count: i64, -// batch_group_count: i64, -// precision_config: []const PrecisionAttribute.Precision = &.{}, -// }; +pub const ConvolutionOpts = struct { + window_strides: []const i64, + pad_value: []const i64, + pad_shape: []const i64 = &.{}, + lhs_dilation: []const i64, + rhs_dilation: []const i64, + window_reversal: []const bool, + input_batch_dimension: i64, + input_feature_dimension: i64, + input_spatial_dimensions: []const i64, + kernel_input_feature_dimension: i64, + kernel_output_feature_dimension: i64, + kernel_spatial_dimensions: []const i64, + output_batch_dimension: i64, + output_feature_dimension: i64, + output_spatial_dimensions: []const i64, + feature_group_count: i64, + batch_group_count: i64, + precision_config: []const PrecisionAttribute.Precision = &.{}, +}; -// pub fn convolution( -// ctx: *mlir.Context, -// lhs: *const mlir.Value, -// rhs: *const mlir.Value, -// opts: ConvolutionOpts, -// res_type: *const mlir.Type, -// location: *const mlir.Location, -// ) *mlir.Operation { -// var max_precisions: [2]mlir.Attribute = undefined; -// for (opts.precision_config, 0..) |p, i| { -// max_precisions[i] = PrecisionAttribute.init(ctx, p).asAttr(); -// } -// var window_reversal: [3]i32 = undefined; -// for (opts.window_reversal, 0..) |w, i| { -// window_reversal[i] = @intCast(@intFromBool(w)); -// } -// return mlir.Operation.make(ctx, "stablehlo.convolution", .{ -// .operands = .{ .flat = &.{ lhs, rhs } }, -// .results = &.{res_type}, -// .attributes = &.{ -// .{ "window_strides", .dense(ctx, .i64, opts.window_strides) }, -// .{ "padding", .denseElements(ctx, opts.pad_shape, .i64, opts.pad_value) }, -// .{ "lhs_dilation", .dense(ctx, .i64, opts.lhs_dilation) }, -// .{ "rhs_dilation", .dense(ctx, .i64, opts.rhs_dilation) }, -// .{ "window_reversal", .dense(ctx, .bool, window_reversal[0..opts.window_reversal.len]) }, -// .{ -// "dimension_numbers", ConvDimensionNumbersAttribute.init(ctx, .{ -// .input_batch_dimension = opts.input_batch_dimension, -// .input_feature_dimension = opts.input_feature_dimension, -// .input_spatial_dimensions = opts.input_spatial_dimensions, -// .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, -// .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, -// .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, -// .output_batch_dimension = opts.output_batch_dimension, -// .output_feature_dimension = opts.output_feature_dimension, -// .output_spatial_dimensions = opts.output_spatial_dimensions, -// }).asAttr(), -// }, -// .{ "feature_group_count", .int(ctx, .i64, opts.feature_group_count) }, -// .{ "batch_group_count", .int(ctx, .i64, opts.batch_group_count) }, -// .{ "precision_config", .array(ctx, &max_precisions) }, -// }, -// .location = location, -// }); -// } +pub fn convolution( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + opts: ConvolutionOpts, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var max_precisions: [2]*const mlir.Attribute = undefined; + for (opts.precision_config, 0..) |p, i| { + max_precisions[i] = precisionAttribute(ctx, p); + } + var window_reversal: [3]i32 = undefined; + for (opts.window_reversal, 0..) |w, i| { + window_reversal[i] = @intCast(@intFromBool(w)); + } + return mlir.Operation.make(ctx, "stablehlo.convolution", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "window_strides", mlir.denseArrayAttribute(ctx, .i64, opts.window_strides)), + .named(ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(opts.pad_shape, mlir.integerType(ctx, .i64)).shaped(), opts.pad_value)), + .named(ctx, "lhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.lhs_dilation)), + .named(ctx, "rhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.rhs_dilation)), + .named(ctx, "window_reversal", mlir.denseArrayAttribute(ctx, .bool, window_reversal[0..opts.window_reversal.len])), + .named(ctx, "dimension_numbers", convDimensionNumbersAttribute(ctx, .{ + .input_batch_dimension = opts.input_batch_dimension, + .input_feature_dimension = opts.input_feature_dimension, + .input_spatial_dimensions = opts.input_spatial_dimensions, + .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, + .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, + .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, + .output_batch_dimension = opts.output_batch_dimension, + .output_feature_dimension = opts.output_feature_dimension, + .output_spatial_dimensions = opts.output_spatial_dimensions, + })), + .named(ctx, "feature_group_count", mlir.integerAttribute(ctx, .i64, opts.feature_group_count)), + .named(ctx, "batch_group_count", mlir.integerAttribute(ctx, .i64, opts.batch_group_count)), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &max_precisions)), + }, + .location = location, + }); +} pub const CustomCallOpts = struct { pub const ApiVersion = enum(i32) { @@ -1110,92 +1108,96 @@ pub fn gatherDimensionNumbersAttribute( )); } -// pub const ConvDimensionNumbersAttribute = struct { -// _inner: c.MlirAttribute, +pub const ConvDimensionNumbersAttribute = opaque { + const M = mlir.Methods(ConvDimensionNumbersAttribute, mlir.Attribute); -// pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; -// const Self = ConvDimensionNumbersAttribute; -// pub const asAttr = mlir.Attribute.fromAny(Self); -// pub const eql = mlir.Attribute.eqlAny(Self); + pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); -// pub fn init(ctx: *mlir.Context, args: struct { -// input_batch_dimension: i64, -// input_feature_dimension: i64, -// input_spatial_dimensions: []const i64, -// kernel_input_feature_dimension: i64, -// kernel_output_feature_dimension: i64, -// kernel_spatial_dimensions: []const i64, -// output_batch_dimension: i64, -// output_feature_dimension: i64, -// output_spatial_dimensions: []const i64, -// }) Self { -// return .{ -// ._inner = c.stablehloConvDimensionNumbersGet( -// ctx._inner, -// args.input_batch_dimension, -// args.input_feature_dimension, -// @intCast(args.input_spatial_dimensions.len), -// args.input_spatial_dimensions.ptr, -// args.kernel_input_feature_dimension, -// args.kernel_output_feature_dimension, -// @intCast(args.kernel_spatial_dimensions.len), -// args.kernel_spatial_dimensions.ptr, -// args.output_batch_dimension, -// args.output_feature_dimension, -// @intCast(args.output_spatial_dimensions.len), -// args.output_spatial_dimensions.ptr, -// ), -// }; -// } + pub const InitArgs = struct { + input_batch_dimension: i64, + input_feature_dimension: i64, + input_spatial_dimensions: []const i64, + kernel_input_feature_dimension: i64, + kernel_output_feature_dimension: i64, + kernel_spatial_dimensions: []const i64, + output_batch_dimension: i64, + output_feature_dimension: i64, + output_spatial_dimensions: []const i64, + }; -// pub fn getInputBatchDimension(self: Self) i64 { -// return c.stablehloConvDimensionNumbersGetInputBatchDimension(self._inner); -// } + pub fn init(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttribute { + return @ptrCast(c.stablehloConvDimensionNumbersGet( + ctx.ptr(), + args.input_batch_dimension, + args.input_feature_dimension, + @intCast(args.input_spatial_dimensions.len), + args.input_spatial_dimensions.ptr, + args.kernel_input_feature_dimension, + args.kernel_output_feature_dimension, + @intCast(args.kernel_spatial_dimensions.len), + args.kernel_spatial_dimensions.ptr, + args.output_batch_dimension, + args.output_feature_dimension, + @intCast(args.output_spatial_dimensions.len), + args.output_spatial_dimensions.ptr, + ).ptr); + } -// pub fn getInputFeatureDimension(self: Self) i64 { -// return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self._inner); -// } + pub fn getInputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetInputBatchDimension(self.ptr()); + } -// pub fn getInputSpatialDimensionsSize(self: Self) usize { -// return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self._inner)); -// } + pub fn getInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self.ptr()); + } -// pub fn getInputSpatialDimensionsElem(self: Self, pos: usize) i64 { -// return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self._inner, @intCast(pos)); -// } + pub fn getInputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self.ptr())); + } -// pub fn getKernelInputFeatureDimension(self: Self) i64 { -// return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self._inner); -// } + pub fn getInputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self.ptr(), @intCast(pos)); + } -// pub fn getKernelOutputFeatureDimension(self: Self) i64 { -// return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self._inner); -// } + pub fn getKernelInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self.ptr()); + } -// pub fn getKernelSpatialDimensionsSize(self: Self) usize { -// return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self._inner)); -// } + pub fn getKernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self.ptr()); + } -// pub fn getKernelSpatialDimensionsElem(self: Self, pos: usize) i64 { -// return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self._inner, @intCast(pos)); -// } + pub fn getKernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self.ptr())); + } -// pub fn getOutputBatchDimension(self: Self) i64 { -// return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self._inner); -// } + pub fn getKernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self.ptr(), @intCast(pos)); + } -// pub fn getOutputFeatureDimension(self: Self) i64 { -// return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self._inner); -// } + pub fn getOutputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self.ptr()); + } -// pub fn getOutputSpatialDimensionsSize(self: Self) usize { -// return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self._inner)); -// } + pub fn getOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self.ptr()); + } -// pub fn getOutputSpatialDimensionsElem(self: Self, pos: usize) i64 { -// return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self._inner, @intCast(pos)); -// } -// }; + pub fn getOutputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self.ptr())); + } + + pub fn getOutputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self.ptr(), @intCast(pos)); + } +}; + +pub fn convDimensionNumbersAttribute(ctx: *mlir.Context, args: ConvDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(ConvDimensionNumbersAttribute.init(ctx, args)); +} pub const OutputOperandAliasAttribute = opaque { const M = mlir.Methods(OutputOperandAliasAttribute, c.MlirAttribute); From c78c8402c49785126b87816bcea6ab8ca01b2120 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 15:03:12 +0200 Subject: [PATCH 34/47] stablehlo: added missing methods --- mlir/dialects/stablehlo/stablehlo.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 4bcfc1b7..75d7b08f 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -1263,6 +1263,7 @@ pub const ComparisonDirection = opaque { pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Direction = enum { EQ, @@ -1293,6 +1294,7 @@ pub const CompareType = opaque { pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { SIGNED, @@ -1321,6 +1323,7 @@ pub const TransposeAttribute = opaque { pub const isAFn = c.stablehloAttributeIsATransposeAttr; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { NO_TRANSPOSE, @@ -1348,6 +1351,7 @@ pub const FftType = opaque { pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { FFT, From c79427e3f49faab3f3776757ed861ad9d3b260d7 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 16:22:11 +0200 Subject: [PATCH 35/47] stablehlo: reorder parameters for consistency --- mlir/dialects/stablehlo/stablehlo.zig | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 75d7b08f..cbf8d825 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -169,7 +169,6 @@ pub fn dot_general( lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, - location: *const mlir.Location, opts: struct { lhs_batching_dimensions: []const i64, rhs_batching_dimensions: []const i64, @@ -177,6 +176,7 @@ pub fn dot_general( rhs_contracting_dimensions: []const i64, dot_precision: DotPrecision, }, + location: *const mlir.Location, ) *mlir.Operation { const precisions: [2]*const mlir.Attribute = @splat(precisionAttribute(ctx, opts.dot_precision.precision())); const attributes = [3]mlir.NamedAttribute{ @@ -244,8 +244,8 @@ pub fn transpose( ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, - location: *const mlir.Location, opts: struct { permutation: []const i64 }, + location: *const mlir.Location, ) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.transpose", .{ .operands = .{ .flat = &.{value} }, @@ -308,7 +308,6 @@ pub fn gather( value: *const mlir.Value, indices: *const mlir.Value, slice_sizes: []const i64, - location: *const mlir.Location, args: struct { offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -318,6 +317,7 @@ pub fn gather( index_vector_dim: i64, indices_are_sorted: bool = false, }, + location: *const mlir.Location, ) *mlir.Operation { return mlir.Operation.make( ctx, @@ -548,7 +548,7 @@ pub const PadOpts = struct { interior: []const i64, }; -pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, location: *const mlir.Location, opts: PadOpts) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.pad", .{ .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, @@ -568,7 +568,7 @@ pub const TriangularSolveOpts = struct { transpose_a: TransposeAttribute.Type, }; -pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, location: *const mlir.Location, opts: TriangularSolveOpts) *mlir.Operation { +pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, opts: TriangularSolveOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ .operands = .{ .flat = &.{ value, other } }, .result_type_inference = true, @@ -587,7 +587,7 @@ pub const FftOpts = struct { length: []const i64, }; -pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location, opts: FftOpts) *mlir.Operation { +pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, opts: FftOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.fft", .{ .operands = .{ .flat = &.{value} }, .result_type_inference = true, @@ -669,7 +669,14 @@ pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape }); } -pub fn rng_bit_generator(ctx: *mlir.Context, rng_algorithm: RngAlgorithm.Type, initial_state: *const mlir.Value, res_state_type: *const mlir.Type, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn rng_bit_generator( + ctx: *mlir.Context, + rng_algorithm: RngAlgorithm.Type, + initial_state: *const mlir.Value, + res_state_type: *const mlir.Type, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ .operands = .{ .flat = &.{initial_state} }, .results = &.{ res_state_type, res_type }, @@ -736,8 +743,8 @@ pub fn convolution( ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, - opts: ConvolutionOpts, res_type: *const mlir.Type, + opts: ConvolutionOpts, location: *const mlir.Location, ) *mlir.Operation { var max_precisions: [2]*const mlir.Attribute = undefined; From c936aec3476bfb41aa5974057d1ec094ae5ba3e4 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 16:31:28 +0200 Subject: [PATCH 36/47] mlir: swap new mlir module --- mlir/BUILD.bazel | 21 +- mlir/dialects/BUILD.bazel | 2 +- mlir/dialects/func.zig | 2 +- mlir/dialects/stablehlo/BUILD.bazel | 2 +- mlir/dialects/stablehlo/stablehlo.zig | 2 +- mlir/main.zig | 2 +- mlir/mlir.zig | 2795 +++++++++++-------------- mlir/registry.zig | 1466 ------------- 8 files changed, 1249 insertions(+), 3043 deletions(-) mode change 100755 => 100644 mlir/mlir.zig delete mode 100644 mlir/registry.zig diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 73f1d12d..e91a2c91 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -28,33 +28,14 @@ zig_test( deps = [":mlir"], ) -zig_library( - name = "mlir2", - main = "registry.zig", - visibility = ["//visibility:public"], - deps = [ - ":c", - "//stdx", - ], -) - zig_binary( name = "main", main = "main.zig", visibility = ["//visibility:public"], deps = [ - ":mlir2", + ":mlir", "//mlir/dialects", # ":c", # "//stdx", ], ) - -zig_test( - name = "test2", - deps = [ - ":mlir2", - # ":c", - # "//stdx", - ], -) diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 34400ca9..8b07057c 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -13,7 +13,7 @@ zig_library( main = "dialects.zig", visibility = ["//visibility:public"], deps = [ - "//mlir:mlir2", + "//mlir:mlir", "//mlir/dialects/stablehlo", "//stdx", ], diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index d316f160..c2af12a5 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const mlir = @import("mlir2"); +const mlir = @import("mlir"); const stdx = @import("stdx"); const FuncOpArgs = struct { diff --git a/mlir/dialects/stablehlo/BUILD.bazel b/mlir/dialects/stablehlo/BUILD.bazel index 442e8c7b..0596357a 100644 --- a/mlir/dialects/stablehlo/BUILD.bazel +++ b/mlir/dialects/stablehlo/BUILD.bazel @@ -7,7 +7,7 @@ zig_library( visibility = ["//mlir/dialects:__subpackages__"], deps = [ "//mlir:c", - "//mlir:mlir2", + "//mlir:mlir", "//stdx", "@stablehlo//:stablehlo_dialect_capi", ], diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index cbf8d825..8789c7b0 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -1,7 +1,7 @@ const std = @import("std"); const c = @import("c"); -const mlir = @import("mlir2"); +const mlir = @import("mlir"); const stringRef = mlir.stringRef; const stdx = @import("stdx"); diff --git a/mlir/main.zig b/mlir/main.zig index 4bf94117..8eee2bbe 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -1,7 +1,7 @@ const std = @import("std"); const dialects = @import("mlir/dialects"); -const mlir = @import("mlir2"); +const mlir = @import("mlir"); pub fn main() !void { const allocator = std.heap.c_allocator; diff --git a/mlir/mlir.zig b/mlir/mlir.zig old mode 100755 new mode 100644 index a222c40a..7536e701 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const c = @import("c"); const stdx = @import("stdx"); @@ -7,9 +6,7 @@ const stdx = @import("stdx"); const log = std.log.scoped(.mlir); test { - std.testing.refAllDecls(@This()); - - _ = try Context.init(); + std.testing.refAllDeclsRecursive(@This()); } const Error = error{ @@ -19,954 +16,1055 @@ const Error = error{ MlirUnexpected, /// A resource/executor was not found. NotFound, - OutOfMemory, -}; +} || std.mem.Allocator.Error; pub inline fn stringRef(str: []const u8) c.MlirStringRef { return .{ .data = str.ptr, .length = str.len }; } -pub inline fn fromStringRef(str: c.MlirStringRef) []const u8 { +pub inline fn string(str: c.MlirStringRef) []const u8 { // Note: mlir.StringRef need not to be null terminated. return str.data[0..str.length]; } +pub const StringCallbackCtx = struct { + writer: *std.Io.Writer, + err: ?std.Io.Writer.Error = null, +}; + +pub fn stringCallback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { + const ctx: *StringCallbackCtx = @ptrCast(@alignCast(ctx_)); + ctx.writer.writeAll(string(str)) catch |err| { + ctx.err = err; + }; +} + pub fn registerPasses(comptime passes: []const u8) void { @field(c, "mlirRegister" ++ passes ++ "Passes")(); } -pub fn successOr(res: c.MlirLogicalResult, err: anytype) !void { - return if (res.value == 0) err; +fn isPtrConst(comptime T: type) bool { + return @typeInfo(T).pointer.is_const; } -/// Alternative to MlirWrapperType -pub const MlirStrCallback = fn (c.MlirStringRef, ?*anyopaque) callconv(.c) void; - -pub const Registry = struct { - _inner: c.MlirDialectRegistry, - - pub const deinit = helpers.deinit(Registry, c.mlirDialectRegistryDestroy); - - pub fn init() !Registry { - return helpers.init(Registry, c.mlirDialectRegistryCreate(), c.mlirDialectRegistryIsNull) orelse Error.MlirUnexpected; - } -}; - -pub const Context = struct { - _inner: c.MlirContext, - const Self = Context; - pub const deinit = helpers.deinit(Context, c.mlirContextDestroy); - pub const wrapOr = helpers.wrapOr(Context, c.mlirContextIsNull); - - pub fn init() !Self { - return Self.wrapOr(c.mlirContextCreate()) orelse Error.MlirUnexpected; - } - - pub fn initWithRegistry(registry: Registry, threadingEnabled: bool) !Self { - return Self.wrapOr( - c.mlirContextCreateWithRegistry(registry._inner, threadingEnabled), - ) orelse Error.InvalidMlir; - } - - pub fn setMultiThreading(self: *Self, enabled: bool) void { - c.mlirContextEnableMultithreading(self._inner, enabled); - } - - pub fn appendDialectRegistry(self: *Self, registry: Registry) void { - c.mlirContextAppendDialectRegistry(self._inner, registry._inner); - } - - pub fn loadAllAvailableDialects(self: *Self) void { - c.mlirContextLoadAllAvailableDialects(self._inner); - } - - pub fn numRegisteredDialects(self: Self) usize { - return @intCast(c.mlirContextGetNumRegisteredDialects(self._inner)); - } - - pub fn numLoadedDialects(self: Self) usize { - return @intCast(c.mlirContextGetNumLoadedDialects(self._inner)); - } - - pub fn isRegisteredOperation(self: Self, op: [:0]const u8) bool { - return c.mlirContextIsRegisteredOperation(self._inner, stringRef(op)); - } - - pub fn location(self: Self, src: std.builtin.SourceLocation) Location { - return Location.fromSrc(self, src); - } -}; - -pub const Module = struct { - _inner: c.MlirModule, - - pub const deinit = helpers.deinit(Module, c.mlirModuleDestroy); - pub const wrapOr = helpers.wrapOr(Module, c.mlirModuleIsNull); - - const Self = Module; - - pub fn init(loc: Location) Self { - return .{ ._inner = c.mlirModuleCreateEmpty(loc._inner) }; - } - - pub fn parse(ctx: Context, source: [:0]const u8) !Module { - return Module.wrapOr( - c.mlirModuleCreateParse(ctx._inner, stringRef(source)), - ) orelse Error.InvalidMlir; - } - - pub fn fromOperation(operation: Operation) Module { - return .{ ._inner = c.mlirModuleFromOperation(operation._inner) }; - } - - pub fn context(self: Module) Context { - return .{ ._inner = c.mlirModuleGetContext(self._inner) }; - } - - pub fn getBody(self: Module) Block { - return .{ ._inner = c.mlirModuleGetBody(self._inner) }; - } - - pub fn op(self: Module) Operation { - return .{ ._inner = c.mlirModuleGetOperation(self._inner) }; - } - - pub fn hash(self: Module, hasher: *std.hash.XxHash64) void { - return self.op().hash(hasher); - } -}; - -pub const PassManager = struct { - _inner: c.MlirPassManager, - - pub const deinit = helpers.deinit(PassManager, c.mlirPassManagerDestroy); - pub const wrapOr = helpers.wrapOr(PassManager, c.mlirPassManagerIsNull); - - const Self = PassManager; - - pub fn init(ctx: Context) !Self { - return Self.wrapOr( - c.mlirPassManagerCreate(ctx._inner), - ) orelse Error.MlirUnexpected; - } - - pub fn initOnOperation(ctx: Context, op: [:0]const u8) !Self { - return Self.wrapOr( - c.mlirPassManagerCreateOnOperation(ctx._inner, stringRef(op)), - ) orelse Error.MlirUnexpected; - } - - pub fn asOpPassManager(self: Self) OpPassManager { - return .{ ._inner = c.mlirPassManagerGetAsOpPassManager(self._inner) }; - } - - pub fn enableIRPrinting(self: *Self) void { - c.mlirPassManagerEnableIRPrinting(self._inner); - } - - pub fn runOnOp(self: *Self, op: Operation) error{InvalidMlir}!void { - if (c.mlirPassManagerRunOnOp(self._inner, op._inner).value == 0) { - return Error.InvalidMlir; - } - } -}; - -fn _mlir_passpipeline_error(err: c.MlirStringRef, ctx: ?*anyopaque) callconv(.c) void { - _ = ctx; - std.debug.print(">>ERROR: {s}\n", .{err.data}); +fn CastedPtr(comptime T1: type, comptime T2: type) type { + return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; } -pub const OpPassManager = struct { - _inner: c.MlirOpPassManager, - - pub fn addPipeline(self: *OpPassManager, pipeline: [:0]const u8) error{OutOfMemory}!void { - if (c.mlirOpPassManagerAddPipeline( - self._inner, - stringRef(pipeline), - &_mlir_passpipeline_error, - null, - ).value == 0) { - return Error.OutOfMemory; - } - } -}; - -pub const Identifier = struct { - _inner: c.MlirIdentifier, - const Self = Identifier; - - pub fn get(ctx: Context, str_: [:0]const u8) Self { - return .{ ._inner = c.mlirIdentifierGet(ctx._inner, stringRef(str_)) }; - } - - pub fn context(self: Self) Context { - return .{ ._inner = c.mlirIdentifierGetContext(self._inner) }; - } - - pub fn str(self: Self) []const u8 { - return fromStringRef(c.mlirIdentifierStr(self._inner)); - } - - pub fn equals(self: Self, other: Self) bool { - return c.mlirIdentifierEqual(self._inner, other._inner); - } -}; - -pub const AttrTuple = struct { [:0]const u8, Attribute }; - -pub const Attribute = struct { - _inner: c.MlirAttribute, - - pub const dump = helpers.dump(Attribute, c.mlirAttributeDump); - pub const eql = helpers.eql(Attribute, c.mlirAttributeEqual); - pub const format = helpers.format(Attribute, c.mlirAttributePrint); - pub const wrapOr = helpers.wrapOr(Attribute, c.mlirAttributeIsNull); - - pub fn wrap(c_attr: c.MlirAttribute) Attribute { - return .{ ._inner = c_attr }; - } - - pub fn parse(ctx: Context, attr: [:0]const u8) !Attribute { - return Attribute.wrapOr( - c.mlirAttributeParseGet(ctx._inner, stringRef(attr)), - ) orelse Error.InvalidMlir; - } - - pub fn fromAny(SpecificAttr: type) fn (x: SpecificAttr) Attribute { - return struct { - fn cast(x: SpecificAttr) Attribute { - return .{ ._inner = x._inner }; - } - }.cast; - } - - pub fn isA(self: Attribute, SpecificAttr: type) bool { - return SpecificAttr.is_a_fn(self._inner); - } - - // utilities function to built common attributes. - // All attributes are upcasted to the Attribute type, making it easier to chain construct, - // but losing type information. - - pub fn null_() Attribute { - return .wrap(c.mlirAttributeGetNull()); - } - - pub fn string(ctx: Context, str: []const u8) Attribute { - return StringAttribute.init(ctx, str).asAttr(); - } - - pub fn type_(t: Type) Attribute { - return TypeAttribute.init(t).asAttr(); - } - - pub fn unit(ctx: Context) Attribute { - return .wrap(c.mlirUnitAttrGet(ctx._inner)); - } - - pub fn boolean(ctx: Context, value: bool) Attribute { - return BoolAttribute.init(ctx, value).asAttr(); - } - - pub fn i1FromBool(ctx: Context, value: bool) Attribute { - return IntegerAttribute(.i1).init(ctx, @intFromBool(value)).asAttr(); - } - - pub fn int(ctx: Context, comptime int_type: IntegerTypes, value: i64) Attribute { - return IntegerAttribute(int_type).init(ctx, value).asAttr(); - } - - pub fn float(ctx: Context, comptime float_type: FloatTypes, value: f64) Attribute { - return FloatAttribute(float_type).init(ctx, value).asAttr(); - } - - pub fn array(ctx: Context, attrs: []const Attribute) Attribute { - return ArrayAttribute.init(ctx, attrs).asAttr(); - } - - pub fn dense(ctx: Context, comptime dt: DenseArrayTypes, values: []const dt.ZigType()) Attribute { - return DenseArrayAttribute(dt).init(ctx, values).asAttr(); - } - - /// Use a tensor as an attribute. - /// The tensor is specified by dims, dtype and a flat slice of values. - pub fn denseElements(ctx: Context, dims: []const i64, comptime dt: DenseElementsAttributeTypes, values: []const dt.ZigType()) Attribute { - return DenseElementsAttribute(dt).init(.tensor(dims, dt.mlirType(ctx)), values).asAttr(); - } - - pub fn denseElementsFromBytes(ctx: Context, dims: []const i64, dt: DenseElementsAttributeTypes, raw_bytes: []const u8) Attribute { - const shape: Type = .tensor(dims, dt.mlirType(ctx)); - return .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( - shape._inner, - @intCast(raw_bytes.len), - raw_bytes.ptr, - ) }; - } - - pub fn symbol(ctx: Context, flat_name: [:0]const u8) Attribute { - return FlatSymbolRefAttribute.init(ctx, flat_name).asAttr(); - } - - pub fn named(attr: Attribute, ctx: Context, name: [:0]const u8) NamedAttribute { - return NamedAttribute.named(ctx, name, attr); - } - - pub fn dict(ctx: Context, named_attrs: []const AttrTuple) Attribute { - var attr_buf: [32]NamedAttribute = undefined; - stdx.debug.assert(named_attrs.len <= attr_buf.len, ".dict helper only support up to {} attribute, got {}. You will need to call mlir.DictionaryAttribute directly", .{ attr_buf.len, named_attrs.len }); - - const attrs = attr_buf[0..named_attrs.len]; - for (attrs, named_attrs) |*attr, tuple| { - attr.* = .named(ctx, tuple[0], tuple[1]); - } - - return DictionaryAttribute.init(ctx, attrs).asAttr(); - } -}; - -pub const NamedAttribute = extern struct { - _inner: c.MlirNamedAttribute, - - pub fn wrap(c_named_attribute: c.MlirNamedAttribute) NamedAttribute { - return @bitCast(c_named_attribute); - } - - pub fn named(ctx: Context, name: [:0]const u8, attr: Attribute) NamedAttribute { - return .{ ._inner = .{ - .name = c.mlirIdentifierGet(ctx._inner, stringRef(name)), - .attribute = attr._inner, - } }; - } - - pub fn init(name: Identifier, attr: Attribute) NamedAttribute { - return .{ ._inner = .{ - .name = name._inner, - .attribute = attr._inner, - } }; - } -}; - -pub const StringAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAString; - const Self = StringAttribute; - pub const asAttr = Attribute.fromAny(Self); - pub const eql = Attribute.eqlAny(Self); - - pub fn init(ctx: Context, str: []const u8) Self { - return .{ ._inner = c.mlirStringAttrGet(ctx._inner, stringRef(str)) }; - } - - pub fn value(self: Self) []const u8 { - return fromStringRef(c.mlirStringAttrGetValue(self._inner)); - } -}; - -pub const BoolAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsABool; - const Self = BoolAttribute; - pub const asAttr = Attribute.fromAny(Self); - pub const eql = Attribute.eqlAny(Self); - - pub fn init(ctx: Context, value_: bool) Self { - return .{ ._inner = c.mlirBoolAttrGet(ctx._inner, if (value_) 1 else 0) }; - } - - pub fn value(self: Self) bool { - return c.mlirBoolAttrGetValue(self._inner); - } -}; - -pub const TypeAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAType; - pub const eql = Attribute.eqlAny(TypeAttribute); - - pub fn init(type_: Type) TypeAttribute { - return .{ ._inner = c.mlirTypeAttrGet(type_._inner) }; - } - - pub fn typ(self: TypeAttribute) Type { - return .{ ._inner = c.mlirAttributeGetType(self._inner) }; - } - - pub const asAttr = Attribute.fromAny(TypeAttribute); -}; - -pub const ArrayAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAArray; - const Self = ArrayAttribute; - pub const asAttr = Attribute.fromAny(Self); - pub const eql = Attribute.eqlAny(Self); - - pub fn init(ctx: Context, attrs: []const Attribute) Self { - return .{ ._inner = c.mlirArrayAttrGet(ctx._inner, @intCast(attrs.len), @ptrCast(attrs.ptr)) }; - } - - pub fn size(self: Self) usize { - return @intCast(c.mlirArrayAttrGetNumElements(self._inner)); - } - - pub fn get(self: Self, index: usize) Attribute { - return .{ ._inner = c.mlirArrayAttrGetElement(self._inner, @intCast(index)) }; - } -}; - -pub fn IntegerAttribute(comptime it: IntegerTypes) type { - const ZigType, const getter = comptime switch (it) { - .i1, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, - .si4, .si8, .si16, .si32, .si64 => .{ i64, c.mlirIntegerAttrGetValueSInt }, - .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, - .unknown => @compileError("IntegerAttribute(unknown)"), - }; - +pub fn Methods(comptime T: type, comptime NativeT: type) type { return struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAInteger; + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; + pub const Ptr = if (IsPtrConst) *const T else *T; - pub const IntegerTypeType = IntegerType(it); - const IntAttr = @This(); - - pub const asAttr = Attribute.fromAny(IntAttr); - pub const eql = Attribute.eqlAny(IntAttr); - - pub fn init(ctx: Context, value: i64) IntAttr { - return .{ ._inner = c.mlirIntegerAttrGet( - IntegerType(it).init(ctx)._inner, - value, - ) }; + pub fn ptr(self: anytype) NativeT { + return .{ .ptr = @ptrCast(@constCast(self)) }; } - pub fn get(value: IntAttr) ZigType { - return @intCast(getter(value._inner)); + fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return struct { + fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var ctx: StringCallbackCtx = .{ .writer = writer }; + f(self.ptr(), stringCallback, &ctx); + if (ctx.err) |err| { + return err; + } + } + }.format; + } + + pub fn eql(f: anytype) fn (self: *const T, other: *const T) bool { + return struct { + fn eql(self: *const T, other: *const T) bool { + return f(self.ptr(), other.ptr()); + } + }.eql; + } + + pub fn isA(self: *const T, OtherType: type) ?CastedPtr(@TypeOf(self), OtherType) { + return if (OtherType.isAFn(self.ptr())) @ptrCast(self) else null; } }; } -pub fn FloatAttribute(comptime ft: FloatTypes) type { - return struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAFloat; - const FloatAttr = @This(); - pub const asAttr = Attribute.fromAny(FloatAttr); +pub const DialectRegistry = opaque { + const M = Methods(DialectRegistry, c.MlirDialectRegistry); - pub fn init(ctx: Context, value: f64) FloatAttr { - return .{ ._inner = c.mlirFloatAttrDoubleGet( - ctx._inner, - FloatType(ft).init(ctx)._inner, - value, - ) }; - } + pub const ptr = M.ptr; - pub fn get(value: FloatAttr) f64 { - return c.mlirFloatAttrGetValueDouble(value._inner); - } + pub fn init() std.mem.Allocator.Error!*DialectRegistry { + return @ptrCast(c.mlirDialectRegistryCreate().ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *DialectRegistry) void { + c.mlirDialectRegistryDestroy(self.ptr()); + } + + pub fn registerDialect(self: *DialectRegistry, comptime name: []const u8) void { + DialectHandle.fromString(name).insertDialect(self); + } +}; + +pub const Context = opaque { + const M = Methods(Context, c.MlirContext); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirContextEqual); + + pub const InitOptions = struct { + registry: ?*const DialectRegistry = null, + threading: bool = false, }; + + pub fn init(opts: InitOptions) std.mem.Allocator.Error!*Context { + const ctx = if (opts.registry) |r| c.mlirContextCreateWithRegistry( + r.ptr(), + opts.threading, + ) else c.mlirContextCreateWithThreading(opts.threading); + return @ptrCast(ctx.ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *Context) void { + c.mlirContextDestroy(self.ptr()); + } + + pub fn setMultiThreading(self: *Context, enabled: bool) void { + c.mlirContextEnableMultithreading(self.ptr(), enabled); + } + + pub fn appendDialectRegistry(self: *Context, registry: *const DialectRegistry) void { + c.mlirContextAppendDialectRegistry(self.ptr(), registry.ptr()); + } + + pub fn loadAllAvailableDialects(self: *Context) void { + c.mlirContextLoadAllAvailableDialects(self.ptr()); + } + + pub fn numRegisteredDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumRegisteredDialects(self.ptr())); + } + + pub fn numLoadedDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumLoadedDialects(self.ptr())); + } + + pub fn numThreads(self: *const Context) usize { + return @intCast(c.mlirContextGetNumThreads(self.ptr())); + } + + pub fn isRegisteredOperation(self: *const Context, op: []const u8) bool { + return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); + } + + pub fn format(self: *const Context, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ + @typeName(Context), + self.numThreads(), + self.numRegisteredDialects(), + self.numLoadedDialects(), + }); + } +}; + +pub const Location = opaque { + const M = Methods(Location, c.MlirLocation); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirLocationEqual); + pub const format = M.format(c.mlirLocationPrint); + + pub fn fromSrc(ctx: *Context, src: std.builtin.SourceLocation) *const Location { + return fileLineCol( + ctx, + src.file, + src.line, + src.column, + ); + } + + pub fn fileLineCol(ctx: *Context, file: []const u8, line: usize, column: usize) *const Location { + return @ptrCast(c.mlirLocationFileLineColGet( + ctx.ptr(), + stringRef(file), + @intCast(line), + @intCast(column), + ).ptr); + } + + pub fn callSite(callee: *const Location, caller: *const Location) *const Location { + return @ptrCast(c.mlirLocationCallSiteGet(callee.ptr(), caller.ptr()).ptr); + } + + pub fn fused(ctx: *Context, locs: []const *const Location, metadata: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFusedGet( + ctx.ptr(), + @intCast(locs.len), + @ptrCast(locs), + metadata.ptr(), + ).ptr); + } + + pub fn named(self: *const Location, ctx: *Context, loc_name: []const u8) *const Location { + return @ptrCast(c.mlirLocationNameGet(ctx.ptr(), stringRef(loc_name), self.ptr()).ptr); + } + + pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { + var buf: [256]u8 = undefined; + return loc.named(loc, ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { + buf[buf.len - 3 ..].* = "...".*; + break :blk &buf; + }); + } + + pub fn fromAttribute(attribute: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFromAttribute(attribute.ptr()).ptr); + } + + pub fn unknown(ctx: *Context) *const Location { + return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); + } +}; + +pub const Type = opaque { + const M = Methods(Type, c.MlirType); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { + return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); + } +}; + +pub const IndexType = opaque { + const M = Methods(IndexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAIndex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context) *const IndexType { + return @ptrCast(c.mlirIndexTypeGet(ctx.ptr()).ptr); + } +}; + +pub fn indexType(ctx: *Context) *const Type { + return @ptrCast(IndexType.get(ctx)); } -pub const DenseArrayTypes = enum { - bool, - i8, - i16, - i32, - i64, - f32, - f64, - - pub fn ZigType(comptime dt: DenseArrayTypes) type { - return switch (dt) { - .bool => i32, - .i8 => i8, - .i16 => i16, - .i32 => i32, - .i64 => i64, - .f32 => f32, - .f64 => f64, - }; - } -}; - -pub fn DenseArrayAttribute(comptime dt: DenseArrayTypes) type { - const _is_a_fn, const get_fn, const get_element_fn = switch (dt) { - .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, - .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, - .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, - .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, - .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, - .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, - .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, - }; - - return struct { - _inner: c.MlirAttribute, - const Attr = @This(); - const ElementType = dt; - const ElementTypeZig = dt.ZigType(); - - pub const asAttr = Attribute.fromAny(Attr); - pub const eql = Attribute.eqlAny(Attr); - pub const is_a_fn = _is_a_fn; - - pub fn init(ctx: Context, values: []const ElementTypeZig) Attr { - return .{ ._inner = get_fn(ctx._inner, @intCast(values.len), @ptrCast(values.ptr)) }; - } - - pub fn get(self: Attr, pos: usize) ElementTypeZig { - return get_element_fn(self._inner, @intCast(pos)); - } - - pub fn len(self: Attr) usize { - return @intCast(c.mlirDenseArrayGetNumElements(self._inner)); - } - }; -} - -pub const DenseElementsAttributeTypes = enum { - bool, +pub const IntegerTypes = enum { + i1, + i4, i8, i16, i32, i64, + i128, + u1, + u4, u8, u16, u32, u64, + u128, + si1, + si4, + si8, + si16, + si32, + si64, + si128, + + pub fn bitwidth(self: IntegerTypes) usize { + return switch (self) { + .i1, .u1, .si1 => 1, + .i4, .u4, .si4 => 4, + .i8, .u8, .si8 => 8, + .i16, .u16, .si16 => 16, + .i32, .u32, .si32 => 32, + .i64, .u64, .si64 => 64, + .i128, .u128, .si128 => 128, + }; + } + + pub fn signedness(self: IntegerTypes) IntegerType.Signedness { + return switch (self) { + .i1, .i4, .i8, .i16, .i32, .i64, .i128 => .signless, + .si1, .si4, .si8, .si16, .si32, .si64, .si128 => .signed, + .u1, .u4, .u8, .u16, .u32, .u64, .u128 => .unsigned, + }; + } +}; + +pub const IntegerType = opaque { + pub const Signedness = enum { + signless, + signed, + unsigned, + }; + + const M = Methods(IntegerType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { + return exact(ctx, it.bitwidth(), it.signedness()); + } + + fn exact(ctx: *Context, bitwidth: usize, sign: Signedness) *const IntegerType { + return @ptrCast(switch (sign) { + .signless => c.mlirIntegerTypeGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .signed => c.mlirIntegerTypeSignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .unsigned => c.mlirIntegerTypeUnsignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + }); + } +}; + +pub fn integerType(ctx: *Context, it: IntegerTypes) *const Type { + return @ptrCast(IntegerType.get(ctx, it)); +} + +pub const FloatTypes = enum { + f4e2m1fn, + f6e2m3fn, + f6e3m2fn, + f8e3m4, + f8e4m3, + f8e4m3b11fnuz, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + f8e8m0fnu, bf16, f16, f32, f64, - index, - - pub fn ZigType(comptime dt: DenseElementsAttributeTypes) type { - return switch (dt) { - .bool => bool, - .i8 => i8, - .i16 => i16, - .i32 => i32, - .i64 => i64, - .u8 => u8, - .u16 => u16, - .u32 => u32, - .u64 => u64, - .bf16 => u16, - .f16 => f16, - .f32 => f32, - .f64 => f64, - .index => usize, - }; - } - - pub fn mlirType(dt: DenseElementsAttributeTypes, ctx: Context) Type { - return switch (dt) { - .bool => .int(ctx, .i1), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .u8 => .int(ctx, .u8), - .u16 => .int(ctx, .u16), - .u32 => .int(ctx, .u32), - .u64 => .int(ctx, .u64), - .bf16 => .float(ctx, .bf16), - .f16 => .float(ctx, .f16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .index => .index(ctx), - }; - } + // f80, + // f128, + tf32, }; -pub fn DenseElementsAttribute(comptime dt: DenseElementsAttributeTypes) type { - return struct { - _inner: c.MlirAttribute, +pub fn FloatType(comptime ft: FloatTypes) type { + const _isAFn, const getter = switch (ft) { + .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, + .f6e2m3fn => .{ c.mlirTypeIsAFloat6E2M3FN, c.mlirFloat6E2M3FNTypeGet }, + .f6e3m2fn => .{ c.mlirTypeIsAFloat6E3M2FN, c.mlirFloat6E3M2FNTypeGet }, + .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, + .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, + .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, + .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, + .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, + .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, + .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, + .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, + .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, + .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, + .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, + .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, + .tf32 => .{ c.mlirTypeIsATF32, c.mlirTF32TypeGet }, + }; - const Attr = @This(); + return opaque { + const Self = @This(); + const M = Methods(Self, c.MlirType); - pub const is_a_fn = c.mlirAttributeIsADenseElements; - pub const asAttr = Attribute.fromAny(Attr); - pub const eql = Attribute.eqlAny(Attr); + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); - pub fn init(shaped_type: Type, slice: []const dt.ZigType()) Attr { - const raw_bytes = std.mem.sliceAsBytes(slice); - const res: Attr = .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( - shaped_type._inner, - @intCast(raw_bytes.len), - @ptrCast(raw_bytes.ptr), - ) }; - std.debug.assert(Attribute.wrapOr(res._inner) != null); - return res; - } - - pub fn len(self: Attr) usize { - return @intCast(c.mlirElementsAttrGetNumElements(self._inner)); - } - - pub fn items(self: Attr) []const dt.ZigType() { - const raw_bytes: [*]const u8 = c.mlirDenseElementsAttrGetRawData(self._inner) orelse unreachable; - const ptr: [*]const dt.ZigType() = @alignCast(@ptrCast(raw_bytes)); - // Note the mlir API returns us the number of elements, not the number of bytes, - // that's why we track the element type at comptime to allow items to work. - return ptr[0..self.len()]; - } - - pub fn bytes(self: Attr) []const u8 { - return std.mem.sliceAsBytes(self.items()); + pub fn get(ctx: *Context) *const Self { + return @ptrCast(getter(ctx.ptr()).ptr); } }; } -pub const FlatSymbolRefAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAFlatSymbolRef; - const Self = FlatSymbolRefAttribute; - pub const eql = Attribute.eqlAny(Self); +pub fn floatType(ft: FloatTypes, ctx: *Context) *const Type { + return switch (ft) { + inline else => |v| @ptrCast(FloatType(v).get(ctx)), + }; +} - pub fn init(ctx: Context, str: [:0]const u8) Self { - return .{ ._inner = c.mlirFlatSymbolRefAttrGet(ctx._inner, stringRef(str)) }; +pub const Attribute = opaque { + const M = Methods(Attribute, c.MlirAttribute); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, attr: []const u8) Error!*const Attribute { + return @ptrCast(c.mlirAttributeParseGet(ctx.ptr(), stringRef(attr)).ptr orelse return Error.InvalidMlir); } - pub fn value(self: Self) []const u8 { - return fromStringRef(c.mlirFlatSymbolRefAttrGetValue(self._inner)); + pub fn context(self: *const Context) *Context { + return @ptrCast(c.mlirAttributeGetContext(self.ptr()).ptr); } - pub const asAttr = Attribute.fromAny(Self); + pub fn dialect(self: *const Context) *const Dialect { + return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); + } +}; + +pub const StringAttribute = opaque { + const M = Methods(StringAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAString; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, str: []const u8) *const StringAttribute { + return @ptrCast(c.mlirStringAttrGet(ctx.ptr(), stringRef(str)).ptr); + } + + pub fn value(self: *const StringAttribute) []const u8 { + return string(c.mlirStringAttrGetValue(self.ptr())); + } +}; + +pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { + return @ptrCast(StringAttribute.init(ctx, str)); +} + +pub const IntegerAttribute = opaque { + const M = Methods(IntegerAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { + return @ptrCast(c.mlirIntegerAttrGet( + IntegerType.get(ctx, it).ptr(), + @intCast(value_), + ).ptr); + } + + pub fn value(self: *const IntegerAttribute, comptime T: type) T { + const getValue = switch (@typeInfo(T).int.signedness) { + .signed => c.mlirIntegerAttrGetValueInt, + .unsigned => c.mlirIntegerAttrGetValueUInt, + }; + return @intCast(getValue(self.ptr())); + } +}; + +pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, it, value_)); +} + +pub const BoolAttribute = opaque { + const M = Methods(BoolAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsABool; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, value_: bool) *const BoolAttribute { + return @ptrCast(c.mlirBoolAttrGet(ctx.ptr(), @intFromBool(value_)).ptr); + } + + pub fn value(self: *const BoolAttribute) bool { + return c.mlirBoolAttrGetValue(self.ptr()); + } +}; + +pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { + return @ptrCast(BoolAttribute.init(ctx, value)); +} + +pub const TypeAttribute = opaque { + const M = Methods(TypeAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(type_: *const Type) *const TypeAttribute { + return @ptrCast(c.mlirTypeAttrGet(type_.ptr()).ptr); + } + + pub fn value(self: *const TypeAttribute) *const Type { + return @ptrCast(c.mlirTypeAttrGetValue(self.ptr()).ptr); + } +}; + +pub fn typeAttribute(type_: *const Type) *const Attribute { + return @ptrCast(TypeAttribute.init(type_)); +} + +pub const Dialect = struct { + const M = Methods(Dialect, c.MlirDialect); + + pub const ptr = M.ptr; + + pub fn context(self: *const Dialect) *Context { + return @ptrCast(c.mlirDialectGetContext(self.ptr()).ptr); + } + + pub fn namespace(self: *const Dialect) []const u8 { + return string(c.mlirDialectGetNamespace(self.ptr())); + } + + pub fn format(self: *const Dialect, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(Dialect), self.namespace() }); + } +}; + +pub const DialectHandle = opaque { + const M = Methods(DialectHandle, c.MlirDialectHandle); + + pub const ptr = M.ptr; + + pub fn format(self: *const DialectHandle, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(DialectHandle), self.namespace() }); + } + + pub fn namespace(self: *const DialectHandle) []const u8 { + return string(c.mlirDialectHandleGetNamespace(self.ptr())); + } + + pub fn insertDialect(self: *const DialectHandle, registry: *DialectRegistry) void { + c.mlirDialectHandleInsertDialect(self.ptr(), registry.ptr()); + } + + pub fn registerDialect(self: *const DialectHandle, ctx: *Context) void { + c.mlirDialectHandleRegisterDialect(self.ptr(), ctx.ptr()); + } + + pub fn loadDialect(self: *const DialectHandle, ctx: *Context) *Dialect { + return @ptrCast(c.mlirDialectHandleLoadDialect(self.ptr(), ctx.ptr()).ptr); + } + + pub fn fromString(comptime namespace_: []const u8) *const DialectHandle { + return @ptrCast(@field(c, "mlirGetDialectHandle__" ++ namespace_ ++ "__")().ptr); + } +}; + +pub const Identifier = opaque { + const M = Methods(Identifier, c.MlirIdentifier); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirIdentifierEqual); + + pub fn context(self: *const Identifier) *Context { + return @ptrCast(c.mlirIdentifierGetContext(self.ptr()).ptr); + } + + pub fn get(ctx: *Context, str_: []const u8) *const Identifier { + return @ptrCast(c.mlirIdentifierGet(ctx.ptr(), stringRef(str_)).ptr); + } + + pub fn str(self: *const Identifier) []const u8 { + return string(c.mlirIdentifierStr(self.ptr())); + } + + pub fn format(self: *const Identifier, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .str = \"{s}\" }}", .{ @typeName(Identifier), self.str() }); + } +}; + +pub const Module = opaque { + const M = Methods(Module, c.MlirModule); + + pub const ptr = M.ptr; + + pub fn init(loc: *const Location) *Module { + return @constCast(@ptrCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr)); + } + + pub fn deinit(self: *Module) void { + c.mlirModuleDestroy(self.ptr()); + } + + pub fn fromOperation(op: *Operation) *Module { + return @constCast(@ptrCast(c.mlirModuleFromOperation(op.ptr()).ptr)); + } + + pub fn operation(self: *const Module) *Operation { + return @ptrCast(c.mlirModuleGetOperation(self.ptr()).ptr); + } + + pub fn parse(ctx: *Context, source: []const u8) Error!*Module { + return @constCast(@ptrCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir)); + } + + pub fn context(self: *const Module) *Context { + return @ptrCast(c.mlirModuleGetContext(self.ptr()).ptr); + } + + pub fn body(self: *const Module) *Block { + return @ptrCast(c.mlirModuleGetBody(self.ptr()).ptr); + } +}; + +pub const Block = opaque { + const M = Methods(Block, c.MlirBlock); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirBlockEqual); + pub const format = M.format(c.mlirBlockPrint); + + pub fn init(args: []const *const Type, locs: []const *const Location) *Block { + return @ptrCast(c.mlirBlockCreate(@intCast(args.len), @ptrCast(args), @ptrCast(locs)).ptr); + } + + pub fn deinit(self: *Block) void { + std.debug.assert(self.parentOperation() == null); + c.mlirBlockDestroy(self.ptr()); + } + + pub fn argument(self: *const Block, index: usize) *const Value { + return @ptrCast(c.mlirBlockGetArgument(self.ptr(), @intCast(index)).ptr); + } + + pub fn numArguments(self: *const Block) usize { + return @intCast(c.mlirBlockGetNumArguments(self.ptr())); + } + + pub fn addArgument(self: *Block, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockAddArgument(self.ptr(), type_.ptr(), loc.ptr()).ptr); + } + + pub fn insertArgument(self: *Block, index: usize, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockInsertArgument(self.ptr(), @intCast(index), type_.ptr(), loc.ptr()).ptr); + } + + pub fn appendOwnedOperation(self: *Block, op: *Operation) void { + c.mlirBlockAppendOwnedOperation(self.ptr(), op.ptr()); + } + + pub fn appendOwnedOperations(self: *Block, ops: []const *Operation) void { + for (ops) |op| { + self.appendOwnedOperation(op); + } + } + + pub fn parentOperation(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); + } + + pub fn detach(self: *Block) void { + c.mlirBlockDetach(self.ptr()); + } + + pub fn terminator(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetTerminator(self.ptr()).ptr); + } +}; + +pub const NamedAttribute = struct { + inner: c.MlirNamedAttribute, + + pub fn ptr(self: NamedAttribute) c.MlirNamedAttribute { + return self.inner; + } + + pub fn name(self: NamedAttribute) *const Identifier { + return @ptrCast(self.inner.name.ptr); + } + + pub fn attribute(self: NamedAttribute) *const Attribute { + return @ptrCast(self.inner.attribute.ptr); + } + + pub fn named(ctx: *Context, name_: []const u8, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = Identifier.get(ctx, name_).ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn init(name_: *const Identifier, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = name_.ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn format(self: NamedAttribute, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s}{{ .{s} = \"{f}\" }}", .{ + @typeName(NamedAttribute), + self.name().str(), + self.attribute(), + }); + } +}; + +pub const DictionaryAttribute = opaque { + const M = Methods(DictionaryAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADictionary; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attributes: []const NamedAttribute) *const DictionaryAttribute { + return @ptrCast(c.mlirDictionaryAttrGet( + ctx.ptr(), + @intCast(attributes.len), + @ptrCast(attributes), + ).ptr); + } + + pub fn size(self: *const DictionaryAttribute) usize { + return @intCast(c.mlirDictionaryAttrGetNumElements(self.ptr())); + } + + pub fn get(self: *const DictionaryAttribute, pos: usize) NamedAttribute { + return .{ .inner = c.mlirDictionaryAttrGetElement(self.ptr(), @intCast(pos)) }; + } + + pub fn getByName(self: *const DictionaryAttribute, name: []const u8) ?*const Attribute { + return @ptrCast(c.mlirDictionaryAttrGetElementByName(self.ptr(), stringRef(name)).ptr); + } +}; + +pub fn dictionaryAttribute(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { + return @ptrCast(DictionaryAttribute.init(ctx, attributes)); +} + +pub const Value = opaque { + const M = Methods(Value, c.MlirValue); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + pub const isA = M.isA; + + pub fn type_(val: *const Value) *const Type { + return @ptrCast(c.mlirValueGetType(val.ptr()).ptr); + } + + pub fn setType(self: *Value, type__: *const Type) void { + c.mlirValueSetType(self.ptr(), type__.ptr()); + } + + pub fn firstUse(self: *const Value) *const OpOperand { + return @ptrCast(c.mlirValueGetFirstUse(self.ptr()).ptr); + } + + pub fn replaceAllUsesWith(self: *const Value, with: *const Value) void { + c.mlirValueReplaceAllUsesOfWith(self.ptr(), with.ptr()); + } +}; + +pub const OpOperand = opaque { + const M = Methods(OpOperand, c.MlirOpOperand); + + pub const ptr = M.ptr; + + pub fn owner(self: *const OpOperand) *Operation { + return @ptrCast(c.mlirOpOperandGetOwner(self.ptr()).ptr); + } + + pub fn number(self: *const OpOperand) usize { + return @intCast(c.mlirOpOperandGetOperandNumber(self.ptr())); + } + + pub fn nextUse(self: *const OpOperand) ?*const OpOperand { + return @ptrCast(c.mlirOpOperandGetNextUse(self.ptr()).ptr); + } +}; + +pub const OpResult = opaque { + const M = Methods(OpResult, c.MlirValue); + + pub const isAFn = c.mlirValueIsAOpResult; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn owner(self: *const OpResult) *Operation { + return @ptrCast(c.mlirOpResultGetOwner(self.ptr()).ptr); + } + + pub fn resultNumber(self: *const OpResult) usize { + return @intCast(c.mlirOpResultGetResultNumber(self.ptr())); + } +}; + +pub const BlockArgument = opaque { + const M = Methods(BlockArgument, c.MlirValue); + + pub const isAFn = c.mlirValueIsABlockArgument; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn block(arg: *const BlockArgument) *Block { + return @ptrCast(c.mlirBlockArgumentGetOwner(arg.ptr()).ptr); + } + + pub fn number(self: *const BlockArgument) usize { + return @intCast(c.mlirBlockArgumentGetArgNumber(self.ptr())); + } + + pub fn setType(self: *BlockArgument, type_: *const Type) void { + c.mlirBlockArgumentSetType(self.ptr(), type_.ptr()); + } }; pub const OperationState = struct { - _inner: c.MlirOperationState, + inner: c.MlirOperationState, - const Self = OperationState; - - pub fn init(name: [:0]const u8, loc: Location) Self { - return .{ ._inner = c.mlirOperationStateGet(stringRef(name), loc._inner) }; + pub fn init(name: []const u8, location: *const Location) OperationState { + return .{ .inner = c.mlirOperationStateGet(stringRef(name), location.ptr()) }; } - pub fn addResult(self: *Self, type_: Type) void { - c.mlirOperationStateAddResults(&self._inner, 1, &[_]c.MlirType{type_._inner}); + pub fn addResults(self: *OperationState, results: []const *const Type) void { + c.mlirOperationStateAddResults(&self.inner, @intCast(results.len), @ptrCast(results)); } - pub fn addResults(self: *Self, types: []const Type) void { - c.mlirOperationStateAddResults(&self._inner, @intCast(types.len), @ptrCast(types.ptr)); + pub fn addOperands(self: *OperationState, operands: []const *const Value) void { + c.mlirOperationStateAddOperands(&self.inner, @intCast(operands.len), @ptrCast(operands)); } - pub fn addOperand(self: *Self, value: Value) void { - c.mlirOperationStateAddOperands(&self._inner, 1, &[_]c.MlirValue{value._inner}); + pub fn addOwnedRegions(self: *OperationState, regions: []const *Region) void { + c.mlirOperationStateAddOwnedRegions(&self.inner, @intCast(regions.len), @ptrCast(regions)); } - pub fn addOperands(self: *Self, values: []const Value) void { - c.mlirOperationStateAddOperands(&self._inner, @intCast(values.len), @ptrCast(values.ptr)); + pub fn addSuccessors(self: *OperationState, successors: []*const Block) void { + c.mlirOperationStateAddSuccessors(&self.inner, @intCast(successors.len), @ptrCast(successors)); } - pub fn addRegion(self: *Self, region: *Region) void { - c.mlirOperationStateAddOwnedRegions(&self._inner, 1, &[_]c.MlirRegion{region._inner}); + pub fn addAttributes(self: *OperationState, attributes: []const NamedAttribute) void { + c.mlirOperationStateAddAttributes(&self.inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); } - pub fn addRegions(self: *Self, regions: []const Region) void { - c.mlirOperationStateAddOwnedRegions(&self._inner, @intCast(regions.len), @ptrCast(regions.ptr)); - } - - pub fn addAttribute(self: *Self, ctx: Context, name: [:0]const u8, attr: Attribute) void { - c.mlirOperationStateAddAttributes(&self._inner, 1, @ptrCast(&.{ - .{ - .name = Identifier.get(ctx, name)._inner, - .attribute = attr._inner, - }, - })); - } - - pub fn addAttributeRaw(self: *Self, name: Identifier, attr: Attribute) void { - c.mlirOperationStateAddAttributes(&self._inner, 1, @ptrCast(&.{ - .{ - .name = name._inner, - .attribute = attr._inner, - }, - })); - } - - pub fn addAttributes(self: *Self, attributes: []const NamedAttribute) void { - c.mlirOperationStateAddAttributes(&self._inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); - } - - pub fn resultTypeInference(self: *Self, enabled: bool) void { - self._inner.enableResultTypeInference = enabled; + pub fn enableResultTypeInference(self: *OperationState) void { + c.mlirOperationStateEnableResultTypeInference(&self.inner); } }; -pub const DictionaryAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsADictionary; - pub const asAttr = Attribute.fromAny(DictionaryAttribute); - pub const eql = Attribute.eqlAny(DictionaryAttribute); +pub const Operation = opaque { + const M = Methods(Operation, c.MlirOperation); - pub fn init(ctx: Context, attributes: []const NamedAttribute) DictionaryAttribute { - return .{ ._inner = c.mlirDictionaryAttrGet( - ctx._inner, - @intCast(attributes.len), - @ptrCast(attributes.ptr), - ) }; + const MAX_SEGMENTS = 32; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirOperationEqual); + pub const format = M.format(c.mlirOperationPrint); + + pub fn init(state: *OperationState) !*Operation { + defer state.* = undefined; + return @ptrCast(c.mlirOperationCreate(&state.inner).ptr orelse return Error.InvalidMlir); } - pub fn size(self: DictionaryAttribute) usize { - return @intCast(c.mlirDictionaryAttrGetNumElements(self._inner)); + pub fn deinit(self: *Operation) void { + std.debug.assert(self.block() == null); + c.mlirOperationDestroy(self.ptr()); } - pub fn get(self: DictionaryAttribute, pos: usize) NamedAttribute { - return .wrap(c.mlirDictionaryAttrGetElement(self._inner, @bitCast(pos))); + pub fn parse(ctx: *Context, source: []const u8, name_: []const u8) Error!*Operation { + return @ptrCast(c.mlirOperationCreateParse(ctx.ptr(), stringRef(source), stringRef(name_)).ptr orelse return Error.InvalidMlir); } - pub fn getByName(self: DictionaryAttribute, name: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirDictionaryAttrGetElementByName(self._inner, name)); - } -}; - -pub const Operation = struct { - const Self = Operation; - _inner: c.MlirOperation, - - pub const dump = helpers.dump(Operation, c.mlirOperationDestroy); - pub const deinit = helpers.deinit(Operation, c.mlirOperationDestroy); - pub const wrapOr = helpers.wrapOr(Operation, c.mlirOperationIsNull); - - pub const eql = Attribute.eqlAny(Self); - - pub fn init(state: *OperationState) !Self { - return Self.wrapOr(c.mlirOperationCreate(&state._inner)) orelse Error.InvalidMlir; + pub fn name(self: *const Operation) []const u8 { + return string(c.mlirOperationGetName(self.ptr())); } - pub fn make(ctx: Context, op_name: [:0]const u8, args: struct { - operands: ?[]const Value = null, - variadic_operands: ?[]const []const Value = null, - tt_variadic_operands: ?[]const []const Value = null, - results: ?[]const Type = null, - variadic_results: ?[]const []const Type = null, - result_type_inference: ?bool = null, - n_regions: usize = 0, - attributes: ?[]const AttrTuple = null, - blocks: ?[]const Block = null, - verify: bool = true, - location: Location, - }) Self { - var state = OperationState.init(op_name, args.location); - std.debug.assert(!(args.operands != null and args.variadic_operands != null)); - if (args.operands) |operands| { - state.addOperands(operands); - } else if (args.variadic_operands) |operands_segments| { - const MAX_SEGMENTS = 32; - var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + pub fn clone(self: *const Operation) *Operation { + return @ptrCast(c.mlirOperationClone(self.ptr()).ptr); + } - for (operands_segments) |operands| { - state.addOperands(operands); - segments.appendAssumeCapacity(@intCast(operands.len)); - } - state.addAttribute(ctx, "operandSegmentSizes", .denseElements(ctx, &.{@intCast(segments.len)}, .i32, segments.constSlice())); - } else if (args.tt_variadic_operands) |operands_segments| { - // stablehlo and triton seems to disagree on the expected type of operandSegmentSizes, let's fix that. - const MAX_SEGMENTS = 32; - var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + pub fn block(self: *const Operation) ?*Block { + return @ptrCast(c.mlirOperationGetBlock(self.ptr()).ptr); + } - for (operands_segments) |operands| { - state.addOperands(operands); - segments.appendAssumeCapacity(@intCast(operands.len)); - } - state.addAttribute(ctx, "operandSegmentSizes", .dense(ctx, .i32, segments.constSlice())); - } - if (args.result_type_inference) |enable| { - state.resultTypeInference(enable); - } - std.debug.assert(!(args.results != null and args.variadic_results != null)); - if (args.results) |results| { - state.addResults(results); - } else if (args.variadic_results) |result_segments| { - for (result_segments) |results| { - state.addResults(results); - } - } - for (0..args.n_regions) |_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - state.addRegion(®ion_); - } - if (args.attributes) |attrs| { - for (attrs) |attr| { - state.addAttributeRaw( - Identifier.get(ctx, attr[0]), - attr[1], - ); - } - } - if (args.blocks) |blocks_| { - for (blocks_) |block_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - region_.appendBlock(block_); - state.addRegion(®ion_); - } - } + pub fn appendTo(self: *Operation, block_: *Block) *Operation { + block_.appendOwnedOperation(self); + return self; + } - const new_op = Operation.init(&state) catch { - @panic("Failed to create MLIR operation"); + pub fn parent(self: *const Operation) ?*Operation { + return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); + } + + pub fn removeFromParent(self: *Operation) void { + c.mlirOperationRemoveFromParent(self.ptr()); + } + + pub fn numOperands(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumOperands(self.ptr())); + } + + pub fn operand(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetOperand(self.ptr(), @intCast(index)).ptr); + } + + pub fn setOperand(self: *Operation, index: usize, value: *const Value) void { + c.mlirOperationSetOperand(self.ptr(), @intCast(index), value.ptr()); + } + + pub fn numResults(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumResults(self.ptr())); + } + + pub fn result(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetResult(self.ptr(), @intCast(index)).ptr); + } + + pub fn nextInBlock(self: *const Operation) *const Operation { + return @ptrCast(c.mlirOperationGetNextInBlock(self.ptr()).ptr); + } + + pub fn context(self: *const Operation) *Context { + return @ptrCast(c.mlirOperationGetContext(self.ptr()).ptr); + } + + pub const PrintFlags = struct { + elide_large_elements_attrs: ?usize = null, + debug_info: bool = false, + debug_info_pretty_form: bool = true, + print_generic_op_form: bool = false, + use_local_scope: bool = false, + assume_verified: bool = false, + + const Ctx = struct { + self: *const Operation, + flags: PrintFlags, }; + }; + + fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + const flags = c.mlirOpPrintingFlagsCreate(); + defer c.mlirOpPrintingFlagsDestroy(flags); + + if (fctx.flags.elide_large_elements_attrs) |v| { + c.mlirOpPrintingFlagsElideLargeElementsAttrs(flags, @intCast(v)); + } + c.mlirOpPrintingFlagsEnableDebugInfo( + flags, + fctx.flags.debug_info, + fctx.flags.debug_info_pretty_form, + ); + if (fctx.flags.print_generic_op_form) { + c.mlirOpPrintingFlagsPrintGenericOpForm(flags); + } + if (fctx.flags.use_local_scope) { + c.mlirOpPrintingFlagsUseLocalScope(flags); + } + if (fctx.flags.assume_verified) { + c.mlirOpPrintingFlagsAssumeVerified(flags); + } + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationPrintWithFlags(fctx.self.ptr(), flags, stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } + } + + pub fn fmt(self: *const Operation, flags: PrintFlags) std.fmt.Alt(PrintFlags.Ctx, formatWithFlags) { + return .{ .data = .{ + .self = self, + .flags = flags, + } }; + } + + pub const MakeArgs = struct { + pub const AttrTuple = struct { []const u8, *const Attribute }; + + pub const Operands = union(enum) { + flat: []const *const Value, + variadic: []const []const *const Value, + }; + + pub const ResultTypes = union(enum) { + flat: []const *const Type, + variadic: []const []const *const Type, + }; + + operands: ?Operands = null, + results: ?ResultTypes = null, + result_type_inference: bool = false, + attributes: ?[]const NamedAttribute = null, + blocks: ?[]const *Block = null, + verify: bool = true, + location: ?*const Location = null, + }; + + pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { + var state: OperationState = .init(name_, args.location orelse .unknown(ctx)); + if (args.result_type_inference) { + state.enableResultTypeInference(); + } + if (args.operands) |operands| switch (operands) { + .flat => |v| state.addOperands(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_operands| { + state.addOperands(segment_operands); + sizes.appendAssumeCapacity(@intCast(segment_operands.len)); + } + state.addAttributes(&.{ + .named(ctx, "operandSegmentSizes", denseArrayAttribute(ctx, .i32, sizes.constSlice())), + }); + }, + }; + if (args.results) |results| switch (results) { + .flat => |v| state.addResults(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_results| { + state.addResults(segment_results); + sizes.appendAssumeCapacity(@intCast(segment_results.len)); + } + state.addAttributes(&.{ + .named(ctx, "resultSegmentSizes", denseArrayAttribute(ctx, .i32, sizes.constSlice())), + }); + }, + }; + if (args.attributes) |attrs| { + state.addAttributes(attrs); + } + if (args.blocks) |blocks| { + const region = Region.init(); + for (blocks) |block_| { + region.appendOwnedBlock(block_); + } + state.addOwnedRegions(&.{region}); + } + const new_op = try Operation.init(&state); + errdefer new_op.deinit(); if (args.verify and new_op.verify() == false) { - log.err("Failed to verify MLIR operation:\n{f}", .{new_op.mlirFormatter(.{ .debug_info = true })}); - @panic("Failed to verify MLIR operation"); + log.err("Failed to verify MLIR operation:\n{f}", .{ + new_op.fmt(.{ + .print_generic_op_form = true, + .debug_info = true, + }), + }); + return Error.InvalidMlir; } return new_op; } - pub fn initParse(ctx: Context, str: [:0]const u8) !Self { - return Self.wrapOr( - c.mlirOperationCreateParse(ctx._inner, stringRef(str), stringRef("pouet")), - ) orelse Error.InvalidMlir; - } - - pub fn clone(self: Self) !Self { - return Self.wrapOr( - c.mlirOperationClone(self._inner), - ) orelse Error.InvalidMlir; - } - - pub fn name(self: Self) Identifier { - return .{ ._inner = c.mlirOperationGetName(self._inner) }; - } - - pub fn removeFromParent(self: *Self) void { - c.mlirOperationRemoveFromParent(self._inner); - } - - pub fn numOperands(self: Self) usize { - return @intCast(c.mlirOperationGetNumOperands(self._inner)); - } - - pub fn operand(self: Self, index: usize) Value { - return .{ ._inner = c.mlirOperationGetOperand(self._inner, @intCast(index)) }; - } - - pub fn setOperand(self: *Self, index: usize, value: Value) void { - c.mlirOperationSetOperand(self._inner, @intCast(index), value._inner); - } - - pub fn numResults(self: Self) usize { - return @intCast(c.mlirOperationGetNumResults(self._inner)); - } - - pub fn result(self: Self, index: usize) Value { - return .{ ._inner = c.mlirOperationGetResult(self._inner, @intCast(index)) }; - } - - pub fn nextInBlock(self: Self) Self { - return .{ ._inner = c.mlirOperationGetNextInBlock(self._inner) }; - } - - // pub fn previousInBlock(self: Self) Self { - // return .{ ._inner = c.mlirOperationGetPrevInBlock(self._inner) }; - // } - - pub fn block(self: Self) ?Block { - return Block.wrapOr(c.mlirOperationGetBlock(self._inner)); - } - - pub fn parent(self: Self) ?Self { - return Self.wrapOr(c.mlirOperationGetParentOperation(self._inner)); - } - - pub fn region(self: Self, index: usize) Region { - return .{ ._inner = c.mlirOperationGetRegion(self._inner, @intCast(index)) }; - } - - pub fn context(self: Self) Context { - return .{ ._inner = c.mlirOperationGetContext(self._inner) }; - } - - pub fn writeBytecode(self: Self, writer: anytype) void { - var writer_context = .{ .writer = writer }; - const WriterContext = @TypeOf(writer_context); - - c.mlirOperationWriteBytecode( - self._inner, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch unreachable; - } - }).callback, - &writer_context, - ); - } - - pub fn writeBytecodeWithConfig(self: Self, writer: anytype, config: struct { - desiredEmitedVersion: ?i64 = null, - }) !void { - const cfg = c.mlirBytecodeWriterConfigCreate(); - defer c.mlirBytecodeWriterConfigDestroy(cfg); - if (config.desiredEmitedVersion) |v| { - c.mlirBytecodeWriterConfigDesiredEmitVersion(cfg, v); - } - - const WriterContext = struct { - writer: @TypeOf(writer), - write_error: ?@TypeOf(writer).Error = null, + pub fn make(ctx: *Context, op_name: []const u8, args: MakeArgs) *Operation { + return try_make(ctx, op_name, args) catch { + @panic("Failed to create MLIR operation"); }; - var writer_context: WriterContext = .{ .writer = writer }; - - try successOr(c.mlirOperationWriteBytecodeWithConfig( - self._inner, - cfg, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch |err| { - inner_writer_context.write_error = err; - }; - } - }).callback, - &writer_context, - ), error.InvalidMlirBytecodeVersion); - - if (writer_context.write_error) |err| return err; } - /// Enable a full dump of the IR. - /// Usage `std.log.debug("{}", .{ module.op().mlirFormatter(.{}) }); - pub fn mlirFormatter(self: Operation, flags: OpPrintingFlags) MlirFormatter { - return .{ .op = self, .flags = flags }; + pub fn verify(self: *const Operation) bool { + return c.mlirOperationVerify(self.ptr()); } - const MlirFormatter = struct { - op: Operation, - flags: OpPrintingFlags, - - pub fn format(self: @This(), writer: anytype) !void { - self.op.print(writer, self.flags); - } - }; - - pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) void { - const pflags = flags.create(); - defer c.mlirOpPrintingFlagsDestroy(pflags); - - c.mlirOperationPrintWithFlags( - self._inner, - pflags, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const _writer: *std.Io.Writer = @ptrCast(@alignCast(ctx_)); - _writer.writeAll(str.data[0..str.length]) catch @panic("Mlir print failed"); - } - }).callback, - writer, - ); - } - - pub fn verify(self: Self) bool { - return c.mlirOperationVerify(self._inner); - } - - pub fn getLocation(self: Self) Location { - return .{ ._inner = c.mlirOperationGetLocation(self._inner) }; + pub fn location(self: *const Operation) *const Location { + return @ptrCast(c.mlirOperationGetLocation(self.ptr()).ptr); } pub const WalkOrder = enum(c.MlirWalkOrder) { @@ -980,7 +1078,7 @@ pub const Operation = struct { skip = c.MlirWalkResultSkip, }; - pub fn walk(self: Self, order: WalkOrder, ctx: anytype, walkfn: fn (ctx: anytype, op: Operation) WalkResult) void { + pub fn walk(self: *Operation, order: WalkOrder, ctx: anytype, walkfn: fn (ctx_: @TypeOf(ctx), op: *Operation) WalkResult) void { var inner_ctx = .{ .ctx = ctx }; const ContextType = @TypeOf(inner_ctx); @@ -989,7 +1087,7 @@ pub const Operation = struct { (struct { pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @intFromEnum(walkfn(inner_ctx_.ctx, .{ ._inner = op })); + return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, @@ -997,779 +1095,372 @@ pub const Operation = struct { ); } - pub fn getAttribute(self: Self, pos: usize) NamedAttribute { - return .{ ._inner = c.mlirOperationGetAttribute(self._inner, @intCast(pos)) }; - } + pub const BytecodeWriterConfig = struct { + desired_emit_version: ?usize = null, - pub fn getAttributeByName(self: Self, name_: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirOperationGetAttributeByName(self._inner, stringRef(name_))); - } - - pub fn setAttributeByName(self: Self, name_: [:0]const u8, attr: Attribute) void { - c.mlirOperationSetAttributeByName(self._inner, stringRef(name_), attr._inner); - } - - pub fn removeAttributeByName(self: Self, name_: [:0]const u8) bool { - return c.mlirOperationRemoveAttributeByName(self._inner, stringRef(name_)); - } - - /// Hash the canonicalized IR, without debug information that can change across builds. - pub fn hash(op: Operation, hasher: *std.hash.XxHash64) void { - // Note: before we where using op.writeBytecode(writer), - // but it crashes on some inputs, notably for unused variables. - // So we use the text representation of the mlir. - // See https://github.com/zml/zml/issues/97. - const flags = OpPrintingFlags.create(.{ .debug_info = false }); - defer c.mlirOpPrintingFlagsDestroy(flags); - - c.mlirOperationPrintWithFlags( - op._inner, - flags, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const _hasher: *std.hash.XxHash64 = @ptrCast(@alignCast(ctx_)); - _hasher.update(str.data[0..str.length]); - } - }).callback, - hasher, - ); - } -}; - -pub const OpPrintingFlags = struct { - elide_large_elements_attrs: ?usize = null, - debug_info: bool = false, - debug_info_pretty_form: bool = true, - print_generic_op_form: bool = false, - use_local_scope: bool = false, - assume_verified: bool = false, - - pub fn create(self: OpPrintingFlags) c.MlirOpPrintingFlags { - const pflags = c.mlirOpPrintingFlagsCreate(); - if (self.elide_large_elements_attrs) |v| { - c.mlirOpPrintingFlagsElideLargeElementsAttrs(pflags, @intCast(v)); - } - c.mlirOpPrintingFlagsEnableDebugInfo(pflags, self.debug_info, self.debug_info_pretty_form); - if (self.print_generic_op_form) { - c.mlirOpPrintingFlagsPrintGenericOpForm(pflags); - } - if (self.use_local_scope) { - c.mlirOpPrintingFlagsUseLocalScope(pflags); - } - if (self.assume_verified) { - c.mlirOpPrintingFlagsAssumeVerified(pflags); - } - return pflags; - } -}; - -pub const OpOperand = struct { - _inner: c.MlirOpOperand, - const Self = OpOperand; - - pub fn owner(self: Self) Operation { - return .{ ._inner = c.mlirOpOperandGetOwner(self._inner) }; - } - - pub fn number(self: Self) usize { - return @intCast(c.mlirOpOperandGetOperandNumber(self._inner)); - } - - pub fn nextUse(self: Self) ?Self { - return Self.wrapOr( - c.mlirOpOperandGetNextUse(self._inner), - ); - } -}; - -pub const Region = struct { - _inner: c.MlirRegion, - - pub const eql = helpers.eql(Region, c.mlirBlockEqual); - pub const deinit = helpers.deinit(Region, c.mlirRegionDestroy); - pub const wrapOr = helpers.wrapOr(Region, c.mlirRegionIsNull); - - const Self = Region; - - pub fn init() !Self { - return Self.wrapOr(c.mlirRegionCreate()) orelse Error.InvalidMlir; - } - - pub fn appendBlock(self: *Self, block: Block) void { - c.mlirRegionAppendOwnedBlock(self._inner, block._inner); - } - - pub fn insertBlock(self: *Self, index: isize, block: Block) void { - c.mlirRegionInsertOwnedBlock(self._inner, index, block._inner); - } - - pub fn insertBlockBefore(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockBefore(self._inner, reference._inner, block._inner); - } - - pub fn insertBlockAfter(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockAfter(self._inner, reference._inner, block._inner); - } - - pub fn firstBlock(self: Self) Block { - return .{ ._inner = c.mlirRegionGetFirstBlock(self._inner) }; - } -}; - -pub const Value = struct { - _inner: c.MlirValue, - - pub const dump = helpers.dump(Value, c.mlirValueDump); - pub const eql = helpers.eql(Value, c.mlirValueEqual); - pub const format = helpers.format(Value, c.mlirValuePrint).format; - pub const wrapOr = helpers.wrapOr(Value, c.mlirValueIsNull); - - pub fn getType(val: Value) Type { - return .{ ._inner = c.mlirValueGetType(val._inner) }; - } - - pub fn setType(val: *Value, typ: Type) void { - c.mlirValueSetType(val._inner, typ._inner); - } - - pub fn firstUse(val: Value) OpOperand { - return .{ ._inner = c.mlirValueGetFirstUse(val._inner) }; - } - - pub fn replaceAllUsesWith(val: Value, with: Value) void { - c.mlirValueReplaceAllUsesOfWith(val._inner, with._inner); - } - - pub fn owner(val: Value) Operation { - return .{ ._inner = c.mlirOpResultGetOwner(val._inner) }; - } - - pub fn isABlockArgument(val: Value) bool { - return c.mlirValueIsABlockArgument(val._inner); - } - - pub fn isAOpResult(val: Value) bool { - return c.mlirValueIsAOpResult(val._inner); - } - - pub const Kind = union(enum) { - block_argument: BlockArgument, - op_result: Operation, - null, + const Ctx = struct { + self: *const Operation, + config: ?BytecodeWriterConfig, + }; }; - pub fn kind(val: Value) Kind { - if (val.isAOpResult()) { - return .{ .op_result = val.owner() }; - } - if (val.isABlockArgument()) { - return .{ .block_argument = .{ ._inner = val._inner } }; - } - // From MLIR docs: - // https://mlir.llvm.org/doxygen/classmlir_1_1Value.html#details - // > An SSA value is either a BlockArgument or the result of an operation. - return .null; - } -}; - -pub const BlockArgument = struct { - _inner: c.MlirValue, - - pub fn block(arg: BlockArgument) Block { - return .{ ._inner = c.mlirBlockArgumentGetOwner(arg._inner) }; - } - - pub fn number(arg: BlockArgument) usize { - return @bitCast(c.mlirBlockArgumentGetArgNumber(arg._inner)); - } - - pub fn format(self: BlockArgument, writer: anytype) !void { - const value = Value{ ._inner = self._inner }; - return value.format(writer); - } -}; - -pub const Type = struct { - _inner: c.MlirType, - - pub const dump = helpers.eql(Type, c.mlirTypeDump); - pub const eql = helpers.eql(Type, c.mlirTypeEqual); - pub const format = helpers.format(Type, c.mlirTypePrint); - pub const wrapOr = helpers.wrapOr(Type, c.mlirTypeIsNull); - - pub fn parse(ctx: Context, str: [:0]const u8) !Type { - return Type.wrapOr( - c.mlirTypeParseGet(ctx._inner, stringRef(str)), - ) orelse Error.InvalidMlir; - } - - pub fn as(generic: Type, SpecificType: type) ?SpecificType { - if (@hasDecl(SpecificType, "is_a_fn")) { - return if (SpecificType.is_a_fn(generic._inner)) - .{ ._inner = generic._inner } - else - null; - } - @compileError("Mlir subclass of type need `is_a_fn` attribute: " ++ @typeName(SpecificType)); - } - - pub fn fromAny(SpecificType: type) fn (x: SpecificType) Type { - stdx.debug.assertComptime(@hasDecl(SpecificType, "asType"), "Type.fromAny expects a type subclass, got: {}. Missing `asAttr` declaration.", .{SpecificType}); - return struct { - fn cast(x: SpecificType) Type { - return .{ ._inner = x._inner }; + pub fn writeBytecode(self: *const Operation, config: ?BytecodeWriterConfig, writer: *std.Io.Writer) !void { + if (config) |config_| { + const bc_config = c.mlirBytecodeWriterConfigCreate(); + defer c.mlirBytecodeWriterConfigDestroy(bc_config); + if (config_.desired_emit_version) |version| { + c.mlirBytecodeWriterConfigDesiredEmitVersion(bc_config, @intCast(version)); } - }.cast; - } - - pub fn eqlAny(SpecificType: type) fn (SpecificType, SpecificType) bool { - return struct { - fn eql(a: SpecificType, b: SpecificType) bool { - return a.asType().eql(b.asType()); + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationWriteBytecodeWithConfig(self.ptr(), bc_config, stringCallback, &sctx); + if (sctx.err) |err| { + return err; } - }.eql; + return; + } + + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationWriteBytecode(self.ptr(), stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } } - pub fn formatAny(SpecificType: type) fn (SpecificType, SpecificType) type { - return struct { - pub fn format(self: SpecificType, writer: anytype) !void { - return try Type.format(self.asType(), writer); - } - }; + fn fmtBytecodeImpl(ctx: BytecodeWriterConfig.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + try ctx.self.writeBytecode(ctx.config, writer); } - pub fn index(ctx: Context) Type { - return IndexType.init(ctx).asType(); - } - - pub fn int(ctx: Context, int_type: IntegerTypes) Type { - return switch (int_type) { - .unknown => @panic("Unknown integer type"), - inline else => |t| IntegerType(t).init(ctx).asType(), - }; - } - - pub fn float(ctx: Context, float_type: FloatTypes) Type { - return switch (float_type) { - inline else => |t| FloatType(t).init(ctx).asType(), - }; - } - - pub fn complex(ctx: Context, complex_type: ComplexTypes) Type { - return switch (complex_type) { - .unknown => @panic("Unknown complex type can't be created like this"), // What's the point ? - inline else => |t| ComplexType(t).init(ctx).asType(), - }; - } - - pub fn tuple(ctx: Context, types: []const Type) Type { - return (TupleType.init(ctx, types) catch unreachable).asType(); - } - - pub fn function(ctx: Context, args: []const Type, results: []const Type) Type { - return (FunctionType.init(ctx, args, results) catch unreachable).asType(); - } - - pub fn tensor(dimensions: []const i64, elem_type: Type) Type { - return RankedTensorType.init(dimensions, elem_type).asType(); + pub fn fmtBytecode(self: *const Operation, config: ?BytecodeWriterConfig) std.fmt.Alt(BytecodeWriterConfig.Ctx, fmtBytecodeImpl) { + return .{ .data = .{ + .self = self, + .config = config, + } }; } }; -pub const IndexType = struct { - _inner: c.MlirType, +pub const Region = opaque { + const M = Methods(Region, c.MlirRegion); - pub const asType = Type.fromAny(IndexType); - pub const eql = Type.eqlAny(IndexType); - pub const format = Type.formatAny(IndexType).format; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirRegionEqual); - pub fn init(ctx: Context) IndexType { - return .{ ._inner = c.mlirIndexTypeGet(ctx._inner) }; + pub fn init() *Region { + return @ptrCast(c.mlirRegionCreate().ptr); + } + + pub fn deinit(self: *Region) void { + c.mlirRegionDestroy(self.ptr()); + } + + pub fn appendOwnedBlock(self: *Region, block: *Block) void { + c.mlirRegionAppendOwnedBlock(self.ptr(), block.ptr()); + } + + pub fn insertOwnedBlock(self: *Region, index: usize, block: *Block) void { + c.mlirRegionInsertOwnedBlock(self.ptr(), @intCast(index), block.ptr()); + } + + pub fn insertOwnedBlockBefore(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockBefore(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn insertOwnedBlockAfter(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockAfter(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn firstBlock(self: *const Region) ?*Block { + return @ptrCast(c.mlirRegionGetFirstBlock(self.ptr()).ptr); + } + + pub fn takeBody(self: *Region, from: *Region) void { + c.mlirRegionTakeBody(self.ptr(), from.ptr()); + } + + pub fn nextInOperation(self: *const Region) *Region { + return @ptrCast(c.mlirRegionGetNextInOperation(self.ptr()).ptr); } }; -pub const IntegerTypes = enum { - i1, - i4, +pub const ShapedType = opaque { + const M = Methods(ShapedType, c.MlirType); + + pub const MAX_RANK = 16; + + pub const isAFn = c.mlirTypeIsAShaped; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn elementType(self: *const ShapedType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const ShapedType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const ShapedType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } +}; + +pub const RankedTensorType = opaque { + const M = Methods(RankedTensorType, c.MlirType); + + pub const isAFn = c.mlirTypeIsARankedTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(dimensions: []const i64, elemType: *const Type) *const RankedTensorType { + return @ptrCast(c.mlirRankedTensorTypeGet( + @intCast(dimensions.len), + @ptrCast(dimensions), + elemType.ptr(), + c.mlirAttributeGetNull(), + ).ptr); + } + + pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return get( + dims.constSlice(), + other.elementType(), + ); + } + + pub fn elementType(self: *const RankedTensorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const RankedTensorType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const RankedTensorType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn shaped(self: *const RankedTensorType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type)); +} + +pub const DenseElementsAttribute = opaque { + const M = Methods(DenseElementsAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADenseElements; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(shaped_type: *const ShapedType, values: anytype) *const DenseElementsAttribute { + const bytes = std.mem.sliceAsBytes(values); + return @ptrCast(c.mlirDenseElementsAttrRawBufferGet( + shaped_type.ptr(), + @intCast(bytes.len), + @ptrCast(bytes), + ).ptr orelse unreachable); + } +}; + +pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: anytype) *const Attribute { + return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); +} + +pub const DenseArrayTypes = enum { + bool, i8, i16, i32, i64, - si4, - si8, - si16, - si32, - si64, - u4, - u8, - u16, - u32, - u64, - - unknown, -}; - -pub fn IntegerType(comptime it: IntegerTypes) type { - const Config = switch (it) { - .i1 => .{ 1, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i4 => .{ 4, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i8 => .{ 8, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i16 => .{ 16, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i32 => .{ 32, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i64 => .{ 64, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .si4 => .{ 4, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si8 => .{ 8, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si16 => .{ 16, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si32 => .{ 32, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si64 => .{ 64, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .u4 => .{ 4, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u8 => .{ 8, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u16 => .{ 16, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u32 => .{ 32, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u64 => .{ 64, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .unknown => .{ 0, null, null }, - }; - - return struct { - _inner: c.MlirType, - - const Int = @This(); - pub const is_a_fn = switch (it) { - .unknown => c.mlirTypeIsAInteger, - else => typeIsAIntegerExact, - }; - - pub const asType = Type.fromAny(Int); - pub const eql = Type.eqlAny(Int); - pub const format = helpers.format(Int, c.mlirTypePrint); - - fn typeIsAIntegerExact(typ: c.MlirType) callconv(.c) bool { - const bit_width = Config[0]; - const is_sign = Config[2]; - return c.mlirTypeIsAInteger(typ) and (c.mlirIntegerTypeGetWidth(typ) == bit_width) and is_sign(typ); - } - - pub const BitWidth = Config[0]; - - pub const init = if (it != .unknown) struct { - pub fn init(ctx: Context) Int { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner, BitWidth) }; - } - }.init else {}; - }; -} - -pub const FloatTypes = enum { - f8e4m3b11fnuz, - f8e4m3fn, - f8e4m3fnuz, - f8e5m2, - f8e5m2fnuz, - bf16, - f16, f32, f64, - pub fn asType(self: FloatTypes, ctx: Context) Type { - return switch (self) { - inline else => |ft| FloatType(ft).init(ctx).asType(), + fn ZigType(comptime ElementType: @This()) type { + return switch (ElementType) { + .bool => i32, + .i8 => i8, + .i16 => i16, + .i32 => i32, + .i64 => i64, + .f32 => f32, + .f64 => f64, }; } }; -pub fn FloatType(comptime ft: FloatTypes) type { - const Config = switch (ft) { - .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, - .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, - .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, - .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, - .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, - .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, - .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, - .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, - .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, +pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { + const ElementTypeZig = ElementType.ZigType(); + + const _isAFn, const getFn, const getElementFn = switch (ElementType) { + .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, + .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, + .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, + .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, + .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, + .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, + .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, }; return struct { - _inner: c.MlirType, - const Self = @This(); - pub const is_a_fn = Config[0]; + const M = Methods(@This(), c.MlirAttribute); - pub const asType = Type.fromAny(Self); - pub const eql = Type.eqlAny(Self); - pub const format = helpers.format(Self, c.mlirTypePrint); + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); - pub fn init(ctx: Context) Self { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner) }; + pub fn init(ctx: *Context, values: []const ElementTypeZig) *const Self { + return @ptrCast(getFn(ctx.ptr(), @intCast(values.len), @ptrCast(values)).ptr); + } + + pub fn element(self: *const Self, pos: usize) ElementTypeZig { + return getElementFn(self.ptr(), @intCast(pos)); + } + + pub fn numElements(self: *const Self) usize { + return @intCast(c.mlirDenseArrayGetNumElements(self.ptr())); } }; } -pub const ComplexTypes = enum { - c64, - c128, - - unknown, -}; - -pub fn ComplexType(comptime ct: ComplexTypes) type { - return struct { - _inner: c.MlirType, - const Complex = @This(); - - fn mlirC64TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF32TypeGet(ctx)); - } - - fn mlirC128TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF64TypeGet(ctx)); - } - - fn mlirTypeIsAC64(typ: c.MlirType) callconv(.c) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF32(element_type); - } - - fn mlirTypeIsAC128(typ: c.MlirType) callconv(.c) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF64(element_type); - } - - const Config = switch (ct) { - .c64 => .{ mlirTypeIsAC64, mlirC64TypeGet }, - .c128 => .{ mlirTypeIsAC128, mlirC128TypeGet }, - .unknown => .{ c.mlirTypeIsAComplex, null }, - }; - - fn typeIsAUnknownComplex(typ: c.MlirType) callconv(.c) bool { - return c.mlirTypeIsAComplex(typ); - } - - pub const is_a_fn = Config[0]; - - pub const asType = Type.fromAny(Complex); - pub const eql = Type.eqlAny(Complex); - pub const format = Type.formatAny(Complex).format; - pub const ComplexTypeType: ComplexTypes = ct; - - pub const init = if (ct != .unknown) struct { - pub fn init(ctx: Context) Complex { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner) }; - } - }.init else {}; - }; +pub fn denseArrayAttribute(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { + return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); } -pub const TupleType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsATuple; +pub const ArrayAttribute = opaque { + const M = Methods(ArrayAttribute, c.MlirAttribute); - const Self = TupleType; + pub const isAFn = c.mlirAttributeIsAArray; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); - pub fn init(ctx: Context, elements: []const Type) !Self { - return Self.wrapOr(c.mlirTupleTypeGet( - ctx._inner, - @intCast(elements.len), - @ptrCast(elements.ptr), - )) orelse Error.InvalidMlir; + pub fn init(ctx: *Context, attrs: []const *const Attribute) *const ArrayAttribute { + return @ptrCast(c.mlirArrayAttrGet(ctx.ptr(), @intCast(attrs.len), @ptrCast(attrs)).ptr); } - pub fn getNumTypes(self: Self) usize { - return @intCast(c.mlirTupleTypeGetNumTypes(self._inner)); + pub fn element(self: *const ArrayAttribute, index: usize) Attribute { + return @ptrCast(c.mlirArrayAttrGetElement(self.ptr(), @intCast(index)).ptr); } - pub fn getElementType(self: Self, index: usize) Type { - return .{ ._inner = c.mlirTupleTypeGetType(self._inner, @intCast(index)) }; + pub fn numElements(self: *const ArrayAttribute) usize { + return @intCast(c.mlirArrayAttrGetNumElements(self.ptr())); } - - pub const asType = Type.fromAny(Self); }; -pub const FunctionType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsAFunction; - const Self = FunctionType; - pub const asType = Type.fromAny(Self); - pub const eql = Type.eqlAny(IndexType); +pub fn arrayAttribute(ctx: *Context, attrs: []const *const Attribute) *const Attribute { + return @ptrCast(ArrayAttribute.init(ctx, attrs)); +} - pub fn init(ctx: Context, args: []const Type, results: []const Type) !Self { - const func = Type.wrapOr(c.mlirFunctionTypeGet( - ctx._inner, +pub const FunctionType = opaque { + const M = Methods(FunctionType, c.MlirType); + pub const isAFn = c.mlirTypeIsAFunction; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(ctx: *Context, args: []const *const Type, results: []const *const Type) *const FunctionType { + return @ptrCast(c.mlirFunctionTypeGet( + ctx.ptr(), @intCast(args.len), @ptrCast(args.ptr), @intCast(results.len), @ptrCast(results.ptr), - )) orelse return Error.InvalidMlir; - return func.as(Self).?; + ).ptr); } }; -pub const RankedTensorType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsARankedTensor; - pub const eql = Type.eqlAny(RankedTensorType); - pub const format = helpers.format(Type, c.mlirTypePrint); +pub fn functionType(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { + return @ptrCast(FunctionType.init(ctx, args, results)); +} - pub fn init(dimensions: []const i64, elemType: Type) RankedTensorType { - return .{ ._inner = c.mlirRankedTensorTypeGet( - @intCast(dimensions.len), - @ptrCast(dimensions.ptr), - elemType._inner, - c.mlirAttributeGetNull(), - ) }; +pub const FlatSymbolRefAttribute = opaque { + const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAFlatSymbolRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, symbol_: []const u8) *const FlatSymbolRefAttribute { + return @ptrCast(c.mlirFlatSymbolRefAttrGet(ctx.ptr(), stringRef(symbol_)).ptr); } - pub fn getElementType(self: RankedTensorType) Type { - return .{ ._inner = c.mlirShapedTypeGetElementType(self._inner) }; - } - - pub fn getRank(self: RankedTensorType) usize { - return @intCast(c.mlirShapedTypeGetRank(self._inner)); - } - - pub fn getDimension(self: RankedTensorType, dim: usize) i64 { - return c.mlirShapedTypeGetDimSize(self._inner, @intCast(dim)); - } - - pub const asType = Type.fromAny(RankedTensorType); -}; - -pub const Dialect = struct { - _inner: c.MlirDialect, - - const Self = Dialect; - - pub fn getContext(self: Self) Context { - return .{ ._inner = c.mlirDialectGetContext(self._inner) }; - } - - pub fn getNamespace(self: Self) []const u8 { - return fromStringRef(c.mlirDialectGetNamespace(self._inner)); + pub fn symbol(self: *const FlatSymbolRefAttribute) []const u8 { + return string(c.mlirFlatSymbolRefAttrGetValue(self.ptr())); } }; -pub const DialectHandle = struct { - _inner: c.MlirDialectHandle, +pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { + return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); +} - pub fn getNamespace(self: DialectHandle) []const u8 { - return fromStringRef(c.mlirDialectHandleGetNamespace(self._inner)); - } +pub const UnitAttribute = opaque { + const M = Methods(UnitAttribute, c.MlirAttribute); - pub fn insertDialect(self: DialectHandle, registry: Registry) void { - c.mlirDialectHandleInsertDialect(self._inner, registry._inner); - } + pub const isAFn = c.mlirAttributeIsAUnit; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); - pub fn registerDialect(self: DialectHandle, ctx: Context) void { - c.mlirDialectHandleRegisterDialect(self._inner, ctx._inner); - } - - pub fn loadDialect(self: DialectHandle, ctx: Context) Dialect { - return .{ ._inner = c.mlirDialectHandleLoadDialect(self._inner, ctx._inner) }; - } - - pub fn fromString(comptime namespace: []const u8) DialectHandle { - return .{ ._inner = @field(c, "mlirGetDialectHandle__" ++ namespace ++ "__")() }; + pub fn get(ctx: *Context) *const UnitAttribute { + return @ptrCast(c.mlirUnitAttrGet(ctx.ptr()).ptr); } }; -pub const Location = struct { - _inner: c.MlirLocation, +pub fn unitAttribute(ctx: *Context) *const Attribute { + return @ptrCast(UnitAttribute.get(ctx)); +} - pub const eql = helpers.eql(Type, c.mlirLocationEqual); - pub const format = helpers.format(Location, c.mlirLocationPrint); +pub const MemRefType = opaque { + const M = Methods(MemRefType, c.MlirType); - pub fn fromSrc(ctx: Context, src: std.builtin.SourceLocation) Location { - return .{ ._inner = c.mlirLocationFileLineColGet( - ctx._inner, - stringRef(src.file), - @intCast(src.line), - @intCast(src.column), - ) }; + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { + return @ptrCast(c.mlirMemRefTypeGet( + element_type.ptr(), + @intCast(shape.len), + @ptrCast(shape), + if (layout) |l| l.ptr() else .{}, + if (memory_space) |m| m.ptr() else .{}, + ).ptr); } - pub fn fileLineCol(ctx: Context, file: []const u8, line: usize, column: usize) Location { - return .{ ._inner = c.mlirLocationFileLineColGet( - ctx._inner, - stringRef(file), - @intCast(line), - @intCast(column), - ) }; - } - - pub fn callSite(callee: Location, caller: Location) Location { - return .{ ._inner = c.mlirLocationCallSiteGet(callee._inner, caller._inner) }; - } - - pub fn fused(ctx: Context, locations: []const Location, metadata: Attribute) Location { - return .{ ._inner = c.mlirLocationFusedGet( - ctx._inner, - @intCast(locations.len), - @ptrCast(locations.ptr), - metadata._inner, - ) }; - } - - pub fn named(loc: Location, ctx: Context, loc_name: [:0]const u8) Location { - return .{ ._inner = c.mlirLocationNameGet(ctx._inner, stringRef(loc_name), loc._inner) }; - } - - pub fn namedFmt(loc: Location, ctx: Context, comptime fmt: [:0]const u8, args: anytype) Location { - var buf: [256]u8 = undefined; - var stream = std.io.fixedBufferStream(&buf); - std.fmt.format(stream.writer(), fmt, args) catch { - buf[256 - 3 ..].* = "...".*; - }; - return loc.named(ctx, @ptrCast(stream.getWritten())); - } - - pub fn unknown(ctx: Context) Location { - return .{ ._inner = c.mlirLocationUnknownGet(ctx._inner) }; - } -}; - -pub const Block = struct { - _inner: c.MlirBlock, - - pub const wrapOr = helpers.wrapOr(Block, c.mlirBlockIsNull); - pub const deinit = helpers.deinit(Block, c.mlirBlockDestroy); - - pub const eql = helpers.eql(Block, c.mlirBlockEqual); - - pub fn init(args: []const Type, locs: []const Location) !Block { - const block = Block.wrapOr( - c.mlirBlockCreate(@intCast(args.len), @ptrCast(args.ptr), @ptrCast(locs.ptr)), + pub fn fromShaped(other: *const ShapedType) *const MemRefType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return init( + other.elementType(), + dims.constSlice(), + null, + null, ); - return block orelse error.InvalidMlir; } - pub fn argument(self: Block, index: usize) Value { - return .{ ._inner = c.mlirBlockGetArgument(self._inner, @intCast(index)) }; - } - - pub fn numArguments(self: Block) usize { - return @intCast(c.mlirBlockGetNumArguments(self._inner)); - } - - pub fn addArgument(self: *Block, typ: Type, loc: Location) Value { - return .{ ._inner = c.mlirBlockAddArgument(self._inner, typ._inner, loc._inner) }; - } - - pub fn insertArgument(self: *Block, index: usize, typ: Type, loc: Location) Value { - return .{ ._inner = c.mlirBlockInsertArgument(self._inner, @intCast(index), typ._inner, loc._inner) }; - } - - pub fn equals(self: Block, other: Block) bool { - return c.mlirBlockEqual(self._inner, other._inner); - } - - pub fn appendOperation(self: Block, op: Operation) void { - c.mlirBlockAppendOwnedOperation(self._inner, op._inner); - } - - pub fn appendOperations(self: *Block, ops: []const Operation) void { - for (ops) |op| { - c.mlirBlockAppendOwnedOperation(self._inner, op._inner); - } - } - - pub const RecursiveOpts = enum { open, hermetic }; - - pub fn appendValueRecursive(self: Block, value: Value, opt: RecursiveOpts) void { - switch (value.kind()) { - .op_result => |parent_op| self.appendOperationRecursive(parent_op, opt), - .block_argument => |arg| { - // Hermetic blocks are not allowed to use arguments from other blocks. - stdx.debug.assert(opt == .open or self.eql(arg.block()), "Can't add {f} from {*} block to {*} block", .{ arg, arg.block()._inner.ptr, self._inner.ptr }); - }, - .null => @panic("InvalidMlir"), - } - } - - pub fn appendOperationRecursive(self: Block, op: Operation, opt: RecursiveOpts) void { - if (op.block()) |prev_block| { - // Hermetic blocks are not allowed to reference values from other blocks. - stdx.debug.assert(opt == .open or self.equals(prev_block), "Can't add {} from {*} block to {*} block", .{ op, prev_block._inner.ptr, self._inner.ptr }); - return; - } - for (0..op.numOperands()) |i| { - self.appendValueRecursive(op.operand(i), opt); - } - self.appendOperation(op); + pub fn shaped(self: *const MemRefType) *const ShapedType { + return @ptrCast(self); } }; -pub const helpers = struct { - pub fn eql(T: type, equal_fn: fn (@FieldType(T, "_inner"), @FieldType(T, "_inner")) callconv(.c) bool) fn (T, T) bool { - return struct { - fn eql(a: T, b: T) bool { - return equal_fn(a._inner, b._inner); - } - }.eql; - } - - pub fn deinit(T: type, deinit_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (*T) void { - return struct { - fn deinit(a: *T) void { - deinit_fn(a._inner); - a.* = undefined; - } - }.deinit; - } - - pub fn dump(T: type, dump_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (T) void { - return struct { - fn dump(a: T) void { - return dump_fn(a._inner); - } - }.dump; - } - - pub fn isNull(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (T) bool { - return struct { - fn isNull(a: T) bool { - return is_null_fn(a._inner); - } - }.isNull; - } - - pub fn format(Any: type, print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void) type { - return struct { - pub fn format(self: Any, writer: *std.Io.Writer) !void { - const WriterWithErr = struct { - writer: *std.Io.Writer, - err: ?std.Io.Writer.Error = null, - fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { - var ctx: *@This() = @alignCast(@ptrCast(opaque_ctx)); - if (ctx.err) |_| return; - _ = ctx.writer.write(mlir_str.data[0..mlir_str.length]) catch |err| { - ctx.err = err; - return; - }; - } - }; - - var context: WriterWithErr = .{ .writer = writer }; - print_fn(self._inner, &WriterWithErr.printCallback, &context); - if (context.err) |err| return err; - } - }; - } - - pub fn wrapOr(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (@FieldType(T, "_inner")) ?T { - return struct { - fn wrapOr(inner: @FieldType(T, "_inner")) ?T { - if (is_null_fn(inner)) return null; - return .{ ._inner = inner }; - } - }.wrapOr; - } - - pub fn init(T: type, inner: @FieldType(T, "_inner"), is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) ?T { - if (is_null_fn(inner)) return null; - return .{ ._inner = inner }; - } -}; +pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { + return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); +} diff --git a/mlir/registry.zig b/mlir/registry.zig deleted file mode 100644 index 7536e701..00000000 --- a/mlir/registry.zig +++ /dev/null @@ -1,1466 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const stdx = @import("stdx"); - -const log = std.log.scoped(.mlir); - -test { - std.testing.refAllDeclsRecursive(@This()); -} - -const Error = error{ - /// Invalid Mlir was created. - InvalidMlir, - /// Another Mlir error. Check the log for more context. - MlirUnexpected, - /// A resource/executor was not found. - NotFound, -} || std.mem.Allocator.Error; - -pub inline fn stringRef(str: []const u8) c.MlirStringRef { - return .{ .data = str.ptr, .length = str.len }; -} - -pub inline fn string(str: c.MlirStringRef) []const u8 { - // Note: mlir.StringRef need not to be null terminated. - return str.data[0..str.length]; -} - -pub const StringCallbackCtx = struct { - writer: *std.Io.Writer, - err: ?std.Io.Writer.Error = null, -}; - -pub fn stringCallback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const ctx: *StringCallbackCtx = @ptrCast(@alignCast(ctx_)); - ctx.writer.writeAll(string(str)) catch |err| { - ctx.err = err; - }; -} - -pub fn registerPasses(comptime passes: []const u8) void { - @field(c, "mlirRegister" ++ passes ++ "Passes")(); -} - -fn isPtrConst(comptime T: type) bool { - return @typeInfo(T).pointer.is_const; -} - -fn CastedPtr(comptime T1: type, comptime T2: type) type { - return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; -} - -pub fn Methods(comptime T: type, comptime NativeT: type) type { - return struct { - pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; - pub const Ptr = if (IsPtrConst) *const T else *T; - - pub fn ptr(self: anytype) NativeT { - return .{ .ptr = @ptrCast(@constCast(self)) }; - } - - fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return struct { - fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { - var ctx: StringCallbackCtx = .{ .writer = writer }; - f(self.ptr(), stringCallback, &ctx); - if (ctx.err) |err| { - return err; - } - } - }.format; - } - - pub fn eql(f: anytype) fn (self: *const T, other: *const T) bool { - return struct { - fn eql(self: *const T, other: *const T) bool { - return f(self.ptr(), other.ptr()); - } - }.eql; - } - - pub fn isA(self: *const T, OtherType: type) ?CastedPtr(@TypeOf(self), OtherType) { - return if (OtherType.isAFn(self.ptr())) @ptrCast(self) else null; - } - }; -} - -pub const DialectRegistry = opaque { - const M = Methods(DialectRegistry, c.MlirDialectRegistry); - - pub const ptr = M.ptr; - - pub fn init() std.mem.Allocator.Error!*DialectRegistry { - return @ptrCast(c.mlirDialectRegistryCreate().ptr orelse return std.mem.Allocator.Error.OutOfMemory); - } - - pub fn deinit(self: *DialectRegistry) void { - c.mlirDialectRegistryDestroy(self.ptr()); - } - - pub fn registerDialect(self: *DialectRegistry, comptime name: []const u8) void { - DialectHandle.fromString(name).insertDialect(self); - } -}; - -pub const Context = opaque { - const M = Methods(Context, c.MlirContext); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirContextEqual); - - pub const InitOptions = struct { - registry: ?*const DialectRegistry = null, - threading: bool = false, - }; - - pub fn init(opts: InitOptions) std.mem.Allocator.Error!*Context { - const ctx = if (opts.registry) |r| c.mlirContextCreateWithRegistry( - r.ptr(), - opts.threading, - ) else c.mlirContextCreateWithThreading(opts.threading); - return @ptrCast(ctx.ptr orelse return std.mem.Allocator.Error.OutOfMemory); - } - - pub fn deinit(self: *Context) void { - c.mlirContextDestroy(self.ptr()); - } - - pub fn setMultiThreading(self: *Context, enabled: bool) void { - c.mlirContextEnableMultithreading(self.ptr(), enabled); - } - - pub fn appendDialectRegistry(self: *Context, registry: *const DialectRegistry) void { - c.mlirContextAppendDialectRegistry(self.ptr(), registry.ptr()); - } - - pub fn loadAllAvailableDialects(self: *Context) void { - c.mlirContextLoadAllAvailableDialects(self.ptr()); - } - - pub fn numRegisteredDialects(self: *const Context) usize { - return @intCast(c.mlirContextGetNumRegisteredDialects(self.ptr())); - } - - pub fn numLoadedDialects(self: *const Context) usize { - return @intCast(c.mlirContextGetNumLoadedDialects(self.ptr())); - } - - pub fn numThreads(self: *const Context) usize { - return @intCast(c.mlirContextGetNumThreads(self.ptr())); - } - - pub fn isRegisteredOperation(self: *const Context, op: []const u8) bool { - return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); - } - - pub fn format(self: *const Context, writer: *std.Io.Writer) !void { - try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ - @typeName(Context), - self.numThreads(), - self.numRegisteredDialects(), - self.numLoadedDialects(), - }); - } -}; - -pub const Location = opaque { - const M = Methods(Location, c.MlirLocation); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirLocationEqual); - pub const format = M.format(c.mlirLocationPrint); - - pub fn fromSrc(ctx: *Context, src: std.builtin.SourceLocation) *const Location { - return fileLineCol( - ctx, - src.file, - src.line, - src.column, - ); - } - - pub fn fileLineCol(ctx: *Context, file: []const u8, line: usize, column: usize) *const Location { - return @ptrCast(c.mlirLocationFileLineColGet( - ctx.ptr(), - stringRef(file), - @intCast(line), - @intCast(column), - ).ptr); - } - - pub fn callSite(callee: *const Location, caller: *const Location) *const Location { - return @ptrCast(c.mlirLocationCallSiteGet(callee.ptr(), caller.ptr()).ptr); - } - - pub fn fused(ctx: *Context, locs: []const *const Location, metadata: *const Attribute) *const Location { - return @ptrCast(c.mlirLocationFusedGet( - ctx.ptr(), - @intCast(locs.len), - @ptrCast(locs), - metadata.ptr(), - ).ptr); - } - - pub fn named(self: *const Location, ctx: *Context, loc_name: []const u8) *const Location { - return @ptrCast(c.mlirLocationNameGet(ctx.ptr(), stringRef(loc_name), self.ptr()).ptr); - } - - pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { - var buf: [256]u8 = undefined; - return loc.named(loc, ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { - buf[buf.len - 3 ..].* = "...".*; - break :blk &buf; - }); - } - - pub fn fromAttribute(attribute: *const Attribute) *const Location { - return @ptrCast(c.mlirLocationFromAttribute(attribute.ptr()).ptr); - } - - pub fn unknown(ctx: *Context) *const Location { - return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); - } -}; - -pub const Type = opaque { - const M = Methods(Type, c.MlirType); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { - return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); - } -}; - -pub const IndexType = opaque { - const M = Methods(IndexType, c.MlirType); - - pub const isAFn = c.mlirTypeIsAIndex; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - - pub fn get(ctx: *Context) *const IndexType { - return @ptrCast(c.mlirIndexTypeGet(ctx.ptr()).ptr); - } -}; - -pub fn indexType(ctx: *Context) *const Type { - return @ptrCast(IndexType.get(ctx)); -} - -pub const IntegerTypes = enum { - i1, - i4, - i8, - i16, - i32, - i64, - i128, - u1, - u4, - u8, - u16, - u32, - u64, - u128, - si1, - si4, - si8, - si16, - si32, - si64, - si128, - - pub fn bitwidth(self: IntegerTypes) usize { - return switch (self) { - .i1, .u1, .si1 => 1, - .i4, .u4, .si4 => 4, - .i8, .u8, .si8 => 8, - .i16, .u16, .si16 => 16, - .i32, .u32, .si32 => 32, - .i64, .u64, .si64 => 64, - .i128, .u128, .si128 => 128, - }; - } - - pub fn signedness(self: IntegerTypes) IntegerType.Signedness { - return switch (self) { - .i1, .i4, .i8, .i16, .i32, .i64, .i128 => .signless, - .si1, .si4, .si8, .si16, .si32, .si64, .si128 => .signed, - .u1, .u4, .u8, .u16, .u32, .u64, .u128 => .unsigned, - }; - } -}; - -pub const IntegerType = opaque { - pub const Signedness = enum { - signless, - signed, - unsigned, - }; - - const M = Methods(IntegerType, c.MlirType); - - pub const isAFn = c.mlirTypeIsAInteger; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - - fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { - return exact(ctx, it.bitwidth(), it.signedness()); - } - - fn exact(ctx: *Context, bitwidth: usize, sign: Signedness) *const IntegerType { - return @ptrCast(switch (sign) { - .signless => c.mlirIntegerTypeGet(ctx.ptr(), @intCast(bitwidth)).ptr, - .signed => c.mlirIntegerTypeSignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, - .unsigned => c.mlirIntegerTypeUnsignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, - }); - } -}; - -pub fn integerType(ctx: *Context, it: IntegerTypes) *const Type { - return @ptrCast(IntegerType.get(ctx, it)); -} - -pub const FloatTypes = enum { - f4e2m1fn, - f6e2m3fn, - f6e3m2fn, - f8e3m4, - f8e4m3, - f8e4m3b11fnuz, - f8e4m3fn, - f8e4m3fnuz, - f8e5m2, - f8e5m2fnuz, - f8e8m0fnu, - bf16, - f16, - f32, - f64, - // f80, - // f128, - tf32, -}; - -pub fn FloatType(comptime ft: FloatTypes) type { - const _isAFn, const getter = switch (ft) { - .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, - .f6e2m3fn => .{ c.mlirTypeIsAFloat6E2M3FN, c.mlirFloat6E2M3FNTypeGet }, - .f6e3m2fn => .{ c.mlirTypeIsAFloat6E3M2FN, c.mlirFloat6E3M2FNTypeGet }, - .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, - .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, - .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, - .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, - .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, - .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, - .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, - .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, - .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, - .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, - .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, - .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, - .tf32 => .{ c.mlirTypeIsATF32, c.mlirTF32TypeGet }, - }; - - return opaque { - const Self = @This(); - const M = Methods(Self, c.MlirType); - - pub const isAFn = _isAFn; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - - pub fn get(ctx: *Context) *const Self { - return @ptrCast(getter(ctx.ptr()).ptr); - } - }; -} - -pub fn floatType(ft: FloatTypes, ctx: *Context) *const Type { - return switch (ft) { - inline else => |v| @ptrCast(FloatType(v).get(ctx)), - }; -} - -pub const Attribute = opaque { - const M = Methods(Attribute, c.MlirAttribute); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn parse(ctx: *Context, attr: []const u8) Error!*const Attribute { - return @ptrCast(c.mlirAttributeParseGet(ctx.ptr(), stringRef(attr)).ptr orelse return Error.InvalidMlir); - } - - pub fn context(self: *const Context) *Context { - return @ptrCast(c.mlirAttributeGetContext(self.ptr()).ptr); - } - - pub fn dialect(self: *const Context) *const Dialect { - return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); - } -}; - -pub const StringAttribute = opaque { - const M = Methods(StringAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsAString; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(ctx: *Context, str: []const u8) *const StringAttribute { - return @ptrCast(c.mlirStringAttrGet(ctx.ptr(), stringRef(str)).ptr); - } - - pub fn value(self: *const StringAttribute) []const u8 { - return string(c.mlirStringAttrGetValue(self.ptr())); - } -}; - -pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { - return @ptrCast(StringAttribute.init(ctx, str)); -} - -pub const IntegerAttribute = opaque { - const M = Methods(IntegerAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsAInteger; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { - return @ptrCast(c.mlirIntegerAttrGet( - IntegerType.get(ctx, it).ptr(), - @intCast(value_), - ).ptr); - } - - pub fn value(self: *const IntegerAttribute, comptime T: type) T { - const getValue = switch (@typeInfo(T).int.signedness) { - .signed => c.mlirIntegerAttrGetValueInt, - .unsigned => c.mlirIntegerAttrGetValueUInt, - }; - return @intCast(getValue(self.ptr())); - } -}; - -pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { - return @ptrCast(IntegerAttribute.init(ctx, it, value_)); -} - -pub const BoolAttribute = opaque { - const M = Methods(BoolAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsABool; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(ctx: *Context, value_: bool) *const BoolAttribute { - return @ptrCast(c.mlirBoolAttrGet(ctx.ptr(), @intFromBool(value_)).ptr); - } - - pub fn value(self: *const BoolAttribute) bool { - return c.mlirBoolAttrGetValue(self.ptr()); - } -}; - -pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { - return @ptrCast(BoolAttribute.init(ctx, value)); -} - -pub const TypeAttribute = opaque { - const M = Methods(TypeAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsAType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(type_: *const Type) *const TypeAttribute { - return @ptrCast(c.mlirTypeAttrGet(type_.ptr()).ptr); - } - - pub fn value(self: *const TypeAttribute) *const Type { - return @ptrCast(c.mlirTypeAttrGetValue(self.ptr()).ptr); - } -}; - -pub fn typeAttribute(type_: *const Type) *const Attribute { - return @ptrCast(TypeAttribute.init(type_)); -} - -pub const Dialect = struct { - const M = Methods(Dialect, c.MlirDialect); - - pub const ptr = M.ptr; - - pub fn context(self: *const Dialect) *Context { - return @ptrCast(c.mlirDialectGetContext(self.ptr()).ptr); - } - - pub fn namespace(self: *const Dialect) []const u8 { - return string(c.mlirDialectGetNamespace(self.ptr())); - } - - pub fn format(self: *const Dialect, writer: *std.Io.Writer) !void { - try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(Dialect), self.namespace() }); - } -}; - -pub const DialectHandle = opaque { - const M = Methods(DialectHandle, c.MlirDialectHandle); - - pub const ptr = M.ptr; - - pub fn format(self: *const DialectHandle, writer: *std.Io.Writer) !void { - try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(DialectHandle), self.namespace() }); - } - - pub fn namespace(self: *const DialectHandle) []const u8 { - return string(c.mlirDialectHandleGetNamespace(self.ptr())); - } - - pub fn insertDialect(self: *const DialectHandle, registry: *DialectRegistry) void { - c.mlirDialectHandleInsertDialect(self.ptr(), registry.ptr()); - } - - pub fn registerDialect(self: *const DialectHandle, ctx: *Context) void { - c.mlirDialectHandleRegisterDialect(self.ptr(), ctx.ptr()); - } - - pub fn loadDialect(self: *const DialectHandle, ctx: *Context) *Dialect { - return @ptrCast(c.mlirDialectHandleLoadDialect(self.ptr(), ctx.ptr()).ptr); - } - - pub fn fromString(comptime namespace_: []const u8) *const DialectHandle { - return @ptrCast(@field(c, "mlirGetDialectHandle__" ++ namespace_ ++ "__")().ptr); - } -}; - -pub const Identifier = opaque { - const M = Methods(Identifier, c.MlirIdentifier); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirIdentifierEqual); - - pub fn context(self: *const Identifier) *Context { - return @ptrCast(c.mlirIdentifierGetContext(self.ptr()).ptr); - } - - pub fn get(ctx: *Context, str_: []const u8) *const Identifier { - return @ptrCast(c.mlirIdentifierGet(ctx.ptr(), stringRef(str_)).ptr); - } - - pub fn str(self: *const Identifier) []const u8 { - return string(c.mlirIdentifierStr(self.ptr())); - } - - pub fn format(self: *const Identifier, writer: *std.Io.Writer) !void { - try writer.print("{s}{{ .str = \"{s}\" }}", .{ @typeName(Identifier), self.str() }); - } -}; - -pub const Module = opaque { - const M = Methods(Module, c.MlirModule); - - pub const ptr = M.ptr; - - pub fn init(loc: *const Location) *Module { - return @constCast(@ptrCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr)); - } - - pub fn deinit(self: *Module) void { - c.mlirModuleDestroy(self.ptr()); - } - - pub fn fromOperation(op: *Operation) *Module { - return @constCast(@ptrCast(c.mlirModuleFromOperation(op.ptr()).ptr)); - } - - pub fn operation(self: *const Module) *Operation { - return @ptrCast(c.mlirModuleGetOperation(self.ptr()).ptr); - } - - pub fn parse(ctx: *Context, source: []const u8) Error!*Module { - return @constCast(@ptrCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir)); - } - - pub fn context(self: *const Module) *Context { - return @ptrCast(c.mlirModuleGetContext(self.ptr()).ptr); - } - - pub fn body(self: *const Module) *Block { - return @ptrCast(c.mlirModuleGetBody(self.ptr()).ptr); - } -}; - -pub const Block = opaque { - const M = Methods(Block, c.MlirBlock); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirBlockEqual); - pub const format = M.format(c.mlirBlockPrint); - - pub fn init(args: []const *const Type, locs: []const *const Location) *Block { - return @ptrCast(c.mlirBlockCreate(@intCast(args.len), @ptrCast(args), @ptrCast(locs)).ptr); - } - - pub fn deinit(self: *Block) void { - std.debug.assert(self.parentOperation() == null); - c.mlirBlockDestroy(self.ptr()); - } - - pub fn argument(self: *const Block, index: usize) *const Value { - return @ptrCast(c.mlirBlockGetArgument(self.ptr(), @intCast(index)).ptr); - } - - pub fn numArguments(self: *const Block) usize { - return @intCast(c.mlirBlockGetNumArguments(self.ptr())); - } - - pub fn addArgument(self: *Block, type_: *const Type, loc: *const Location) *const Value { - return @ptrCast(c.mlirBlockAddArgument(self.ptr(), type_.ptr(), loc.ptr()).ptr); - } - - pub fn insertArgument(self: *Block, index: usize, type_: *const Type, loc: *const Location) *const Value { - return @ptrCast(c.mlirBlockInsertArgument(self.ptr(), @intCast(index), type_.ptr(), loc.ptr()).ptr); - } - - pub fn appendOwnedOperation(self: *Block, op: *Operation) void { - c.mlirBlockAppendOwnedOperation(self.ptr(), op.ptr()); - } - - pub fn appendOwnedOperations(self: *Block, ops: []const *Operation) void { - for (ops) |op| { - self.appendOwnedOperation(op); - } - } - - pub fn parentOperation(self: *const Block) ?*Operation { - return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); - } - - pub fn detach(self: *Block) void { - c.mlirBlockDetach(self.ptr()); - } - - pub fn terminator(self: *const Block) ?*Operation { - return @ptrCast(c.mlirBlockGetTerminator(self.ptr()).ptr); - } -}; - -pub const NamedAttribute = struct { - inner: c.MlirNamedAttribute, - - pub fn ptr(self: NamedAttribute) c.MlirNamedAttribute { - return self.inner; - } - - pub fn name(self: NamedAttribute) *const Identifier { - return @ptrCast(self.inner.name.ptr); - } - - pub fn attribute(self: NamedAttribute) *const Attribute { - return @ptrCast(self.inner.attribute.ptr); - } - - pub fn named(ctx: *Context, name_: []const u8, attr: *const Attribute) NamedAttribute { - return .{ - .inner = .{ - .name = Identifier.get(ctx, name_).ptr(), - .attribute = attr.ptr(), - }, - }; - } - - pub fn init(name_: *const Identifier, attr: *const Attribute) NamedAttribute { - return .{ - .inner = .{ - .name = name_.ptr(), - .attribute = attr.ptr(), - }, - }; - } - - pub fn format(self: NamedAttribute, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("{s}{{ .{s} = \"{f}\" }}", .{ - @typeName(NamedAttribute), - self.name().str(), - self.attribute(), - }); - } -}; - -pub const DictionaryAttribute = opaque { - const M = Methods(DictionaryAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsADictionary; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(ctx: *Context, attributes: []const NamedAttribute) *const DictionaryAttribute { - return @ptrCast(c.mlirDictionaryAttrGet( - ctx.ptr(), - @intCast(attributes.len), - @ptrCast(attributes), - ).ptr); - } - - pub fn size(self: *const DictionaryAttribute) usize { - return @intCast(c.mlirDictionaryAttrGetNumElements(self.ptr())); - } - - pub fn get(self: *const DictionaryAttribute, pos: usize) NamedAttribute { - return .{ .inner = c.mlirDictionaryAttrGetElement(self.ptr(), @intCast(pos)) }; - } - - pub fn getByName(self: *const DictionaryAttribute, name: []const u8) ?*const Attribute { - return @ptrCast(c.mlirDictionaryAttrGetElementByName(self.ptr(), stringRef(name)).ptr); - } -}; - -pub fn dictionaryAttribute(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { - return @ptrCast(DictionaryAttribute.init(ctx, attributes)); -} - -pub const Value = opaque { - const M = Methods(Value, c.MlirValue); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirValueEqual); - pub const format = M.format(c.mlirValuePrint); - pub const isA = M.isA; - - pub fn type_(val: *const Value) *const Type { - return @ptrCast(c.mlirValueGetType(val.ptr()).ptr); - } - - pub fn setType(self: *Value, type__: *const Type) void { - c.mlirValueSetType(self.ptr(), type__.ptr()); - } - - pub fn firstUse(self: *const Value) *const OpOperand { - return @ptrCast(c.mlirValueGetFirstUse(self.ptr()).ptr); - } - - pub fn replaceAllUsesWith(self: *const Value, with: *const Value) void { - c.mlirValueReplaceAllUsesOfWith(self.ptr(), with.ptr()); - } -}; - -pub const OpOperand = opaque { - const M = Methods(OpOperand, c.MlirOpOperand); - - pub const ptr = M.ptr; - - pub fn owner(self: *const OpOperand) *Operation { - return @ptrCast(c.mlirOpOperandGetOwner(self.ptr()).ptr); - } - - pub fn number(self: *const OpOperand) usize { - return @intCast(c.mlirOpOperandGetOperandNumber(self.ptr())); - } - - pub fn nextUse(self: *const OpOperand) ?*const OpOperand { - return @ptrCast(c.mlirOpOperandGetNextUse(self.ptr()).ptr); - } -}; - -pub const OpResult = opaque { - const M = Methods(OpResult, c.MlirValue); - - pub const isAFn = c.mlirValueIsAOpResult; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirValueEqual); - pub const format = M.format(c.mlirValuePrint); - - pub fn owner(self: *const OpResult) *Operation { - return @ptrCast(c.mlirOpResultGetOwner(self.ptr()).ptr); - } - - pub fn resultNumber(self: *const OpResult) usize { - return @intCast(c.mlirOpResultGetResultNumber(self.ptr())); - } -}; - -pub const BlockArgument = opaque { - const M = Methods(BlockArgument, c.MlirValue); - - pub const isAFn = c.mlirValueIsABlockArgument; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirValueEqual); - pub const format = M.format(c.mlirValuePrint); - - pub fn block(arg: *const BlockArgument) *Block { - return @ptrCast(c.mlirBlockArgumentGetOwner(arg.ptr()).ptr); - } - - pub fn number(self: *const BlockArgument) usize { - return @intCast(c.mlirBlockArgumentGetArgNumber(self.ptr())); - } - - pub fn setType(self: *BlockArgument, type_: *const Type) void { - c.mlirBlockArgumentSetType(self.ptr(), type_.ptr()); - } -}; - -pub const OperationState = struct { - inner: c.MlirOperationState, - - pub fn init(name: []const u8, location: *const Location) OperationState { - return .{ .inner = c.mlirOperationStateGet(stringRef(name), location.ptr()) }; - } - - pub fn addResults(self: *OperationState, results: []const *const Type) void { - c.mlirOperationStateAddResults(&self.inner, @intCast(results.len), @ptrCast(results)); - } - - pub fn addOperands(self: *OperationState, operands: []const *const Value) void { - c.mlirOperationStateAddOperands(&self.inner, @intCast(operands.len), @ptrCast(operands)); - } - - pub fn addOwnedRegions(self: *OperationState, regions: []const *Region) void { - c.mlirOperationStateAddOwnedRegions(&self.inner, @intCast(regions.len), @ptrCast(regions)); - } - - pub fn addSuccessors(self: *OperationState, successors: []*const Block) void { - c.mlirOperationStateAddSuccessors(&self.inner, @intCast(successors.len), @ptrCast(successors)); - } - - pub fn addAttributes(self: *OperationState, attributes: []const NamedAttribute) void { - c.mlirOperationStateAddAttributes(&self.inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); - } - - pub fn enableResultTypeInference(self: *OperationState) void { - c.mlirOperationStateEnableResultTypeInference(&self.inner); - } -}; - -pub const Operation = opaque { - const M = Methods(Operation, c.MlirOperation); - - const MAX_SEGMENTS = 32; - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirOperationEqual); - pub const format = M.format(c.mlirOperationPrint); - - pub fn init(state: *OperationState) !*Operation { - defer state.* = undefined; - return @ptrCast(c.mlirOperationCreate(&state.inner).ptr orelse return Error.InvalidMlir); - } - - pub fn deinit(self: *Operation) void { - std.debug.assert(self.block() == null); - c.mlirOperationDestroy(self.ptr()); - } - - pub fn parse(ctx: *Context, source: []const u8, name_: []const u8) Error!*Operation { - return @ptrCast(c.mlirOperationCreateParse(ctx.ptr(), stringRef(source), stringRef(name_)).ptr orelse return Error.InvalidMlir); - } - - pub fn name(self: *const Operation) []const u8 { - return string(c.mlirOperationGetName(self.ptr())); - } - - pub fn clone(self: *const Operation) *Operation { - return @ptrCast(c.mlirOperationClone(self.ptr()).ptr); - } - - pub fn block(self: *const Operation) ?*Block { - return @ptrCast(c.mlirOperationGetBlock(self.ptr()).ptr); - } - - pub fn appendTo(self: *Operation, block_: *Block) *Operation { - block_.appendOwnedOperation(self); - return self; - } - - pub fn parent(self: *const Operation) ?*Operation { - return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); - } - - pub fn removeFromParent(self: *Operation) void { - c.mlirOperationRemoveFromParent(self.ptr()); - } - - pub fn numOperands(self: *const Operation) usize { - return @intCast(c.mlirOperationGetNumOperands(self.ptr())); - } - - pub fn operand(self: *const Operation, index: usize) *const Value { - return @ptrCast(c.mlirOperationGetOperand(self.ptr(), @intCast(index)).ptr); - } - - pub fn setOperand(self: *Operation, index: usize, value: *const Value) void { - c.mlirOperationSetOperand(self.ptr(), @intCast(index), value.ptr()); - } - - pub fn numResults(self: *const Operation) usize { - return @intCast(c.mlirOperationGetNumResults(self.ptr())); - } - - pub fn result(self: *const Operation, index: usize) *const Value { - return @ptrCast(c.mlirOperationGetResult(self.ptr(), @intCast(index)).ptr); - } - - pub fn nextInBlock(self: *const Operation) *const Operation { - return @ptrCast(c.mlirOperationGetNextInBlock(self.ptr()).ptr); - } - - pub fn context(self: *const Operation) *Context { - return @ptrCast(c.mlirOperationGetContext(self.ptr()).ptr); - } - - pub const PrintFlags = struct { - elide_large_elements_attrs: ?usize = null, - debug_info: bool = false, - debug_info_pretty_form: bool = true, - print_generic_op_form: bool = false, - use_local_scope: bool = false, - assume_verified: bool = false, - - const Ctx = struct { - self: *const Operation, - flags: PrintFlags, - }; - }; - - fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { - const flags = c.mlirOpPrintingFlagsCreate(); - defer c.mlirOpPrintingFlagsDestroy(flags); - - if (fctx.flags.elide_large_elements_attrs) |v| { - c.mlirOpPrintingFlagsElideLargeElementsAttrs(flags, @intCast(v)); - } - c.mlirOpPrintingFlagsEnableDebugInfo( - flags, - fctx.flags.debug_info, - fctx.flags.debug_info_pretty_form, - ); - if (fctx.flags.print_generic_op_form) { - c.mlirOpPrintingFlagsPrintGenericOpForm(flags); - } - if (fctx.flags.use_local_scope) { - c.mlirOpPrintingFlagsUseLocalScope(flags); - } - if (fctx.flags.assume_verified) { - c.mlirOpPrintingFlagsAssumeVerified(flags); - } - var sctx: StringCallbackCtx = .{ .writer = writer }; - c.mlirOperationPrintWithFlags(fctx.self.ptr(), flags, stringCallback, &sctx); - if (sctx.err) |err| { - return err; - } - } - - pub fn fmt(self: *const Operation, flags: PrintFlags) std.fmt.Alt(PrintFlags.Ctx, formatWithFlags) { - return .{ .data = .{ - .self = self, - .flags = flags, - } }; - } - - pub const MakeArgs = struct { - pub const AttrTuple = struct { []const u8, *const Attribute }; - - pub const Operands = union(enum) { - flat: []const *const Value, - variadic: []const []const *const Value, - }; - - pub const ResultTypes = union(enum) { - flat: []const *const Type, - variadic: []const []const *const Type, - }; - - operands: ?Operands = null, - results: ?ResultTypes = null, - result_type_inference: bool = false, - attributes: ?[]const NamedAttribute = null, - blocks: ?[]const *Block = null, - verify: bool = true, - location: ?*const Location = null, - }; - - pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { - var state: OperationState = .init(name_, args.location orelse .unknown(ctx)); - if (args.result_type_inference) { - state.enableResultTypeInference(); - } - if (args.operands) |operands| switch (operands) { - .flat => |v| state.addOperands(v), - .variadic => |segments| { - var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; - for (segments) |segment_operands| { - state.addOperands(segment_operands); - sizes.appendAssumeCapacity(@intCast(segment_operands.len)); - } - state.addAttributes(&.{ - .named(ctx, "operandSegmentSizes", denseArrayAttribute(ctx, .i32, sizes.constSlice())), - }); - }, - }; - if (args.results) |results| switch (results) { - .flat => |v| state.addResults(v), - .variadic => |segments| { - var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; - for (segments) |segment_results| { - state.addResults(segment_results); - sizes.appendAssumeCapacity(@intCast(segment_results.len)); - } - state.addAttributes(&.{ - .named(ctx, "resultSegmentSizes", denseArrayAttribute(ctx, .i32, sizes.constSlice())), - }); - }, - }; - if (args.attributes) |attrs| { - state.addAttributes(attrs); - } - if (args.blocks) |blocks| { - const region = Region.init(); - for (blocks) |block_| { - region.appendOwnedBlock(block_); - } - state.addOwnedRegions(&.{region}); - } - const new_op = try Operation.init(&state); - errdefer new_op.deinit(); - if (args.verify and new_op.verify() == false) { - log.err("Failed to verify MLIR operation:\n{f}", .{ - new_op.fmt(.{ - .print_generic_op_form = true, - .debug_info = true, - }), - }); - return Error.InvalidMlir; - } - return new_op; - } - - pub fn make(ctx: *Context, op_name: []const u8, args: MakeArgs) *Operation { - return try_make(ctx, op_name, args) catch { - @panic("Failed to create MLIR operation"); - }; - } - - pub fn verify(self: *const Operation) bool { - return c.mlirOperationVerify(self.ptr()); - } - - pub fn location(self: *const Operation) *const Location { - return @ptrCast(c.mlirOperationGetLocation(self.ptr()).ptr); - } - - pub const WalkOrder = enum(c.MlirWalkOrder) { - pre_order = c.MlirWalkPreOrder, - post_order = c.MlirWalkPostOrder, - }; - - pub const WalkResult = enum(c.MlirWalkResult) { - advance = c.MlirWalkResultAdvance, - interrupt = c.MlirWalkResultInterrupt, - skip = c.MlirWalkResultSkip, - }; - - pub fn walk(self: *Operation, order: WalkOrder, ctx: anytype, walkfn: fn (ctx_: @TypeOf(ctx), op: *Operation) WalkResult) void { - var inner_ctx = .{ .ctx = ctx }; - const ContextType = @TypeOf(inner_ctx); - - c.mlirOperationWalk( - self._inner, - (struct { - pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { - const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); - } - }).callback, - &inner_ctx, - @intFromEnum(order), - ); - } - - pub const BytecodeWriterConfig = struct { - desired_emit_version: ?usize = null, - - const Ctx = struct { - self: *const Operation, - config: ?BytecodeWriterConfig, - }; - }; - - pub fn writeBytecode(self: *const Operation, config: ?BytecodeWriterConfig, writer: *std.Io.Writer) !void { - if (config) |config_| { - const bc_config = c.mlirBytecodeWriterConfigCreate(); - defer c.mlirBytecodeWriterConfigDestroy(bc_config); - if (config_.desired_emit_version) |version| { - c.mlirBytecodeWriterConfigDesiredEmitVersion(bc_config, @intCast(version)); - } - var sctx: StringCallbackCtx = .{ .writer = writer }; - c.mlirOperationWriteBytecodeWithConfig(self.ptr(), bc_config, stringCallback, &sctx); - if (sctx.err) |err| { - return err; - } - return; - } - - var sctx: StringCallbackCtx = .{ .writer = writer }; - c.mlirOperationWriteBytecode(self.ptr(), stringCallback, &sctx); - if (sctx.err) |err| { - return err; - } - } - - fn fmtBytecodeImpl(ctx: BytecodeWriterConfig.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { - try ctx.self.writeBytecode(ctx.config, writer); - } - - pub fn fmtBytecode(self: *const Operation, config: ?BytecodeWriterConfig) std.fmt.Alt(BytecodeWriterConfig.Ctx, fmtBytecodeImpl) { - return .{ .data = .{ - .self = self, - .config = config, - } }; - } -}; - -pub const Region = opaque { - const M = Methods(Region, c.MlirRegion); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirRegionEqual); - - pub fn init() *Region { - return @ptrCast(c.mlirRegionCreate().ptr); - } - - pub fn deinit(self: *Region) void { - c.mlirRegionDestroy(self.ptr()); - } - - pub fn appendOwnedBlock(self: *Region, block: *Block) void { - c.mlirRegionAppendOwnedBlock(self.ptr(), block.ptr()); - } - - pub fn insertOwnedBlock(self: *Region, index: usize, block: *Block) void { - c.mlirRegionInsertOwnedBlock(self.ptr(), @intCast(index), block.ptr()); - } - - pub fn insertOwnedBlockBefore(self: *Region, reference: *const Block, block: *Block) void { - c.mlirRegionInsertOwnedBlockBefore(self.ptr(), reference.ptr(), block.ptr()); - } - - pub fn insertOwnedBlockAfter(self: *Region, reference: *const Block, block: *Block) void { - c.mlirRegionInsertOwnedBlockAfter(self.ptr(), reference.ptr(), block.ptr()); - } - - pub fn firstBlock(self: *const Region) ?*Block { - return @ptrCast(c.mlirRegionGetFirstBlock(self.ptr()).ptr); - } - - pub fn takeBody(self: *Region, from: *Region) void { - c.mlirRegionTakeBody(self.ptr(), from.ptr()); - } - - pub fn nextInOperation(self: *const Region) *Region { - return @ptrCast(c.mlirRegionGetNextInOperation(self.ptr()).ptr); - } -}; - -pub const ShapedType = opaque { - const M = Methods(ShapedType, c.MlirType); - - pub const MAX_RANK = 16; - - pub const isAFn = c.mlirTypeIsAShaped; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub fn elementType(self: *const ShapedType) *const Type { - return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); - } - - pub fn rank(self: *const ShapedType) usize { - return @intCast(c.mlirShapedTypeGetRank(self.ptr())); - } - - pub fn dimension(self: *const ShapedType, dim: usize) i64 { - return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); - } -}; - -pub const RankedTensorType = opaque { - const M = Methods(RankedTensorType, c.MlirType); - - pub const isAFn = c.mlirTypeIsARankedTensor; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - - pub fn get(dimensions: []const i64, elemType: *const Type) *const RankedTensorType { - return @ptrCast(c.mlirRankedTensorTypeGet( - @intCast(dimensions.len), - @ptrCast(dimensions), - elemType.ptr(), - c.mlirAttributeGetNull(), - ).ptr); - } - - pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { - var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; - for (0..other.rank()) |i| { - dims.appendAssumeCapacity(other.dimension(i)); - } - return get( - dims.constSlice(), - other.elementType(), - ); - } - - pub fn elementType(self: *const RankedTensorType) *const Type { - return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); - } - - pub fn rank(self: *const RankedTensorType) usize { - return @intCast(c.mlirShapedTypeGetRank(self.ptr())); - } - - pub fn dimension(self: *const RankedTensorType, dim: usize) i64 { - return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); - } - - pub fn shaped(self: *const RankedTensorType) *const ShapedType { - return @ptrCast(self); - } -}; - -pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { - return @ptrCast(RankedTensorType.get(dimensions, elem_type)); -} - -pub const DenseElementsAttribute = opaque { - const M = Methods(DenseElementsAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsADenseElements; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(shaped_type: *const ShapedType, values: anytype) *const DenseElementsAttribute { - const bytes = std.mem.sliceAsBytes(values); - return @ptrCast(c.mlirDenseElementsAttrRawBufferGet( - shaped_type.ptr(), - @intCast(bytes.len), - @ptrCast(bytes), - ).ptr orelse unreachable); - } -}; - -pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: anytype) *const Attribute { - return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); -} - -pub const DenseArrayTypes = enum { - bool, - i8, - i16, - i32, - i64, - f32, - f64, - - fn ZigType(comptime ElementType: @This()) type { - return switch (ElementType) { - .bool => i32, - .i8 => i8, - .i16 => i16, - .i32 => i32, - .i64 => i64, - .f32 => f32, - .f64 => f64, - }; - } -}; - -pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { - const ElementTypeZig = ElementType.ZigType(); - - const _isAFn, const getFn, const getElementFn = switch (ElementType) { - .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, - .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, - .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, - .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, - .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, - .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, - .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, - }; - - return struct { - const Self = @This(); - - const M = Methods(@This(), c.MlirAttribute); - - pub const isAFn = _isAFn; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(ctx: *Context, values: []const ElementTypeZig) *const Self { - return @ptrCast(getFn(ctx.ptr(), @intCast(values.len), @ptrCast(values)).ptr); - } - - pub fn element(self: *const Self, pos: usize) ElementTypeZig { - return getElementFn(self.ptr(), @intCast(pos)); - } - - pub fn numElements(self: *const Self) usize { - return @intCast(c.mlirDenseArrayGetNumElements(self.ptr())); - } - }; -} - -pub fn denseArrayAttribute(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { - return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); -} - -pub const ArrayAttribute = opaque { - const M = Methods(ArrayAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsAArray; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(ctx: *Context, attrs: []const *const Attribute) *const ArrayAttribute { - return @ptrCast(c.mlirArrayAttrGet(ctx.ptr(), @intCast(attrs.len), @ptrCast(attrs)).ptr); - } - - pub fn element(self: *const ArrayAttribute, index: usize) Attribute { - return @ptrCast(c.mlirArrayAttrGetElement(self.ptr(), @intCast(index)).ptr); - } - - pub fn numElements(self: *const ArrayAttribute) usize { - return @intCast(c.mlirArrayAttrGetNumElements(self.ptr())); - } -}; - -pub fn arrayAttribute(ctx: *Context, attrs: []const *const Attribute) *const Attribute { - return @ptrCast(ArrayAttribute.init(ctx, attrs)); -} - -pub const FunctionType = opaque { - const M = Methods(FunctionType, c.MlirType); - pub const isAFn = c.mlirTypeIsAFunction; - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - - pub fn init(ctx: *Context, args: []const *const Type, results: []const *const Type) *const FunctionType { - return @ptrCast(c.mlirFunctionTypeGet( - ctx.ptr(), - @intCast(args.len), - @ptrCast(args.ptr), - @intCast(results.len), - @ptrCast(results.ptr), - ).ptr); - } -}; - -pub fn functionType(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { - return @ptrCast(FunctionType.init(ctx, args, results)); -} - -pub const FlatSymbolRefAttribute = opaque { - const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsAFlatSymbolRef; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(ctx: *Context, symbol_: []const u8) *const FlatSymbolRefAttribute { - return @ptrCast(c.mlirFlatSymbolRefAttrGet(ctx.ptr(), stringRef(symbol_)).ptr); - } - - pub fn symbol(self: *const FlatSymbolRefAttribute) []const u8 { - return string(c.mlirFlatSymbolRefAttrGetValue(self.ptr())); - } -}; - -pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { - return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); -} - -pub const UnitAttribute = opaque { - const M = Methods(UnitAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsAUnit; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn get(ctx: *Context) *const UnitAttribute { - return @ptrCast(c.mlirUnitAttrGet(ctx.ptr()).ptr); - } -}; - -pub fn unitAttribute(ctx: *Context) *const Attribute { - return @ptrCast(UnitAttribute.get(ctx)); -} - -pub const MemRefType = opaque { - const M = Methods(MemRefType, c.MlirType); - - pub const isAFn = c.mlirTypeIsAInteger; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - - pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { - return @ptrCast(c.mlirMemRefTypeGet( - element_type.ptr(), - @intCast(shape.len), - @ptrCast(shape), - if (layout) |l| l.ptr() else .{}, - if (memory_space) |m| m.ptr() else .{}, - ).ptr); - } - - pub fn fromShaped(other: *const ShapedType) *const MemRefType { - var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; - for (0..other.rank()) |i| { - dims.appendAssumeCapacity(other.dimension(i)); - } - return init( - other.elementType(), - dims.constSlice(), - null, - null, - ); - } - - pub fn shaped(self: *const MemRefType) *const ShapedType { - return @ptrCast(self); - } -}; - -pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { - return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); -} From fc3cd785fb5b380d82da21353dcacb0274250765 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 28 Aug 2025 17:36:20 +0200 Subject: [PATCH 37/47] wip --- mlir/mlir.zig | 77 +++++++++++++++++++++++++++++++++++++++++++++++++- zml/mlirx.zig | 55 ++++++++++++++++-------------------- zml/module.zig | 70 +++++++++++++++++++++++---------------------- zml/tensor.zig | 2 +- 4 files changed, 139 insertions(+), 65 deletions(-) diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 7536e701..7f13df69 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -39,6 +39,13 @@ pub fn stringCallback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void }; } +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: StringCallbackCtx = .{ .writer = &writer }; + _ = @call(.auto, streamFn, args ++ .{ stringCallback, &sctx }); + return writer.buffered(); +} + pub fn registerPasses(comptime passes: []const u8) void { @field(c, "mlirRegister" ++ passes ++ "Passes")(); } @@ -104,6 +111,45 @@ pub const DialectRegistry = opaque { } }; +pub const PassManager = opaque { + const M = Methods(PassManager, c.MlirPassManager); + + pub const ptr = M.ptr; + + pub fn init(ctx: *Context) *PassManager { + return @ptrCast(c.mlirPassManagerCreate(ctx.ptr()).ptr); + } + + pub fn deinit(self: *const PassManager) void { + c.mlirPassManagerDestroy(self.ptr()); + } + + pub fn asOpPassManager(self: *PassManager) *OpPassManager { + return @ptrCast(c.mlirPassManagerGetAsOpPassManager(self.ptr()).ptr); + } +}; + +pub const OpPassManager = opaque { + const M = Methods(OpPassManager, c.MlirOpPassManager); + + pub const ptr = M.ptr; + + pub fn addPipeline(self: *OpPassManager, pipeline_name: []const u8) !void { + var buffer: [1024]u8 = undefined; + const result = stringFromStream(&buffer, c.mlirOpPassManagerAddPipeline, .{ self.ptr(), stringRef(pipeline_name) }); + if (result.len != 0) { + log.err("mlir error: {s}", .{result}); + return error.MlirUnexpected; + } + } +}; + +pub const Pass = opaque { + const M = Methods(Pass, c.MlirPass); + + pub const ptr = M.ptr; +}; + pub const Context = opaque { const M = Methods(Context, c.MlirContext); @@ -385,12 +431,37 @@ pub fn FloatType(comptime ft: FloatTypes) type { }; } -pub fn floatType(ft: FloatTypes, ctx: *Context) *const Type { +pub fn floatType(ctx: *Context, ft: FloatTypes) *const Type { return switch (ft) { inline else => |v| @ptrCast(FloatType(v).get(ctx)), }; } +pub const ComplexTypes = enum { + c64, + c128, +}; + +pub const ComplexType = opaque { + const M = Methods(ComplexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAComplex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context, complex_type: ComplexTypes) *const ComplexType { + return @ptrCast(c.mlirComplexTypeGet(switch (complex_type) { + .c64 => floatType(ctx, .f32).ptr(), + .c128 => floatType(ctx, .f64).ptr(), + }).ptr); + } +}; + +pub fn complexType(ctx: *Context, complex_type: ComplexTypes) *const Type { + return @ptrCast(ComplexType.get(ctx, complex_type)); +} + pub const Attribute = opaque { const M = Methods(Attribute, c.MlirAttribute); @@ -927,6 +998,10 @@ pub const Operation = opaque { return @ptrCast(c.mlirOperationGetContext(self.ptr()).ptr); } + pub fn setAttributeByName(self: *Operation, name_: []const u8, attribute: *const Attribute) void { + c.mlirOperationSetAttributeByName(self.ptr(), stringRef(name_), attribute.ptr()); + } + pub const PrintFlags = struct { elide_large_elements_attrs: ?usize = null, debug_info: bool = false, diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 28f40383..46c85fa4 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -7,11 +7,6 @@ const Shape = @import("shape.zig").Shape; const mlirx = @This(); -/// Returns the mlir.Type corresponding to a given zml.Shape. -pub fn tensorType(ctx: mlir.Context, sh: Shape) mlir.Type { - return .tensor(sh.dims(), mlirx.Type.fromDType(ctx, sh.dtype())); -} - pub fn denseElementAttrType(dt: dtype.DataType) ?mlir.DenseElementsAttributeTypes { return switch (dt) { .bool => .bool, @@ -32,34 +27,34 @@ pub fn denseElementAttrType(dt: dtype.DataType) ?mlir.DenseElementsAttributeType } pub const Type = struct { - pub fn fromDType(ctx: mlir.Context, dt: dtype.DataType) mlir.Type { + pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { return switch (dt) { - .bool => .int(ctx, .i1), - .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), - .f8e4m3fn => .float(ctx, .f8e4m3fn), - .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), - .f8e5m2 => .float(ctx, .f8e5m2), - .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), - .bf16 => .float(ctx, .bf16), - .f16 => .float(ctx, .f16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .i4 => .int(ctx, .i4), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .u4 => .int(ctx, .u4), - .u8 => .int(ctx, .u8), - .u16 => .int(ctx, .u16), - .u32 => .int(ctx, .u32), - .u64 => .int(ctx, .u64), - .c64 => .complex(ctx, .c64), - .c128 => .complex(ctx, .c128), + .bool => mlir.integerType(ctx, .i1), + .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), + .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), + .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), + .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), + .bf16 => mlir.floatType(ctx, .bf16), + .f16 => mlir.floatType(ctx, .f16), + .f32 => mlir.floatType(ctx, .f32), + .f64 => mlir.floatType(ctx, .f64), + .i4 => mlir.integerType(ctx, .i4), + .i8 => mlir.integerType(ctx, .i8), + .i16 => mlir.integerType(ctx, .i16), + .i32 => mlir.integerType(ctx, .i32), + .i64 => mlir.integerType(ctx, .i64), + .u4 => mlir.integerType(ctx, .u4), + .u8 => mlir.integerType(ctx, .u8), + .u16 => mlir.integerType(ctx, .u16), + .u32 => mlir.integerType(ctx, .u32), + .u64 => mlir.integerType(ctx, .u64), + .c64 => mlir.complexType(ctx, .c64), + .c128 => mlir.complexType(ctx, .c128), }; } - pub fn toDType(mlir_type: mlir.Type) dtype.DataType { + pub fn toDType(mlir_type: *const mlir.Type) dtype.DataType { const mapping = .{ .{ .bool, mlir.IntegerType(.i1) }, @@ -91,7 +86,7 @@ pub const Type = struct { inline for (mapping) |entry| { const dt, const mlirT = entry; - if (mlirT.is_a_fn(mlir_type._inner)) { + if (mlirT.isAFn(mlir_type._inner)) { return dt; } } diff --git a/zml/module.zig b/zml/module.zig index 4e40882e..9f3858d8 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -29,10 +29,10 @@ pub const MlirFn = struct { name: []const u8, args_shapes: []Shape, res_tensors: *const anyopaque, - res_types: []mlir.Type, + res_types: []*const mlir.Type, res_shapes: []Shape, res_donations: []Tensor._Donation, - mlir_fn: mlir.Operation, + mlir_fn: *mlir.Operation, pub const Kind = enum { main, @@ -40,16 +40,18 @@ pub const MlirFn = struct { }; }; +const RecursiveOpts = enum { open, hermetic }; + pub const CompilationContext = struct { _platform: Platform, _name: []const u8, _arena: std.heap.ArenaAllocator, - _mlir_ctx: mlir.Context, - _mlir_registry: mlir.Registry, - _mlir_canonicalizer: mlir.PassManager, + _mlir_ctx: *mlir.Context, + _mlir_registry: *mlir.DialectRegistry, + _mlir_canonicalizer: *mlir.PassManager, - _module: mlir.Module, + _module: *mlir.Module, _blocks: stdx.BoundedArray(TaggedBlock, 64) = .{}, _fn_cache: FnCache = .{}, @@ -61,16 +63,16 @@ pub const CompilationContext = struct { _previous: ?*CompilationContext = null, threadlocal var _current: ?*CompilationContext = null; - const TaggedBlock = struct { mlir.Block, mlir.Block.RecursiveOpts }; - const TensorToBlockArg = std.AutoHashMapUnmanaged(Tensor._Id, struct { mlir.Value, Tensor._Donation }); + const TaggedBlock = struct { *mlir.Block, RecursiveOpts }; + const TensorToBlockArg = std.AutoHashMapUnmanaged(Tensor._Id, struct { *const mlir.Value, Tensor._Donation }); const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); pub fn init(allocator_: std.mem.Allocator, full_name: []const u8, platform: Platform) !CompilationContext { - const mlir_registry = mlir.Registry.init() catch unreachable; + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); } - var mlir_ctx = mlir.Context.initWithRegistry(mlir_registry, false) catch unreachable; + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; mlir_ctx.loadAllAvailableDialects(); // Too long module names create too long file paths and files failed to create. @@ -79,11 +81,11 @@ pub const CompilationContext = struct { const max_name_len = @divFloor(std.fs.max_path_bytes, 2) - 17; const name = full_name[0..@min(max_name_len, full_name.len)]; - const loc = mlir_ctx.location(@src()).named(mlir_ctx, "main"); + const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); const module = mlir.Module.init(loc); - module.op().setAttributeByName("sym_name", .string(mlir_ctx, "zml")); + module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); - var canonicalizer = try mlir.PassManager.init(mlir_ctx); + var canonicalizer = mlir.PassManager.init(mlir_ctx); { var opm = canonicalizer.asOpPassManager(); try opm.addPipeline("canonicalize"); @@ -138,7 +140,7 @@ pub const CompilationContext = struct { return self._platform.target; } - pub fn mlirCtx(self: *const CompilationContext) mlir.Context { + pub fn mlirCtx(self: *const CompilationContext) *mlir.Context { return self._mlir_ctx; } @@ -164,12 +166,12 @@ pub const CompilationContext = struct { // Run in a dedicated thread because compilation relies on `threadlocal`. const f = try asynk.callBlocking(CompilationContext.emitMlir, .{ self, func, &tensor_args, CompilationContext.EmitMlirOpts{ .name = "main", .kind = .main } }); const module = self._module; - module.getBody().appendOperation(f.mlir_fn); + _ = f.mlir_fn.appendTo(module.body()); const sharding = self._platform.sharding(); const mlir_ctx = self._mlir_ctx; - module.op().setAttributeByName("mhlo.num_replicas", .int(mlir_ctx, .i32, sharding.num_replicas)); - module.op().setAttributeByName("mhlo.num_partitions", .int(mlir_ctx, .i32, sharding.num_partitions)); + module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(mlir_ctx, .i32, sharding.num_replicas)); + module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(mlir_ctx, .i32, sharding.num_partitions)); const module_hash = computeModuleHash(self._platform, module); var module_dir: ?[]const u8 = null; @@ -244,8 +246,8 @@ pub const CompilationContext = struct { return if (self._blocks.len > 0) self._blocks.get(self._blocks.len - 1) else null; } - pub fn openBlock(self: *CompilationContext, kind: mlir.Block.RecursiveOpts, args: []const mlir.Type, locs: []const mlir.Location) !TaggedBlock { - const block: TaggedBlock = .{ try mlir.Block.init(args, locs), kind }; + pub fn openBlock(self: *CompilationContext, kind: RecursiveOpts, args: []const *const mlir.Type, locs: []const *const mlir.Location) !TaggedBlock { + const block: TaggedBlock = .{ mlir.Block.init(args, locs), kind }; self.pushBlock(block); return block; } @@ -336,17 +338,19 @@ pub const CompilationContext = struct { const tensor_count = meta.count(Tensor, args); const mlir_ctx = self.mlirCtx(); - const loc = mlir_ctx.location(@src()); + const loc: *const mlir.Location = .fromSrc(mlir_ctx, @src()); - const locations = try arena.alloc(mlir.Location, tensor_count); - @memset(locations, mlir.Location.unknown(mlir_ctx)); + const locations = try arena.alloc(*const mlir.Location, tensor_count); + @memset(locations, .unknown(mlir_ctx)); var input_shapes: std.array_list.Managed(Shape) = try .initCapacity(res_allocator, tensor_count); meta.collect(Tensor.shape, {}, &input_shapes, args) catch unreachable; stdx.debug.internalAssert(input_shapes.items.len == tensor_count, "args have changed ?", .{}); - const input_types = try arena.alloc(mlir.Type, tensor_count); - for (input_types, input_shapes.items) |*t, sh| t.* = mlirx.tensorType(mlir_ctx, sh); + const input_types = try arena.alloc(*const mlir.Type, tensor_count); + for (input_types, input_shapes.items) |*t, sh| { + t.* = mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlir_ctx, sh.dtype())); + } const og_block_args = self._block_args; defer { @@ -364,7 +368,7 @@ pub const CompilationContext = struct { // Those are returned to caller so we don't put them in the arena, but in the module allocator. const fn_res = try res_allocator.create(ReturnT); - const fn_res_types = try res_allocator.alloc(mlir.Type, out_tensor_count); + const fn_res_types = try res_allocator.alloc(*const mlir.Type, out_tensor_count); const fn_res_shapes = try res_allocator.alloc(Shape, out_tensor_count); const fn_res_donations = try res_allocator.alloc(Tensor._Donation, out_tensor_count); const fn_res_output_memory_kind = try res_allocator.alloc(Buffer.Memory, out_tensor_count); @@ -382,10 +386,10 @@ pub const CompilationContext = struct { break :forward @call(.auto, func, args.*); }; - var fn_res_values: [out_tensor_count]mlir.Value = undefined; + var fn_res_values: [out_tensor_count]*const mlir.Value = undefined; self.extractValuesAndTypes(fn_res, &fn_res_values, fn_res_types, fn_res_shapes, fn_res_donations, fn_res_output_memory_kind); - const fn_ret = dialect.func.return_(mlir_ctx, &fn_res_values, loc); + const fn_ret = dialect.func.returns(mlir_ctx, &fn_res_values, loc); fn_body[0].appendOperationRecursive(fn_ret, fn_body[1]); } @@ -690,10 +694,10 @@ pub const CompilationContext = struct { /// This is done so that we have a mapping between the arguments of the kernel associated with a module and the actual Tensors /// stored in the Module. /// Caller need to allocate required memory in self._block_args. - pub fn mapBlockArguments(self: *CompilationContext, v: anytype, block: mlir.Block, start: usize) usize { + pub fn mapBlockArguments(self: *CompilationContext, v: anytype, block: *mlir.Block, start: usize) usize { const LocalContext = struct { index: usize, - block: mlir.Block, + block: *mlir.Block, self: *CompilationContext, }; var context = LocalContext{ .self = self, .block = block, .index = start }; @@ -736,8 +740,8 @@ pub const CompilationContext = struct { pub fn extractValuesAndTypes( self: *const CompilationContext, v: anytype, - values: []mlir.Value, - types: []mlir.Type, + values: []*const mlir.Value, + types: []*const mlir.Type, shapes: []Shape, donations: []Tensor._Donation, output_memory_kind: []Buffer.Memory, @@ -746,8 +750,8 @@ pub const CompilationContext = struct { const LocalContext = struct { self: *const CompilationContext, index: usize = 0, - values: []mlir.Value, - types: []mlir.Type, + values: []*const mlir.Value, + types: []*const mlir.Type, shapes: []Shape, donations: []Tensor._Donation, output_memory_kind: []Buffer.Memory, diff --git a/zml/tensor.zig b/zml/tensor.zig index 26f670d7..2dbfe9a2 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -42,7 +42,7 @@ pub const Tensor = struct { _output_memory_kind: Memory = .device, pub const _Donation = union(enum) { no_buffer, input_buffer, arg: u16 }; - pub const _Id = union(enum) { mlir: mlir.Value, buffer_id: u64, arg_id: u64 }; + pub const _Id = union(enum) { mlir: *const mlir.Value, buffer_id: u64, arg_id: u64 }; pub const MAX_RANK = Shape.MAX_RANK; /// Returns the current compilation context. From 5eac1f7ae435dc7b91e6a90d77cf2fb71ce7932f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 29 Aug 2025 17:04:25 +0200 Subject: [PATCH 38/47] wip: using Tensor pointer address --- BUILD.bazel | 4 +- mlir/BUILD.bazel | 3 +- mlir/main.zig | 240 +++++++++++++++++++++++++++++------------------ 3 files changed, 154 insertions(+), 93 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index f891cd2d..344438f7 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -12,8 +12,8 @@ zls_completion( deps = [ # "//async", # "//examples/llama", - # "//stdx", - # "//zml", + "//stdx", + "//zml", "//mlir:main", # "//async", # "//stdx", diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index e91a2c91..a07c486b 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -34,8 +34,9 @@ zig_binary( visibility = ["//visibility:public"], deps = [ ":mlir", + "//zml", "//mlir/dialects", # ":c", - # "//stdx", + "//stdx", ], ) diff --git a/mlir/main.zig b/mlir/main.zig index 8eee2bbe..03360c78 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -1,109 +1,169 @@ const std = @import("std"); +const stdx = @import("stdx"); +const zml = @import("zml"); + +const TensorOriginKind = enum { + argument, + value, +}; + +const TensorOrigin = union(TensorOriginKind) { + argument: void, + value: usize, +}; + +var global_compilation_context: ?*CompilationContext = null; + +pub const Tensor = struct { + shape_: ?zml.Shape = null, + tensor_origin: TensorOrigin = .{ .argument = {} }, + + pub fn init(shape_: zml.Shape) Tensor { + return .{ .shape_ = shape_ }; + } + + pub fn matmul(self: *const Tensor, other: *const Tensor) Tensor { + const self_value = self.getValue(); + const other_value = other.getValue(); + + std.debug.print("self: {} - other: {}\n", .{ self_value, other_value }); + return .{}; + } + + pub fn add(self: *const Tensor, other: *const Tensor) Tensor { + _ = self; // autofix + _ = other; // autofix + return .{ .tensor_origin = .{ .value = 1 } }; + } + + pub fn relu(self: *const Tensor) Tensor { + _ = self; // autofix + return .{ .tensor_origin = .{ .value = 1 } }; + } + + pub fn flattenAll(self: *const Tensor) Tensor { + _ = self; // autofix + return .{ .tensor_origin = .{ .value = 1 } }; + } + + pub fn convert(self: *const Tensor, dtype: zml.DataType) Tensor { + _ = self; // autofix + _ = dtype; // autofix + return .{ .tensor_origin = .{ .value = 1 } }; + } + + pub fn argMax(self: *const Tensor, dim: i64) struct { indices: Tensor } { + _ = self; // autofix + _ = dim; // autofix + return .{ .indices = .{ .tensor_origin = .{ .value = 1 } } }; + } + + pub fn getValue(self: *const Tensor) usize { + return switch (self.tensor_origin) { + .argument => b: { + const entry = global_compilation_context.?.mapping.get(self).?; + break :b entry * 1000; + }, + .value => |v| v, + }; + } +}; + +/// Model definition +const Mnist = struct { + fc1: Layer = .{}, + fc2: Layer = .{}, + + const Layer = struct { + weight: Tensor = .{}, + bias: Tensor = .{}, + + pub fn forward(self: *const Layer, input: *const Tensor) Tensor { + return self.weight.matmul(input).add(&self.bias).relu(); + } + }; + + pub fn init() Mnist { + return .{}; + } + + /// just two linear layers + relu activation + pub fn forward(self: *const Mnist, input: *const Tensor) Tensor { + // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); + var x = input.flattenAll().convert(.f32); + const layers: []const *const Layer = &.{ &self.fc1, &self.fc2 }; + for (layers) |layer| { + x = layer.forward(&x); + } + return x.argMax(0).indices.convert(.u8); + } +}; const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); pub fn main() !void { const allocator = std.heap.c_allocator; - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - // const handle = mlir.DialectHandle.fromString("func"); + var mnist: Mnist = .init(); + _ = &mnist; - inline for (.{ "memref", "func", "stablehlo" }) |d| { - registry.registerDialect(d); + // TODO(Corentin): Hydrate the model using the config/bufferstore + + const input: Tensor = .init(zml.Shape.init(.{ 28, 28 }, .u8)); + + compileModel(allocator, Mnist.forward, &mnist, .{&input}); +} + +const CompilationContext = struct { + allocator: std.mem.Allocator, + + mapping: std.AutoArrayHashMapUnmanaged(*const Tensor, usize), + + pub fn init(allocator: std.mem.Allocator) CompilationContext { + return .{ + .allocator = allocator, + .mapping = .{}, + }; } - // mlir.registerPasses("AllStablehlo"); + pub fn deinit(self: *CompilationContext) void { + self.mapping.deinit(self.allocator); + } - const ctx = try mlir.Context.init(.{ - .registry = registry, - .threading = true, - }); - defer ctx.deinit(); + pub fn appendToMapping(self: *CompilationContext, tensor: *const Tensor, unique_id: usize) void { + const gop = self.mapping.getOrPut(self.allocator, tensor) catch unreachable; + std.debug.assert(gop.found_existing == false); - std.debug.print(">>> {f}\n", .{ctx}); - ctx.loadAllAvailableDialects(); - std.debug.print(">>> {f}\n", .{ctx}); + gop.value_ptr.* = unique_id; + } +}; - const unk = mlir.Location.unknown(ctx); - _ = unk; // autofix +const Context = struct { + compilation_context: *CompilationContext, + current_id: usize = 0, +}; - var serialized: std.Io.Writer.Allocating = .init(allocator); - defer serialized.deinit(); +fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) void { + var compilation_context: CompilationContext = .init(allocator); + defer compilation_context.deinit(); - // const stablehlo_version = blk: { - // if (dialects.stablehlo.currentVersion()) |requested_version| { - // break :blk dialects.stablehlo.stablehloGetSmallerVersion(requested_version, dialects.stablehlo.getCurrentVersion()); - // } - // break :blk dialects.stablehlo.minimumVersion(); - // }; + global_compilation_context = &compilation_context; + defer global_compilation_context = null; - // const attr = mlir.integerAttribute(ctx, @as(i32, 5)); - // _ = attr; // autofix + var context: Context = .{ .compilation_context = &compilation_context }; + zml.meta.visit(struct { + fn cb(inner_context: *Context, tensor: *const Tensor) void { + inner_context.compilation_context.appendToMapping(tensor, inner_context.current_id); + inner_context.current_id += 1; + } + }.cb, &context, model); + zml.meta.visit(struct { + fn cb(inner_context: *Context, tensor: *const Tensor) void { + inner_context.compilation_context.appendToMapping(tensor, inner_context.current_id); + inner_context.current_id += 1; + } + }.cb, &context, &args); - // const one: i32 = 1; - // const two: i32 = 2; - - // const dict = mlir.DictionaryAttribute.init(ctx, &.{ - // .named(ctx, "pouet", mlir.integerAttribute(ctx, one)), - // .named(ctx, "pouet2", mlir.integerAttribute(ctx, two)), - // }); - // _ = dict; // autofix - - const module = mlir.Module.init(.unknown(ctx)); - defer module.deinit(); - - const tensor_i32 = mlir.rankedTensorType(&.{ 4096, 4096 }, mlir.integerType(ctx, .i32)); - - _ = dialects.func.func(ctx, .{ - .name = "pouet", - .visibility = .private, - .block = blk: { - const body = mlir.Block.init(&.{ tensor_i32, tensor_i32 }, &.{ .unknown(ctx), .unknown(ctx) }); - const add_op = dialects.stablehlo.add(ctx, body.argument(0), body.argument(1), .unknown(ctx)).appendTo(body); - const mul_op = dialects.stablehlo.multiply(ctx, add_op.result(0), body.argument(1), .unknown(ctx)).appendTo(body); - _ = dialects.func.return_(ctx, mul_op.result(0), .unknown(ctx)).appendTo(body); - break :blk body; - }, - .location = .unknown(ctx), - }).appendTo(module.body()); - - std.debug.print(">>>>>>\n{f}\n", .{module.operation().fmt(.{ .print_generic_op_form = false })}); - std.debug.print(">>>>>>\n{f}\n", .{module.operation().fmt(.{ .print_generic_op_form = true })}); - - // dialects.stablehlo.serializePortableArtifact2(module, dialects.stablehlo.minimumVersion(), &serialized.writer) catch |err| { - // std.debug.print("failed to serialize to portable artifact: {}\n", .{err}); - // return err; - // }; - - // const body = mod.body(); - // _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); - // _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); - // _ = body.addArgument(mlir.integerType(ctx, 64, .signless), .unknown(ctx)); - - // body.appendOwnedOperation(op); - - // op.deinit(); - // body.detach(); - // body.deinit(); - - // std.debug.print(">>>>>> {any}\n", .{op.block().?.parentOperation()}); - - // std.debug.print(">>>>>> {any}\n", .{op.block().?.parentOperation()}); - - // std.debug.print(">>> {f}\n", .{body}); - // const attr = mlir.integerAttribute(ctx, @as(i32, 5)); - // const attr2: *const mlir.Attribute = @ptrCast(attr); - - // std.debug.print("coucou 2 {any} {f}\n", .{ - // ctx.isRegisteredOperation("func.func"), - // attr2, - // }); - - // var buffer: [256]u8 = undefined; - // const vv = "coucou\n"; - // var writer = std.fs.File.stdout().writer(&buffer); - // try writer.interface.writeAll(vv); - // try writer.interface.flush(); + _ = @call(.auto, func, .{model} ++ args); } From 6c11cb3b9e6b12c89ec9649e8f74c9451d8b6a49 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 1 Sep 2025 17:36:44 +0200 Subject: [PATCH 39/47] wip: made mnist compile --- mlir/BUILD.bazel | 9 + mlir/dialects/stablehlo/stablehlo.zig | 2 +- mlir/main.zig | 531 ++++++++++++++++++++++---- 3 files changed, 471 insertions(+), 71 deletions(-) diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index a07c486b..65829673 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -30,6 +30,14 @@ zig_test( zig_binary( name = "main", + args = [ + "$(location @mnist//:mnist.safetensors)", + "$(location @mnist//:t10k-images.idx3-ubyte)", + ], + data = [ + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", + ], main = "main.zig", visibility = ["//visibility:public"], deps = [ @@ -38,5 +46,6 @@ zig_binary( "//mlir/dialects", # ":c", "//stdx", + "//async", ], ) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 8789c7b0..24d4574e 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -217,7 +217,7 @@ pub fn constant( .operands = .{ .flat = &.{} }, .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, .attributes = &.{ - .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.init(dims, elem_type).shaped(), raw_bytes)), + .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.get(dims, elem_type).shaped(), raw_bytes)), }, .location = location, }); diff --git a/mlir/main.zig b/mlir/main.zig index 03360c78..725109c7 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -1,101 +1,376 @@ +const asynk = @import("async"); const std = @import("std"); const stdx = @import("stdx"); const zml = @import("zml"); -const TensorOriginKind = enum { +const Shape = zml.Shape; + +var global_compilation_context: ?*CompilationContext = null; +pub const MAX_RANK: u8 = 8; + +pub const TensorOriginType = enum { argument, value, }; -const TensorOrigin = union(TensorOriginKind) { +pub const TensorOrigin = union(TensorOriginType) { argument: void, - value: usize, + value: *const mlir.Value, }; -var global_compilation_context: ?*CompilationContext = null; - pub const Tensor = struct { - shape_: ?zml.Shape = null, + var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; + + id: usize, + _shape: zml.Shape, tensor_origin: TensorOrigin = .{ .argument = {} }, + pub const tensor = init; + pub fn init(shape_: zml.Shape) Tensor { - return .{ .shape_ = shape_ }; + return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; } - pub fn matmul(self: *const Tensor, other: *const Tensor) Tensor { - const self_value = self.getValue(); - const other_value = other.getValue(); - - std.debug.print("self: {} - other: {}\n", .{ self_value, other_value }); - return .{}; + pub fn shape(self: Tensor) Shape { + return self._shape; } - pub fn add(self: *const Tensor, other: *const Tensor) Tensor { - _ = self; // autofix - _ = other; // autofix - return .{ .tensor_origin = .{ .value = 1 } }; + pub fn dtype(self: Tensor) zml.DataType { + return self._shape.dtype(); } - pub fn relu(self: *const Tensor) Tensor { - _ = self; // autofix - return .{ .tensor_origin = .{ .value = 1 } }; + pub fn rank(self: Tensor) u4 { + return self._shape.rank(); } - pub fn flattenAll(self: *const Tensor) Tensor { - _ = self; // autofix - return .{ .tensor_origin = .{ .value = 1 } }; + pub fn dim(self: Tensor, axis_: anytype) i64 { + return self._shape.dim(axis_); } - pub fn convert(self: *const Tensor, dtype: zml.DataType) Tensor { - _ = self; // autofix - _ = dtype; // autofix - return .{ .tensor_origin = .{ .value = 1 } }; + pub fn dims(self: *const Tensor) []const i64 { + return self._shape.dims(); } - pub fn argMax(self: *const Tensor, dim: i64) struct { indices: Tensor } { - _ = self; // autofix - _ = dim; // autofix - return .{ .indices = .{ .tensor_origin = .{ .value = 1 } } }; + pub fn count(self: Tensor) usize { + return self._shape.count(); } - pub fn getValue(self: *const Tensor) usize { + pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { + //stdx.debug.assert(lhs.rank() >= 1 and rhs.rank() >= 1, "Can't matmul({f}, {f}) ! The two tensors need to have at least rank 1.", .{ lhs, rhs }); + + const contracting = [_][2]i8{.{ -1, if (rhs.rank() >= 2) rhs.rank() - 2 else 0 }}; + if (lhs.rank() == 1 or rhs.rank() <= 2) { + // When lhs is a vector or rhs is small the torch semantics match the dot_general semantics and life is easy. + return lhs.dotGeneral(rhs, &contracting, &.{}); + } + + //stdx.debug.assert(lhs.rank() == 2, "Can't matmul({f}, {f}) ! One of the two tensors need to have a rank less than 2.", .{ lhs, rhs }); + + // Pytorch treats the extra dimensions of rhs has batching dimensions, + // and implicitly broadcast lhs along those. + // We make this broadcasting explicit. + var left_shape = rhs.shape(); + left_shape._dims.set(left_shape.axis(-2), lhs.dim(-2)); + left_shape._tags.set(left_shape.axis(-2), lhs.shape().tag(-2)); + left_shape._dims.set(left_shape.axis(-1), lhs.dim(-1)); + left_shape._tags.set(left_shape.axis(-1), lhs.shape().tag(-1)); + const lhs_broad = lhs.broadcastLeft(left_shape); + + const n_batching_axes = rhs.rank() - lhs.rank(); + var batching: [Tensor.MAX_RANK][2]i8 = undefined; + for (0..n_batching_axes) |i| { + batching[i] = .{ @intCast(i), @intCast(i) }; + } + return lhs_broad.dotGeneral(rhs, &contracting, batching[0..n_batching_axes]); + } + + pub fn dotGeneral( + lhs: Tensor, + rhs: Tensor, + contracting_axes: []const [2]i8, + batching_axes: []const [2]i8, + ) Tensor { + stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); + + const Axes = stdx.BoundedArray(i64, MAX_RANK); + + var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + // Validate batching axes + var lhs_batching_axes: Axes = .{}; + var rhs_batching_axes: Axes = .{}; + for (batching_axes) |b_axes| { + const l, const r = b_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + var t = lhs._shape.tag(l); + if (t == Shape.TagUnknown) t = rhs._shape.tag(r); + res_shape = res_shape.appendDim(lhs._shape.dim(l), t); + lhs_batching_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_batching_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + // Validate contracting axes + var lhs_contracting_axes: Axes = .{}; + var rhs_contracting_axes: Axes = .{}; + for (contracting_axes) |c_axes| { + const l, const r = c_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + // Result shape is obtained by concatenating batching dimensions, (already done) + // then dimensions from lhs axes that aren't contracting nor batching, + // then dimensions from rhs axes that aren't contracting nor batching. + for (0..lhs.rank()) |l| { + if (std.mem.indexOfScalar(i64, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, lhs_batching_axes.constSlice(), @intCast(l))) |_| { + continue; + } + res_shape = res_shape.appendDim(lhs._shape.dim(l), lhs._shape.tag(l)); + } + for (0..rhs.rank()) |r| { + if (std.mem.indexOfScalar(i64, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, rhs_batching_axes.constSlice(), @intCast(r))) |_| { + continue; + } + res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); + } + + const mlir_ctx = lhs.getContext().mlir_ctx; + const loc = lhs.getContext().location(@src(), "dot({f},{f},contracting={any},batching={any}", .{ lhs, rhs, contracting_axes, batching_axes }); + const op = dialects.stablehlo.dot_general( + mlir_ctx, + lhs.value(), + rhs.value(), + mlir.rankedTensorType(res_shape.dims(), zml.mlir.Type.fromDType(mlir_ctx, res_shape.dtype())), + .{ + .lhs_batching_dimensions = lhs_batching_axes.constSlice(), + .rhs_batching_dimensions = rhs_batching_axes.constSlice(), + .lhs_contracting_dimensions = lhs_contracting_axes.constSlice(), + .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), + .precision = .fast, + }, + loc, + ); + CompilationContext.current().currentBlock().appendOwnedOperation(op); + return _result(res_shape, op.result(0)); + } + + pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { + const res: Tensor = .{ + ._shape = sh, + .tensor_origin = .{ .value = val }, + .id = Tensor.current_id.fetchAdd(1, .seq_cst), + }; + + //if (builtin.mode == .Debug) { + // // Check that the MLIR value actually have the same shape. + // const other = fromMlirValue(val); + // stdx.debug.internalAssert(sh.eql(other._shape), "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); + //} + + return res; + } + + pub fn constant(dimz: anytype, val: zml.Data) Tensor { + const sh = Shape.init(dimz, val.dtype()); + const ctx = CompilationContext.current().mlir_ctx; + //const loc = CompilationContext.current().location(@src(), "dims={f}, value={}", .{ sh, val }); + const loc = mlir.Location.unknown(ctx); + + var constant_op = switch (val.dtype()) { + inline .bool, .i8, .i16, .i32, .i64, .u8, .u16, .u32, .u64, .bf16, .f16, .f32, .f64 => |t| dialects.stablehlo.constant(ctx, &.{}, zml.mlir.Type.fromDType(ctx, t), val.constSlice(), loc), + inline else => b: { + const val_f32 = val.as(f32); + break :b dialects.stablehlo.constant(ctx, &.{}, mlir.floatType(ctx, .f32), std.mem.asBytes(&val_f32), loc); + }, + }; + CompilationContext.current().currentBlock().appendOwnedOperation(constant_op); + + if (sh.rank() > 0) { + constant_op = dialects.stablehlo.broadcast_in_dim(ctx, constant_op.result(0), &.{}, mlir.rankedTensorType(sh.dims(), zml.mlir.Type.fromDType(ctx, sh.dtype())), loc); + CompilationContext.current().currentBlock().appendOwnedOperation(constant_op); + } + return _result(sh, constant_op.result(0)).convert(val.dtype()); + } + + pub fn add(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "add", dialects.stablehlo.add)(self, other); + } + + pub fn maximum(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "maximum", dialects.stablehlo.maximum)(self, other); + } + + pub fn relu(self: Tensor) Tensor { + return self.maximum(Tensor.constant(self.dims(), self.dtype().zero())); + } + + pub fn flattenAll(self: Tensor) Tensor { + // TODO: rename to just flatten, once flatten is moved to torch + return self.reshape(.{self.count()}); + } + + pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { + const output_shape = self._shape.reshape(output_shape_); + const ctx = self.getContext().mlir_ctx; + const tensor_type = mlir.rankedTensorType(output_shape.dims(), zml.mlir.Type.fromDType(ctx, output_shape.dtype())); + //const loc = self.getContext().location(@src(), "reshape({f})", .{output_shape}); + const reshape_op = dialects.stablehlo.reshape(self.getContext().mlir_ctx, self.value(), tensor_type, .unknown(ctx)); + CompilationContext.current().currentBlock().appendOwnedOperation(reshape_op); + return _result(output_shape, reshape_op.result(0)); + } + + pub fn convert(self: Tensor, to: zml.DataType) Tensor { + if (to == self.dtype()) { + return self; + } + //const loc = self.getContext().location(@src(), "convert({f},to={s})", .{ self, @tagName(to) }); + + const mlir_ctx = self.getContext().mlir_ctx; + const res_type = mlir.rankedTensorType(self.dims(), zml.mlir.Type.fromDType(mlir_ctx, to)); + const op = dialects.stablehlo.convert(mlir_ctx, self.value(), res_type, .unknown(mlir_ctx)); + CompilationContext.current().currentBlock().appendOwnedOperation(op); + return _result(self._shape.withDtype(to), op.result(0)); + } + + pub const ArgMaxRes = struct { + values: Tensor, + indices: Tensor, + + fn cmp(left: ArgMaxRes, right: ArgMaxRes) ArgMaxRes { + const left_val = left.values; + const right_val = right.values; + const left_idx = left.indices; + const right_idx = right.indices; + + const left_gt_right = left_val.cmp(.GT, right_val); + const is_nan = left_val.cmp(.NE, left_val); + const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(left_val, right_val); + + // If left_val == right_val: keep the smallest idx. + const is_same = left_val.cmp(.EQ, right_val); + const is_first = left_idx.cmp(.LT, right_idx); + const is_same_but_first = is_same.logical(.AND, is_first); + const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + const max_idx = keep_left_idx.select(left_idx, right_idx); + + return .{ .values = max_val, .indices = max_idx }; + } + }; + + //pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { + // const a = x.axis(axis_); + // const dt: zml.DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; + + // return ops.reduce( + // ArgMaxRes.cmp, + // .{ .values = x, .indices = Tensor.arange(.{ .end = x.dim(a) }, dt).broadcast(x.shape(), &.{a}) }, + // .{ .values = Tensor.constant(&.{}, x.dtype().minValue()), .indices = Tensor.scalar(0, dt) }, + // &.{a}, + // ); + //} + + fn binaryOp( + src: std.builtin.SourceLocation, + op_name: []const u8, + comptime op_fn: fn (*mlir.Context, *const mlir.Value, *const mlir.Value, *const mlir.Location) *mlir.Operation, + ) fn (Tensor, Tensor) Tensor { + _ = op_name; // autofix + _ = src; // autofix + return struct { + pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { + //stdx.debug.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {f} and {f}", .{ op_name, self, other }); + + //if (self.rank() == 0 and other.rank() != 0) { + // return binaryOpHelper(self.broad(other._shape), other); + //} + + //if (self.rank() != 0 and other.rank() == 0) { + // return binaryOpHelper(self, other.broad(self._shape)); + //} + + //stdx.debug.assert(self._shape.eql(other._shape), "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); + + const ctx = self.getContext(); + //const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other }); + const ret = op_fn(ctx.mlir_ctx, self.value(), other.value(), .unknown(ctx.mlir_ctx)); + CompilationContext.current().currentBlock().appendOwnedOperation(ret); + return _result(self._shape, ret.result(0)); + } + }.binaryOpHelper; + } + + pub fn value(self: Tensor) *const mlir.Value { return switch (self.tensor_origin) { .argument => b: { - const entry = global_compilation_context.?.mapping.get(self).?; - break :b entry * 1000; + const argument_index = self.getContext().getArgumentIndex(self.id); + break :b self.getContext().currentBlock().argument(argument_index); }, .value => |v| v, }; } + + pub fn getContext(self: Tensor) *CompilationContext { + _ = self; // autofix + return global_compilation_context.?; + } + + pub fn format( + self: Tensor, + writer: *std.Io.Writer, + ) !void { + // TODO(0.15.0) handle format + // const bare_fmt = fmt.len == 1 and fmt[0] == '_'; + try writer.print("{s}({f})", .{ @typeName(Tensor), self._shape }); + } }; /// Model definition const Mnist = struct { - fc1: Layer = .{}, - fc2: Layer = .{}, + fc1: Layer, + fc2: Layer, const Layer = struct { - weight: Tensor = .{}, - bias: Tensor = .{}, + weight: Tensor, + bias: Tensor, - pub fn forward(self: *const Layer, input: *const Tensor) Tensor { - return self.weight.matmul(input).add(&self.bias).relu(); + pub fn forward(self: Layer, input: Tensor) Tensor { + _ = self; // autofix + //return self.weight.matmul(input).add(self.bias).relu(); + return input.relu(); } }; - pub fn init() Mnist { - return .{}; + pub fn init(buffer_store: zml.aio.BufferStore) Mnist { + return .{ + .fc1 = .{ + .weight = .init(buffer_store.get("fc1.weight").?.shape()), + .bias = .init(buffer_store.get("fc1.bias").?.shape()), + }, + .fc2 = .{ + .weight = .init(buffer_store.get("fc2.weight").?.shape()), + .bias = .init(buffer_store.get("fc2.bias").?.shape()), + }, + }; } /// just two linear layers + relu activation - pub fn forward(self: *const Mnist, input: *const Tensor) Tensor { + pub fn forward(self: Mnist, input: Tensor) Tensor { // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); var x = input.flattenAll().convert(.f32); - const layers: []const *const Layer = &.{ &self.fc1, &self.fc2 }; + const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { - x = layer.forward(&x); + x = layer.forward(x); } - return x.argMax(0).indices.convert(.u8); + //return x.argMax(0).indices.convert(.u8); + return x; } }; @@ -103,39 +378,100 @@ const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { const allocator = std.heap.c_allocator; - var mnist: Mnist = .init(); - _ = &mnist; + const process_args = try std.process.argsAlloc(allocator); + defer std.process.argsFree(allocator, process_args); + const pt_model = process_args[1]; + const t10kfilename = process_args[2]; + _ = t10kfilename; // autofix - // TODO(Corentin): Hydrate the model using the config/bufferstore + // Read model shapes. + // Note this works because Mnist struct uses the same layer names as the pytorch model + var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); + defer buffer_store.deinit(); + + const mnist: Mnist = .init(buffer_store); const input: Tensor = .init(zml.Shape.init(.{ 28, 28 }, .u8)); - compileModel(allocator, Mnist.forward, &mnist, .{&input}); + compileModel(allocator, Mnist.forward, mnist, .{input}); } const CompilationContext = struct { allocator: std.mem.Allocator, - mapping: std.AutoArrayHashMapUnmanaged(*const Tensor, usize), + mlir_registry: *mlir.DialectRegistry, + mlir_ctx: *mlir.Context, + module: *mlir.Module, + + blocks: stdx.BoundedArray(*mlir.Block, 16) = .{}, + mappings: stdx.BoundedArray(std.AutoArrayHashMapUnmanaged(usize, usize), 16) = .{}, pub fn init(allocator: std.mem.Allocator) CompilationContext { + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; + inline for (.{ "func", "stablehlo" }) |d| { + mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); + } + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + //const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); + const module = mlir.Module.init(.unknown(mlir_ctx)); + module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); + return .{ .allocator = allocator, - .mapping = .{}, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .module = module, }; } pub fn deinit(self: *CompilationContext) void { - self.mapping.deinit(self.allocator); + self.module.deinit(); + self.mlir_ctx.deinit(); + self.mlir_registry.deinit(); } - pub fn appendToMapping(self: *CompilationContext, tensor: *const Tensor, unique_id: usize) void { - const gop = self.mapping.getOrPut(self.allocator, tensor) catch unreachable; - std.debug.assert(gop.found_existing == false); + //pub fn registerTensor(self: *CompilationContext, tensor_id: usize, argument_id: usize) !void { + // const gop = try self.mapping.getOrPut(self.allocator, tensor_id); + // std.debug.assert(!gop.found_existing); - gop.value_ptr.* = unique_id; + // gop.value_ptr.* = argument_id; + //} + + pub fn currentBlock(self: *const CompilationContext) *mlir.Block { + return self.blocks.get(self.blocks.len - 1); + } + + pub fn currentMapping(self: *CompilationContext) *std.AutoArrayHashMapUnmanaged(usize, usize) { + return &self.mappings.slice()[self.blocks.len - 1]; + } + + pub fn getArgumentIndex(self: *CompilationContext, tensor_id: usize) usize { + return self.currentMapping().get(tensor_id).?; + } + + pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { + self.blocks.appendAssumeCapacity(block); + self.mappings.appendAssumeCapacity(.{}); + } + + pub fn popBlock(self: *CompilationContext) void { + _ = self.blocks.pop(); + var maybe_popped = self.mappings.pop(); + if (maybe_popped) |*popped| { + popped.deinit(self.allocator); + } + } + + pub fn current() *CompilationContext { + return global_compilation_context.?; } }; @@ -148,22 +484,77 @@ fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: std var compilation_context: CompilationContext = .init(allocator); defer compilation_context.deinit(); - global_compilation_context = &compilation_context; - defer global_compilation_context = null; + //var context: Context = .{ .compilation_context = &compilation_context }; + //zml.meta.visit(struct { + // fn cb(inner_context: *Context, tensor: *const Tensor) void { + // std.debug.print("Registering {}\n", .{tensor.id}); + // inner_context.compilation_context.registerTensor(tensor.id, inner_context.current_id) catch unreachable; + // inner_context.current_id += 1; + // } + //}.cb, &context, &model); + //zml.meta.visit(struct { + // fn cb(inner_context: *Context, tensor: *const Tensor) void { + // std.debug.print("Registering {}\n", .{tensor.id}); + // inner_context.compilation_context.registerTensor(tensor.id, inner_context.current_id) catch unreachable; + // inner_context.current_id += 1; + // } + //}.cb, &context, &args); - var context: Context = .{ .compilation_context = &compilation_context }; + emitMlir(&compilation_context, func, .{model} ++ args); +} + +fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) void { + var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); + defer arena.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + compilation_context.pushBlock(block); + defer compilation_context.popBlock(); + + var input_types_list = std.array_list.Managed(*const mlir.Type).init(arena.allocator()); + defer input_types_list.deinit(); + + const LocalContext = struct { + compilation_context: *CompilationContext, + input_types_list: std.array_list.Managed(*const mlir.Type), + current_argument_id: usize = 0, + }; + var context: LocalContext = .{ + .compilation_context = compilation_context, + .input_types_list = input_types_list, + }; zml.meta.visit(struct { - fn cb(inner_context: *Context, tensor: *const Tensor) void { - inner_context.compilation_context.appendToMapping(tensor, inner_context.current_id); - inner_context.current_id += 1; - } - }.cb, &context, model); - zml.meta.visit(struct { - fn cb(inner_context: *Context, tensor: *const Tensor) void { - inner_context.compilation_context.appendToMapping(tensor, inner_context.current_id); - inner_context.current_id += 1; + fn cb(inner_context: *LocalContext, tensor: *const Tensor) void { + _ = inner_context.compilation_context.currentBlock().addArgument( + mlir.rankedTensorType(tensor.dims(), zml.mlir.Type.fromDType(inner_context.compilation_context.mlir_ctx, tensor.dtype())), + .unknown(inner_context.compilation_context.mlir_ctx), + ); + inner_context.compilation_context.currentMapping().put(inner_context.compilation_context.allocator, tensor.id, inner_context.current_argument_id) catch unreachable; + inner_context.input_types_list.append(mlir.rankedTensorType(tensor.dims(), zml.mlir.Type.fromDType(inner_context.compilation_context.mlir_ctx, tensor.dtype()))) catch unreachable; + inner_context.current_argument_id += 1; } }.cb, &context, &args); - _ = @call(.auto, func, .{model} ++ args); + { + global_compilation_context = compilation_context; + defer global_compilation_context = null; + const result = @call(.auto, func, args); + const fn_op = dialects.func.returns(compilation_context.mlir_ctx, &.{result.value()}, .unknown(compilation_context.mlir_ctx)); + compilation_context.currentBlock().appendOwnedOperation(fn_op); + } + + const mlir_fn = dialects.func.func(compilation_context.mlir_ctx, .{ + .name = "mnist", + .args = null, + .args_attributes = null, + .results = null, + .results_attributes = null, + .block = compilation_context.currentBlock(), + .location = .unknown(compilation_context.mlir_ctx), + }); + _ = mlir_fn; // autofix + + std.debug.print("{f}\n", .{compilation_context.currentBlock()}); } From 0d1aacac251f61351be797c88739ddf921f4830a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 2 Sep 2025 14:10:40 +0200 Subject: [PATCH 40/47] WIP --- mlir/dialects/func.zig | 6 +- mlir/main.zig | 228 ++++++++++++++++++--------- mlir/mlir.zig | 112 +++++++++++--- platforms/BUILD.bazel | 6 + runtimes/neuron/BUILD.bazel | 1 + stdx/io.zig | 4 - third_party/zls/zls.bzl | 8 +- zml/dtype.zig | 298 ++++++++++++++++++------------------ 8 files changed, 416 insertions(+), 247 deletions(-) diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index c2af12a5..b9f8b7f5 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -76,9 +76,9 @@ pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Va }); } -pub fn return_(ctx: *mlir.Context, values: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.return", .{ - .operands = .{ .flat = &.{values} }, + .operands = .{ .flat = &.{value} }, .location = location, .verify = false, }); @@ -86,7 +86,7 @@ pub fn return_(ctx: *mlir.Context, values: *const mlir.Value, location: *const m pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.return", .{ - .operands = .{ .variadic = &.{values} }, + .operands = .{ .flat = values }, .verify = false, .location = location, }); diff --git a/mlir/main.zig b/mlir/main.zig index 725109c7..71563e3a 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -1,8 +1,10 @@ -const asynk = @import("async"); const std = @import("std"); + +const asynk = @import("async"); +const dialects = @import("mlir/dialects"); +const mlir = @import("mlir"); const stdx = @import("stdx"); const zml = @import("zml"); - const Shape = zml.Shape; var global_compilation_context: ?*CompilationContext = null; @@ -22,11 +24,27 @@ pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; id: usize, + auto_broadcast: bool = false, _shape: zml.Shape, tensor_origin: TensorOrigin = .{ .argument = {} }, pub const tensor = init; + fn mlirType(self: Tensor, mlir_ctx: *mlir.Context) *const mlir.Type { + return mlir.rankedTensorType( + self.dims(), + zml.mlir.Type.fromDType(mlir_ctx, self.dtype()), + ); + } + + fn mlirCtx() *mlir.Context { + return CompilationContext.current().mlir_ctx; + } + + fn currentBlock() *mlir.Block { + return CompilationContext.current().currentBlock(); + } + pub fn init(shape_: zml.Shape) Tensor { return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; } @@ -55,6 +73,12 @@ pub const Tensor = struct { return self._shape.count(); } + pub fn autoBroadcast(self: Tensor) Tensor { + var ret = self; + ret.auto_broadcast = true; + return ret; + } + pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { //stdx.debug.assert(lhs.rank() >= 1 and rhs.rank() >= 1, "Can't matmul({f}, {f}) ! The two tensors need to have at least rank 1.", .{ lhs, rhs }); @@ -84,6 +108,15 @@ pub const Tensor = struct { return lhs_broad.dotGeneral(rhs, &contracting, batching[0..n_batching_axes]); } + pub const DotArgs = struct { + contract_on: []const [2]i8, + batch_on: ?[]const [2]i8 = null, + }; + + pub fn dot(lhs: Tensor, rhs: Tensor, args: DotArgs) Tensor { + return lhs.dotGeneral(rhs, args.contract_on, args.batch_on orelse &.{}); + } + pub fn dotGeneral( lhs: Tensor, rhs: Tensor, @@ -176,26 +209,15 @@ pub const Tensor = struct { return res; } - pub fn constant(dimz: anytype, val: zml.Data) Tensor { - const sh = Shape.init(dimz, val.dtype()); - const ctx = CompilationContext.current().mlir_ctx; - //const loc = CompilationContext.current().location(@src(), "dims={f}, value={}", .{ sh, val }); - const loc = mlir.Location.unknown(ctx); - - var constant_op = switch (val.dtype()) { - inline .bool, .i8, .i16, .i32, .i64, .u8, .u16, .u32, .u64, .bf16, .f16, .f32, .f64 => |t| dialects.stablehlo.constant(ctx, &.{}, zml.mlir.Type.fromDType(ctx, t), val.constSlice(), loc), - inline else => b: { - const val_f32 = val.as(f32); - break :b dialects.stablehlo.constant(ctx, &.{}, mlir.floatType(ctx, .f32), std.mem.asBytes(&val_f32), loc); - }, - }; - CompilationContext.current().currentBlock().appendOwnedOperation(constant_op); - - if (sh.rank() > 0) { - constant_op = dialects.stablehlo.broadcast_in_dim(ctx, constant_op.result(0), &.{}, mlir.rankedTensorType(sh.dims(), zml.mlir.Type.fromDType(ctx, sh.dtype())), loc); - CompilationContext.current().currentBlock().appendOwnedOperation(constant_op); - } - return _result(sh, constant_op.result(0)).convert(val.dtype()); + pub fn constant(val: zml.DataType.Value) Tensor { + const op = dialects.stablehlo.constant( + mlirCtx(), + &.{}, + zml.mlir.Type.fromDType(mlirCtx(), val.dtype()), + val.constSlice(), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(.init(&.{}, val.dtype()), op.result(0)).autoBroadcast(); } pub fn add(self: Tensor, other: Tensor) Tensor { @@ -207,7 +229,7 @@ pub const Tensor = struct { } pub fn relu(self: Tensor) Tensor { - return self.maximum(Tensor.constant(self.dims(), self.dtype().zero())); + return self.maximum(.constant(self.dtype().zero())); } pub fn flattenAll(self: Tensor) Tensor { @@ -215,13 +237,63 @@ pub const Tensor = struct { return self.reshape(.{self.count()}); } + pub fn broadcast(self: Tensor, output_shape: Shape, axes_: []const i64) Tensor { + stdx.debug.assert(axes_.len == self.rank(), "broadcast expects axes_ to map all axes from self to axes of the output shape, got broadcast({f}, {f}, {any})", .{ self, output_shape, axes_ }); + for (0.., axes_) |self_ax, other_ax| { + const d = self.dim(self_ax); + stdx.debug.assert(d == 1 or d == output_shape.dim(other_ax), "broadcast expects shape axes to either be 1-sized or to match the target size. got broadcast({f}, {f}, {any}), error on self axis {d} mapping to other axis {d}", .{ self, output_shape, axes_, self_ax, other_ax }); + } + + const res_shape = output_shape.withDtype(self.dtype()); + if (std.mem.eql(i64, self.dims(), output_shape.dims())) { + // No broadcast needed. We don't emit a new stablehlo value + // but we propagate output_shape tags. + return _result(res_shape, self.value()); + } + const ctx = self.getContext(); + const result_type = mlir.rankedTensorType( + res_shape.dims(), + zml.mlir.Type.fromDType(ctx.mlir_ctx, res_shape.dtype()), + ); + const broadcast_op = dialects.stablehlo.broadcast_in_dim(ctx.mlir_ctx, self.value(), axes_, result_type, .unknown(ctx.mlir_ctx)).appendTo(ctx.currentBlock()); + return _result(res_shape, broadcast_op.result(0)); + } + + pub fn broad(self: Tensor, other: Shape) Tensor { + // TODO: broad is too restrictive because sometime you only want to specify one specific axis + // Note: if you code below, make sure to update Shape.canBroadcastTo. + stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self, other }); + + // Already the right shape + if (std.mem.eql(i64, self.dims(), other.dims())) return self; + + // Non ambiguous broadcasting + // TODO: broad is error prone because of this: + // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } + if (self._shape.rank() == 0 or self._shape.rank() == other.rank()) { + const all_axes = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + return self.broadcast(other, all_axes[0..self.rank()]); + } + + // check that each axis of self maps to an axis of other + var axes_: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (self._shape.tags()) |t| { + axes_.appendAssumeCapacity(@intCast(other.axis(t))); + } + return self.broadcast(other, axes_.constSlice()); + } + pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); const ctx = self.getContext().mlir_ctx; const tensor_type = mlir.rankedTensorType(output_shape.dims(), zml.mlir.Type.fromDType(ctx, output_shape.dtype())); //const loc = self.getContext().location(@src(), "reshape({f})", .{output_shape}); - const reshape_op = dialects.stablehlo.reshape(self.getContext().mlir_ctx, self.value(), tensor_type, .unknown(ctx)); - CompilationContext.current().currentBlock().appendOwnedOperation(reshape_op); + const reshape_op = dialects.stablehlo.reshape( + self.getContext().mlir_ctx, + self.value(), + tensor_type, + .unknown(ctx), + ).appendTo(CompilationContext.current().currentBlock()); return _result(output_shape, reshape_op.result(0)); } @@ -233,8 +305,12 @@ pub const Tensor = struct { const mlir_ctx = self.getContext().mlir_ctx; const res_type = mlir.rankedTensorType(self.dims(), zml.mlir.Type.fromDType(mlir_ctx, to)); - const op = dialects.stablehlo.convert(mlir_ctx, self.value(), res_type, .unknown(mlir_ctx)); - CompilationContext.current().currentBlock().appendOwnedOperation(op); + const op = dialects.stablehlo.convert( + mlir_ctx, + self.value(), + res_type, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentBlock()); return _result(self._shape.withDtype(to), op.result(0)); } @@ -276,6 +352,7 @@ pub const Tensor = struct { // &.{a}, // ); //} + // fn binaryOp( src: std.builtin.SourceLocation, @@ -286,6 +363,7 @@ pub const Tensor = struct { _ = src; // autofix return struct { pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { + const other_ = if (other.auto_broadcast) other.broad(self.shape()) else other; //stdx.debug.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {f} and {f}", .{ op_name, self, other }); //if (self.rank() == 0 and other.rank() != 0) { @@ -300,8 +378,7 @@ pub const Tensor = struct { const ctx = self.getContext(); //const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other }); - const ret = op_fn(ctx.mlir_ctx, self.value(), other.value(), .unknown(ctx.mlir_ctx)); - CompilationContext.current().currentBlock().appendOwnedOperation(ret); + const ret = op_fn(mlirCtx(), self.value(), other_.value(), .unknown(ctx.mlir_ctx)).appendTo(currentBlock()); return _result(self._shape, ret.result(0)); } }.binaryOpHelper; @@ -334,29 +411,31 @@ pub const Tensor = struct { /// Model definition const Mnist = struct { - fc1: Layer, - fc2: Layer, + layers: [2]Layer, const Layer = struct { weight: Tensor, bias: Tensor, pub fn forward(self: Layer, input: Tensor) Tensor { - _ = self; // autofix - //return self.weight.matmul(input).add(self.bias).relu(); - return input.relu(); + return self.weight.dot( + input, + .{ .contract_on = .{ 0, 1 } }, + ).matmul(input).add(self.bias).relu(); } }; pub fn init(buffer_store: zml.aio.BufferStore) Mnist { return .{ - .fc1 = .{ - .weight = .init(buffer_store.get("fc1.weight").?.shape()), - .bias = .init(buffer_store.get("fc1.bias").?.shape()), - }, - .fc2 = .{ - .weight = .init(buffer_store.get("fc2.weight").?.shape()), - .bias = .init(buffer_store.get("fc2.bias").?.shape()), + .layers = .{ + .{ + .weight = .init(buffer_store.get("fc1.weight").?.shape()), + .bias = .init(buffer_store.get("fc1.bias").?.shape()), + }, + .{ + .weight = .init(buffer_store.get("fc2.weight").?.shape()), + .bias = .init(buffer_store.get("fc2.bias").?.shape()), + }, }, }; } @@ -365,8 +444,7 @@ const Mnist = struct { pub fn forward(self: Mnist, input: Tensor) Tensor { // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); var x = input.flattenAll().convert(.f32); - const layers: []const Layer = &.{ self.fc1, self.fc2 }; - for (layers) |layer| { + for (self.layers) |layer| { x = layer.forward(x); } //return x.argMax(0).indices.convert(.u8); @@ -374,9 +452,6 @@ const Mnist = struct { } }; -const dialects = @import("mlir/dialects"); -const mlir = @import("mlir"); - pub fn main() !void { try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); } @@ -413,6 +488,7 @@ const CompilationContext = struct { mappings: stdx.BoundedArray(std.AutoArrayHashMapUnmanaged(usize, usize), 16) = .{}, pub fn init(allocator: std.mem.Allocator) CompilationContext { + mlir.registerPasses("Transforms"); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); @@ -500,40 +576,37 @@ fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: std // } //}.cb, &context, &args); - emitMlir(&compilation_context, func, .{model} ++ args); + emitMlir(&compilation_context, func, .{model} ++ args) catch unreachable; } -fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) void { +fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !void { var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); defer arena.deinit(); + const module = mlir.Module.init(.unknown(compilation_context.mlir_ctx)); + errdefer module.deinit(); + const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); compilation_context.pushBlock(block); defer compilation_context.popBlock(); - var input_types_list = std.array_list.Managed(*const mlir.Type).init(arena.allocator()); - defer input_types_list.deinit(); - const LocalContext = struct { compilation_context: *CompilationContext, - input_types_list: std.array_list.Managed(*const mlir.Type), current_argument_id: usize = 0, }; var context: LocalContext = .{ .compilation_context = compilation_context, - .input_types_list = input_types_list, }; zml.meta.visit(struct { - fn cb(inner_context: *LocalContext, tensor: *const Tensor) void { - _ = inner_context.compilation_context.currentBlock().addArgument( - mlir.rankedTensorType(tensor.dims(), zml.mlir.Type.fromDType(inner_context.compilation_context.mlir_ctx, tensor.dtype())), - .unknown(inner_context.compilation_context.mlir_ctx), + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + _ = ctx_.compilation_context.currentBlock().addArgument( + tensor.mlirType(ctx_.compilation_context.mlir_ctx), + .unknown(ctx_.compilation_context.mlir_ctx), ); - inner_context.compilation_context.currentMapping().put(inner_context.compilation_context.allocator, tensor.id, inner_context.current_argument_id) catch unreachable; - inner_context.input_types_list.append(mlir.rankedTensorType(tensor.dims(), zml.mlir.Type.fromDType(inner_context.compilation_context.mlir_ctx, tensor.dtype()))) catch unreachable; - inner_context.current_argument_id += 1; + ctx_.compilation_context.currentMapping().put(ctx_.compilation_context.allocator, tensor.id, ctx_.current_argument_id) catch unreachable; + ctx_.current_argument_id += 1; } }.cb, &context, &args); @@ -541,20 +614,37 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar global_compilation_context = compilation_context; defer global_compilation_context = null; const result = @call(.auto, func, args); - const fn_op = dialects.func.returns(compilation_context.mlir_ctx, &.{result.value()}, .unknown(compilation_context.mlir_ctx)); - compilation_context.currentBlock().appendOwnedOperation(fn_op); + _ = dialects.func.return_(compilation_context.mlir_ctx, result.value(), .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentBlock()); } const mlir_fn = dialects.func.func(compilation_context.mlir_ctx, .{ - .name = "mnist", - .args = null, - .args_attributes = null, - .results = null, - .results_attributes = null, + .name = "main", .block = compilation_context.currentBlock(), .location = .unknown(compilation_context.mlir_ctx), }); - _ = mlir_fn; // autofix - std.debug.print("{f}\n", .{compilation_context.currentBlock()}); + var pass_manager = mlir.PassManager.init(compilation_context.mlir_ctx); + pass_manager.enableIRPrinting(.{ + .printBeforeAll = true, + }); + { + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + try opm.addPipeline(pass); + } + } + + pass_manager.runOnOp(mlir_fn) catch |err| { + std.debug.print("Failed to optimize module: {any}\n", .{err}); + return err; + }; + + _ = mlir_fn.appendTo(module.body()); + + std.debug.print("{f}\n", .{module}); } diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 7f13df69..d55c0ecd 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -127,12 +127,59 @@ pub const PassManager = opaque { pub fn asOpPassManager(self: *PassManager) *OpPassManager { return @ptrCast(c.mlirPassManagerGetAsOpPassManager(self.ptr()).ptr); } + + pub fn runOnOp(self: *const PassManager, op: *Operation) !void { + if (c.mlirLogicalResultIsFailure(c.mlirPassManagerRunOnOp(self.ptr(), op.ptr()))) { + // return error.MlirUnexpected; + } + } + + pub const EnableIRPrintingOpts = struct { + printBeforeAll: bool = false, + printAfterAll: bool = false, + printModuleScope: bool = false, + printAfterOnlyOnChange: bool = false, + printAfterOnlyOnFailure: bool = false, + flags: Operation.PrintFlags = .{}, + treePrintingPath: []const u8 = "", + }; + + pub fn enableIRPrinting(self: *PassManager, opts: EnableIRPrintingOpts) void { + const flags = opts.flags.createNative(); + defer c.mlirOpPrintingFlagsDestroy(flags); + + c.mlirPassManagerEnableIRPrinting( + self.ptr(), + opts.printBeforeAll, + opts.printAfterAll, + opts.printModuleScope, + opts.printAfterOnlyOnChange, + opts.printAfterOnlyOnFailure, + flags, + stringRef(opts.treePrintingPath), + ); + } + + // pub extern fn mlirPassManagerEnableIRPrinting(passManager: MlirPassManager, printBeforeAll: bool, printAfterAll: bool, printModuleScope: bool, printAfterOnlyOnChange: bool, printAfterOnlyOnFailure: bool, flags: MlirOpPrintingFlags, treePrintingPath: MlirStringRef) void; + // pub extern fn mlirPassManagerEnableVerifier(passManager: MlirPassManager, enable: bool) void; + }; pub const OpPassManager = opaque { const M = Methods(OpPassManager, c.MlirOpPassManager); pub const ptr = M.ptr; + pub const format = M.format(c.mlirPrintPassPipeline); + + pub fn addPass(self: *OpPassManager, pass: *Pass) void { + c.mlirOpPassManagerAddOwnedPass(self.ptr(), pass.ptr()); + } + + pub fn addPasses(self: *OpPassManager, passes: []const *Pass) void { + for (passes) |pass| { + self.addPass(pass); + } + } pub fn addPipeline(self: *OpPassManager, pipeline_name: []const u8) !void { var buffer: [1024]u8 = undefined; @@ -142,12 +189,25 @@ pub const OpPassManager = opaque { return error.MlirUnexpected; } } + + // pub fn parsePipeline(self: *OpPassManager, pipeline: []const u8) !void { + // var buffer: [1024]u8 = undefined; + // const result = stringFromStream(&buffer, c.mlirOpPassManagerParsePipeline, .{ self.ptr(), stringRef(pipeline) }); + // if (result.len != 0) { + // log.err("mlir error: {s}", .{result}); + // return error.MlirUnexpected; + // } + // } }; pub const Pass = opaque { const M = Methods(Pass, c.MlirPass); pub const ptr = M.ptr; + + pub fn create(comptime name: @TypeOf(.enum_literal)) *Pass { + return @ptrCast(@field(c, "mlirCreate" ++ @tagName(name))().ptr); + } }; pub const Context = opaque { @@ -651,15 +711,19 @@ pub const Module = opaque { pub const ptr = M.ptr; pub fn init(loc: *const Location) *Module { - return @constCast(@ptrCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr)); + return @ptrCast(@constCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr)); } pub fn deinit(self: *Module) void { c.mlirModuleDestroy(self.ptr()); } + pub fn format(self: *const Module, writer: *std.Io.Writer) !void { + return self.operation().format(writer); + } + pub fn fromOperation(op: *Operation) *Module { - return @constCast(@ptrCast(c.mlirModuleFromOperation(op.ptr()).ptr)); + return @ptrCast(@constCast(c.mlirModuleFromOperation(op.ptr()).ptr)); } pub fn operation(self: *const Module) *Operation { @@ -667,7 +731,7 @@ pub const Module = opaque { } pub fn parse(ctx: *Context, source: []const u8) Error!*Module { - return @constCast(@ptrCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir)); + return @ptrCast(@constCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir)); } pub fn context(self: *const Module) *Context { @@ -1014,29 +1078,33 @@ pub const Operation = opaque { self: *const Operation, flags: PrintFlags, }; + + fn createNative(self: PrintFlags) c.MlirOpPrintingFlags { + const flags = c.mlirOpPrintingFlagsCreate(); + if (self.elide_large_elements_attrs) |v| { + c.mlirOpPrintingFlagsElideLargeElementsAttrs(flags, @intCast(v)); + } + c.mlirOpPrintingFlagsEnableDebugInfo( + flags, + self.debug_info, + self.debug_info_pretty_form, + ); + if (self.print_generic_op_form) { + c.mlirOpPrintingFlagsPrintGenericOpForm(flags); + } + if (self.use_local_scope) { + c.mlirOpPrintingFlagsUseLocalScope(flags); + } + if (self.assume_verified) { + c.mlirOpPrintingFlagsAssumeVerified(flags); + } + return flags; + } }; fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { - const flags = c.mlirOpPrintingFlagsCreate(); + const flags = fctx.flags.createNative(); defer c.mlirOpPrintingFlagsDestroy(flags); - - if (fctx.flags.elide_large_elements_attrs) |v| { - c.mlirOpPrintingFlagsElideLargeElementsAttrs(flags, @intCast(v)); - } - c.mlirOpPrintingFlagsEnableDebugInfo( - flags, - fctx.flags.debug_info, - fctx.flags.debug_info_pretty_form, - ); - if (fctx.flags.print_generic_op_form) { - c.mlirOpPrintingFlagsPrintGenericOpForm(flags); - } - if (fctx.flags.use_local_scope) { - c.mlirOpPrintingFlagsUseLocalScope(flags); - } - if (fctx.flags.assume_verified) { - c.mlirOpPrintingFlagsAssumeVerified(flags); - } var sctx: StringCallbackCtx = .{ .writer = writer }; c.mlirOperationPrintWithFlags(fctx.self.ptr(), flags, stringCallback, &sctx); if (sctx.err) |err| { diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 45fddc0b..52c509b6 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -4,6 +4,9 @@ platform( "@platforms//cpu:x86_64", "@platforms//os:linux", ], + flags = [ + "--force_pic=true", + ], visibility = ["//visibility:public"], ) @@ -13,6 +16,9 @@ platform( "@platforms//cpu:aarch64", "@platforms//os:linux", ], + flags = [ + "--force_pic=true", + ], visibility = ["//visibility:public"], ) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 184925a5..7a41737c 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -83,6 +83,7 @@ zig_shared_library( "-fno-stack-check", "-fPIC", ], + features = ["pic"], main = "libneuronxla.zig", shared_lib_name = "libneuronxla.so", visibility = ["@libpjrt_neuron//:__subpackages__"], diff --git a/stdx/io.zig b/stdx/io.zig index 390647b8..e69de29b 100644 --- a/stdx/io.zig +++ b/stdx/io.zig @@ -1,4 +0,0 @@ -const std = @import("std"); - -pub const BufferedAnyWriter = std.io.BufferedWriter(4096, std.io.AnyWriter); -pub const BufferedAnyReader = std.io.BufferedReader(4096, std.io.AnyReader); diff --git a/third_party/zls/zls.bzl b/third_party/zls/zls.bzl index 4cc5caa1..9e28da65 100644 --- a/third_party/zls/zls.bzl +++ b/third_party/zls/zls.bzl @@ -1,24 +1,24 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_VERSION = "0.14.0" +_VERSION = "0.15.0" _ARCH = { "x86_64-linux": struct( - sha256 = "661f8d402ba3dc9b04b6e9bc3026495be7b838d2f18d148db2bd98bd699c1360", + sha256 = "508bfe3fd637d2a02f07f3fc7da8900351f407116b03685c5dae26b4f01a30de", exec_compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", ], ), "x86_64-macos": struct( - sha256 = "baee69e4645deeccb42970b4a01f573592209dc1cf72e32893c59ca06af511dc", + sha256 = "46c31838bfef5adcc7fee82428c3ec2b9abbfae38242639afea5f242ee133d93", exec_compatible_with = [ "@platforms//os:macos", "@platforms//cpu:x86_64", ], ), "aarch64-macos": struct( - sha256 = "dfb627e1f9603583678f552d8035a12dce878215c0a507b32d6f1b9d074d6c4d", + sha256 = "76c7a23190f67e67970024065f689c2c49b3c7b0fc16876fb24ef199fb05fc2a", exec_compatible_with = [ "@platforms//os:macos", "@platforms//cpu:aarch64", diff --git a/zml/dtype.zig b/zml/dtype.zig index b0deb8f4..033c6032 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -1,9 +1,15 @@ const std = @import("std"); const floats = @import("floats.zig"); +pub const Float8E4M3B11FNUZ = floats.Float8E4M3B11FNUZ; +pub const Float8E4M3FN = floats.Float8E4M3FN; +pub const Float8E4M3FNUZ = floats.Float8E4M3FNUZ; +pub const Float8E5M2 = floats.Float8E5M2; +pub const Float8E5M2FNUZ = floats.Float8E5M2FNUZ; +pub const BFloat16 = floats.BFloat16; -const C64 = std.math.Complex(f32); -const C128 = std.math.Complex(f64); +pub const C64 = std.math.Complex(f32); +pub const C128 = std.math.Complex(f64); test { std.testing.refAllDecls(@This()); @@ -70,12 +76,12 @@ pub const DataType = enum(u8) { pub fn fromZigType(comptime T: type) DataType { return switch (T) { - floats.Float8E4M3B11FNUZ => .f8e4m3b11fnuz, - floats.Float8E4M3FN => .f8e4m3fn, - floats.Float8E4M3FNUZ => .f8e4m3fnuz, - floats.Float8E5M2 => .f8e5m2, - floats.Float8E5M2FNUZ => .f8e5m2fnuz, - floats.BFloat16 => .bf16, + Float8E4M3B11FNUZ => .f8e4m3b11fnuz, + Float8E4M3FN => .f8e4m3fn, + Float8E4M3FNUZ => .f8e4m3fnuz, + Float8E5M2 => .f8e5m2, + Float8E5M2FNUZ => .f8e5m2fnuz, + BFloat16 => .bf16, f16 => .f16, f32 => .f32, f64 => .f64, @@ -112,7 +118,7 @@ pub const DataType = enum(u8) { } pub fn toZigType(comptime dtype: DataType) type { - return @FieldType(Data, @tagName(dtype)); + return @FieldType(Value, @tagName(dtype)); } pub fn isSignedInt(dtype: DataType) bool { @@ -170,165 +176,165 @@ pub const DataType = enum(u8) { try std.testing.expectEqual(DataType.c64.resolvePeerType(.f32), null); } - pub fn zero(dtype: DataType) Data { - return Data.init(dtype, 0); + pub fn zero(dtype: DataType) Value { + return .init(dtype, 0); } - pub fn one(dtype: DataType) Data { - return Data.init(dtype, 1); + pub fn one(dtype: DataType) Value { + return .init(dtype, 1); } - pub fn minValue(dtype: DataType) Data { + pub fn minValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = false }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).zero), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).minus_inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), -std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.minInt(@FieldType(Data, @tagName(tag)))), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).zero), + inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).minus_inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), -std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.minInt(@FieldType(Value, @tagName(tag)))), inline else => |tag| @panic("Unsupported type: " ++ @tagName(tag)), }; } - pub fn maxValue(dtype: DataType) Data { + pub fn maxValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = true }, inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.maxInt(@FieldType(Value, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), }; } - pub fn constant(dtype: DataType, value: anytype) Data { - return Data.init(dtype, value); + pub fn constant(dtype: DataType, value: anytype) Value { + return .init(dtype, value); } -}; -pub const Data = union(DataType) { - bool: bool, - f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, - f8e4m3fn: floats.Float8E4M3FN, - f8e4m3fnuz: floats.Float8E4M3FNUZ, - f8e5m2: floats.Float8E5M2, - f8e5m2fnuz: floats.Float8E5M2FNUZ, - bf16: floats.BFloat16, - f16: f16, - f32: f32, - f64: f64, - i4: i4, - i8: i8, - i16: i16, - i32: i32, - i64: i64, - u4: u4, - u8: u8, - u16: u16, - u32: u32, - u64: u64, - c64: C64, - c128: C128, + pub const Value = union(DataType) { + bool: bool, + f8e4m3b11fnuz: Float8E4M3B11FNUZ, + f8e4m3fn: Float8E4M3FN, + f8e4m3fnuz: Float8E4M3FNUZ, + f8e5m2: Float8E5M2, + f8e5m2fnuz: Float8E5M2FNUZ, + bf16: BFloat16, + f16: f16, + f32: f32, + f64: f64, + i4: i4, + i8: i8, + i16: i16, + i32: i32, + i64: i64, + u4: u4, + u8: u8, + u16: u16, + u32: u32, + u64: u64, + c64: C64, + c128: C128, - /// Creates `Data` from a `value`. - /// - /// If the `dtype` and `@TypeOf(value)` are incompatible - /// or a cast from `value` to `FieldType(dtype)` would - /// be lossy, a panic occurs. - pub fn init(dtype_: DataType, value: anytype) Data { - const T = @TypeOf(value); - const Ti = @typeInfo(T); + /// Creates `Data` from a `value`. + /// + /// If the `dtype` and `@TypeOf(value)` are incompatible + /// or a cast from `value` to `FieldType(dtype)` would + /// be lossy, a panic occurs. + pub fn init(dtype_: DataType, value: anytype) Value { + const T = @TypeOf(value); + const Ti = @typeInfo(T); - return switch (dtype_) { - .bool => switch (Ti) { - .bool => .{ .bool = value }, - .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, - else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), - }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), - else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), - }, - inline .f16, .f32, .f64 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @floatFromInt(value)), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { - .comptime_int => blk: { - const OutT = @FieldType(Data, @tagName(tag)); - if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { - break :blk @unionInit(Data, @tagName(tag), @intCast(value)); - } else { - @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); - } + return switch (dtype_) { + .bool => switch (Ti) { + .bool => .{ .bool = value }, + .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, + else => @panic("Could not create Value of type bool from value of type " ++ @typeName(T)), }, - .int => @unionInit(Data, @tagName(tag), @intCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - .c64 => switch (T) { - C64 => .{ .c64 = value }, - C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - else => @panic("Could not create Data of type c64 from value of type " ++ @typeName(T)), - }, - .c128 => switch (T) { - C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - C128 => .{ .c128 = value }, - else => @panic("Could not create Data of type c128 from value of type " ++ @typeName(T)), - }, - }; - } - - test init { - try std.testing.expectEqual(20.0, Data.init(.f16, 20).f16); - try std.testing.expectEqual(20.5, Data.init(.f16, 20.5).f16); - try std.testing.expectEqual(20, Data.init(.f16, @as(u8, 20)).f16); - try std.testing.expectEqual(-20, Data.init(.f16, @as(i8, -20)).f16); - try std.testing.expectEqual(2000.5, Data.init(.f16, @as(f32, 2000.5)).f16); - - try std.testing.expectEqual(true, Data.init(.bool, true).bool); - - try std.testing.expectEqual(10, Data.init(.u8, 10).u8); - try std.testing.expectEqual(10, Data.init(.u8, @as(u16, 10)).u8); - - try std.testing.expectEqual(10, Data.init(.i8, 10).i8); - try std.testing.expectEqual(10, Data.init(.i8, @as(u16, 10)).i8); - try std.testing.expectEqual(-10, Data.init(.i8, -10).i8); - try std.testing.expectEqual(-10, Data.init(.i8, @as(i16, -10)).i8); - - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C64.init(1, 2)).c64); - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C128.init(1, 2)).c64); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C128.init(1, 2)).c128); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C64.init(1, 2)).c128); - } - - pub fn dtype(self: Data) DataType { - return std.meta.activeTag(self); - } - - pub fn constSlice(data: *const Data) []const u8 { - return switch (data.*) { - inline else => |*value| std.mem.asBytes(value), - }; - } - - pub fn as(self: Data, comptime T: type) T { - // TODO allow more lossless conversions - switch (@typeInfo(T)) { - .bool => return self.bool, - .float => switch (self) { - inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), - else => {}, - }, - .int => switch (self) { - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), - else => {}, - }, - else => {}, + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatFromInt(value))), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatCast(value))), + else => @panic("Could not create Value of type bf16 from value of type " ++ @typeName(T)), + }, + inline .f16, .f32, .f64 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @floatFromInt(value)), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @floatCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + .comptime_int => blk: { + const OutT = @FieldType(Value, @tagName(tag)); + if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { + break :blk @unionInit(Value, @tagName(tag), @intCast(value)); + } else { + @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + } + }, + .int => @unionInit(Value, @tagName(tag), @intCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + .c64 => switch (T) { + C64 => .{ .c64 = value }, + C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + else => @panic("Could not create Value of type c64 from value of type " ++ @typeName(T)), + }, + .c128 => switch (T) { + C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + C128 => .{ .c128 = value }, + else => @panic("Could not create Value of type c128 from value of type " ++ @typeName(T)), + }, + }; } - std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); - } + + test init { + try std.testing.expectEqual(20.0, Value.init(.f16, 20).f16); + try std.testing.expectEqual(20.5, Value.init(.f16, 20.5).f16); + try std.testing.expectEqual(20, Value.init(.f16, @as(u8, 20)).f16); + try std.testing.expectEqual(-20, Value.init(.f16, @as(i8, -20)).f16); + try std.testing.expectEqual(2000.5, Value.init(.f16, @as(f32, 2000.5)).f16); + + try std.testing.expectEqual(true, Value.init(.bool, true).bool); + + try std.testing.expectEqual(10, Value.init(.u8, 10).u8); + try std.testing.expectEqual(10, Value.init(.u8, @as(u16, 10)).u8); + + try std.testing.expectEqual(10, Value.init(.i8, 10).i8); + try std.testing.expectEqual(10, Value.init(.i8, @as(u16, 10)).i8); + try std.testing.expectEqual(-10, Value.init(.i8, -10).i8); + try std.testing.expectEqual(-10, Value.init(.i8, @as(i16, -10)).i8); + + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C64.init(1, 2)).c64); + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C128.init(1, 2)).c64); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C128.init(1, 2)).c128); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C64.init(1, 2)).c128); + } + + pub fn dtype(self: Value) DataType { + return std.meta.activeTag(self); + } + + pub fn constSlice(data: *const Value) []const u8 { + return switch (data.*) { + inline else => |*value| std.mem.asBytes(value), + }; + } + + pub fn as(self: Value, comptime T: type) T { + // TODO allow more lossless conversions + switch (@typeInfo(T)) { + .bool => return self.bool, + .float => switch (self) { + inline .f16, .f32, .f64 => |v| return @floatCast(v), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), + else => {}, + }, + .int => switch (self) { + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), + else => {}, + }, + else => {}, + } + std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); + } + }; }; pub fn mantissaSize(dtype: DataType) usize { @@ -342,3 +348,5 @@ pub fn mantissaSize(dtype: DataType) usize { else => @panic("Can't get mantissa size for a non-float dtype"), }; } + +pub const Data = DataType.Value; From 73b096f5268071e819290ac515930199f56bec8a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 5 Sep 2025 18:47:23 +0200 Subject: [PATCH 41/47] wip: final version of reduce Ideas for dot --- mlir/dialects/stablehlo/stablehlo.zig | 18 +- mlir/main.zig | 755 +++++++++++++++++++++++++- zml/mlirx.zig | 2 + 3 files changed, 745 insertions(+), 30 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 24d4574e..80dbfcf1 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -154,7 +154,7 @@ pub fn dotAlgorithmAttribute(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, el ctx.ptr(), element_type.ptr(), element_type.ptr(), - mlir.floatType(dot_algorithm.accumulation, ctx).ptr(), + mlir.floatType(ctx, dot_algorithm.accumulation).ptr(), dot_algorithm.component_count, dot_algorithm.component_count, dot_algorithm.num_primitive_operations, @@ -1460,3 +1460,19 @@ pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const return error.InvalidMlirBytecodeVersion; } } + +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .flat = &.{value} }, + .location = location, + .verify = false, + }); +} + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} diff --git a/mlir/main.zig b/mlir/main.zig index 71563e3a..44d9f4e9 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -69,6 +69,10 @@ pub const Tensor = struct { return self._shape.dims(); } + pub fn axis(self: Tensor, axis_: anytype) u3 { + return self._shape.axis(axis_); + } + pub fn count(self: Tensor) usize { return self._shape.count(); } @@ -101,7 +105,7 @@ pub const Tensor = struct { const lhs_broad = lhs.broadcastLeft(left_shape); const n_batching_axes = rhs.rank() - lhs.rank(); - var batching: [Tensor.MAX_RANK][2]i8 = undefined; + var batching: [MAX_RANK][2]i8 = undefined; for (0..n_batching_axes) |i| { batching[i] = .{ @intCast(i), @intCast(i) }; } @@ -117,6 +121,33 @@ pub const Tensor = struct { return lhs.dotGeneral(rhs, args.contract_on, args.batch_on orelse &.{}); } + const ArgsKind = enum { + simple, + contracting_only, + full, + }; + + fn isFullArgsKind(comptime T: type) bool { + return std.meta.fieldIndex(T, "contracting") != null and std.meta.fieldIndex(T, "batching") != null; + } + + fn getArgsKind(comptime T: type) ArgsKind { + const type_info = @typeInfo(T); + return switch (type_info) { + .enum_literal => .simple, + .@"struct" => if (isFullArgsKind(T)) .full else .contracting_only, + else => unreachable, + }; + } + + pub fn dot2(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { + _ = lhs; // autofix + _ = rhs; // autofix + const args_kind = getArgsKind(@TypeOf(args)); + std.debug.print("{}\n", .{args_kind}); + unreachable; + } + pub fn dotGeneral( lhs: Tensor, rhs: Tensor, @@ -174,7 +205,8 @@ pub const Tensor = struct { } const mlir_ctx = lhs.getContext().mlir_ctx; - const loc = lhs.getContext().location(@src(), "dot({f},{f},contracting={any},batching={any}", .{ lhs, rhs, contracting_axes, batching_axes }); + //const loc = lhs.getContext().location(@src(), "dot({f},{f},contracting={any},batching={any}", .{ lhs, rhs, contracting_axes, batching_axes }); + const loc = mlir.Location.unknown(mlirCtx()); const op = dialects.stablehlo.dot_general( mlir_ctx, lhs.value(), @@ -185,7 +217,7 @@ pub const Tensor = struct { .rhs_batching_dimensions = rhs_batching_axes.constSlice(), .lhs_contracting_dimensions = lhs_contracting_axes.constSlice(), .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), - .precision = .fast, + .dot_precision = .fast, }, loc, ); @@ -283,6 +315,48 @@ pub const Tensor = struct { return self.broadcast(other, axes_.constSlice()); } + pub fn broadcastLeft(self: Tensor, output_shape: Shape) Tensor { + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastLeft expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); + + const a = output_shape.rank() - self.rank(); + if (self.rank() == output_shape.rank() and std.mem.eql(i64, self.dims(), output_shape.dims())) { + return self; + } + + return self.broadcast(output_shape, Shape.range(output_shape.rank(), output_shape.dtype()).dims()[a..]); + } + + pub const ArangeArgs = struct { + start: i64 = 0, + end: i64, + step: i64 = 1, + }; + + pub fn arange(args: ArangeArgs, dt: zml.DataType) Tensor { + stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); + stdx.debug.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); + + const ctx = CompilationContext.current(); + //const loc = ctx.location(@src(), "arange({}, dtype={})", .{ args, dt }); + const loc = mlir.Location.unknown(ctx.mlir_ctx); + + const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; + const sh = Shape.init(.{n_steps}, dt); + var op = dialects.stablehlo.iota(ctx.mlir_ctx, 0, mlir.rankedTensorType(sh.dims(), zml.mlir.Type.fromDType(ctx.mlir_ctx, sh.dtype())), loc).appendTo(CompilationContext.current().currentBlock()); + var res = _result(sh, op.result(0)); + _ = &res; + + //if (args.step != 1) { + // res = res.scale(args.step); + //} + + //if (args.start != 0) { + // res = res.addConstant(args.start); + //} + + return res; + } + pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); const ctx = self.getContext().mlir_ctx; @@ -314,11 +388,38 @@ pub const Tensor = struct { return _result(self._shape.withDtype(to), op.result(0)); } + pub fn select(bool_tensor: Tensor, on_true: Tensor, on_false: Tensor) Tensor { + stdx.debug.assert(bool_tensor.dtype() == .bool, "select expects input tensor type to be a boolean, got {}", .{bool_tensor.dtype()}); + stdx.debug.assert(on_true.dtype() == on_false.dtype(), "select expects 'on_true' and 'on_false' tensor types to be equal, got {} and {}", .{ on_true.dtype(), on_false.dtype() }); + + if (bool_tensor.rank() != 0 and on_true.rank() == 0) { + return bool_tensor.select(on_true.broad(bool_tensor.shape()), on_false); + } + if (bool_tensor.rank() != 0 and on_false.rank() == 0) { + return bool_tensor.select(on_true, on_false.broad(bool_tensor.shape())); + } + + stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); + stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); + + //const loc = bool_tensor.mlirCtx().location(@src()); + const loc = mlir.Location.unknown(mlirCtx()); + const op = dialects.stablehlo.select( + mlirCtx(), + bool_tensor.value(), + on_true.value(), + on_false.value(), + loc, + ).appendTo(CompilationContext.current().currentBlock()); + + return _result(on_true._shape, op.result(0)); + } + pub const ArgMaxRes = struct { values: Tensor, indices: Tensor, - fn cmp(left: ArgMaxRes, right: ArgMaxRes) ArgMaxRes { + fn cmpOld2(left: ArgMaxRes, right: ArgMaxRes) ArgMaxRes { const left_val = left.values; const right_val = right.values; const left_idx = left.indices; @@ -339,6 +440,62 @@ pub const Tensor = struct { return .{ .values = max_val, .indices = max_idx }; } + + fn cmpOld(left: [2]Tensor, right: [2]Tensor) [2]Tensor { + const left_val = left[0]; + const right_val = right[0]; + const left_idx = left[1]; + const right_idx = right[1]; + + const left_gt_right = left_val.cmp(.GT, right_val); + const is_nan = left_val.cmp(.NE, left_val); + const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(left_val, right_val); + + // If left_val == right_val: keep the smallest idx. + const is_same = left_val.cmp(.EQ, right_val); + const is_first = left_idx.cmp(.LT, right_idx); + const is_same_but_first = is_same.logical(.AND, is_first); + const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + const max_idx = keep_left_idx.select(left_idx, right_idx); + + return .{ max_val, max_idx }; + } + + fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { + const left_gt_right = values.left.cmp(.GT, values.right); + const is_nan = values.left.cmp(.NE, values.left); + const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); + + // If values.left == values.right: keep the smallest idx. + const is_same = values.left.cmp(.EQ, values.right); + const is_first = indices.left.cmp(.LT, indices.right); + const is_same_but_first = is_same.logical(.AND, is_first); + const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + const max_idx = keep_left_idx.select(indices.left, indices.right); + + return .{ max_val, max_idx }; + } + + fn inputs(x: Tensor, dt: zml.DataType, axis_: i8) ArgMaxRes { + return .{ .values = x, .indices = Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{axis_}) }; + } + + fn inits(x: Tensor, dt: zml.DataType) ArgMaxRes { + return .{ .values = Tensor.constant(x.dtype().minValue()), .indices = Tensor.constant(dt.zero()) }; + } + + fn operand(x: Tensor, dt: zml.DataType) ArgMaxRes { + return .{ .values = .init(.init(&.{}, x.dtype())), .indices = .init(.init(&.{}, dt)) }; + } + }; + + pub const ReduceArgs = struct { + left: Tensor, + right: Tensor, }; //pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { @@ -352,7 +509,549 @@ pub const Tensor = struct { // &.{a}, // ); //} - // + + pub const LogicalOp = enum { OR, XOR, AND }; + pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { + return switch (logical_op) { + .OR => binaryOp(@src(), "or", dialects.stablehlo.or_)(self, other), + .XOR => binaryOp(@src(), "xor", dialects.stablehlo.xor)(self, other), + .AND => binaryOp(@src(), "and", dialects.stablehlo.and_)(self, other), + }; + } + + fn getComparisonType(dt: zml.DataType) dialects.stablehlo.CompareType.Type { + return switch (dt) { + .i4, .i8, .i16, .i32, .i64 => .SIGNED, + .bool, .u4, .u8, .u16, .u32, .u64 => .UNSIGNED, + .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .FLOAT, + .c64, .c128 => @panic("Can't compare complex numbers"), + }; + } + + pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); + + if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); + if (self.rank() != 0 and other.rank() == 0) return self.cmp(direction, other.broadcast(self._shape, &.{})); + + stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); + + //const loc = self.getContext().location(@src(), "cmp(.{s})", .{@tagName(direction)}); + const loc = mlir.Location.unknown(mlirCtx()); + const op = dialects.stablehlo.compare( + mlirCtx(), + self.value(), + other.value(), + direction, + getComparisonType(self.dtype()), + loc, + ).appendTo(CompilationContext.current().currentBlock()); + + return _result(self._shape.withDtype(.bool), op.result(0)); + } + + pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { + const a = x.axis(axis_); + const dt: zml.DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; + + //return reduce(ArgMaxRes.cmp, .inputs(x, dt, a), .inits(x, dt), .operand(x, dt), .operand(x, dt), &.{a}); + const values, const indices = reduce( + .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, + .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, + &.{a}, + struct { + fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { + const left_gt_right = values.left.cmp(.GT, values.right); + const is_nan = values.left.cmp(.NE, values.left); + const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); + + // If values.left == values.right: keep the smallest idx. + const is_same = values.left.cmp(.EQ, values.right); + const is_first = indices.left.cmp(.LT, indices.right); + const is_same_but_first = is_same.logical(.AND, is_first); + const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + const max_idx = keep_left_idx.select(indices.left, indices.right); + + return .{ max_val, max_idx }; + } + }.cmp, + .{}, + ); + return .{ .values = values, .indices = indices }; + } + + fn collectValues(allocator: std.mem.Allocator, v: anytype) []const *const mlir.Value { + const LocalContext = struct { + list: *std.ArrayList(*const mlir.Value), + allocator: std.mem.Allocator, + }; + + var list: std.ArrayList(*const mlir.Value) = .{}; + var context: LocalContext = .{ .list = &list, .allocator = allocator }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *const Tensor) void { + context_.list.append(context_.allocator, t.value()) catch unreachable; + } + }.cb, &context, v); + + return list.toOwnedSlice(allocator) catch unreachable; + } + + fn collectShapes(allocator: std.mem.Allocator, v: anytype) []const zml.Shape { + const LocalContext = struct { + list: *std.ArrayList(zml.Shape), + allocator: std.mem.Allocator, + }; + + var list: std.ArrayList(zml.Shape) = .{}; + var context: LocalContext = .{ .list = &list, .allocator = allocator }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *const Tensor) void { + context_.list.append(context_.allocator, t.shape()) catch unreachable; + } + }.cb, &context, v); + + return list.toOwnedSlice(allocator) catch unreachable; + } + + fn mapToArguments(compilation_context: *CompilationContext, v: anytype, starting_index: usize) usize { + const LocalContext = struct { + current_argument_index: usize, + compilation_context: *CompilationContext, + }; + + var context: LocalContext = .{ + .current_argument_index = starting_index, + .compilation_context = compilation_context, + }; + + zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *const Tensor) void { + _ = context_.compilation_context.currentBlock().addArgument( + mlir.rankedTensorType(t.dims(), zml.mlir.Type.fromDType(mlirCtx(), t.dtype())), + .unknown(mlirCtx()), + ); + context_.compilation_context.currentMapping().put(context_.compilation_context.allocator, t.id, context_.current_argument_index) catch unreachable; + context_.current_argument_index += 1; + } + }.cb, &context, v); + + return context.current_argument_index; + } + + fn ReduceResult(comptime T: anytype) type { + const type_info = @typeInfo(T); + return switch (type_info.@"struct".fields.len) { + 0 => unreachable, + 1 => type_info.@"struct".fields[0].type, + else => |len| @Type(.{ .array = .{ + .len = len, + .sentinel_ptr = null, + .child = type_info.@"struct".fields[0].type, + } }), + }; + } + + fn reduce(inputs: anytype, inits: anytype, axes: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), zml.mlir.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), zml.mlir.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + inline for (0..inits.len) |i| { + CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, args[i].left.id, i) catch unreachable; + CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + const result_values = collectValues(arena.allocator(), &result); + + _ = dialects.stablehlo.returns(mlirCtx(), result_values, .unknown(mlirCtx())).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes)), + }, + .verify = false, + .location = .unknown(mlirCtx()), + }).appendTo(CompilationContext.current().currentBlock()); + + inline for (0..result.len) |i| { + const val = reduce_op.result(i); + const reduced_shape = inputs[i].shape().removeMany(axes); + + result[i] = ._result(reduced_shape, val).broad(inputs[i].shape()); + } + + return result; + } + + fn reduceOld(comptime func: anytype, inputs: anytype, inits: anytype, axes: []const i64) ReduceResult(@TypeOf(inputs)) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const reduce_block = b: { + const reduce_block = mlir.Block.init(&.{}, &.{}); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + var lhs: [inputs.len]Tensor = undefined; + var rhs: [inputs.len]Tensor = undefined; + inline for (0..inits.len) |i| { + lhs[i] = Tensor.init(inits[i].shape()); + rhs[i] = Tensor.init(inits[i].shape()); + } + const arg_count = mapToArguments(CompilationContext.current(), &lhs, 0); + _ = mapToArguments(CompilationContext.current(), &rhs, arg_count); + + var result = if (inputs.len == 1) func(lhs[0], rhs[0]) else func(lhs, rhs); + + const result_values = collectValues(arena.allocator(), &result); + + _ = dialects.stablehlo.returns(mlirCtx(), result_values, .unknown(mlirCtx())).appendTo(reduce_block); + break :b reduce_block; + }; + + const input_values = collectValues(arena.allocator(), &inputs); + const init_values = collectValues(arena.allocator(), &inits); + + const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ input_values, init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes)), + }, + .verify = false, + .location = .unknown(mlirCtx()), + }).appendTo(CompilationContext.current().currentBlock()); + + var broadcasting_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (0..MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes, @intCast(i)) == null) { + broadcasting_axes.append(@intCast(i)) catch unreachable; + } + } + + const LocalContext = struct { + axes: []const i64, + broadcasting_axes: []const i64, + n_reduced: u8, + current_id: usize = 0, + reduce_op: *mlir.Operation, + }; + var context2: LocalContext = .{ + .axes = axes, + .broadcasting_axes = broadcasting_axes.constSlice(), + .n_reduced = @intCast(axes.len), + .reduce_op = reduce_op, + }; + var result: ReduceResult(@TypeOf(inputs)) = inputs; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *Tensor) void { + const val = context_.reduce_op.result(context_.current_id); + var reduced_shape = t.shape(); + for (context_.axes) |a| { + reduced_shape = reduced_shape.setDim(a, 1); + } + + const broadcast_op = dialects.stablehlo.broadcast_in_dim( + mlirCtx(), + val, + context_.broadcasting_axes[0 .. t.rank() - context_.n_reduced], + mlir.rankedTensorType(reduced_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), reduced_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(CompilationContext.current().currentBlock()); + t.* = Tensor._result(reduced_shape, broadcast_op.result(0)); + context_.current_id += 1; + } + }.cb, &context2, &result); + + return result; + } + + fn reduceOld2( + comptime func: anytype, + inputs: stdx.meta.FnParam(func, 0), + inits: stdx.meta.FnParam(func, 0), + lhs: stdx.meta.FnParam(func, 0), + rhs: stdx.meta.FnParam(func, 0), + axes: []const i64, + ) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const reduce_block = b: { + const reduce_block = mlir.Block.init(&.{}, &.{}); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const arg_count = mapToArguments(CompilationContext.current(), &lhs, 0); + _ = mapToArguments(CompilationContext.current(), &rhs, arg_count); + + var result = func(lhs, rhs); + + const result_values = collectValues(arena.allocator(), &result); + + _ = dialects.stablehlo.returns(mlirCtx(), result_values, .unknown(mlirCtx())).appendTo(reduce_block); + break :b reduce_block; + }; + + const input_values = collectValues(arena.allocator(), &inputs); + const init_values = collectValues(arena.allocator(), &inits); + + const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ input_values, init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes)), + }, + .verify = false, + .location = .unknown(mlirCtx()), + }).appendTo(CompilationContext.current().currentBlock()); + + var broadcasting_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (0..MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes, @intCast(i)) == null) { + broadcasting_axes.append(@intCast(i)) catch unreachable; + } + } + + const LocalContext = struct { + axes: []const i64, + broadcasting_axes: []const i64, + n_reduced: u8, + current_id: usize = 0, + reduce_op: *mlir.Operation, + }; + var context2: LocalContext = .{ + .axes = axes, + .broadcasting_axes = broadcasting_axes.constSlice(), + .n_reduced = @intCast(axes.len), + .reduce_op = reduce_op, + }; + var result = inputs; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *Tensor) void { + const val = context_.reduce_op.result(context_.current_id); + var reduced_shape = t.shape(); + for (context_.axes) |a| { + reduced_shape = reduced_shape.setDim(a, 1); + } + + const broadcast_op = dialects.stablehlo.broadcast_in_dim( + mlirCtx(), + val, + context_.broadcasting_axes[0 .. t.rank() - context_.n_reduced], + mlir.rankedTensorType(reduced_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), reduced_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(CompilationContext.current().currentBlock()); + t.* = Tensor._result(reduced_shape, broadcast_op.result(0)); + context_.current_id += 1; + } + }.cb, &context2, &result); + + return result; + } + + fn reduceOld3(comptime func: anytype, inputs: stdx.meta.FnParam(func, 0), inits: stdx.meta.FnParam(func, 0), axes: []const i64) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const input_values, const input_shapes = b: { + var values_list: std.ArrayList(*const mlir.Value) = .{}; + var shapes_list: std.ArrayList(zml.Shape) = .{}; + const LocalContext = struct { + values_list: *std.ArrayList(*const mlir.Value), + shapes_list: *std.ArrayList(zml.Shape), + allocator: std.mem.Allocator, + }; + var context: LocalContext = .{ + .values_list = &values_list, + .shapes_list = &shapes_list, + .allocator = arena.allocator(), + }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *const Tensor) void { + context_.values_list.append(context_.allocator, t.value()) catch unreachable; + context_.shapes_list.append(context_.allocator, t.shape()) catch unreachable; + } + }.cb, &context, &inputs); + break :b .{ + values_list.toOwnedSlice(arena.allocator()) catch unreachable, + shapes_list.toOwnedSlice(arena.allocator()) catch unreachable, + }; + }; + const init_values = b: { + var list: std.ArrayList(*const mlir.Value) = .{}; + const LocalContext = struct { + list: *std.ArrayList(*const mlir.Value), + allocator: std.mem.Allocator, + }; + var context: LocalContext = .{ .list = &list, .allocator = arena.allocator() }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *const Tensor) void { + context_.list.append(context_.allocator, t.value()) catch unreachable; + } + }.cb, &context, &inits); + break :b list.toOwnedSlice(arena.allocator()) catch unreachable; + }; + + const reduce_block, var result = b2: { + const reduce_block = mlir.Block.init(&.{}, &.{}); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const LocalContext = struct { + current_argument_id: usize = 0, + }; + var context: LocalContext = .{}; + + var lhs = inputs; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *Tensor) void { + const dt = t.dtype(); + t.* = .init(.init(&.{}, dt)); + _ = CompilationContext.current().currentBlock().addArgument( + mlir.rankedTensorType(t.dims(), zml.mlir.Type.fromDType(CompilationContext.current().mlir_ctx, t.dtype())), + .unknown(CompilationContext.current().mlir_ctx), + ); + CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, t.id, context_.current_argument_id) catch unreachable; + context_.current_argument_id += 1; + } + }.cb, &context, &lhs); + + var rhs = inits; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *Tensor) void { + const dt = t.dtype(); + t.* = .init(.init(&.{}, dt)); + _ = CompilationContext.current().currentBlock().addArgument( + mlir.rankedTensorType(t.dims(), zml.mlir.Type.fromDType(CompilationContext.current().mlir_ctx, t.dtype())), + .unknown(CompilationContext.current().mlir_ctx), + ); + CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, t.id, context_.current_argument_id) catch unreachable; + context_.current_argument_id += 1; + } + }.cb, &context, &rhs); + + var result = func(lhs, rhs); + const result_values = b: { + var list: std.ArrayList(*const mlir.Value) = .{}; + const LocalContext2 = struct { + list: *std.ArrayList(*const mlir.Value), + allocator: std.mem.Allocator, + }; + var context2: LocalContext2 = .{ .list = &list, .allocator = arena.allocator() }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext2, t: *const Tensor) void { + context_.list.append(context_.allocator, t.value()) catch unreachable; + } + }.cb, &context2, &result); + break :b list.toOwnedSlice(arena.allocator()) catch unreachable; + }; + _ = dialects.stablehlo.returns(mlir_ctx, result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + + break :b2 .{ reduce_block, result }; + }; + const reduce_op = mlir.Operation.make(CompilationContext.current().mlir_ctx, "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ input_values, init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(CompilationContext.current().mlir_ctx, "dimensions", mlir.denseArrayAttribute(CompilationContext.current().mlir_ctx, .i64, axes)), + }, + .verify = false, + .location = .unknown(CompilationContext.current().mlir_ctx), + }).appendTo(CompilationContext.current().currentBlock()); + + var broadcasting_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (0..MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes, @intCast(i)) == null) { + broadcasting_axes.append(@intCast(i)) catch unreachable; + } + } + + const LocalContext2 = struct { + axes: []const i64, + broadcasting_axes: []const i64, + n_reduced: u8, + current_id: usize = 0, + reduce_op: *mlir.Operation, + input_shapes: []const zml.Shape, + }; + var context2: LocalContext2 = .{ + .axes = axes, + .broadcasting_axes = broadcasting_axes.constSlice(), + .n_reduced = @intCast(axes.len), + .reduce_op = reduce_op, + .input_shapes = input_shapes, + }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext2, t: *Tensor) void { + const mlir_ctx_ = CompilationContext.current().mlir_ctx; + + const val = context_.reduce_op.result(context_.current_id); + var reduced_shape = context_.input_shapes[context_.current_id]; + for (context_.axes) |a| { + reduced_shape = reduced_shape.setDim(a, 1); + } + const broadcast_op = dialects.stablehlo.broadcast_in_dim( + mlir_ctx_, + val, + context_.broadcasting_axes[0 .. context_.input_shapes[context_.current_id].rank() - context_.n_reduced], + mlir.rankedTensorType(reduced_shape.dims(), zml.mlir.Type.fromDType(mlir_ctx_, reduced_shape.dtype())), + .unknown(mlir_ctx_), + ).appendTo(CompilationContext.current().currentBlock()); + t.* = Tensor._result(reduced_shape, broadcast_op.result(0)); + context_.current_id += 1; + } + }.cb, &context2, &result); + + return result; + } fn binaryOp( src: std.builtin.SourceLocation, @@ -418,10 +1117,7 @@ const Mnist = struct { bias: Tensor, pub fn forward(self: Layer, input: Tensor) Tensor { - return self.weight.dot( - input, - .{ .contract_on = .{ 0, 1 } }, - ).matmul(input).add(self.bias).relu(); + return self.weight.matmul(input).add(self.bias).relu(); } }; @@ -447,8 +1143,8 @@ const Mnist = struct { for (self.layers) |layer| { x = layer.forward(x); } - //return x.argMax(0).indices.convert(.u8); - return x; + return x.argMax(0).indices.convert(.u8); + //return x; } }; @@ -471,10 +1167,23 @@ pub fn asyncMain() !void { defer buffer_store.deinit(); const mnist: Mnist = .init(buffer_store); + _ = mnist; // autofix const input: Tensor = .init(zml.Shape.init(.{ 28, 28 }, .u8)); + _ = input; // autofix - compileModel(allocator, Mnist.forward, mnist, .{input}); + //compileModel(allocator, Mnist.forward, mnist, .{input}); + + compile(allocator, testDot, .{ Tensor.init(.init(.{ .m = 256, .n = 128 }, .f32)), Tensor.init(.init(.{ .b = 16, .n = 128, .k = 256 }, .f32)) }); +} + +fn testDot(a: Tensor, b: Tensor) Tensor { + _ = a.dot2(b, .n); + _ = a.dot2(b, .{ .n = .k }); + _ = a.dot2(b, .{ .contracting = .{ .n = .k }, .batching = .{.b} }); + _ = a.dot2(b, .{ .contracting = .{.n}, .batching = .{.b} }); + unreachable; + //return a.dotGeneral(b, &.{.{ 1, 1 }}, &.{}); } const CompilationContext = struct { @@ -557,26 +1266,14 @@ const Context = struct { }; fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) void { + compile(allocator, func, .{model} ++ args); +} + +fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx.meta.FnArgs(func)) void { var compilation_context: CompilationContext = .init(allocator); defer compilation_context.deinit(); - //var context: Context = .{ .compilation_context = &compilation_context }; - //zml.meta.visit(struct { - // fn cb(inner_context: *Context, tensor: *const Tensor) void { - // std.debug.print("Registering {}\n", .{tensor.id}); - // inner_context.compilation_context.registerTensor(tensor.id, inner_context.current_id) catch unreachable; - // inner_context.current_id += 1; - // } - //}.cb, &context, &model); - //zml.meta.visit(struct { - // fn cb(inner_context: *Context, tensor: *const Tensor) void { - // std.debug.print("Registering {}\n", .{tensor.id}); - // inner_context.compilation_context.registerTensor(tensor.id, inner_context.current_id) catch unreachable; - // inner_context.current_id += 1; - // } - //}.cb, &context, &args); - - emitMlir(&compilation_context, func, .{model} ++ args) catch unreachable; + emitMlir(&compilation_context, func, args) catch unreachable; } fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !void { diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 46c85fa4..e18eb04e 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -54,6 +54,8 @@ pub const Type = struct { }; } + // TODO(Corentin): Maybe remove that as it now requires a mlir context. + // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. pub fn toDType(mlir_type: *const mlir.Type) dtype.DataType { const mapping = .{ .{ .bool, mlir.IntegerType(.i1) }, From c26c75961db2d4b20e2b0f2ae20053f0d749f68c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 9 Oct 2025 13:37:06 +0200 Subject: [PATCH 42/47] wip: big changes --- mlir/BUILD.bazel | 36 +- mlir/dialects/stablehlo/stablehlo.zig | 8 +- mlir/llama.zig | 260 +++ mlir/main.zig | 2509 +++++++++++++++++++------ mlir/mlir.zig | 20 +- mlir/writer.zig | 190 ++ pjrt/pjrt.zig | 20 +- zml/pjrtx.zig | 34 +- zml/shape.zig | 13 +- zml/zml.zig | 2 + 10 files changed, 2460 insertions(+), 632 deletions(-) create mode 100644 mlir/llama.zig create mode 100644 mlir/writer.zig diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 65829673..63ef0239 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -30,15 +30,18 @@ zig_test( zig_binary( name = "main", - args = [ - "$(location @mnist//:mnist.safetensors)", - "$(location @mnist//:t10k-images.idx3-ubyte)", - ], - data = [ - "@mnist//:mnist.safetensors", - "@mnist//:t10k-images.idx3-ubyte", - ], + #args = [ + # "$(location @mnist//:mnist.safetensors)", + # "$(location @mnist//:t10k-images.idx3-ubyte)", + #], + #data = [ + # "@mnist//:mnist.safetensors", + # "@mnist//:t10k-images.idx3-ubyte", + #], main = "main.zig", + srcs = [ + "llama.zig", + ], visibility = ["//visibility:public"], deps = [ ":mlir", @@ -49,3 +52,20 @@ zig_binary( "//async", ], ) + +zig_library( + name = "writer", + main = "writer.zig", + visibility = ["//visibility:public"], + deps = [ + ":c", + "//zml", + "//stdx", + "//async", + ], +) + +zig_test( + name = "test_writer", + deps = [":writer"], +) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 80dbfcf1..d4f0ae72 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -1384,7 +1384,7 @@ pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { var writer = std.Io.Writer.fixed(buf); var sctx: mlir.StringCallbackCtx = .{ .writer = &writer }; - @call(.auto, streamFn, args ++ .{ mlir.stringCallback, &sctx }); + _ = @call(.auto, streamFn, args ++ .{ mlir.stringCallback, &sctx }); return writer.buffered(); } @@ -1439,7 +1439,7 @@ pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u if (sctx.err) |err| { return err; } - if (result.ptr == null) { + if (result.value == 0) { return error.InvalidMlirBytecodeVersion; } } @@ -1463,7 +1463,7 @@ pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.return", .{ - .operands = .{ .flat = &.{value} }, + .operands = .{ .flat = &.{&.{value}} }, .location = location, .verify = false, }); @@ -1471,7 +1471,7 @@ pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const ml pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.return", .{ - .operands = .{ .flat = values }, + .operands = .{ .variadic = &.{values} }, .verify = false, .location = location, }); diff --git a/mlir/llama.zig b/mlir/llama.zig new file mode 100644 index 00000000..90f67974 --- /dev/null +++ b/mlir/llama.zig @@ -0,0 +1,260 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = @import("stdx"); + +const Tensor = @import("main.zig").Tensor; +const TensorDescriptor = @import("main.zig").TensorDescriptor; +const BufferStore4 = @import("main.zig").BufferStore4; +const BufferStore5 = @import("main.zig").BufferStore5; +const Bufferized = @import("main.zig").Bufferized; +const loadBuffersFromId = @import("main.zig").loadBuffersFromId; +const multiTransfer = @import("main.zig").multiTransfer; +const singleTransfer = @import("main.zig").singleTransfer; +const structTransfer = @import("main.zig").structTransfer; +const initBufferizedFrom = @import("main.zig").initBufferizedFrom; +const compile = @import("main.zig").compile; + +pub const LlamaLM = struct { + pub const Config = struct { + bos_token_id: u32, + eos_token_id: stdx.json.Union(union(enum) { + int: u32, + ints: []u32, + }), + num_hidden_layers: usize, + num_attention_heads: usize, + num_key_value_heads: usize, + rope_theta: f32, + max_position_embeddings: usize, + rms_norm_eps: f32, + hf_rope_impl: bool = true, + //rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, + }; + + pub const Options = struct { + //sampling_strategy: ?zml.nn.SamplingStrategy, + max_seq_len: usize, + }; + + lm_head: ?Linear, + model: Llama, + + // Options controlling generation + //gen_opts: zml.nn.SamplingStrategy = .{}, + config: Config, + + pub fn init(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, config: LlamaLM.Config) !LlamaLM { + return .{ + .lm_head = null, + .model = try .init(allocator, buffer_store.withPrefix("model"), config), + .config = config, + }; + } + + pub fn deinit(self: LlamaLM, allocator: std.mem.Allocator) void { + allocator.free(self.model.layers); + } + + pub fn loadBuffers(allocator: std.mem.Allocator, llama: LlamaLM, buffer_store: BufferStore5.View, platform: zml.Platform) !LlamaLMBufferized { + return .{ + .lm_head = null, + .model = try Llama.loadBuffers(allocator, llama.model, buffer_store.withPrefix("model"), platform), + }; + } +}; + +pub const LlamaLMBufferized = struct { + lm_head: ?Bufferized(Linear), + model: Bufferized(Llama), + + pub fn deinit(self: LlamaLMBufferized, allocator: std.mem.Allocator) void { + allocator.free(self.model.layers); + } +}; + +pub const Linear = struct { + weight: Tensor, + bias: ?Tensor = null, + tag: zml.Shape.Tag, + + pub fn initWithTag(buffer_store: BufferStore5.View, tag: anytype) Linear { + return .{ + .weight = buffer_store.createTensorWithTags("weight", .{ .dout, tag }), + .bias = buffer_store.maybeCreateTensorWithTags("bias", .{.dout}), + .tag = zml.Shape.toTag(tag), + }; + } +}; + +pub const Llama = struct { + embed_tokens: struct { weight: Tensor }, + norm: RmsNorm, + layers: []TransformerLayer, + + max_seq_len: u32, + num_heads: i64, + num_kv_heads: i64, + + pub fn init(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, config: LlamaLM.Config) !Llama { + const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + var buffer: [16]u8 = undefined; + const number_prefix = try std.fmt.bufPrint(&buffer, "layers.{d}", .{i}); + layer.* = .init(buffer_store.withPrefix(number_prefix), config); + } + + return .{ + .embed_tokens = .{ .weight = buffer_store.createTensorWithTags("embed_tokens.weight", .{ .x, .y }) }, + .norm = .init(buffer_store.withPrefix("norm"), config), + .layers = layers, + .max_seq_len = 256, + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + }; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, model: Llama, buffer_store: BufferStore5.View, platform: zml.Platform) !Bufferized(Llama) { + std.debug.print("Loading Llama\n", .{}); + const layers = try allocator.alloc(Bufferized(TransformerLayer), model.layers.len); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + var buffer: [16]u8 = undefined; + const number_prefix = try std.fmt.bufPrint(&buffer, "layers.{d}", .{i}); + std.debug.print("Loading layer {d}\n", .{i}); + layer.* = try TransformerLayer.loadBuffers(allocator, model.layers[i], buffer_store.withPrefix(number_prefix), platform); + } + + return .{ + .embed_tokens = try loadBuffersFromId(allocator, model.embed_tokens, buffer_store.withPrefix("embed_tokens"), platform), + .layers = layers, + .norm = try loadBuffersFromId(allocator, model.norm, buffer_store.withPrefix("norm"), platform), + }; + } +}; + +const RmsNorm = struct { + weight: Tensor, + eps: f32 = 1e-5, + + pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config) RmsNorm { + return .{ + .weight = buffer_store.createTensorWithTags("weight", .{.d}), + .eps = config.rms_norm_eps, + }; + } +}; + +pub const TransformerLayer = struct { + input_layernorm: RmsNorm, + self_attn: SelfAttn, + post_attention_layernorm: RmsNorm, + mlp: Mlp, + + pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config) TransformerLayer { + return .{ + .input_layernorm = .init(buffer_store.withPrefix("input_layernorm"), config), + .self_attn = SelfAttn.init(buffer_store.withPrefix("self_attn"), config), + .post_attention_layernorm = .init(buffer_store.withPrefix("post_attention_layernorm"), config), + .mlp = .init(buffer_store.withPrefix("mlp")), + }; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, layer: TransformerLayer, buffer_store: BufferStore5.View, platform: zml.Platform) !Bufferized(TransformerLayer) { + return .{ + .input_layernorm = try loadBuffersFromId(allocator, layer.input_layernorm, buffer_store.withPrefix("input_layernorm"), platform), + .self_attn = try SelfAttn.loadBuffers(allocator, layer.self_attn, buffer_store.withPrefix("self_attn"), platform), + .post_attention_layernorm = try loadBuffersFromId(allocator, layer.post_attention_layernorm, buffer_store.withPrefix("post_attention_layernorm"), platform), + .mlp = try loadBuffersFromId(allocator, layer.mlp, buffer_store.withPrefix("mlp"), platform), + }; + } +}; + +pub const SelfAttn = struct { + qkv_proj: Linear, + //q_proj: Linear, + //k_proj: Linear, + //v_proj: Linear, + + o_proj: Linear, + num_heads: i64, + num_kv_heads: i64, + + pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config) SelfAttn { + const q_weight_shape = buffer_store.getShape("q_proj.weight").?; + const k_weight_shape = buffer_store.getShape("k_proj.weight").?; + const v_weight_shape = buffer_store.getShape("v_proj.weight").?; + const qkv_weight_shape = zml.Shape.concatenate(&.{ q_weight_shape, k_weight_shape, v_weight_shape }, 0); + return .{ + .qkv_proj = .{ .weight = Tensor.init(qkv_weight_shape).withTags(.{ .dout, .d }), .tag = zml.Shape.toTag(.d) }, + .o_proj = .initWithTag(buffer_store.withPrefix("o_proj"), .d), + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + }; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, self_attn: SelfAttn, buffer_store: BufferStore5.View, platform: zml.Platform) !Bufferized(SelfAttn) { + const q_proj = Linear.initWithTag(buffer_store.withPrefix("q_proj"), .d); + const k_proj = Linear.initWithTag(buffer_store.withPrefix("k_proj"), .d); + const v_proj = Linear.initWithTag(buffer_store.withPrefix("v_proj"), .d); + + const exe2 = try compile(allocator, mergeQkv2, .{ q_proj, k_proj, v_proj }, platform); + defer exe2.deinit(); + + var args = try exe2.args(allocator); + defer args.deinit(allocator); + + var results = try exe2.results(allocator); + defer results.deinit(allocator); + + const q_proj_buffers = try loadBuffersFromId(allocator, q_proj, buffer_store.withPrefix("q_proj"), platform); + const k_proj_buffers = try loadBuffersFromId(allocator, k_proj, buffer_store.withPrefix("k_proj"), platform); + const v_proj_buffers = try loadBuffersFromId(allocator, v_proj, buffer_store.withPrefix("v_proj"), platform); + + args.set(.{ q_proj_buffers, k_proj_buffers, v_proj_buffers }); + exe2.call(args, &results); + + var qkv_proj: Bufferized(Linear) = undefined; + initBufferizedFrom(self_attn.qkv_proj, &qkv_proj); + + results.fill(&qkv_proj); + errdefer qkv_proj.weight.deinit(); + errdefer if (qkv_proj.bias != null) qkv_proj.bias.?.deinit(); + + const o_proj = try loadBuffersFromId(allocator, self_attn.o_proj, buffer_store.withPrefix("o_proj"), platform); + + return .{ + .qkv_proj = qkv_proj, + .o_proj = o_proj, + }; + } +}; + +fn mergeQkv(q: Tensor, k: Tensor, v: Tensor) Tensor { + return Tensor.concatenate(&.{ q, k, v }, 0); +} + +fn mergeQkv2(q: Linear, k: Linear, v: Linear) Linear { + return .{ + .weight = Tensor.concatenate(&.{ q.weight, k.weight, v.weight }, 0), + .bias = if (q.bias != null) Tensor.concatenate(&.{ q.bias.?, k.bias.?, v.bias.? }, 0) else null, + .tag = q.tag, + }; +} + +const Mlp = struct { + up_proj: Linear, // (dim -> hidden_dim) + gate_proj: Linear, // (dim -> hidden_dim) + down_proj: Linear, // (hidden_dim -> dim) + + pub fn init(buffer_store: BufferStore5.View) Mlp { + return .{ + .up_proj = .initWithTag(buffer_store.withPrefix("up_proj"), .d), + .gate_proj = .initWithTag(buffer_store.withPrefix("gate_proj"), .d), + .down_proj = .initWithTag(buffer_store.withPrefix("down_proj"), .d), + }; + } +}; diff --git a/mlir/main.zig b/mlir/main.zig index 44d9f4e9..772701d7 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -1,11 +1,18 @@ const std = @import("std"); +const llama = @import("llama.zig"); + +const c = zml.c; const asynk = @import("async"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const stdx = @import("stdx"); +const upb = zml.upb; const zml = @import("zml"); +const pjrt = zml.pjrt; const Shape = zml.Shape; +const EnumLiteral = @TypeOf(.enum_literal); +const Platform = zml.Platform; var global_compilation_context: ?*CompilationContext = null; pub const MAX_RANK: u8 = 8; @@ -28,8 +35,6 @@ pub const Tensor = struct { _shape: zml.Shape, tensor_origin: TensorOrigin = .{ .argument = {} }, - pub const tensor = init; - fn mlirType(self: Tensor, mlir_ctx: *mlir.Context) *const mlir.Type { return mlir.rankedTensorType( self.dims(), @@ -49,32 +54,84 @@ pub const Tensor = struct { return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; } + /// Returns the shape of a Tensor. pub fn shape(self: Tensor) Shape { return self._shape; } + /// Returns the datatype of a Tensor. pub fn dtype(self: Tensor) zml.DataType { return self._shape.dtype(); } + /// Returns the rank of a Tensor. pub fn rank(self: Tensor) u4 { return self._shape.rank(); } + /// Returns the number of element of a Tensor. + pub fn count(self: Tensor) usize { + return self._shape.count(); + } + + /// Returns the size in bytes of a Tensor. + pub fn byteSize(self: Tensor) usize { + return self._shape.byteSize(); + } + + /// Returns the dimension of axis 'axis_'. + /// + /// 'axis_' can be an integer or a tag. pub fn dim(self: Tensor, axis_: anytype) i64 { return self._shape.dim(axis_); } + /// Returns the dimensions of a Tensor as a slice. pub fn dims(self: *const Tensor) []const i64 { return self._shape.dims(); } + /// Returns the index of axis 'axis_'. + /// + /// 'axis_' can be an integer or a tag. pub fn axis(self: Tensor, axis_: anytype) u3 { return self._shape.axis(axis_); } - pub fn count(self: Tensor) usize { - return self._shape.count(); + /// Returns the indices of each of the given axes. + /// + /// 'axis_' can be an integer or a tag. + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, Tensor.MAX_RANK) { + return self._shape.axes(axes_); + } + + /// Returns a Tensor tagged with the tags in 'tagz'. + pub fn withTags(self: Tensor, tagz: anytype) Tensor { + var res = self; + res._shape = self._shape.withTags(tagz); + return res; + } + + /// Returns a Tensor tagged partially with the tags in 'tagz'. + /// + /// If 'tagz' is of length n, the n last dimensions of the Tensor will be tagged. + pub fn withPartialTags(self: Tensor, tagz: anytype) Tensor { + var res = self; + res._shape = self._shape.withPartialTags(tagz); + return res; + } + + /// Returns a Tensor with new tag names. + pub fn rename(self: Tensor, renames: anytype) Tensor { + var res = self; + res._shape = self._shape.rename(renames); + return res; + } + + pub fn renameAxis(self: Tensor, ax: i8, name: EnumLiteral) Tensor { + var res = self; + res._shape._tags.set(self.axis(ax), @tagName(name).ptr); + return res; } pub fn autoBroadcast(self: Tensor) Tensor { @@ -112,15 +169,6 @@ pub const Tensor = struct { return lhs_broad.dotGeneral(rhs, &contracting, batching[0..n_batching_axes]); } - pub const DotArgs = struct { - contract_on: []const [2]i8, - batch_on: ?[]const [2]i8 = null, - }; - - pub fn dot(lhs: Tensor, rhs: Tensor, args: DotArgs) Tensor { - return lhs.dotGeneral(rhs, args.contract_on, args.batch_on orelse &.{}); - } - const ArgsKind = enum { simple, contracting_only, @@ -140,12 +188,12 @@ pub const Tensor = struct { }; } - pub fn dot2(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { - _ = lhs; // autofix - _ = rhs; // autofix - const args_kind = getArgsKind(@TypeOf(args)); - std.debug.print("{}\n", .{args_kind}); - unreachable; + pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { + //const args_kind = getArgsKind(@TypeOf(args)); + //std.debug.print("{}\n", .{args_kind}); + const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); + const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); + return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, &.{}); } pub fn dotGeneral( @@ -256,6 +304,10 @@ pub const Tensor = struct { return binaryOp(@src(), "add", dialects.stablehlo.add)(self, other); } + pub fn mul(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "mul", dialects.stablehlo.multiply)(self, other); + } + pub fn maximum(self: Tensor, other: Tensor) Tensor { return binaryOp(@src(), "maximum", dialects.stablehlo.maximum)(self, other); } @@ -418,98 +470,8 @@ pub const Tensor = struct { pub const ArgMaxRes = struct { values: Tensor, indices: Tensor, - - fn cmpOld2(left: ArgMaxRes, right: ArgMaxRes) ArgMaxRes { - const left_val = left.values; - const right_val = right.values; - const left_idx = left.indices; - const right_idx = right.indices; - - const left_gt_right = left_val.cmp(.GT, right_val); - const is_nan = left_val.cmp(.NE, left_val); - const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(left_val, right_val); - - // If left_val == right_val: keep the smallest idx. - const is_same = left_val.cmp(.EQ, right_val); - const is_first = left_idx.cmp(.LT, right_idx); - const is_same_but_first = is_same.logical(.AND, is_first); - const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); - const max_idx = keep_left_idx.select(left_idx, right_idx); - - return .{ .values = max_val, .indices = max_idx }; - } - - fn cmpOld(left: [2]Tensor, right: [2]Tensor) [2]Tensor { - const left_val = left[0]; - const right_val = right[0]; - const left_idx = left[1]; - const right_idx = right[1]; - - const left_gt_right = left_val.cmp(.GT, right_val); - const is_nan = left_val.cmp(.NE, left_val); - const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(left_val, right_val); - - // If left_val == right_val: keep the smallest idx. - const is_same = left_val.cmp(.EQ, right_val); - const is_first = left_idx.cmp(.LT, right_idx); - const is_same_but_first = is_same.logical(.AND, is_first); - const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); - const max_idx = keep_left_idx.select(left_idx, right_idx); - - return .{ max_val, max_idx }; - } - - fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { - const left_gt_right = values.left.cmp(.GT, values.right); - const is_nan = values.left.cmp(.NE, values.left); - const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(values.left, values.right); - - // If values.left == values.right: keep the smallest idx. - const is_same = values.left.cmp(.EQ, values.right); - const is_first = indices.left.cmp(.LT, indices.right); - const is_same_but_first = is_same.logical(.AND, is_first); - const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); - const max_idx = keep_left_idx.select(indices.left, indices.right); - - return .{ max_val, max_idx }; - } - - fn inputs(x: Tensor, dt: zml.DataType, axis_: i8) ArgMaxRes { - return .{ .values = x, .indices = Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{axis_}) }; - } - - fn inits(x: Tensor, dt: zml.DataType) ArgMaxRes { - return .{ .values = Tensor.constant(x.dtype().minValue()), .indices = Tensor.constant(dt.zero()) }; - } - - fn operand(x: Tensor, dt: zml.DataType) ArgMaxRes { - return .{ .values = .init(.init(&.{}, x.dtype())), .indices = .init(.init(&.{}, dt)) }; - } }; - pub const ReduceArgs = struct { - left: Tensor, - right: Tensor, - }; - - //pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { - // const a = x.axis(axis_); - // const dt: zml.DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; - - // return ops.reduce( - // ArgMaxRes.cmp, - // .{ .values = x, .indices = Tensor.arange(.{ .end = x.dim(a) }, dt).broadcast(x.shape(), &.{a}) }, - // .{ .values = Tensor.constant(&.{}, x.dtype().minValue()), .indices = Tensor.scalar(0, dt) }, - // &.{a}, - // ); - //} - pub const LogicalOp = enum { OR, XOR, AND }; pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { return switch (logical_op) { @@ -554,7 +516,6 @@ pub const Tensor = struct { const a = x.axis(axis_); const dt: zml.DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; - //return reduce(ArgMaxRes.cmp, .inputs(x, dt, a), .inits(x, dt), .operand(x, dt), .operand(x, dt), &.{a}); const values, const indices = reduce( .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, @@ -582,79 +543,12 @@ pub const Tensor = struct { return .{ .values = values, .indices = indices }; } - fn collectValues(allocator: std.mem.Allocator, v: anytype) []const *const mlir.Value { - const LocalContext = struct { - list: *std.ArrayList(*const mlir.Value), - allocator: std.mem.Allocator, - }; + pub const ReduceArgs = struct { + left: Tensor, + right: Tensor, + }; - var list: std.ArrayList(*const mlir.Value) = .{}; - var context: LocalContext = .{ .list = &list, .allocator = allocator }; - zml.meta.visit(struct { - fn cb(context_: *LocalContext, t: *const Tensor) void { - context_.list.append(context_.allocator, t.value()) catch unreachable; - } - }.cb, &context, v); - - return list.toOwnedSlice(allocator) catch unreachable; - } - - fn collectShapes(allocator: std.mem.Allocator, v: anytype) []const zml.Shape { - const LocalContext = struct { - list: *std.ArrayList(zml.Shape), - allocator: std.mem.Allocator, - }; - - var list: std.ArrayList(zml.Shape) = .{}; - var context: LocalContext = .{ .list = &list, .allocator = allocator }; - zml.meta.visit(struct { - fn cb(context_: *LocalContext, t: *const Tensor) void { - context_.list.append(context_.allocator, t.shape()) catch unreachable; - } - }.cb, &context, v); - - return list.toOwnedSlice(allocator) catch unreachable; - } - - fn mapToArguments(compilation_context: *CompilationContext, v: anytype, starting_index: usize) usize { - const LocalContext = struct { - current_argument_index: usize, - compilation_context: *CompilationContext, - }; - - var context: LocalContext = .{ - .current_argument_index = starting_index, - .compilation_context = compilation_context, - }; - - zml.meta.visit(struct { - fn cb(context_: *LocalContext, t: *const Tensor) void { - _ = context_.compilation_context.currentBlock().addArgument( - mlir.rankedTensorType(t.dims(), zml.mlir.Type.fromDType(mlirCtx(), t.dtype())), - .unknown(mlirCtx()), - ); - context_.compilation_context.currentMapping().put(context_.compilation_context.allocator, t.id, context_.current_argument_index) catch unreachable; - context_.current_argument_index += 1; - } - }.cb, &context, v); - - return context.current_argument_index; - } - - fn ReduceResult(comptime T: anytype) type { - const type_info = @typeInfo(T); - return switch (type_info.@"struct".fields.len) { - 0 => unreachable, - 1 => type_info.@"struct".fields[0].type, - else => |len| @Type(.{ .array = .{ - .len = len, - .sentinel_ptr = null, - .child = type_info.@"struct".fields[0].type, - } }), - }; - } - - fn reduce(inputs: anytype, inits: anytype, axes: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); defer arena.deinit(); @@ -685,9 +579,12 @@ pub const Tensor = struct { var result = @call(.auto, func, args ++ context); - const result_values = collectValues(arena.allocator(), &result); + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } - _ = dialects.stablehlo.returns(mlirCtx(), result_values, .unknown(mlirCtx())).appendTo(reduce_block); + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); break :b .{ reduce_block, result }; }; var input_values: [inputs.len]*const mlir.Value = undefined; @@ -703,7 +600,7 @@ pub const Tensor = struct { .result_type_inference = true, .blocks = &.{reduce_block}, .attributes = &.{ - .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes)), + .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes_)), }, .verify = false, .location = .unknown(mlirCtx()), @@ -711,348 +608,17 @@ pub const Tensor = struct { inline for (0..result.len) |i| { const val = reduce_op.result(i); - const reduced_shape = inputs[i].shape().removeMany(axes); + var reduced_shape: Shape = inputs[i].shape(); + for (axes_) |a| { + reduced_shape = reduced_shape.setDim(a, 1); + } - result[i] = ._result(reduced_shape, val).broad(inputs[i].shape()); + result[i] = Tensor._result(result[i].shape(), val).broad(reduced_shape); } return result; } - fn reduceOld(comptime func: anytype, inputs: anytype, inits: anytype, axes: []const i64) ReduceResult(@TypeOf(inputs)) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const reduce_block = b: { - const reduce_block = mlir.Block.init(&.{}, &.{}); - errdefer reduce_block.deinit(); - - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); - - var lhs: [inputs.len]Tensor = undefined; - var rhs: [inputs.len]Tensor = undefined; - inline for (0..inits.len) |i| { - lhs[i] = Tensor.init(inits[i].shape()); - rhs[i] = Tensor.init(inits[i].shape()); - } - const arg_count = mapToArguments(CompilationContext.current(), &lhs, 0); - _ = mapToArguments(CompilationContext.current(), &rhs, arg_count); - - var result = if (inputs.len == 1) func(lhs[0], rhs[0]) else func(lhs, rhs); - - const result_values = collectValues(arena.allocator(), &result); - - _ = dialects.stablehlo.returns(mlirCtx(), result_values, .unknown(mlirCtx())).appendTo(reduce_block); - break :b reduce_block; - }; - - const input_values = collectValues(arena.allocator(), &inputs); - const init_values = collectValues(arena.allocator(), &inits); - - const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ - .operands = .{ .variadic = &.{ input_values, init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes)), - }, - .verify = false, - .location = .unknown(mlirCtx()), - }).appendTo(CompilationContext.current().currentBlock()); - - var broadcasting_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; - for (0..MAX_RANK) |i| { - if (std.mem.indexOfScalar(i64, axes, @intCast(i)) == null) { - broadcasting_axes.append(@intCast(i)) catch unreachable; - } - } - - const LocalContext = struct { - axes: []const i64, - broadcasting_axes: []const i64, - n_reduced: u8, - current_id: usize = 0, - reduce_op: *mlir.Operation, - }; - var context2: LocalContext = .{ - .axes = axes, - .broadcasting_axes = broadcasting_axes.constSlice(), - .n_reduced = @intCast(axes.len), - .reduce_op = reduce_op, - }; - var result: ReduceResult(@TypeOf(inputs)) = inputs; - zml.meta.visit(struct { - fn cb(context_: *LocalContext, t: *Tensor) void { - const val = context_.reduce_op.result(context_.current_id); - var reduced_shape = t.shape(); - for (context_.axes) |a| { - reduced_shape = reduced_shape.setDim(a, 1); - } - - const broadcast_op = dialects.stablehlo.broadcast_in_dim( - mlirCtx(), - val, - context_.broadcasting_axes[0 .. t.rank() - context_.n_reduced], - mlir.rankedTensorType(reduced_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), reduced_shape.dtype())), - .unknown(mlirCtx()), - ).appendTo(CompilationContext.current().currentBlock()); - t.* = Tensor._result(reduced_shape, broadcast_op.result(0)); - context_.current_id += 1; - } - }.cb, &context2, &result); - - return result; - } - - fn reduceOld2( - comptime func: anytype, - inputs: stdx.meta.FnParam(func, 0), - inits: stdx.meta.FnParam(func, 0), - lhs: stdx.meta.FnParam(func, 0), - rhs: stdx.meta.FnParam(func, 0), - axes: []const i64, - ) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const reduce_block = b: { - const reduce_block = mlir.Block.init(&.{}, &.{}); - errdefer reduce_block.deinit(); - - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); - - const arg_count = mapToArguments(CompilationContext.current(), &lhs, 0); - _ = mapToArguments(CompilationContext.current(), &rhs, arg_count); - - var result = func(lhs, rhs); - - const result_values = collectValues(arena.allocator(), &result); - - _ = dialects.stablehlo.returns(mlirCtx(), result_values, .unknown(mlirCtx())).appendTo(reduce_block); - break :b reduce_block; - }; - - const input_values = collectValues(arena.allocator(), &inputs); - const init_values = collectValues(arena.allocator(), &inits); - - const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ - .operands = .{ .variadic = &.{ input_values, init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes)), - }, - .verify = false, - .location = .unknown(mlirCtx()), - }).appendTo(CompilationContext.current().currentBlock()); - - var broadcasting_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; - for (0..MAX_RANK) |i| { - if (std.mem.indexOfScalar(i64, axes, @intCast(i)) == null) { - broadcasting_axes.append(@intCast(i)) catch unreachable; - } - } - - const LocalContext = struct { - axes: []const i64, - broadcasting_axes: []const i64, - n_reduced: u8, - current_id: usize = 0, - reduce_op: *mlir.Operation, - }; - var context2: LocalContext = .{ - .axes = axes, - .broadcasting_axes = broadcasting_axes.constSlice(), - .n_reduced = @intCast(axes.len), - .reduce_op = reduce_op, - }; - var result = inputs; - zml.meta.visit(struct { - fn cb(context_: *LocalContext, t: *Tensor) void { - const val = context_.reduce_op.result(context_.current_id); - var reduced_shape = t.shape(); - for (context_.axes) |a| { - reduced_shape = reduced_shape.setDim(a, 1); - } - - const broadcast_op = dialects.stablehlo.broadcast_in_dim( - mlirCtx(), - val, - context_.broadcasting_axes[0 .. t.rank() - context_.n_reduced], - mlir.rankedTensorType(reduced_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), reduced_shape.dtype())), - .unknown(mlirCtx()), - ).appendTo(CompilationContext.current().currentBlock()); - t.* = Tensor._result(reduced_shape, broadcast_op.result(0)); - context_.current_id += 1; - } - }.cb, &context2, &result); - - return result; - } - - fn reduceOld3(comptime func: anytype, inputs: stdx.meta.FnParam(func, 0), inits: stdx.meta.FnParam(func, 0), axes: []const i64) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const mlir_ctx = CompilationContext.current().mlir_ctx; - - const input_values, const input_shapes = b: { - var values_list: std.ArrayList(*const mlir.Value) = .{}; - var shapes_list: std.ArrayList(zml.Shape) = .{}; - const LocalContext = struct { - values_list: *std.ArrayList(*const mlir.Value), - shapes_list: *std.ArrayList(zml.Shape), - allocator: std.mem.Allocator, - }; - var context: LocalContext = .{ - .values_list = &values_list, - .shapes_list = &shapes_list, - .allocator = arena.allocator(), - }; - zml.meta.visit(struct { - fn cb(context_: *LocalContext, t: *const Tensor) void { - context_.values_list.append(context_.allocator, t.value()) catch unreachable; - context_.shapes_list.append(context_.allocator, t.shape()) catch unreachable; - } - }.cb, &context, &inputs); - break :b .{ - values_list.toOwnedSlice(arena.allocator()) catch unreachable, - shapes_list.toOwnedSlice(arena.allocator()) catch unreachable, - }; - }; - const init_values = b: { - var list: std.ArrayList(*const mlir.Value) = .{}; - const LocalContext = struct { - list: *std.ArrayList(*const mlir.Value), - allocator: std.mem.Allocator, - }; - var context: LocalContext = .{ .list = &list, .allocator = arena.allocator() }; - zml.meta.visit(struct { - fn cb(context_: *LocalContext, t: *const Tensor) void { - context_.list.append(context_.allocator, t.value()) catch unreachable; - } - }.cb, &context, &inits); - break :b list.toOwnedSlice(arena.allocator()) catch unreachable; - }; - - const reduce_block, var result = b2: { - const reduce_block = mlir.Block.init(&.{}, &.{}); - errdefer reduce_block.deinit(); - - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); - - const LocalContext = struct { - current_argument_id: usize = 0, - }; - var context: LocalContext = .{}; - - var lhs = inputs; - zml.meta.visit(struct { - fn cb(context_: *LocalContext, t: *Tensor) void { - const dt = t.dtype(); - t.* = .init(.init(&.{}, dt)); - _ = CompilationContext.current().currentBlock().addArgument( - mlir.rankedTensorType(t.dims(), zml.mlir.Type.fromDType(CompilationContext.current().mlir_ctx, t.dtype())), - .unknown(CompilationContext.current().mlir_ctx), - ); - CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, t.id, context_.current_argument_id) catch unreachable; - context_.current_argument_id += 1; - } - }.cb, &context, &lhs); - - var rhs = inits; - zml.meta.visit(struct { - fn cb(context_: *LocalContext, t: *Tensor) void { - const dt = t.dtype(); - t.* = .init(.init(&.{}, dt)); - _ = CompilationContext.current().currentBlock().addArgument( - mlir.rankedTensorType(t.dims(), zml.mlir.Type.fromDType(CompilationContext.current().mlir_ctx, t.dtype())), - .unknown(CompilationContext.current().mlir_ctx), - ); - CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, t.id, context_.current_argument_id) catch unreachable; - context_.current_argument_id += 1; - } - }.cb, &context, &rhs); - - var result = func(lhs, rhs); - const result_values = b: { - var list: std.ArrayList(*const mlir.Value) = .{}; - const LocalContext2 = struct { - list: *std.ArrayList(*const mlir.Value), - allocator: std.mem.Allocator, - }; - var context2: LocalContext2 = .{ .list = &list, .allocator = arena.allocator() }; - zml.meta.visit(struct { - fn cb(context_: *LocalContext2, t: *const Tensor) void { - context_.list.append(context_.allocator, t.value()) catch unreachable; - } - }.cb, &context2, &result); - break :b list.toOwnedSlice(arena.allocator()) catch unreachable; - }; - _ = dialects.stablehlo.returns(mlir_ctx, result_values, .unknown(mlir_ctx)).appendTo(reduce_block); - - break :b2 .{ reduce_block, result }; - }; - const reduce_op = mlir.Operation.make(CompilationContext.current().mlir_ctx, "stablehlo.reduce", .{ - .operands = .{ .variadic = &.{ input_values, init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(CompilationContext.current().mlir_ctx, "dimensions", mlir.denseArrayAttribute(CompilationContext.current().mlir_ctx, .i64, axes)), - }, - .verify = false, - .location = .unknown(CompilationContext.current().mlir_ctx), - }).appendTo(CompilationContext.current().currentBlock()); - - var broadcasting_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; - for (0..MAX_RANK) |i| { - if (std.mem.indexOfScalar(i64, axes, @intCast(i)) == null) { - broadcasting_axes.append(@intCast(i)) catch unreachable; - } - } - - const LocalContext2 = struct { - axes: []const i64, - broadcasting_axes: []const i64, - n_reduced: u8, - current_id: usize = 0, - reduce_op: *mlir.Operation, - input_shapes: []const zml.Shape, - }; - var context2: LocalContext2 = .{ - .axes = axes, - .broadcasting_axes = broadcasting_axes.constSlice(), - .n_reduced = @intCast(axes.len), - .reduce_op = reduce_op, - .input_shapes = input_shapes, - }; - zml.meta.visit(struct { - fn cb(context_: *LocalContext2, t: *Tensor) void { - const mlir_ctx_ = CompilationContext.current().mlir_ctx; - - const val = context_.reduce_op.result(context_.current_id); - var reduced_shape = context_.input_shapes[context_.current_id]; - for (context_.axes) |a| { - reduced_shape = reduced_shape.setDim(a, 1); - } - const broadcast_op = dialects.stablehlo.broadcast_in_dim( - mlir_ctx_, - val, - context_.broadcasting_axes[0 .. context_.input_shapes[context_.current_id].rank() - context_.n_reduced], - mlir.rankedTensorType(reduced_shape.dims(), zml.mlir.Type.fromDType(mlir_ctx_, reduced_shape.dtype())), - .unknown(mlir_ctx_), - ).appendTo(CompilationContext.current().currentBlock()); - t.* = Tensor._result(reduced_shape, broadcast_op.result(0)); - context_.current_id += 1; - } - }.cb, &context2, &result); - - return result; - } - fn binaryOp( src: std.builtin.SourceLocation, op_name: []const u8, @@ -1083,6 +649,28 @@ pub const Tensor = struct { }.binaryOpHelper; } + pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { + if (tensors.len == 1) return tensors[0]; + stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); + var buffer: [32]*const mlir.Value = undefined; + std.debug.assert(tensors.len <= buffer.len); + std.debug.assert(tensors.len > 0); + const a = tensors[0].axis(axis_); + // TODO(Corendos): Check that tensor axes match. + + var concatenated_dim: i64 = 0; + for (tensors, 0..) |t, i| { + buffer[i] = t.value(); + concatenated_dim += t.dim(a); + } + + const res_shape = tensors[0]._shape.set(a, concatenated_dim); + const ctx = tensors[0].getContext(); + const op = dialects.stablehlo.concatenate(ctx.mlir_ctx, buffer[0..tensors.len], a, .unknown(ctx.mlir_ctx)).appendTo(currentBlock()); + // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); + return _result(res_shape, op.result(0)); + } + pub fn value(self: Tensor) *const mlir.Value { return switch (self.tensor_origin) { .argument => b: { @@ -1108,6 +696,105 @@ pub const Tensor = struct { } }; +const MoE = struct { + experts: Mlp, + router: zml.nn.Linear, + + pub fn init(tensor_descriptor: TensorDescriptor.View) MoE { + return .{ + .experts = .{ + .gate_up_proj = .{ + // We need to bitcast the scale cause safetensors doesn't encode f8 types correctly + .scale = tensor_descriptor.getWithTags("gate_up_proj_scales", .{ .expert, .out, .d }).?, + // We don't bitcast here because PJRT doesn't handle packed host buffers + .blocks = tensor_descriptor.getWithTags("gate_up_proj_blocks", .{ .expert, .out, .d, .d_block }), + .blocks_dtype = .f4e2m1, + .bias = tensor_descriptor.getWithTags("gate_up_proj_bias", .{ .expert, .d }), + }, + .down_proj = .{ + .blocks = tensor_descriptor.getWithTags("down_proj_blocks", .{ .expert, .out, .d, .d_block }), + .blocks_dtype = .f4e2m1, + .scale = tensor_descriptor.getWithTags("down_proj_scales", .{ .expert, .out, .d }), + .bias = tensor_descriptor.getWithTags("down_proj_bias", .{ .expert, .d }), + }, + }, + .router = .{ + .weight = tensor_descriptor.getWithTags("router.weight", .{ .dout, .d }).?, + .bias = tensor_descriptor.getWithTags("router.bias", .{.d}).?, + }, + }; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, moe: MoE, buffer_store: BufferStore4.View, platform: Platform) !Bufferized(MoE) { + var transfer, const entries = try multiTransfer(allocator, .{ + moe.experts.gate_up_proj.scale.shape(), + moe.experts.gate_up_proj.blocks.shape(), + moe.experts.gate_up_proj.bias.shape(), + moe.experts.down_proj.scale.shape(), + moe.experts.down_proj.blocks.shape(), + moe.experts.down_proj.bias.shape(), + moe.router.weight.shape(), + moe.router.bias.shape(), + }, platform); + defer transfer.deinit(allocator, platform); + + const gate_up_proj_scales, const gate_up_proj_blocks, const gate_up_proj_bias, const down_proj_scales, const down_proj_blocks, const down_proj_bias, const router_weight, const router_bias = entries; + buffer_store.get("gate_up_proj_scales").?.reader().stream(gate_up_proj_scales.writer); + buffer_store.get("gate_up_proj_blocks").?.reader().stream(gate_up_proj_blocks.writer); + buffer_store.get("gate_up_proj_bias").?.reader().stream(gate_up_proj_bias.writer); + buffer_store.get("down_proj_scales").?.reader().stream(down_proj_scales.writer); + buffer_store.get("down_proj_blocks").?.reader().stream(down_proj_blocks.writer); + buffer_store.get("down_proj_bias").?.reader().stream(down_proj_bias.writer); + buffer_store.get("router.weight").?.reader().stream(router_weight.writer); + buffer_store.get("router.bias").?.reader().stream(router_bias.writer); + + return .{ + .experts = .{ + .gate_up_proj = .{ + .scale = gate_up_proj_scales, + .blocks = gate_up_proj_blocks, + .bias = gate_up_proj_bias, + }, + .down_proj = .{ + .scale = down_proj_scales, + .blocks = down_proj_blocks, + .bias = down_proj_bias, + }, + }, + .router = .{ + .weight = router_weight, + .bias = router_bias, + }, + }; + } +}; + +pub const BlockScaledLinear = struct { + blocks: Tensor, + scale: Tensor, + bias: ?Tensor = null, + blocks_dtype: zml.DataType, +}; + +pub const Mlp = struct { + gate_up_proj: BlockScaledLinear, // {.out = intermediate_size * 2, .d = hidden_size / block_size, .d_block = block_size } + down_proj: BlockScaledLinear, // {.out = hidden_size * 2, .d = intermediate_size / block_size, .d_block = block_size } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const dt = x.dtype(); + var gate, var up = zml.nn.splitRealImg(self.gate_up_proj.forward(x), .interleaved); + gate = .minimum(gate, .scalar(7, dt)); + up = .clamp(up, .scalar(-7, dt), .scalar(7, dt)); + + const out = gate.quickGelu().mul(up.addConstant(1)); + return zml.call(self.down_proj, .forward, .{out}); + } + + pub fn format(self: Mlp, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Mlp(gate_up_proj=.{f}, down_proj=.{f})", .{ self.gate_up_proj, self.down_proj }); + } +}; + /// Model definition const Mnist = struct { layers: [2]Layer, @@ -1116,81 +803,721 @@ const Mnist = struct { weight: Tensor, bias: Tensor, + pub fn init(tensor_descriptor: TensorDescriptor.View) Layer { + return .{ + .weight = tensor_descriptor.getWithTags("weight", .{ .dout, .d }).?, + .bias = tensor_descriptor.getWithTags("bias", .{.d}).?, + }; + } + pub fn forward(self: Layer, input: Tensor) Tensor { - return self.weight.matmul(input).add(self.bias).relu(); + return self.weight.dot(input, .d).add(self.bias).relu(); + } + + pub fn loadBuffers(allocator: std.mem.Allocator, layer: Layer, buffer_store: BufferStore4.View, platform: Platform) !Bufferized(Layer) { + var transfer, const entries = try multiTransfer(allocator, .{ layer.weight.shape(), layer.bias.shape() }, platform); + defer transfer.deinit(allocator, platform); + + const weight, const bias = entries; + buffer_store.get("weight").?.reader().stream(weight.writer); + buffer_store.get("bias").?.reader().stream(bias.writer); + + return .{ + .weight = weight.buffer, + .bias = bias.buffer, + }; } }; - pub fn init(buffer_store: zml.aio.BufferStore) Mnist { - return .{ - .layers = .{ - .{ - .weight = .init(buffer_store.get("fc1.weight").?.shape()), - .bias = .init(buffer_store.get("fc1.bias").?.shape()), - }, - .{ - .weight = .init(buffer_store.get("fc2.weight").?.shape()), - .bias = .init(buffer_store.get("fc2.bias").?.shape()), - }, - }, - }; + pub fn init(tensor_descriptor: TensorDescriptor.View) Mnist { + return .{ .layers = .{ + Layer.init(tensor_descriptor.withPrefix("fc1")), + Layer.init(tensor_descriptor.withPrefix("fc2")), + } }; } /// just two linear layers + relu activation pub fn forward(self: Mnist, input: Tensor) Tensor { - // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); - var x = input.flattenAll().convert(.f32); + var x = input.flattenAll().convert(.f32).withTags(.{.d}); + std.debug.print("x.shape: {f}\n", .{x.shape()}); for (self.layers) |layer| { - x = layer.forward(x); + x = layer.forward(x).withTags(.{.d}); } return x.argMax(0).indices.convert(.u8); - //return x; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, model: Mnist, buffer_store: BufferStore4.View, platform: Platform) !Bufferized(Mnist) { + return .{ + .layers = .{ + try Layer.loadBuffers(allocator, model.layers[0], buffer_store.withPrefix("fc1"), platform), + //try Layer.loadBuffers(allocator, model.layers[1], buffer_store.withPrefix("fc2"), platform), + //try autoLoad(allocator, model.layers[1], buffer_store.withPrefix("fc2"), platform), + try loadBuffersFromId(allocator, model.layers[1], buffer_store.withPrefix("fc2"), platform), + }, + }; } }; +pub const TransferEntry = struct { + buffer: zml.Buffer, + writer: Transfer.Writer, +}; + +fn TransferReturnType(comptime ShapeType: type) type { + if (ShapeType == zml.Shape) { + return std.meta.Tuple(&.{ Transfer, TransferEntry }); + } + const type_info = @typeInfo(ShapeType); + return switch (type_info) { + .@"struct" => |struct_type_info| b: { + if (!struct_type_info.is_tuple) unreachable; + inline for (struct_type_info.fields) |field| { + if (field.type != zml.Shape) unreachable; + } + const types = [1]type{Transfer} ++ [_]type{TransferEntry} ** struct_type_info.fields.len; + break :b std.meta.Tuple(&types); + }, + else => unreachable, + }; +} + +pub fn singleTransfer(allocator: std.mem.Allocator, shape: Shape, platform: Platform) !struct { Transfer, TransferEntry } { + var transfer = try Transfer.init(allocator, &.{shape}, platform); + errdefer transfer.deinit(allocator, platform); + + const entry = .{ + .buffer = zml.Buffer.fromPjrtBuffers(platform, shape, &.{transfer.get(0).buffer}), + .writer = transfer.get(0).writer(), + }; + + return .{ transfer, entry }; +} + +pub fn multiTransfer(allocator: std.mem.Allocator, shapes: anytype, platform: Platform) !struct { Transfer, [shapes.len]TransferEntry } { + var shapes_array: [shapes.len]zml.Shape = undefined; + inline for (shapes, 0..) |shape, index| shapes_array[index] = shape; + + var transfer = try Transfer.init(allocator, &shapes_array, platform); + errdefer transfer.deinit(allocator, platform); + + var entries: [shapes.len]TransferEntry = undefined; + inline for (shapes, 0..) |shape, index| { + entries[index].buffer = zml.Buffer.fromPjrtBuffers(platform, shape, &.{transfer.get(index).buffer}); + entries[index].writer = transfer.get(index).writer(); + } + + return .{ transfer, entries }; +} + +fn autoLoad(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferStore4.View, platform: Platform) !Bufferized(@TypeOf(model)) { + const Model = @TypeOf(model); + var result: Bufferized(Model) = undefined; + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const shapes = try collectShapes(arena.allocator(), &model); + + var transfer: Transfer = try .init(arena.allocator(), shapes, platform); + defer transfer.deinit(arena.allocator(), platform); + + const readers = try arena.allocator().alloc(BufferStore4.Reader, shapes.len); + + { + var index: usize = 0; + const type_info = @typeInfo(Model); + switch (type_info) { + .@"struct" => |struct_type_info| { + inline for (struct_type_info.fields) |field| { + const reader = buffer_store.get(field.name).?.reader(); + readers[index] = reader; + index += 1; + } + }, + else => unreachable, + } + } + + const LocalContext = struct { + readers: []BufferStore4.Reader, + shapes: []const Shape, + platform: Platform, + transfer: *Transfer, + index: usize = 0, + }; + var context: LocalContext = .{ .readers = readers, .shapes = shapes, .platform = platform, .transfer = &transfer }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { + const writer = context_.transfer.get(context_.index).writer(); + context_.readers[context_.index].stream(writer); + + buffer.* = zml.Buffer.fromPjrtBuffers(context_.platform, context_.shapes[context_.index], &.{context_.transfer.get(context_.index).buffer}); + context_.index += 1; + } + }.cb, &context, &result); + + return result; +} + +pub fn loadBuffersFromId(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferStore5.View, platform: Platform) !Bufferized(@TypeOf(model)) { + const Model = @TypeOf(model); + var result: Bufferized(Model) = undefined; + initBufferizedFrom(model, &result); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const shapes = try collectShapes(arena.allocator(), &model); + std.debug.print("Found {d} shapes\n", .{shapes.len}); + + var transfer: Transfer = try .init(arena.allocator(), shapes, platform); + defer transfer.deinit(arena.allocator(), platform); + + const readers = try collectReaders(arena.allocator(), buffer_store, &model); + std.debug.print("Found {d} readers\n", .{readers.len}); + + const LocalContext = struct { + readers: []BufferStore5.Reader, + shapes: []const Shape, + platform: Platform, + transfer: *Transfer, + index: usize = 0, + }; + var context: LocalContext = .{ .readers = readers, .shapes = shapes, .platform = platform, .transfer = &transfer }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { + const writer = context_.transfer.get(context_.index).writer(); + context_.readers[context_.index].stream(writer); + + buffer.* = zml.Buffer.fromPjrtBuffers(context_.platform, context_.shapes[context_.index], &.{context_.transfer.get(context_.index).buffer}); + context_.index += 1; + } + }.cb, &context, &result); + + return result; +} + +pub fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model))) void { + const Model = @TypeOf(model); + const type_info = @typeInfo(Bufferized(Model)); + switch (type_info) { + .@"struct" => |struct_type_info| { + if (Bufferized(Model) == zml.Buffer) return; + inline for (struct_type_info.fields) |field| { + initBufferizedFrom(@field(model, field.name), &@field(bufferized_, field.name)); + } + }, + .@"union" => { + switch (model) { + inline else => |v, tag| { + bufferized_.* = @unionInit(Bufferized(Model), @tagName(tag), undefined); + initBufferizedFrom(v, @field(bufferized_, @tagName(tag))); + }, + } + }, + .optional => { + if (model == null) { + bufferized_.* = null; + } else { + bufferized_.* = undefined; + initBufferizedFrom(model.?, &bufferized_.*.?); + } + }, + .void, .int, .@"enum", .bool, .enum_literal, .float, .pointer, .vector => {}, + else => unreachable, + } +} + +pub fn structTransfer(allocator: std.mem.Allocator, model: anytype, platform: Platform) !struct { Transfer, Transferized(@TypeOf(model)) } { + const Model = @TypeOf(model); + var result: Transferized(Model) = undefined; + initTransferizedFrom(model, &result); + + const shapes = try collectShapes(allocator, &model); + defer allocator.free(shapes); + std.debug.print("Found {d} shapes\n", .{shapes.len}); + + var transfer: Transfer = try .init(allocator, shapes, platform); + errdefer transfer.deinit(allocator, platform); + + const LocalContext = struct { + shapes: []const Shape, + platform: Platform, + transfer: *Transfer, + index: usize = 0, + }; + var context: LocalContext = .{ .shapes = shapes, .platform = platform, .transfer = &transfer }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, transfer_entry: *TransferEntry) void { + transfer_entry.buffer = zml.Buffer.fromPjrtBuffers(context_.platform, context_.shapes[context_.index], &.{context_.transfer.get(context_.index).buffer}); + transfer_entry.writer = context_.transfer.get(context_.index).writer(); + context_.index += 1; + } + }.cb, &context, &result); + + return .{ transfer, result }; +} + +pub fn Transferized(comptime T: type) type { + return zml.meta.MapRestrict(Tensor, TransferEntry).map(T); +} + +fn initTransferizedFrom(model: anytype, transferized: *Transferized(@TypeOf(model))) void { + const Model = @TypeOf(model); + const type_info = @typeInfo(Transferized(Model)); + switch (type_info) { + .@"struct" => |struct_type_info| { + if (Transferized(Model) == TransferEntry) return; + inline for (struct_type_info.fields) |field| { + initTransferizedFrom(@field(model, field.name), &@field(transferized, field.name)); + } + }, + .@"union" => { + switch (model) { + inline else => |v, tag| { + transferized.* = @unionInit(Bufferized(Model), @tagName(tag), undefined); + initTransferizedFrom(v, @field(transferized, @tagName(tag))); + }, + } + }, + .optional => { + if (model == null) { + transferized.* = null; + } else { + transferized.* = undefined; + initTransferizedFrom(model.?, &transferized.*.?); + } + }, + .void, .int, .@"enum", .bool, .enum_literal, .float, .pointer, .vector => {}, + else => unreachable, + } +} + +pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), + }; +} + +const Transfer = struct { + entries: []Entry, + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + + pub const Writer = struct { + entry: *Entry, + }; + + pub const Entry = struct { + shape: Shape, + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + buffer_index: usize, + buffer: *pjrt.Buffer, + platform: Platform, + + pub fn writer(entry: *Entry) Writer { + return .{ .entry = entry }; + } + }; + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: Platform) !Transfer { + const shape_specs = try allocator.alloc(pjrt.ShapeSpec, shapes.len); + defer allocator.free(shape_specs); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + for (shape_specs, shapes) |*spec, shape| { + const dims = try arena.allocator().dupe(i64, shape.dims()); + spec.* = pjrt.ShapeSpec.init(dims, bufferTypeFromDtype(shape.dtype())); + } + + const memory = platform.pjrt_client.memoryByKind(platform.pjrt_api, .unpinned_host).?; + + const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); + errdefer transfer_manager.deinit(platform.pjrt_api); + + const count = transfer_manager.bufferCount(platform.pjrt_api) catch unreachable; + for (0..count) |index| { + const buffer_size = transfer_manager.bufferSize(platform.pjrt_api, index) catch unreachable; + std.debug.print("index: {d} - size: {d}\n", .{ index, buffer_size }); + } + + const entries = try allocator.alloc(Entry, shapes.len); + errdefer allocator.free(entries); + + for (entries, shapes, 0..) |*e, shape, index| { + const buffer = try transfer_manager.retrieveBuffer(platform.pjrt_api, index); + e.* = .{ + .shape = shape, + .transfer_manager = transfer_manager, + .buffer_index = index, + .buffer = buffer, + .platform = platform, + }; + } + + return .{ .entries = entries, .transfer_manager = transfer_manager }; + } + + pub fn deinit(self: Transfer, allocator: std.mem.Allocator, platform: Platform) void { + allocator.free(self.entries); + self.transfer_manager.deinit(platform.pjrt_api); + } + + pub fn get(self: *const Transfer, index: usize) *Entry { + return &self.entries[index]; + } +}; + +pub const Linear = struct { + weight: Tensor, + bias: ?Tensor = null, + tag: Shape.Tag, + + pub fn init(weight: Tensor, bias: ?Tensor, dot_tag: anytype) Linear { + return .{ + .weight = weight, + .bias = bias, + .tag = zml.Shape.toTag(dot_tag), + }; + } + + pub fn forward(self: Linear, x: Tensor) Tensor { + var y = x.dot(self.weight.convert(x.dtype()), self.tag); + + // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); + return if (self.bias) |bias| y.add(bias.autoBroadcast()) else y; + } +}; + +const KvCacheKind = enum { + split, + unified, +}; + +const KvCache = union(KvCacheKind) { + split: struct { + k: Tensor, + v: Tensor, + }, + unified: Tensor, +}; + +fn testKvCache(kv_cache: KvCache, input: Tensor) KvCache { + return switch (kv_cache) { + .split => |split_kv_cache| .{ + .split = .{ + .k = split_kv_cache.k.add(input.autoBroadcast()), + .v = split_kv_cache.v.add(input.autoBroadcast()), + }, + }, + .unified => |unified_kv_cache| .{ .unified = unified_kv_cache.add(input.autoBroadcast()) }, + }; +} + +const RmsNorm = struct { + weight: Tensor, + eps: f32 = 1e-5, + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: Tensor) Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + const normalized = zml.nn.rmsNorm(x, .d, self.eps); + return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); + } +}; + +pub fn testLinear2(linear1: Linear, linear2: Linear, x: Tensor) Tensor { + const out1 = linear1.forward(x).rename(.{ .dout = .d }); + std.debug.print("out1: {f}\n", .{out1}); + const out2 = linear2.forward(out1).rename(.{ .dout = .d }); + std.debug.print("out2: {f}\n", .{out1}); + return out2; +} + pub fn main() !void { try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); } +pub fn asyncMain2() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa.deinit(); + //const allocator = std.heap.c_allocator; + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const exe = b: { + const linear1: Linear = .init( + Tensor.init(zml.Shape.init(.{ .dout = 128, .d = 128 }, .f32)), + Tensor.init(zml.Shape.init(.{ .dout = 128 }, .f32)), + .d, + ); + const linear2: Linear = .init( + Tensor.init(zml.Shape.init(.{ .dout = 64, .d = 128 }, .f32)), + Tensor.init(zml.Shape.init(.{ .dout = 64 }, .f32)), + .d, + ); + + const x = Tensor.init(zml.Shape.init(.{ .b = 16, .s = 16, .d = 128 }, .f32)); + + const exe = try compile(allocator, testLinear2, .{ linear1, linear2, x }, platform); + errdefer exe.deinit(); + + break :b exe; + }; + defer exe.deinit(); +} + pub fn asyncMain() !void { - const allocator = std.heap.c_allocator; + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa.deinit(); + //const allocator = std.heap.c_allocator; + const allocator = gpa.allocator(); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); const process_args = try std.process.argsAlloc(allocator); defer std.process.argsFree(allocator, process_args); - const pt_model = process_args[1]; - const t10kfilename = process_args[2]; - _ = t10kfilename; // autofix - // Read model shapes. - // Note this works because Mnist struct uses the same layer names as the pytorch model - var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); + const model_config_path = try std.fs.path.join(allocator, &.{ process_args[1], "config.json" }); + defer allocator.free(model_config_path); + + const model_weights_path = b: { + const simple_path = try std.fs.path.join(allocator, &.{ process_args[1], "model.safetensors" }); + if (asynk.File.access(simple_path, .{})) { + break :b simple_path; + } else |_| { + allocator.free(simple_path); + } + + const sharded_path = try std.fs.path.join(allocator, &.{ process_args[1], "model.safetensors.index.json" }); + break :b sharded_path; + }; + defer allocator.free(model_weights_path); + + const config = blk: { + var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(&config_json_buffer); + var reader = std.json.Reader.init(allocator, &config_reader.interface); + defer reader.deinit(); + const config_obj = try std.json.parseFromTokenSourceLeaky(llama.LlamaLM.Config, arena.allocator(), &reader, .{ .ignore_unknown_fields = true }); + break :blk config_obj; + }; + + var old_buffer_store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); + defer old_buffer_store.deinit(); + + var buffer_store: BufferStore5 = try .fromBufferStore(allocator, old_buffer_store); defer buffer_store.deinit(); - const mnist: Mnist = .init(buffer_store); - _ = mnist; // autofix + var context = try zml.Context.init(); + defer context.deinit(); - const input: Tensor = .init(zml.Shape.init(.{ 28, 28 }, .u8)); - _ = input; // autofix + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + var llama_model = try llama.LlamaLM.init(allocator, buffer_store.view(), config); + defer llama_model.deinit(allocator); + + const llama_buffers = try llama.LlamaLM.loadBuffers(allocator, llama_model, buffer_store.view(), platform); + defer llama_buffers.deinit(allocator); + + //const process_args = try std.process.argsAlloc(allocator); + //defer std.process.argsFree(allocator, process_args); + //const pt_model = process_args[1]; + //const t10kfilename = process_args[2]; + //_ = t10kfilename; // autofix + + //// Read model shapes. + //// Note this works because Mnist struct uses the same layer names as the pytorch model + //var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); + //defer buffer_store.deinit(); + + //var buffer_store2 = try BufferStore2.fromBufferStore(allocator, buffer_store, platform); + //defer buffer_store2.deinit(); + + //var tensor_descriptor = try TensorDescriptor.fromBufferStore(allocator, buffer_store); + //defer tensor_descriptor.deinit(); + + //// Compile MNIST Model + //const mnist: Mnist = .init(tensor_descriptor.view()); + //const input: Tensor = .init(zml.Shape.init(.{ 28, 28 }, .u8)); + + //var buffer_store3 = try BufferStore3.fromTensorDescriptor(allocator, tensor_descriptor, buffer_store, platform); + //defer buffer_store3.deinit(); + + //std.debug.print("{f}\n", .{mnist.layers[0].weight.shape()}); + + //const exe = try compileModel(allocator, Mnist.forward, mnist, .{input}, platform); + //defer exe.deinit(); + + //// Create model, input and output buffers. + //const input_buffer = try zml.Buffer.fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]u8{0} ** (28 * 28))); + //_ = input_buffer; // autofix + ////const mnist_buffer = try Mnist.initBuffer(buffer_store2); + + //const mnist_buffer = try bufferized(allocator, mnist, buffer_store3); + //_ = mnist_buffer; // autofix + + //var buffer_store4: BufferStore4 = try .fromBufferStore(allocator, buffer_store, tensor_descriptor); + //defer buffer_store4.deinit(); + + //const mnist_buffer2 = try Mnist.loadBuffers(allocator, mnist, buffer_store4.view(), platform); + //_ = mnist_buffer2; // autofix + + //var it = tensor_descriptor.map.iterator(); + //while (it.next()) |entry| { + // std.debug.print("key: \"{s}\" - value: {any}\n", .{ entry.key_ptr.*, entry.value_ptr.* }); + //} + + //const mnist_buffer, const arena = try bufferized(mnist, buffer_store2); + + // Create arguments (also prepare them) and results helper. + //var args = try exe.args(allocator); + //defer args.deinit(allocator); + //args.set(.{ mnist_buffer, input_buffer }); + + //var results = try exe.results(allocator); + //defer results.deinit(allocator); + + //exe.call(args, &results); + //const output = results.get(zml.Buffer); + //defer output.deinit(); + + //const linear: Linear = .{ + // .weight = Tensor.init(Shape.init(.{ .d = 128, .d_out = 128 }, .f32)), + // .bias = Tensor.init(Shape.init(.{ .d_out = 128 }, .f32)), + // .tag = Shape.toTag(.d), + //}; + + //const mlp: Mlp = .{ + // .up_proj = .{ + // .weight = Tensor.init(Shape.init(.{ .d_hidden = 128, .d = 128 }, .f32)), + // .bias = Tensor.init(Shape.init(.{ .d_hidden = 128 }, .f32)), + // }, + // .gate_proj = .{ + // .weight = Tensor.init(Shape.init(.{ .d_hidden = 128, .d = 128 }, .f32)), + // .bias = Tensor.init(Shape.init(.{ .d_hidden = 128 }, .f32)), + // }, + // .down_proj = .{ + // .weight = Tensor.init(Shape.init(.{ .d = 128, .d_hidden = 128 }, .f32)), + // .bias = Tensor.init(Shape.init(.{ .d = 128 }, .f32)), + // }, + //}; + //_ = mlp; // autofix + + //const x = Tensor.init(Shape.init(.{ .b = 16, .d = 128 }, .f32)); //compileModel(allocator, Mnist.forward, mnist, .{input}); + //const exe = try compile(allocator, testLinear, .{ linear, x }, platform); + //defer exe.deinit(); - compile(allocator, testDot, .{ Tensor.init(.init(.{ .m = 256, .n = 128 }, .f32)), Tensor.init(.init(.{ .b = 16, .n = 128, .k = 256 }, .f32)) }); + //const linear_buffers = zml.meta.MapRestrict(Tensor, zml.Buffer).map(Linear){ + // .weight = try zml.Buffer.fromBytes(platform, linear.weight.shape(), std.mem.sliceAsBytes(&[_]f32{2} ** (128 * 128))), + // .bias = try zml.Buffer.fromBytes(platform, linear.bias.?.shape(), std.mem.sliceAsBytes(&[_]f32{3} ** (128))), + //}; + //const x_buffer = try zml.Buffer.fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** (16 * 128))); + + //var inputs: [3]zml.Buffer = undefined; + //var outputs: [1]zml.Buffer = undefined; + + //serialize(&.{ linear_buffers, x_buffer }, &inputs); + //exe.call(&inputs, &outputs); + //var result: zml.Buffer = undefined; + //deserialize(&result, outputs[0..1]); + + //var host = try result.toHostAlloc(allocator); + //defer host.deinit(allocator); + + //std.debug.print("{any}\n", .{host.items(f32)}); + + //compile(allocator, testMlp, .{ mlp, x }); + + //compile(allocator, testDot, .{ Tensor.init(.init(.{ .m = 256, .n = 128 }, .f32)), Tensor.init(.init(.{ .b = 16, .n = 128, .k = 256 }, .f32)) }); + + //try testSplitKvCache(allocator, platform); + //try testUnifiedKvCache(allocator, platform); + //try testMnist(allocator, buffer_store2, mnist, platform); } +//fn testUnifiedKvCache(allocator: std.mem.Allocator, platform: Platform) !void { +// const kv_cache: KvCache = .{ +// .unified = Tensor.init(Shape.init(.{ .layer = 10, .h = 16, .hd = 128 }, .f32)), +// }; +// +// const input = Tensor.init(Shape.init(.{ .hd = 128 }, .f32)); +// const exe = try compile(allocator, testKvCache, .{ kv_cache, input }, platform); +// defer exe.deinit(); +// +// var kv_cache_buffers: zml.meta.MapRestrict(Tensor, zml.Buffer).map(KvCache) = .{ +// .unified = try zml.Buffer.fromBytes(platform, kv_cache.unified.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** (10 * 16 * 128))), +// }; +// const input_buffer = try zml.Buffer.fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** 128)); +// +// var inputs: [2]zml.Buffer = undefined; +// var outputs: [1]zml.Buffer = undefined; +// +// serialize(&.{ kv_cache_buffers, input_buffer }, &inputs); +// exe.call(&inputs, &outputs); +// deserialize(&kv_cache_buffers, &outputs); +//} + +//fn testSplitKvCache(allocator: std.mem.Allocator, platform: Platform) !void { +// const kv_cache: KvCache = .{ +// .split = .{ +// .k = Tensor.init(Shape.init(.{ .layer = 10, .h = 8, .hd = 128 }, .f32)), +// .v = Tensor.init(Shape.init(.{ .layer = 10, .h = 8, .hd = 128 }, .f32)), +// }, +// }; +// +// const input = Tensor.init(Shape.init(.{ .hd = 128 }, .f32)); +// const exe = try compile(allocator, testKvCache, .{ kv_cache, input }, platform); +// defer exe.deinit(); +// +// var kv_cache_buffers: zml.meta.MapRestrict(Tensor, zml.Buffer).map(KvCache) = .{ +// .split = .{ +// .k = try zml.Buffer.fromBytes(platform, kv_cache.split.k.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** (10 * 8 * 128))), +// .v = try zml.Buffer.fromBytes(platform, kv_cache.split.k.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** (10 * 8 * 128))), +// }, +// }; +// const input_buffer = try zml.Buffer.fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** 128)); +// +// var inputs: [3]zml.Buffer = undefined; +// var outputs: [2]zml.Buffer = undefined; +// +// serialize(&.{ kv_cache_buffers, input_buffer }, &inputs); +// exe.call(&inputs, &outputs); +// deserialize(&kv_cache_buffers, &outputs); +//} + fn testDot(a: Tensor, b: Tensor) Tensor { - _ = a.dot2(b, .n); - _ = a.dot2(b, .{ .n = .k }); - _ = a.dot2(b, .{ .contracting = .{ .n = .k }, .batching = .{.b} }); - _ = a.dot2(b, .{ .contracting = .{.n}, .batching = .{.b} }); + _ = a.dot(b, .n); + _ = a.dot(b, .{ .n = .k }); + _ = a.dot(b, .{ .contracting = .{ .n = .k }, .batching = .{.b} }); + _ = a.dot(b, .{ .contracting = .{.n}, .batching = .{.b} }); unreachable; //return a.dotGeneral(b, &.{.{ 1, 1 }}, &.{}); } +fn testLinear(linear: Linear, x: Tensor) Tensor { + return linear.forward(x); +} + const CompilationContext = struct { allocator: std.mem.Allocator, mlir_registry: *mlir.DialectRegistry, mlir_ctx: *mlir.Context, + mlir_pass_manager: *mlir.PassManager, + //mlir_op_pass_manager: *mlir.OpPassManager, module: *mlir.Module, blocks: stdx.BoundedArray(*mlir.Block, 16) = .{}, @@ -1209,27 +1536,38 @@ const CompilationContext = struct { const module = mlir.Module.init(.unknown(mlir_ctx)); module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); + const pass_manager = mlir.PassManager.init(mlir_ctx); + pass_manager.enableIRPrinting(.{ + .printBeforeAll = true, + }); + { + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + opm.addPipeline(pass) catch unreachable; + } + } + return .{ .allocator = allocator, .mlir_registry = mlir_registry, .mlir_ctx = mlir_ctx, + .mlir_pass_manager = pass_manager, .module = module, }; } pub fn deinit(self: *CompilationContext) void { + self.mlir_pass_manager.deinit(); self.module.deinit(); self.mlir_ctx.deinit(); self.mlir_registry.deinit(); } - //pub fn registerTensor(self: *CompilationContext, tensor_id: usize, argument_id: usize) !void { - // const gop = try self.mapping.getOrPut(self.allocator, tensor_id); - // std.debug.assert(!gop.found_existing); - - // gop.value_ptr.* = argument_id; - //} - pub fn currentBlock(self: *const CompilationContext) *mlir.Block { return self.blocks.get(self.blocks.len - 1); } @@ -1265,18 +1603,86 @@ const Context = struct { current_id: usize = 0, }; -fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) void { - compile(allocator, func, .{model} ++ args); +fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), platform: Platform) !Exe { + return compile(allocator, func, .{model} ++ args, platform); } -fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx.meta.FnArgs(func)) void { +pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { var compilation_context: CompilationContext = .init(allocator); defer compilation_context.deinit(); - emitMlir(&compilation_context, func, args) catch unreachable; + const result = emitMlir(&compilation_context, func, args) catch unreachable; + + _ = result.func.appendTo(compilation_context.module.body()); + defer compilation_context.allocator.free(result.output_shapes); + defer compilation_context.allocator.free(result.input_shapes); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), platform, compilation_context.module, null) catch unreachable; + + const num_devices = platform.sharding().num_replicas * platform.sharding().num_partitions; + const exe = try Exe.init(allocator, platform, loaded_executable, result.input_shapes, result.output_shapes, num_devices); + errdefer exe.deinit(); + + return exe; } -fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !void { +fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { + const LocalContext = struct { + list: *std.array_list.Managed(Shape), + }; + var list = std.array_list.Managed(Shape).init(allocator); + var context: LocalContext = .{ .list = &list }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + ctx_.list.append(tensor.shape()) catch unreachable; + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectValues(allocator: std.mem.Allocator, v: anytype) ![]*const mlir.Value { + const LocalContext = struct { + list: *std.array_list.Managed(*const mlir.Value), + }; + var list = std.array_list.Managed(*const mlir.Value).init(allocator); + var context: LocalContext = .{ .list = &list }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + ctx_.list.append(tensor.value()) catch unreachable; + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectReaders(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, v: anytype) ![]BufferStore5.Reader { + const LocalContext = struct { + list: *std.array_list.Managed(BufferStore5.Reader), + buffer_store: BufferStore5.View, + }; + var list = std.array_list.Managed(BufferStore5.Reader).init(allocator); + var context: LocalContext = .{ .list = &list, .buffer_store = buffer_store }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + const reader = ctx_.buffer_store.getReaderFromId(tensor.id).?; + ctx_.list.append(reader) catch unreachable; + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +const EmitMlirResult = struct { + func: *mlir.Operation, + input_shapes: []const Shape, + output_shapes: []const Shape, +}; + +fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); defer arena.deinit(); @@ -1307,41 +1713,958 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar } }.cb, &context, &args); - { + const input_shapes = try collectShapes(compilation_context.allocator, &args); + errdefer compilation_context.allocator.free(input_shapes); + + const output_shapes = b: { global_compilation_context = compilation_context; defer global_compilation_context = null; const result = @call(.auto, func, args); - _ = dialects.func.return_(compilation_context.mlir_ctx, result.value(), .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentBlock()); - } + const output_shapes = try collectShapes(compilation_context.allocator, &result); + const output_values = try collectValues(arena.allocator(), &result); + _ = dialects.func.returns(compilation_context.mlir_ctx, output_values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentBlock()); + break :b output_shapes; + }; + errdefer compilation_context.allocator.free(output_shapes); - const mlir_fn = dialects.func.func(compilation_context.mlir_ctx, .{ + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ .name = "main", .block = compilation_context.currentBlock(), .location = .unknown(compilation_context.mlir_ctx), }); - var pass_manager = mlir.PassManager.init(compilation_context.mlir_ctx); - pass_manager.enableIRPrinting(.{ - .printBeforeAll = true, - }); - { - var opm = pass_manager.asOpPassManager(); - const passes: []const []const u8 = &.{ - "canonicalize", - "cse", - "canonicalize", - }; - for (passes) |pass| { - try opm.addPipeline(pass); - } - } - - pass_manager.runOnOp(mlir_fn) catch |err| { - std.debug.print("Failed to optimize module: {any}\n", .{err}); - return err; + compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { + error.MlirUnexpected => { + std.log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_func}); + // user errors should have triggered a panic before we reach this. + @panic("ZML generated invalid mlir. Please open a bug report"); + }, }; - _ = mlir_fn.appendTo(module.body()); - - std.debug.print("{f}\n", .{module}); + return .{ + .func = mlir_func, + .input_shapes = input_shapes, + .output_shapes = output_shapes, + }; } + +fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { + const result = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView(flag) }, + .{ .msg_val = blk: { + const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); + switch (@typeInfo(@TypeOf(value))) { + .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), + .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), + .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), + else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), + } + break :blk @ptrCast(field); + } }, + upb_arena, + ); + + if (result == false) { + return std.mem.Allocator.Error.OutOfMemory; + } +} + +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { + //const tracer = Tracer.init("ai.zml.compilation"); + //const compile_frame = tracer.frameStart("pjrt compilation"); + //defer tracer.frameEnd(compile_frame, "pjrt compilation"); + + const sharding = platform.sharding(); + + var upb_alloc: upb.Allocator = .init(arena); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const options = blk: { + const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); + c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { + const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); + + c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { + const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); + for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { + computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); + } + break :device_assignment_blk device_assignment; + }); + + break :executable_build_options_blk exec_build_options; + }); + + const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); + switch (platform.target) { + .cuda => { + // NVIDIA recommends these settings + // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_libnvptxcompiler", true, upb_arena); + }, + .rocm => { + // Disable Triton GEMM on ROCM. For some reason it's much, much slower when + // enabled on CDNA and it's used on RDNA. Disable it altogether. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + // Use lld from libllvm instead of invoking the ld.lld binary. + // This saves us from having to sandbox it. + try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); + }, + else => {}, + } + + if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { + try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); + if (platform.compilation_options.xla_dump_fusion_visualization) { + try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); + } + if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); + } + } + + break :blk options; + }; + + const loaded_executable = try platform.pjrt_client.compile( + platform.pjrt_api, + arena, + module, + try upb.serialize(options, upb_arena), + ); + errdefer loaded_executable.deinit(); + + return loaded_executable; +} + +const Exe = struct { + platform: Platform, + exe: *pjrt.LoadedExecutable, + + context: ?*pjrt.ExecuteContext = null, + + input_shapes: []const Shape, + output_shapes: []const Shape, + + num_devices: u8, + + arena: std.heap.ArenaAllocator, + + pub fn init( + allocator: std.mem.Allocator, + platform: Platform, + exe: *pjrt.LoadedExecutable, + input_shapes: []const Shape, + output_shapes: []const Shape, + num_devices: u8, + ) !Exe { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); + const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); + + return .{ + .platform = platform, + .exe = exe, + .input_shapes = input_shapes_copy, + .output_shapes = output_shapes_copy, + .num_devices = num_devices, + .arena = arena, + }; + } + + pub fn deinit(self: *const Exe) void { + self.arena.deinit(); + } + + pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { + return Arguments.init(allocator, self.input_shapes, self.num_devices); + } + + pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { + return Results.init(allocator, self.output_shapes, self.num_devices, self.platform); + } + + pub const FlatBuffers = struct { + buffers: []const [*]*pjrt.Buffer, + raw_buffers: []const *pjrt.Buffer, + + num_devices: usize, + + pub fn init(allocator: std.mem.Allocator, count: usize, num_devices: usize) !FlatBuffers { + const raw_buffers = try allocator.alloc(*pjrt.Buffer, num_devices * count); + errdefer allocator.free(raw_buffers); + + const buffers = try allocator.alloc([*]*pjrt.Buffer, num_devices); + errdefer allocator.free(buffers); + + for (0..num_devices) |i| { + buffers[i] = raw_buffers[i * count ..].ptr; + } + + return .{ + .buffers = buffers, + .raw_buffers = raw_buffers, + .num_devices = num_devices, + }; + } + + pub fn deinit(self: *const FlatBuffers, allocator: std.mem.Allocator) void { + allocator.free(self.buffers); + allocator.free(self.raw_buffers); + } + }; + + pub const Arguments = struct { + flat_buffers: FlatBuffers, + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Arguments, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn set(self: *Arguments, v: anytype) void { + return self.setPartial(v, 0); + } + + pub fn setPartial(self: *Arguments, v: anytype, offset: usize) void { + const LocalContext = struct { + self: *Arguments, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = offset }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const zml.Buffer) void { + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + } + }; + + pub const Results = struct { + platform: Platform, + flat_buffers: FlatBuffers, + + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: Platform) !Results { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .platform = platform, + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Results, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn get(self: *Results, comptime T: type) T { + var result: T = undefined; + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { + var shards: zml.Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = zml.Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, &result); + return result; + } + + pub fn fill(self: *Results, v: anytype) void { + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { + var shards: zml.Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = zml.Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, v); + } + }; + + pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { + var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; + const sharding = self.platform.sharding(); + + self.exe.execute(self.platform.pjrt_api, .{ + .arguments = arguments.flat_buffers.buffers, + .num_args = arguments.expected_shapes.len, + .results = results_.flat_buffers.buffers, + .events = events[0..sharding.num_partitions], + // this allows to tell a specific buffer shouldn't be donated, + // even if it has been marked as "can be donated" during compilation. + // TODO: expose it ? + .non_donatable_input_indices = &.{}, + .context = self.context, + }) catch |err| { + std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); + }; + + for (events[0..sharding.num_partitions]) |e| { + if (e) |ev| { + ev.await_(self.platform.pjrt_api) catch unreachable; + } + } + } +}; + +pub const TensorDescriptor = struct { + map: std.StringHashMapUnmanaged(Entry), + allocator: std.mem.Allocator, + + const Entry = struct { + shape: zml.Shape, + associated_tensor_id: ?usize = null, + }; + + pub const View = struct { + tensor_descriptor: *const TensorDescriptor, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + + return .{ + .tensor_descriptor = self.tensor_descriptor, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + fn prefix(self: *const View) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + pub fn get(self: *const View, subkey: []const u8) ?zml.Shape { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + std.debug.print("Trying to get: {s}\n", .{key}); + const entry = self.tensor_descriptor.getPtr(key) orelse return null; + return entry.shape; + } + + pub fn getWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + std.debug.print("Trying to get: {s}\n", .{key}); + const ptr = self.tensor_descriptor.getPtr(key) orelse return null; + ptr.shape = ptr.shape.withTags(tagz).withSharding(.{0}); + const tensor = Tensor.init(ptr.shape); + ptr.associated_tensor_id = tensor.id; + return tensor; + } + }; + + pub fn init(allocator: std.mem.Allocator) !TensorDescriptor { + return .{ + .map = .{}, + .allocator = allocator, + }; + } + + pub fn deinit(self: *TensorDescriptor) void { + var it = self.map.iterator(); + while (it.next()) |entry| { + self.allocator.free(entry.key_ptr.*); + } + self.map.deinit(self.allocator); + } + + pub fn add(self: *TensorDescriptor, key: []const u8, shape: zml.Shape) !void { + const key_copy = try self.allocator.dupe(u8, key); + errdefer self.allocator.free(key_copy); + + const gop = try self.map.getOrPut(self.allocator, key_copy); + if (gop.found_existing) { + return error.AlreadyExists; + } + + errdefer self.map.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = .{ .shape = shape }; + } + + pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore) !TensorDescriptor { + var new_tensor_descriptor: TensorDescriptor = try .init(allocator); + errdefer new_tensor_descriptor.deinit(); + + var it = buffer_store.buffers.iterator(); + while (it.next()) |entry| { + try new_tensor_descriptor.add(entry.key_ptr.*, entry.value_ptr.shape()); + } + + return new_tensor_descriptor; + } + + pub fn withPrefix(self: *const TensorDescriptor, prefix_: []const u8) TensorDescriptor { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + + return .{ + .map = self.map, + .allocator = self.allocator, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + fn prefix(self: *const TensorDescriptor) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + fn getPtr(self: *const TensorDescriptor, key: []const u8) ?*Entry { + return self.map.getPtr(key); + } + + pub fn view(self: *const TensorDescriptor) View { + return .{ .tensor_descriptor = self }; + } +}; + +const BufferStore2 = struct { + buffers: std.StringHashMapUnmanaged(zml.Buffer), + allocator: std.mem.Allocator, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn init(allocator: std.mem.Allocator) !BufferStore2 { + return .{ + .buffers = .{}, + .allocator = allocator, + }; + } + + pub fn deinit(self: *BufferStore2) void { + var it = self.buffers.iterator(); + while (it.next()) |entry| { + entry.value_ptr.deinit(); + self.allocator.free(entry.key_ptr.*); + } + self.buffers.deinit(self.allocator); + } + + pub fn add(self: *BufferStore2, key: []const u8, buffer: zml.Buffer) !void { + const key_copy = try self.allocator.dupe(u8, key); + errdefer self.allocator.free(key_copy); + + const gop = try self.buffers.getOrPut(self.allocator, key_copy); + if (gop.found_existing) { + return error.AlreadyExists; + } + + errdefer self.buffers.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = buffer; + } + + pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore, platform: Platform) !BufferStore2 { + var new_buffer_store: BufferStore2 = try .init(allocator); + errdefer new_buffer_store.deinit(); + + var it = buffer_store.buffers.iterator(); + while (it.next()) |entry| { + try new_buffer_store.add(entry.key_ptr.*, try entry.value_ptr.toDevice(platform)); + } + + return new_buffer_store; + } + + pub fn withPrefix(self: *const BufferStore2, prefix_: []const u8) BufferStore2 { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + + return .{ + .buffers = self.buffers, + .allocator = self.allocator, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + fn prefix(self: *const BufferStore2) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + fn get(self: *const BufferStore2, subkey: []const u8) ?zml.Buffer { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + std.debug.print("Trying to get: {s}\n", .{key}); + return self.buffers.get(key); + } +}; + +const BufferStore3 = struct { + buffers: std.ArrayList(zml.Buffer) = .{}, + id_to_buffer: std.AutoHashMapUnmanaged(usize, usize) = .{}, + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) !BufferStore3 { + return .{ + .allocator = allocator, + }; + } + + pub fn deinit(self: *BufferStore3) void { + self.buffers.deinit(self.allocator); + self.id_to_buffer.deinit(self.allocator); + } + + pub fn fromTensorDescriptor(allocator: std.mem.Allocator, tensor_descriptor: TensorDescriptor, old_buffer_store: zml.aio.BufferStore, platform: Platform) !BufferStore3 { + var buffer_store: BufferStore3 = try .init(allocator); + errdefer for (buffer_store.buffers.items) |*item| item.deinit(); + errdefer buffer_store.deinit(); + + var it = tensor_descriptor.map.iterator(); + while (it.next()) |entry| { + var host_buffer = old_buffer_store.get(entry.key_ptr.*).?; + host_buffer._shape = entry.value_ptr.shape; + const buffer = try host_buffer.toDevice(platform); + errdefer buffer.deinit(); + + try buffer_store.add(buffer, entry.value_ptr.associated_tensor_id.?); + } + return buffer_store; + } + + pub fn add(self: *BufferStore3, buffer: zml.Buffer, id: usize) !void { + const gop = try self.id_to_buffer.getOrPut(self.allocator, id); + if (gop.found_existing) return error.AlreadyExists; + + errdefer self.id_to_buffer.removeByPtr(gop.key_ptr); + + const new_index = self.buffers.items.len; + + try self.buffers.append(self.allocator, buffer); + errdefer _ = self.buffers.pop(); + + gop.value_ptr.* = new_index; + } + + pub fn getFromId(self: *const BufferStore3, id: usize) ?zml.Buffer { + const index = self.id_to_buffer.get(id) orelse return null; + return self.buffers.items[index]; + } +}; + +pub fn bufferized(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferStore3) !Bufferized(@TypeOf(model)) { + const T = @TypeOf(model); + const BufferizedT = Bufferized(T); + var buf: BufferizedT = undefined; + + const LocalContext = struct { + buffer_store: *const BufferStore3, + }; + + var context: LocalContext = .{ .buffer_store = &buffer_store }; + + try zml.meta.mapAlloc(struct { + fn cb(context_: *LocalContext, tensor: Tensor) zml.Buffer { + const buffer = context_.buffer_store.getFromId(tensor.id).?; + return buffer; + } + }.cb, allocator, &context, model, &buf); + + return buf; +} + +pub fn Bufferized(comptime T: type) type { + // TODO: we should strip out the non-buffer fields. + // Currently it's confusing cause the Bufferized struct contains field that are never read. + // Also it will simplify the layout of the Bufferized struct. + // accelerating the calls to execute. + return zml.meta.MapRestrict(Tensor, zml.Buffer).map(T); +} + +pub const ShardWriter = struct { + tensor_shape: Shape, + shard_shape: Shape, + interface: std.Io.Writer, + out: *std.Io.Writer, + + tensor_index: usize = 0, + shard_index: usize = 0, + shard_coordinates: [2]usize, + + fn init(out: *std.Io.Writer, buffer: []u8, tensor_shape: Shape, shard_shape: Shape) ShardWriter { + return .{ + .tensor_shape = tensor_shape, + .shard_shape = shard_shape, + .interface = .{ + .buffer = buffer, + .vtable = &.{ .drain = drain }, + }, + .out = out, + }; + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *ShardWriter = @alignCast(@fieldParentPtr("interface", w)); + _ = self; // autofix + _ = data; // autofix + _ = splat; // autofix + } + + fn computeCoordinates(self: ShardWriter, offset: usize) [2]usize { + const y = offset / self.tensor_shape.dim(0); + const x = offset % self.tensor_shape.dim(0); + + const x_shard = @divFloor(x, self.shard_shape.dim(1)); + const y_shard = @divFloor(y, self.shard_shape.dim(0)); + + return .{ x_shard, y_shard }; + } +}; + +const Slice = struct { + shape: Shape, + data: []const u8, +}; + +pub const BufferStore4 = struct { + slices: std.StringHashMapUnmanaged(Entry), + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) !BufferStore4 { + return .{ .slices = .{}, .allocator = allocator }; + } + + pub fn deinit(self: *BufferStore4) void { + var it = self.slices.iterator(); + while (it.next()) |entry| self.allocator.free(entry.key_ptr.*); + self.slices.deinit(self.allocator); + } + + pub fn add(self: *BufferStore4, key: []const u8, slice: Slice, associated_tensor_id: ?usize) !void { + const key_copy = try self.allocator.dupe(u8, key); + errdefer self.allocator.free(key_copy); + + const gop = try self.slices.getOrPut(self.allocator, key_copy); + if (gop.found_existing) { + return error.AlreadyExists; + } + + errdefer self.slices.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = .{ .slice = slice, .associated_tensor_id = associated_tensor_id }; + } + + pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore, tensor_descriptor: TensorDescriptor) !BufferStore4 { + var new_buffer_store: BufferStore4 = try .init(allocator); + errdefer new_buffer_store.deinit(); + + var it = buffer_store.buffers.iterator(); + while (it.next()) |entry| { + const descriptor_entry = tensor_descriptor.getPtr(entry.key_ptr.*).?; + const slice: Slice = .{ .shape = entry.value_ptr.shape(), .data = entry.value_ptr.bytes() }; + try new_buffer_store.add(entry.key_ptr.*, slice, descriptor_entry.associated_tensor_id); + } + + return new_buffer_store; + } + + pub const Reader = struct { + entry: *const Entry, + pub fn stream(self: Reader, writer: Transfer.Writer) void { + const chunk_size = 1024 * 1024; + const chunk_count = (self.entry.slice.data.len + chunk_size - 1) / chunk_size; + + for (0..chunk_count) |chunk_index| { + const start = chunk_index * chunk_size; + const end = @min((chunk_index + 1) * chunk_size, self.entry.slice.data.len); + const is_last_transfer = chunk_index == chunk_count - 1; + _ = writer.entry.transfer_manager.transferData(writer.entry.platform.pjrt_api, writer.entry.buffer_index, self.entry.slice.data[start..end], @intCast(start), is_last_transfer) catch unreachable; + } + } + }; + + pub const Entry = struct { + slice: Slice, + associated_tensor_id: ?usize = null, + pub fn reader(entry: *const Entry) Reader { + return .{ .entry = entry }; + } + }; + + pub fn get(self: BufferStore4, key: []const u8) ?*Entry { + std.debug.print("Trying to get {s}\n", .{key}); + return self.slices.getPtr(key); + } + + pub fn view(self: *const BufferStore4) View { + return .{ .buffer_store = self }; + } + + pub const View = struct { + buffer_store: *const BufferStore4, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + + return .{ + .buffer_store = self.buffer_store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + fn prefix(self: *const View) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + pub fn get(self: *const View, subkey: []const u8) ?*Entry { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + std.debug.print("Trying to get: {s}\n", .{key}); + return self.buffer_store.get(key); + } + }; +}; + +pub const BufferStore5 = struct { + key_map: std.StringHashMapUnmanaged(usize), + id_map: std.AutoHashMapUnmanaged(usize, usize), + entries: std.ArrayList(Entry), + allocator: std.mem.Allocator, + + pub const Entry = struct { + shape: zml.Shape, + // TODO(Corentin): Replace with reader ? + data: []const u8, + associated_tensor_id: ?usize = null, + }; + + pub fn init(allocator: std.mem.Allocator) !BufferStore5 { + return .{ + .key_map = .empty, + .id_map = .empty, + .entries = .empty, + .allocator = allocator, + }; + } + + pub fn deinit(self: *BufferStore5) void { + var it = self.key_map.iterator(); + while (it.next()) |entry| { + self.allocator.free(entry.key_ptr.*); + } + self.key_map.deinit(self.allocator); + self.id_map.deinit(self.allocator); + self.entries.deinit(self.allocator); + } + + pub fn add(self: *BufferStore5, key: []const u8, shape: zml.Shape, data: []const u8) !void { + const new_entry_index = self.entries.items.len; + const new_entry = try self.entries.addOne(self.allocator); + errdefer _ = self.entries.pop(); + + const key_copy = try self.allocator.dupe(u8, key); + errdefer self.allocator.free(key_copy); + + const gop = try self.key_map.getOrPut(self.allocator, key_copy); + if (gop.found_existing) { + return error.AlreadyExists; + } + errdefer self.key_map.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = new_entry_index; + new_entry.* = .{ .shape = shape, .data = data }; + } + + fn bindIdToKey(self: *BufferStore5, key: []const u8, id: usize) !void { + const index = self.key_map.get(key).?; + + const gop = try self.id_map.getOrPut(self.allocator, id); + if (gop.found_existing) { + stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.value_ptr.* }); + } + errdefer self.id_map.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = index; + } + + pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore) !BufferStore5 { + var new_buffer_store: BufferStore5 = try .init(allocator); + errdefer new_buffer_store.deinit(); + + var it = buffer_store.buffers.iterator(); + while (it.next()) |entry| { + try new_buffer_store.add(entry.key_ptr.*, entry.value_ptr.shape(), entry.value_ptr.bytes()); + } + + return new_buffer_store; + } + + fn getPtrFromKey(self: *const BufferStore5, key: []const u8) ?*Entry { + const index = self.key_map.get(key) orelse return null; + return &self.entries.items[index]; + } + + fn getPtrFromId(self: *const BufferStore5, id: usize) ?*Entry { + const index = self.id_map.get(id) orelse return null; + return &self.entries.items[index]; + } + + pub fn view(self: *BufferStore5) View { + return .{ .store = self }; + } + + pub const Reader = struct { + entry: *const Entry, + pub fn stream(self: Reader, writer: Transfer.Writer) void { + const chunk_size = 1024 * 1024; + const chunk_count = (self.entry.data.len + chunk_size - 1) / chunk_size; + + for (0..chunk_count) |chunk_index| { + const start = chunk_index * chunk_size; + const end = @min((chunk_index + 1) * chunk_size, self.entry.data.len); + const is_last_transfer = chunk_index == chunk_count - 1; + _ = writer.entry.transfer_manager.transferData(writer.entry.platform.pjrt_api, writer.entry.buffer_index, self.entry.data[start..end], @intCast(start), is_last_transfer) catch unreachable; + } + } + }; + + pub const View = struct { + store: *BufferStore5, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + fn prefix(self: *const View) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + + const tensor = Tensor.init(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensor(self: View, subkey: []const u8) Tensor { + return self.maybeCreateTensor(subkey).?; + } + + pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + ptr.shape = ptr.shape.withTags(tagz).withSharding(.{0}); + + const tensor = Tensor.init(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) Tensor { + return self.maybeCreateTensorWithTags(subkey, tagz).?; + } + + pub fn getReader(self: View, subkey: []const u8) ?Reader { + return self.getMaybeReader(subkey).?; + } + + pub fn getMaybeReader(self: View, subkey: []const u8) ?Reader { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return .{ .entry = entry_ptr }; + } + + pub fn getReaderFromId(self: View, id: usize) ?Reader { + const entry_ptr = self.store.getPtrFromId(id) orelse return null; + return .{ .entry = entry_ptr }; + } + + pub fn getShape(self: View, subkey: []const u8) ?Shape { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + }; +}; diff --git a/mlir/mlir.zig b/mlir/mlir.zig index d55c0ecd..b017693d 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -129,8 +129,9 @@ pub const PassManager = opaque { } pub fn runOnOp(self: *const PassManager, op: *Operation) !void { - if (c.mlirLogicalResultIsFailure(c.mlirPassManagerRunOnOp(self.ptr(), op.ptr()))) { - // return error.MlirUnexpected; + const result = c.mlirPassManagerRunOnOp(self.ptr(), op.ptr()); + if (result.value == 0) { + return error.MlirUnexpected; } } @@ -1155,7 +1156,7 @@ pub const Operation = opaque { sizes.appendAssumeCapacity(@intCast(segment_operands.len)); } state.addAttributes(&.{ - .named(ctx, "operandSegmentSizes", denseArrayAttribute(ctx, .i32, sizes.constSlice())), + .named(ctx, "operandSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32)).shaped(), sizes.constSlice())), }); }, }; @@ -1168,7 +1169,7 @@ pub const Operation = opaque { sizes.appendAssumeCapacity(@intCast(segment_results.len)); } state.addAttributes(&.{ - .named(ctx, "resultSegmentSizes", denseArrayAttribute(ctx, .i32, sizes.constSlice())), + .named(ctx, "resultSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32)).shaped(), sizes.constSlice())), }); }, }; @@ -1247,7 +1248,9 @@ pub const Operation = opaque { }; }; - pub fn writeBytecode(self: *const Operation, config: ?BytecodeWriterConfig, writer: *std.Io.Writer) !void { + const WriteBytecodeError = error{InvalidMlirBytecodeVersion} || std.Io.Writer.Error; + + pub fn writeBytecode(self: *const Operation, config: ?BytecodeWriterConfig, writer: *std.Io.Writer) WriteBytecodeError!void { if (config) |config_| { const bc_config = c.mlirBytecodeWriterConfigCreate(); defer c.mlirBytecodeWriterConfigDestroy(bc_config); @@ -1255,7 +1258,12 @@ pub const Operation = opaque { c.mlirBytecodeWriterConfigDesiredEmitVersion(bc_config, @intCast(version)); } var sctx: StringCallbackCtx = .{ .writer = writer }; - c.mlirOperationWriteBytecodeWithConfig(self.ptr(), bc_config, stringCallback, &sctx); + const logical_result = c.mlirOperationWriteBytecodeWithConfig(self.ptr(), bc_config, stringCallback, &sctx); + + if (logical_result.value == 0) { + return WriteBytecodeError.InvalidMlirBytecodeVersion; + } + if (sctx.err) |err| { return err; } diff --git a/mlir/writer.zig b/mlir/writer.zig new file mode 100644 index 00000000..c00d8fa6 --- /dev/null +++ b/mlir/writer.zig @@ -0,0 +1,190 @@ +const std = @import("std"); +const asynk = @import("async"); + +const stdx = @import("stdx"); +const zml = @import("zml"); +const Shape = zml.Shape; + +pub const ShardWriter = struct { + tensor_shape: Shape, + shard_shape: Shape, + interface: std.Io.Writer, + out: *std.Io.Writer, + + fn init(out: *std.Io.Writer, buffer: []u8, tensor_shape: Shape, shard_shape: Shape) ShardWriter { + return .{ + .tensor_shape = tensor_shape, + .shard_shape = shard_shape, + .interface = .{ + .buffer = buffer, + .vtable = &.{ .drain = drain }, + }, + .out = out, + }; + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *ShardWriter = @alignCast(@fieldParentPtr("interface", w)); + return self.out.writeSplat(data, splat); + } +}; + +fn computeCoordinates(tensor_shape: Shape, shard_shape: Shape, offset: usize) [Shape.MAX_RANK]usize { + var coordinates: [Shape.MAX_RANK]usize = @splat(0); + var offset_in_elem = offset / tensor_shape.dtype().sizeOf(); + const strides = tensor_shape.computeStrides(); + for (0..tensor_shape.rank()) |dim| { + const offset_in_dim = offset_in_elem / @as(usize, @intCast(strides.get(dim))); + const coordinate_in_dim = offset_in_dim / @as(usize, @intCast(shard_shape.dim(dim))); + offset_in_elem -= offset_in_dim * @as(usize, @intCast(strides.get(dim))); + coordinates[dim] = coordinate_in_dim; + } + + return coordinates; +} + +test computeCoordinates { + { + const tensor_shape: Shape = .init(.{ 1024, 1024 }, .u8); + const shard_shape: Shape = .init(.{ 512, 512 }, .u8); + const rank = tensor_shape.rank(); + + try std.testing.expectEqualSlices(usize, &[2]usize{ 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 0)[0..rank]); + try std.testing.expectEqualSlices(usize, &[2]usize{ 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 511)[0..rank]); + try std.testing.expectEqualSlices(usize, &[2]usize{ 0, 1 }, computeCoordinates(tensor_shape, shard_shape, 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[2]usize{ 1, 0 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[2]usize{ 1, 1 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 512 + 512)[0..rank]); + } + + { + const tensor_shape: Shape = .init(.{ 1024, 1024, 1024 }, .u8); + const shard_shape: Shape = .init(.{ 512, 512, 512 }, .u8); + const rank = tensor_shape.rank(); + + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 0)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 511)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 0, 1 }, computeCoordinates(tensor_shape, shard_shape, 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 1, 0 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 1, 1 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 512 + 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 1, 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 1024 * 512)[0..rank]); + } +} + +pub const TensorWriter = struct { + shard_writers: []ShardWriter, + temp_writers: []std.Io.Writer.Allocating, + interface: std.Io.Writer, + + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator, tensor_shape: Shape, buffer: []u8, platform: zml.Platform) !TensorWriter { + const device_count = platform.getDevices().len; + var dims: stdx.BoundedArray(i64, Shape.MAX_RANK) = try .init(tensor_shape.rank()); + for (0..tensor_shape.rank()) |dim| { + if (tensor_shape._sharding_info[dim]) { + dims.slice()[dim] = @divExact(tensor_shape.dim(dim), @as(i64, @intCast(device_count))); + } else { + dims.slice()[dim] = tensor_shape.dim(dim); + } + } + const shard_shape = Shape.init(dims.slice(), tensor_shape.dtype()); + + const shard_writers = try allocator.alloc(ShardWriter, device_count); + errdefer allocator.free(shard_writers); + + const temp_writers = try allocator.alloc(std.Io.Writer.Allocating, device_count); + errdefer allocator.free(temp_writers); + + for (0..device_count) |i| { + temp_writers[i] = std.Io.Writer.Allocating.init(allocator); + shard_writers[i] = ShardWriter.init(&temp_writers[i].writer, &.{}, tensor_shape, shard_shape); + } + + return .{ + .shard_writers = shard_writers, + .temp_writers = temp_writers, + .interface = .{ + .buffer = buffer, + .vtable = &.{ + .drain = drain, + }, + }, + .allocator = allocator, + }; + } + + pub fn deinit(self: TensorWriter) void { + self.allocator.free(self.shard_writers); + for (self.temp_writers) |*temp_writer| temp_writer.deinit(); + self.allocator.free(self.temp_writers); + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *TensorWriter = @alignCast(@fieldParentPtr("interface", w)); + var maybe_written: ?usize = null; + for (self.shard_writers) |*shard_writer| { + const shard_written = try shard_writer.interface.writeSplat(data, splat); + if (maybe_written) |written| { + std.debug.assert(written == shard_written); + } else { + maybe_written = shard_written; + } + } + + return maybe_written.?; + } +}; + +pub const RandomReader = struct { + random: std.Random, + interface: std.Io.Reader, + + pub fn init(random: std.Random, buffer: []u8) RandomReader { + return .{ + .random = random, + .interface = .{ + .buffer = buffer, + .vtable = &.{ + .stream = stream, + }, + .seek = 0, + .end = 0, + }, + }; + } + + pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *RandomReader = @alignCast(@fieldParentPtr("interface", r)); + const real_limit = limit.toInt() orelse 1024; + for (0..real_limit) |_| try w.writeByte(self.random.int(u8)); + return real_limit; + } +}; + +test "random reader" { + try asynk.AsyncThread.main(std.heap.c_allocator, testRandomReader); +} + +fn testRandomReader() !void { + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + + var generator = std.Random.DefaultPrng.init(0); + const random = generator.random(); + + var random_reader: RandomReader = .init(random, &.{}); + _ = &random_reader; // autofix + + const tensor_shape: Shape = .init(.{ 16, 16 }, .u8); + var tensor_writer: TensorWriter = try .init(std.testing.allocator, tensor_shape, &.{}, platform); + defer tensor_writer.deinit(); + + _ = try random_reader.interface.stream(&tensor_writer.interface, .limited(16 * 16)); + try tensor_writer.interface.flush(); + + for (tensor_writer.temp_writers, 0..) |*temp_writer, i| { + std.debug.print("shard: {d} - written: {}\n", .{ i, temp_writer.written().len }); + } +} diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 87f3d174..80a57469 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -58,7 +58,7 @@ pub const ApiError = error{ fn InnerMixin(comptime innerT: type) type { return struct { fn inner(self: anytype) *innerT { - return @ptrCast(@constCast(@alignCast(self))); + return @ptrCast(@alignCast(@constCast(self))); } }; } @@ -125,10 +125,10 @@ pub const Api = struct { } pub fn lookupExtension(self: *const Api, comptime ExtensionT: type, ext_id: c_int) ?*const ExtensionT { - var cur: [*c]const c.PJRT_Extension_Base = @alignCast(@ptrCast(self.inner.extension_start)); + var cur: [*c]const c.PJRT_Extension_Base = @ptrCast(@alignCast(self.inner.extension_start)); while (cur != null) : (cur = cur.*.next) { if (cur.*.type == ext_id) { - return @alignCast(@ptrCast(cur)); + return @ptrCast(@alignCast(cur)); } } @@ -264,22 +264,22 @@ pub const ShapeSpec = extern struct { inner: c.PJRT_ShapeSpec, - pub fn init(dims_: []const usize, bt: BufferType) ShapeSpec { + pub fn init(dims_: []const i64, bt: BufferType) ShapeSpec { return .{ .inner = pjrtStruct(c.PJRT_ShapeSpec{ .dims = @ptrCast(@constCast(dims_.ptr)), - .num_dims = dims.len, - .buffer_type = @intFromEnum(bt), + .num_dims = dims_.len, + .element_type = @intFromEnum(bt), }), }; } - pub fn dims(self: ShapeSpec) []usize { + pub fn dims(self: ShapeSpec) []const i64 { return self.inner.dims[0..self.inner.num_dims]; } pub fn bufferType(self: ShapeSpec) BufferType { - return @enumFromInt(self.inner.buffer_type); + return @enumFromInt(self.inner.element_type); } }; @@ -432,7 +432,7 @@ pub const Client = opaque { .client = self.inner(), }) catch unreachable; if (ret.addressable_memories) |memories| { - return @constCast(@ptrCast(memories[0..ret.num_addressable_memories])); + return @ptrCast(@constCast(memories[0..ret.num_addressable_memories])); } return &.{}; } @@ -461,7 +461,7 @@ pub const Client = opaque { pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { const ret = try api.call(.PJRT_Client_CreateBuffersForAsyncHostToDevice, .{ .client = self.inner(), - .shape_specs = @ptrCast(args.shape_specs.ptr), + .shape_specs = @ptrCast(@constCast(args.shape_specs.ptr)), .num_shape_specs = args.shape_specs.len, .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts.ptr)) else null, .num_device_layouts = if (args.device_layouts) |layouts| @intCast(layouts.len) else 0, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 84343a63..0b3cda57 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -23,6 +23,7 @@ pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; pub const ExecuteError = ApiError; pub const Memory = pjrt.Memory; pub const Stream = pjrt.Stream; +pub const ShapeSpec = pjrt.ShapeSpec; const log = std.log.scoped(.zml); @@ -74,12 +75,16 @@ pub const Client = opaque { return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); } - fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { var bytecode: std.array_list.Managed(u8) = .init(allocator); defer bytecode.deinit(); - module.op().writeBytecodeWithConfig(bytecode.writer(), .{ .desiredEmitedVersion = 1 }) catch |err| { + var bytecode_adapter = bytecode.writer().adaptToNewApi(&.{}); + module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode_adapter.new_interface) catch |err| { log.err("failed to write module bytecode: {}", .{err}); - return err; + return switch (err) { + error.InvalidMlirBytecodeVersion => error.InvalidMlirBytecodeVersion, + error.WriteFailed => error.OutOfMemory, + }; }; var serialized_buffer: std.array_list.Managed(u8) = .init(allocator); @@ -87,14 +92,18 @@ pub const Client = opaque { const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { - break :blk dialects.stablehlo.stablehloGetSmallerVersion(requested_version, dialects.stablehlo.getCurrentVersion()); + break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); } - break :blk dialects.stablehlo.getMinimumVersion(); + break :blk dialects.stablehlo.minimumVersion(); }; - dialects.stablehlo.serializePortableArtifact(bytecode.items, stablehlo_version, serialized_buffer.writer()) catch |err| { + var serialized_adapter = serialized_buffer.writer().adaptToNewApi(&.{}); + dialects.stablehlo.serializePortableArtifact(bytecode.items, stablehlo_version, &serialized_adapter.new_interface) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); - return err; + return switch (err) { + error.InvalidMlirBytecodeVersion => error.InvalidMlirBytecodeVersion, + error.WriteFailed => error.OutOfMemory, + }; }; return @ptrCast(try self.inner().compile(api, .{ @@ -104,7 +113,7 @@ pub const Client = opaque { })); } - pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { return try asynk.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); } @@ -126,6 +135,11 @@ pub const Client = opaque { pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); } + + pub const CreateBuffersForAsyncHostToDeviceArgs = pjrt.Client.CreateBuffersForAsyncHostToDeviceArgs; + pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { + return @ptrCast(try self.inner().createBuffersForAsyncHostToDevice(api, args)); + } }; pub const Buffer = opaque { @@ -298,11 +312,11 @@ pub const AsyncHostToDeviceTransferManager = opaque { return @ptrCast(self.inner().device(api)); } - pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) usize { + pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!usize { return self.inner().bufferCount(api); } - pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) usize { + pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!usize { return self.inner().bufferSize(api, buffer_index); } diff --git a/zml/shape.zig b/zml/shape.zig index bd5f2a22..ac86f314 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -180,7 +180,7 @@ pub const Shape = struct { }; } - fn toTag(v: anytype) Tag { + pub fn toTag(v: anytype) Tag { const T = @TypeOf(v); return switch (T) { EnumLiteral => @tagName(v).ptr, @@ -1092,4 +1092,15 @@ pub const Shape = struct { } return res_shape; } + + pub fn concatenate(shapes: []const Shape, ax: anytype) Shape { + // TODO: Check that shapes are compatible + var new_shape = shapes[0]; + var new_dim: i64 = 0; + for (shapes) |s| { + new_dim += s.dim(ax); + } + new_shape = new_shape.setDim(ax, new_dim); + return new_shape; + } }; diff --git a/zml/zml.zig b/zml/zml.zig index b5cd8039..4f8b9244 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -42,6 +42,8 @@ pub const Target = @import("platform.zig").Target; pub const Tensor = @import("tensor.zig").Tensor; pub const testing = @import("testing.zig"); pub const torch = @import("torch.zig"); +pub const upb = @import("upb"); +pub const c = @import("c"); // pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { From 66d51dbc3b2e4b4b508bb1ea32fffbcd17ab3eca Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 20 Oct 2025 13:53:48 +0200 Subject: [PATCH 43/47] wip --- mlir/BUILD.bazel | 1 + mlir/llama.zig | 777 +++++++++++++++-- mlir/main.zig | 1906 +++++++++++++++++++++++++++++++++++++++++- zml/tools/tracer.zig | 2 +- 4 files changed, 2574 insertions(+), 112 deletions(-) diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 63ef0239..dc065330 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -51,6 +51,7 @@ zig_binary( "//stdx", "//async", ], + copts = ["-freference-trace"] ) zig_library( diff --git a/mlir/llama.zig b/mlir/llama.zig index 90f67974..c009827b 100644 --- a/mlir/llama.zig +++ b/mlir/llama.zig @@ -7,11 +7,15 @@ const Tensor = @import("main.zig").Tensor; const TensorDescriptor = @import("main.zig").TensorDescriptor; const BufferStore4 = @import("main.zig").BufferStore4; const BufferStore5 = @import("main.zig").BufferStore5; +const CompilationContext = @import("main.zig").CompilationContext; +const Exe = @import("main.zig").Exe; const Bufferized = @import("main.zig").Bufferized; const loadBuffersFromId = @import("main.zig").loadBuffersFromId; const multiTransfer = @import("main.zig").multiTransfer; const singleTransfer = @import("main.zig").singleTransfer; const structTransfer = @import("main.zig").structTransfer; +const bufferTransfer = @import("main.zig").bufferTransfer; +const autoLoad = @import("main.zig").autoLoad; const initBufferizedFrom = @import("main.zig").initBufferizedFrom; const compile = @import("main.zig").compile; @@ -29,25 +33,37 @@ pub const LlamaLM = struct { max_position_embeddings: usize, rms_norm_eps: f32, hf_rope_impl: bool = true, - //rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, }; pub const Options = struct { - //sampling_strategy: ?zml.nn.SamplingStrategy, + sampling_strategy: ?zml.nn.SamplingStrategy, max_seq_len: usize, + qkv_type: QkvKind, + }; + + pub const LoadingOptions = struct { + merge_qkv_exe: ?Exe = null, + precompute_qkv_exe: ?Exe = null, }; lm_head: ?Linear, model: Llama, // Options controlling generation - //gen_opts: zml.nn.SamplingStrategy = .{}, + gen_opts: zml.nn.SamplingStrategy = .{}, config: Config, - pub fn init(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, config: LlamaLM.Config) !LlamaLM { + pub fn init(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, config: LlamaLM.Config, options: Options) !LlamaLM { + const lm_head: ?Linear = b: { + const lm_head_weight = buffer_store.withPrefix("lm_head").maybeCreateTensorWithTags("weight", .{ .dout, .d }) orelse break :b null; + break :b .{ .weight = lm_head_weight, .tag = zml.Shape.toTag(.d) }; + }; + return .{ - .lm_head = null, - .model = try .init(allocator, buffer_store.withPrefix("model"), config), + .lm_head = lm_head, + .model = try .init(allocator, buffer_store.withPrefix("model"), config, options), + .gen_opts = options.sampling_strategy orelse .{}, .config = config, }; } @@ -56,12 +72,55 @@ pub const LlamaLM = struct { allocator.free(self.model.layers); } - pub fn loadBuffers(allocator: std.mem.Allocator, llama: LlamaLM, buffer_store: BufferStore5.View, platform: zml.Platform) !LlamaLMBufferized { + pub fn loadBuffers(allocator: std.mem.Allocator, llama: LlamaLM, buffer_store: BufferStore5.View, platform: zml.Platform, options: LoadingOptions) !LlamaLMBufferized { return .{ - .lm_head = null, - .model = try Llama.loadBuffers(allocator, llama.model, buffer_store.withPrefix("model"), platform), + .lm_head = if (llama.lm_head != null) try loadBuffersFromId(allocator, llama.lm_head.?, buffer_store.withPrefix("lm_head"), platform) else null, + .model = try Llama.loadBuffers(allocator, llama.model, buffer_store.withPrefix("model"), platform, options), }; } + + /// Predicts the token at `token_index` position. + /// Returns: + /// - updated `tokens`, + /// - updated KV cache + /// - a Rng state to allow for probabilistic generation + pub fn forward( + self: LlamaLM, + tokens_: Tensor, + token_index: Tensor, + kv_cache: KvCache, + rng: Tensor.Rng, + ) struct { Tensor, KvCache, Tensor.Rng } { + stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); + const tokens = tokens_.withPartialTags(.{.s}); + const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache); + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + } + + pub fn sampleTokens( + self: LlamaLM, + lm_head_: ?Linear, + out_: Tensor, + rng: Tensor.Rng, + opts: zml.nn.SamplingStrategy, + ) struct { Tensor, Tensor.Rng } { + const out = out_.withPartialTags(.{ .s, .d }); + + var logits = blk: { + if (lm_head_) |lm_head| { + break :blk lm_head.forward(out).rename(.{ .dout = .d }); + } else { + break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .d); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = sampleTokens2(logits, opts, rng); + return .{ next_tokens, new_rng }; + } }; pub const LlamaLMBufferized = struct { @@ -85,38 +144,54 @@ pub const Linear = struct { .tag = zml.Shape.toTag(tag), }; } + + pub fn forward(self: Linear, x: Tensor) Tensor { + var y = x.dot(self.weight.convert(x.dtype()), self.tag); + + // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); + return if (self.bias) |bias| y.add(bias.autoBroadcast()) else y; + } +}; + +pub const TokenEmbedding = struct { + weight: Tensor, + + pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { + stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); + stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); + return self.weight.gatherValues(0, idx, .{}); + } }; pub const Llama = struct { - embed_tokens: struct { weight: Tensor }, + embed_tokens: TokenEmbedding, norm: RmsNorm, layers: []TransformerLayer, - max_seq_len: u32, - num_heads: i64, - num_kv_heads: i64, - - pub fn init(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, config: LlamaLM.Config) !Llama { + pub fn init(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, config: LlamaLM.Config, options: LlamaLM.Options) !Llama { const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); errdefer allocator.free(layers); for (layers, 0..) |*layer, i| { + //const custom_options = if (i != 0) options else b: { + // var custom_options = options; + // custom_options.qkv_type = .precomputed; + // break :b custom_options; + //}; + const custom_options = options; var buffer: [16]u8 = undefined; const number_prefix = try std.fmt.bufPrint(&buffer, "layers.{d}", .{i}); - layer.* = .init(buffer_store.withPrefix(number_prefix), config); + layer.* = .init(buffer_store.withPrefix(number_prefix), config, custom_options); } return .{ .embed_tokens = .{ .weight = buffer_store.createTensorWithTags("embed_tokens.weight", .{ .x, .y }) }, - .norm = .init(buffer_store.withPrefix("norm"), config), + .norm = .init(buffer_store.withPrefix("norm"), config.rms_norm_eps), .layers = layers, - .max_seq_len = 256, - .num_heads = @intCast(config.num_attention_heads), - .num_kv_heads = @intCast(config.num_key_value_heads), }; } - pub fn loadBuffers(allocator: std.mem.Allocator, model: Llama, buffer_store: BufferStore5.View, platform: zml.Platform) !Bufferized(Llama) { + pub fn loadBuffers(allocator: std.mem.Allocator, model: Llama, buffer_store: BufferStore5.View, platform: zml.Platform, options: LlamaLM.LoadingOptions) !Bufferized(Llama) { std.debug.print("Loading Llama\n", .{}); const layers = try allocator.alloc(Bufferized(TransformerLayer), model.layers.len); errdefer allocator.free(layers); @@ -125,7 +200,7 @@ pub const Llama = struct { var buffer: [16]u8 = undefined; const number_prefix = try std.fmt.bufPrint(&buffer, "layers.{d}", .{i}); std.debug.print("Loading layer {d}\n", .{i}); - layer.* = try TransformerLayer.loadBuffers(allocator, model.layers[i], buffer_store.withPrefix(number_prefix), platform); + layer.* = try TransformerLayer.loadBuffers(allocator, model.layers[i], buffer_store.withPrefix(number_prefix), platform, options); } return .{ @@ -134,18 +209,45 @@ pub const Llama = struct { .norm = try loadBuffersFromId(allocator, model.norm, buffer_store.withPrefix("norm"), platform), }; } + + /// Forward one token, using KV cache for previous tokens. + /// Returns result and updated KV cache. + pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { + const embeds = embed(self.embed_tokens, tokens); + var hidden = embeds; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + std.debug.print("hidden.shape: {f}\n", .{hidden.shape()}); + hidden, updated_kv_cache = layer.forward(hidden, tokens, token_index, updated_kv_cache.atLayer(i)); + } + const output = self.norm.forward(hidden); + + return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; + } + + pub fn embed(embed_tokens_: TokenEmbedding, tokens_: Tensor) Tensor { + return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); + } }; -const RmsNorm = struct { +pub const RmsNorm = struct { weight: Tensor, eps: f32 = 1e-5, - pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config) RmsNorm { + pub fn init(buffer_store: BufferStore5.View, eps: f32) RmsNorm { return .{ .weight = buffer_store.createTensorWithTags("weight", .{.d}), - .eps = config.rms_norm_eps, + .eps = eps, }; } + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: Tensor) Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + const normalized = rmsNorm(x, .d, self.eps); + return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); + } }; pub const TransformerLayer = struct { @@ -154,90 +256,280 @@ pub const TransformerLayer = struct { post_attention_layernorm: RmsNorm, mlp: Mlp, - pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config) TransformerLayer { + pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config, options: LlamaLM.Options) TransformerLayer { return .{ - .input_layernorm = .init(buffer_store.withPrefix("input_layernorm"), config), - .self_attn = SelfAttn.init(buffer_store.withPrefix("self_attn"), config), - .post_attention_layernorm = .init(buffer_store.withPrefix("post_attention_layernorm"), config), + .input_layernorm = .init(buffer_store.withPrefix("input_layernorm"), config.rms_norm_eps), + .self_attn = SelfAttn.init(buffer_store.withPrefix("self_attn"), config, options), + .post_attention_layernorm = .init(buffer_store.withPrefix("post_attention_layernorm"), config.rms_norm_eps), .mlp = .init(buffer_store.withPrefix("mlp")), }; } - pub fn loadBuffers(allocator: std.mem.Allocator, layer: TransformerLayer, buffer_store: BufferStore5.View, platform: zml.Platform) !Bufferized(TransformerLayer) { + pub fn loadBuffers(allocator: std.mem.Allocator, layer: TransformerLayer, buffer_store: BufferStore5.View, platform: zml.Platform, options: LlamaLM.LoadingOptions) !Bufferized(TransformerLayer) { return .{ .input_layernorm = try loadBuffersFromId(allocator, layer.input_layernorm, buffer_store.withPrefix("input_layernorm"), platform), - .self_attn = try SelfAttn.loadBuffers(allocator, layer.self_attn, buffer_store.withPrefix("self_attn"), platform), + .self_attn = try SelfAttn.loadBuffers(allocator, layer.self_attn, buffer_store.withPrefix("self_attn"), platform, options), .post_attention_layernorm = try loadBuffersFromId(allocator, layer.post_attention_layernorm, buffer_store.withPrefix("post_attention_layernorm"), platform), .mlp = try loadBuffersFromId(allocator, layer.mlp, buffer_store.withPrefix("mlp"), platform), }; } + + pub fn forward( + self: TransformerLayer, + x0: Tensor, + tokens: Tensor, + token_index: Tensor, + kv_cache: KvCache, + ) struct { Tensor, KvCache } { + // Self Attention + //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); + //stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); + + const params: SelfAttn.ForwardParams = switch (self.self_attn.proj) { + .merged => .{ .merged = self.input_layernorm.forward(x0) }, + .separate => .{ .separate = self.input_layernorm.forward(x0) }, + .precomputed => .{ .precomputed = .{ .x = x0, .tokens = tokens } }, + }; + const delta0, const updated_kv_cache = self.self_attn.forward(params, token_index, kv_cache); + const x1 = x0.add(delta0); + + // Fully Connected + const x1_normalized = self.post_attention_layernorm.forward(x1); + const x2 = self.mlp.forward(x1_normalized).add(x1).rename(.{ .dout = .d }); + + return .{ x2.reuseBuffer(x0), updated_kv_cache }; + } +}; + +const QkvKind = enum { + merged, + separate, + precomputed, +}; + +pub const SeparateQkv = struct { + q_proj: Linear, + k_proj: Linear, + v_proj: Linear, + + pub fn init(buffer_store: BufferStore5.View) SeparateQkv { + const q_proj = Linear.initWithTag(buffer_store.withPrefix("q_proj"), .d); + const k_proj = Linear.initWithTag(buffer_store.withPrefix("k_proj"), .d); + const v_proj = Linear.initWithTag(buffer_store.withPrefix("v_proj"), .d); + return .{ .q_proj = q_proj, .k_proj = k_proj, .v_proj = v_proj }; + } }; pub const SelfAttn = struct { - qkv_proj: Linear, - //q_proj: Linear, - //k_proj: Linear, - //v_proj: Linear, + pub const Proj = union(QkvKind) { + merged: Linear, + separate: SeparateQkv, + precomputed: Tensor, + }; + proj: Proj, o_proj: Linear, num_heads: i64, num_kv_heads: i64, + rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = 10_000, + }, + + pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config, options: LlamaLM.Options) SelfAttn { + const proj: Proj = switch (options.qkv_type) { + .separate => .{ .separate = SeparateQkv.init(buffer_store) }, + .merged => b: { + const q_weight_shape = buffer_store.getShape("q_proj.weight").?; + const k_weight_shape = buffer_store.getShape("k_proj.weight").?; + const v_weight_shape = buffer_store.getShape("v_proj.weight").?; + const qkv_weight_shape = zml.Shape.concatenate(&.{ q_weight_shape, k_weight_shape, v_weight_shape }, 0); + break :b .{ .merged = .{ .weight = Tensor.init(qkv_weight_shape).withTags(.{ .dout, .d }), .tag = zml.Shape.toTag(.d) } }; + }, + .precomputed => b: { + const q_weight_shape = buffer_store.getShape("q_proj.weight").?; + const k_weight_shape = buffer_store.getShape("k_proj.weight").?; + const v_weight_shape = buffer_store.getShape("v_proj.weight").?; + const embedding_shape = buffer_store.getShapeOpts("model.embed_tokens.weight", .{ .no_prefix = true }).?; + const qkv_weight_shape = zml.Shape.concatenate(&.{ q_weight_shape, k_weight_shape, v_weight_shape }, 0); + const precomputed_qkv_weight_shape = zml.Shape.init(.{ embedding_shape.dim(0), qkv_weight_shape.dim(0) }, qkv_weight_shape.dtype()); + + break :b .{ .precomputed = Tensor.init(precomputed_qkv_weight_shape).withTags(.{ .voc, .d }) }; + }, + }; - pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config) SelfAttn { - const q_weight_shape = buffer_store.getShape("q_proj.weight").?; - const k_weight_shape = buffer_store.getShape("k_proj.weight").?; - const v_weight_shape = buffer_store.getShape("v_proj.weight").?; - const qkv_weight_shape = zml.Shape.concatenate(&.{ q_weight_shape, k_weight_shape, v_weight_shape }, 0); return .{ - .qkv_proj = .{ .weight = Tensor.init(qkv_weight_shape).withTags(.{ .dout, .d }), .tag = zml.Shape.toTag(.d) }, + .proj = proj, .o_proj = .initWithTag(buffer_store.withPrefix("o_proj"), .d), .num_heads = @intCast(config.num_attention_heads), .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }, }; } - pub fn loadBuffers(allocator: std.mem.Allocator, self_attn: SelfAttn, buffer_store: BufferStore5.View, platform: zml.Platform) !Bufferized(SelfAttn) { - const q_proj = Linear.initWithTag(buffer_store.withPrefix("q_proj"), .d); - const k_proj = Linear.initWithTag(buffer_store.withPrefix("k_proj"), .d); - const v_proj = Linear.initWithTag(buffer_store.withPrefix("v_proj"), .d); + pub fn loadBuffers(allocator: std.mem.Allocator, self_attn: SelfAttn, buffer_store: BufferStore5.View, platform: zml.Platform, options: LlamaLM.LoadingOptions) !Bufferized(SelfAttn) { + return switch (self_attn.proj) { + .separate => |separate| .{ + .proj = .{ + .separate = .{ + .q_proj = try loadBuffersFromId(allocator, separate.q_proj, buffer_store.withPrefix("q_proj"), platform), + .k_proj = try loadBuffersFromId(allocator, separate.k_proj, buffer_store.withPrefix("k_proj"), platform), + .v_proj = try loadBuffersFromId(allocator, separate.v_proj, buffer_store.withPrefix("v_proj"), platform), + }, + }, + .o_proj = try loadBuffersFromId(allocator, self_attn.o_proj, buffer_store.withPrefix("o_proj"), platform), + }, + .merged => b: { + const proj: SeparateQkv = .init(buffer_store); - const exe2 = try compile(allocator, mergeQkv2, .{ q_proj, k_proj, v_proj }, platform); - defer exe2.deinit(); + var args = try options.merge_qkv_exe.?.args(allocator); + defer args.deinit(allocator); - var args = try exe2.args(allocator); - defer args.deinit(allocator); + var results = try options.merge_qkv_exe.?.results(allocator); + defer results.deinit(allocator); - var results = try exe2.results(allocator); - defer results.deinit(allocator); + const proj_buffers = try loadBuffersFromId(allocator, proj, buffer_store, platform); - const q_proj_buffers = try loadBuffersFromId(allocator, q_proj, buffer_store.withPrefix("q_proj"), platform); - const k_proj_buffers = try loadBuffersFromId(allocator, k_proj, buffer_store.withPrefix("k_proj"), platform); - const v_proj_buffers = try loadBuffersFromId(allocator, v_proj, buffer_store.withPrefix("v_proj"), platform); + args.set(.{proj_buffers}); + options.merge_qkv_exe.?.call(args, &results); - args.set(.{ q_proj_buffers, k_proj_buffers, v_proj_buffers }); - exe2.call(args, &results); + var qkv_proj: Bufferized(Linear) = undefined; + initBufferizedFrom(self_attn.proj.merged, &qkv_proj); - var qkv_proj: Bufferized(Linear) = undefined; - initBufferizedFrom(self_attn.qkv_proj, &qkv_proj); + results.fill(&qkv_proj); + errdefer qkv_proj.weight.deinit(); + errdefer if (qkv_proj.bias != null) qkv_proj.bias.?.deinit(); - results.fill(&qkv_proj); - errdefer qkv_proj.weight.deinit(); - errdefer if (qkv_proj.bias != null) qkv_proj.bias.?.deinit(); + const o_proj = try loadBuffersFromId(allocator, self_attn.o_proj, buffer_store.withPrefix("o_proj"), platform); - const o_proj = try loadBuffersFromId(allocator, self_attn.o_proj, buffer_store.withPrefix("o_proj"), platform); + break :b .{ + .proj = .{ + .merged = qkv_proj, + }, + .o_proj = o_proj, + }; + }, + .precomputed => b: { + const proj: SeparateQkv = .init(buffer_store); - return .{ - .qkv_proj = qkv_proj, - .o_proj = o_proj, + var args = try options.precompute_qkv_exe.?.args(allocator); + defer args.deinit(allocator); + + var results = try options.precompute_qkv_exe.?.results(allocator); + defer results.deinit(allocator); + + const proj_buffers = try loadBuffersFromId(allocator, proj, buffer_store, platform); + const embedding_buffer = try bufferTransfer(allocator, buffer_store.root(), "model.embed_tokens.weight", platform); + defer embedding_buffer.deinit(); + + const input_layernorm_buffer = try bufferTransfer(allocator, buffer_store.parent(), "input_layernorm.weight", platform); + defer input_layernorm_buffer.deinit(); + + args.set(.{ proj_buffers, input_layernorm_buffer, embedding_buffer }); + options.precompute_qkv_exe.?.call(args, &results); + + var precomputed_qkv = results.get(zml.Buffer); + errdefer precomputed_qkv.deinit(); + + const o_proj = try loadBuffersFromId(allocator, self_attn.o_proj, buffer_store.withPrefix("o_proj"), platform); + + break :b .{ + .proj = .{ + .precomputed = precomputed_qkv, + }, + .o_proj = o_proj, + }; + }, }; } + + const ForwardParams = union(QkvKind) { + merged: Tensor, + separate: Tensor, + precomputed: struct { x: Tensor, tokens: Tensor }, + }; + + /// Self Attention. + /// - If token_index is set, x is assumed to be the representation of one new token, + /// and kv_cache will be read for the previous tokens. + /// - If token_index is not set, x is assumed to be the representation of all tokens + /// since the beginning of the sequence, and kv_cache won't be read. + /// In both case, kv_cache will be updated with the computed key and value. + /// x: {.b, .s, .d } -> .{.b, .s, .d} + pub fn forward( + self: SelfAttn, + params: ForwardParams, + token_index: Tensor, + kv_cache: KvCache, + ) struct { Tensor, KvCache } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + std.debug.assert(@as(QkvKind, params) == @as(QkvKind, self.proj)); + var q, var k, var v = switch (self.proj) { + .separate => b: { + const q = self.proj.separate.q_proj.forward(params.separate).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); + const k = self.proj.separate.k_proj.forward(params.separate).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + const v = self.proj.separate.v_proj.forward(params.separate).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + break :b .{ q, k, v }; + }, + .merged => b: { + const qkv = self.proj.merged.forward(params.merged).splitAxis(-1, .{ .h = self.num_heads + num_kv_heads + num_kv_heads, .hd = .auto }); + const q = qkv.slice1d(.h, .{ .start = 0, .end = self.num_heads }); + const k = qkv.slice1d(.h, .{ .start = self.num_heads, .end = self.num_heads + num_kv_heads }); + const v = qkv.slice1d(.h, .{ .start = self.num_heads + num_kv_heads, .end = self.num_heads + num_kv_heads + num_kv_heads }); + break :b .{ q, k, v }; + }, + .precomputed => b: { + std.debug.print("shape: {f}\n", .{self.proj.precomputed.shape()}); + std.debug.print("tokens.shape: {f}\n", .{params.precomputed.tokens.shape()}); + const qkv = self.proj.precomputed.gatherValues(0, params.precomputed.tokens, .{}).splitAxis(-1, .{ .h = self.num_heads + num_kv_heads + num_kv_heads, .hd = .auto }); + const q = qkv.slice1d(.h, .{ .start = 0, .end = self.num_heads }); + const k = qkv.slice1d(.h, .{ .start = self.num_heads, .end = self.num_heads + num_kv_heads }); + const v = qkv.slice1d(.h, .{ .start = self.num_heads + num_kv_heads, .end = self.num_heads + num_kv_heads + num_kv_heads }); + break :b .{ q, k, v }; + }, + }; + + // Generate the attention mask. + const seq_len = kv_cache.k.dim(.k); + var attn_mask = causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); + + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); + + // In self-attention, .s axis is used both for keys and queries. + const pos_index = b: { + const temp = Tensor.arange(.{ .end = q.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = q.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + q = rope(q, pos_index, self.rope_opts); + k = rope(k, pos_index, self.rope_opts); + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + const dtype = q.dtype(); + const new_kv_cache = kv_cache.update(k, v, token_index); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + const attn_output = sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ self.o_proj.forward(attn), new_kv_cache }; + } }; fn mergeQkv(q: Tensor, k: Tensor, v: Tensor) Tensor { return Tensor.concatenate(&.{ q, k, v }, 0); } -fn mergeQkv2(q: Linear, k: Linear, v: Linear) Linear { +pub fn mergeQkv2(q: Linear, k: Linear, v: Linear) Linear { return .{ .weight = Tensor.concatenate(&.{ q.weight, k.weight, v.weight }, 0), .bias = if (q.bias != null) Tensor.concatenate(&.{ q.bias.?, k.bias.?, v.bias.? }, 0) else null, @@ -245,6 +537,25 @@ fn mergeQkv2(q: Linear, k: Linear, v: Linear) Linear { }; } +pub fn mergeQkv3(proj: SeparateQkv) Linear { + return .{ + .weight = Tensor.concatenate(&.{ proj.q_proj.weight, proj.k_proj.weight, proj.v_proj.weight }, 0), + .bias = if (proj.q_proj.bias != null) Tensor.concatenate(&.{ proj.q_proj.bias.?, proj.k_proj.bias.?, proj.v_proj.bias.? }, 0) else null, + .tag = proj.q_proj.tag, + }; +} + +pub fn precomputeQkv0(proj: SeparateQkv, layer_norm: RmsNorm, embedding: Tensor) Tensor { + const hidden = layer_norm.forward(embedding); + const qkv_proj: Linear = .{ + .weight = Tensor.concatenate(&.{ proj.q_proj.weight, proj.k_proj.weight, proj.v_proj.weight }, 0), + .bias = if (proj.q_proj.bias != null) Tensor.concatenate(&.{ proj.q_proj.bias.?, proj.k_proj.bias.?, proj.v_proj.bias.? }, 0) else null, + .tag = proj.q_proj.tag, + }; + + return qkv_proj.forward(hidden).rename(.{ .dout = .d }); +} + const Mlp = struct { up_proj: Linear, // (dim -> hidden_dim) gate_proj: Linear, // (dim -> hidden_dim) @@ -257,4 +568,338 @@ const Mlp = struct { .down_proj = .initWithTag(buffer_store.withPrefix("down_proj"), .d), }; } + + pub fn forward(self: Mlp, x: Tensor) Tensor { + const proj = self.up_proj.forward(x); + var output = self.gate_proj.forward(x); + output = output.silu().mul(proj).rename(.{ .dout = .d }); + return self.down_proj.forward(output); + } }; + +pub const KvCache = struct { + k: Tensor, + v: Tensor, + layer_index: Tensor, + + pub fn init(kv_shape: zml.Shape) KvCache { + // The KV-cache is initialized with ones to detect reads of uninitialized memory. + return .{ + .k = Tensor.init(kv_shape), + .v = Tensor.init(kv_shape), + .layer_index = Tensor.init(zml.Shape.init(.{}, .u32)), + }; + } + + pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !Bufferized(KvCache) { + return .{ + .k = try zml.Buffer.constant(platform, kv_shape, 0), + .v = try zml.Buffer.constant(platform, kv_shape, 0), + .layer_index = try zml.Buffer.scalar(platform, 0, .u32), + }; + } + + pub fn keys(self: KvCache) Tensor { + return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache) Tensor { + return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + std.debug.print("new_k: {f}\n", .{new_k.shape()}); + std.debug.print("self: {f}\n", .{self.k.shape()}); + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .k = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .k = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } +}; + +pub fn sampleTokens2(activations: Tensor, opts: zml.nn.SamplingStrategy, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { + _ = opts; // autofix + //if (opts.topk <= 1) { + if (true) { + const next_tokens = activations.argMax(.voc).indices.squeeze(.voc); + return .{ next_tokens, rng }; + } + + //const topk = activations.topK(opts.topk, .voc, .{}); + //// After the topk, we don't have .voc values, anymore, only topk. + //var x = topk.values.rename(.{ .voc = .topk }); + //if (opts.temperature != 1.0) { + // x = x.scale(1 / opts.temperature); + //} + + //// Gumbel reparametrization trick: + //// Adding gumbel noise and taking the argmax is equivalent + //// to sampling from the categorical distribution produced by the softmax. + //// https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks + //const next_rng, const gumbel_noise = rng.gumbel(x.shape()); + //x = x.add(gumbel_noise); + //const topk_idx = x.argMax(.topk).indices; + + //// topk_idx is indices into topk.values ! so in the range [0, topk] + //// Convert for the original indices from the full [0, voc] range. + //const next_tokens = topk.indices.gatherValues(.voc, topk_idx.squeeze(.topk), .{}); + //// log.debug("sampleTokens({}) -> {} -> {} -> {}", .{ activations, topk.indices, topk_idx, next_tokens }); + //return .{ next_tokens, next_rng }; +} + +pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x.axis(axis); + // upcast to improve precision + const xf32 = x.convert(.f32); + const mean = xf32.mul(xf32).mean(ax); + const rsqrt = Tensor.rsqrt(mean.addConstant(eps)).convert(x.dtype()); + return x.mul(rsqrt.broad(x.shape())); +} + +/// Return causal attention masks for the given shape. +/// The last dimensions are +pub fn causalAttnMask( + attn_shape_: anytype, + dtype: zml.DataType, + attn_window_len: ?u32, +) Tensor { + const attn_shape = zml.Shape.init(attn_shape_, dtype); + stdx.debug.assert(attn_shape.rank() == 2, "causalAttnMask({f}) shape need to be exactly 2 axes", .{attn_shape}); + const qlen = attn_shape.dim(-2); + const q_idx = Tensor.iota(attn_shape, -2); + const klen = attn_shape.dim(-1); + const k_idx = Tensor.iota(attn_shape, -1); + + // all elements > main diagonal must be 0 + // (q_idx - window_len < k_idx <= q_idx) + var mask = k_idx.cmp(.LE, q_idx); + if (attn_window_len) |window_len| { + if (qlen >= window_len or klen >= window_len) { + const window_mask = q_idx.cmp(.LT, k_idx.addConstant(window_len)); + mask = mask.logical(.AND, window_mask); + } + } + + if (dtype.isFloat()) { + const zeros = Tensor.constant(dtype.zero()).broad(mask.shape()); + const minus_inf = Tensor.constant(dtype.minValue()).broad(mask.shape()); + mask = Tensor.select(mask, zeros, minus_inf); + } else { + mask = mask.convert(dtype); + } + + return mask; +} + +/// Rotary position embedding modify queries and keys tensor before compute Q * K in self attention. +/// This biases a token to look at token near him. +/// The nice thing with rope is that you can cache the modified queries and keys directly. +/// See: https://paperswithcode.com/method/rope +/// +/// Expected shapes of tensor: +/// - x: .{ .s, .hd } where .s is the sequence length and .hd the head dimension +/// - pos_idx: optional tensor which indicates which positions are needed. +/// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. +pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: zml.nn.RopeOpts) Tensor { + stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); + + const idx = if (pos_idx) |idx| blk: { + stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={f}, idx={f})", .{ x, idx }); + break :blk idx; + } else blk: { + stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={f})", .{x}); + break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); + }; + const x_real, const x_imag = splitRealImg(x, opts.layout); + + // compute sin and cos in f32 before downcasting to x type. + const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); + const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); + const cos = inv_freq_pos.cos().convert(x.dtype()).broad(x_real.shape()); + const sin = inv_freq_pos.sin().convert(x.dtype()).broad(x_real.shape()); + + // apply rotation + const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); + const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); + + // flatten last dimensions + return mergeRealImg(y_real, y_imag, opts.layout); +} + +pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: zml.nn.RopeOpts.Layout) Tensor { + return switch (layout) { + .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), + .interleaved => Tensor.concatenate(&.{ + x_real.appendAxes(.{.interleaved_real_img}), + x_imag.appendAxes(.{.interleaved_real_img}), + }, -1).flatten(-2), + }; +} + +/// {exp( - n * ln(10_000) / N ) | n in [0..N] } +pub fn invFreq(N: i64, opts: zml.nn.RopeOpts) Tensor { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + const N_half: usize = @intCast(@divExact(N, 2)); + const inv_freq = arena.allocator().alloc(f32, N_half) catch @panic("OOM"); + _invFreq(opts, inv_freq); + return Tensor.constantTensor(std.mem.sliceAsBytes(inv_freq), zml.Shape.init(.{@divExact(N, 2)}, .f32)); +} + +fn _invFreq(opts: zml.nn.RopeOpts, inv_freq: []f32) void { + const N = inv_freq.len; + // Default frequencies + for (0.., inv_freq) |n, *f| { + f.* = @exp(-@log(opts.freq_base) * stdx.math.divFloat(f32, n, N)); + } + + switch (opts.scaling) { + .default => {}, + .custom => { + stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); + @memcpy(inv_freq, opts.scaling.custom); + }, + .llama3 => |s| { + // https://arxiv.org/pdf/2309.16039 + // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. + // In Llama3 they used a higher base freq and also downscaled low frequencies. + std.debug.assert(s.low_freq_factor < s.high_freq_factor); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.high_freq_factor * (2 * std.math.pi) / M; + const f_low = s.low_freq_factor * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Linear interpolation for middle freq + const lerp: f64 = (inv_freq[n] - f_low) / (f_high - f_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, + } +} + +pub fn splitRealImg(x: Tensor, layout: zml.nn.RopeOpts.Layout) [2]Tensor { + const n = x.dim(-1); + + return switch (layout) { + .sequential => .{ + x.slice1d(-1, .{ .end = @divExact(n, 2) }), + x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), + }, + .interleaved => .{ + x.slice1d(-1, .{ .start = 0, .step = 2 }), + x.slice1d(-1, .{ .start = 1, .step = 2 }), + }, + }; +} + +pub const SdpaOpts = struct { + attn_mask: ?Tensor = null, + scale: ?Tensor = null, + allow_cudnn: bool = true, + // TODO: put a callback instead of all this field, + // so that +}; + +/// Scaled dot product attention. +/// +/// **Shapes**: +/// - q, result: .{ .h, .q, .hd } +/// - k, v: .{ .h, .k, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +/// +/// .h is allowed to differ from queries and keys as long as the key heads +/// can be repeated to match query heads. +pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { + var q, var k, var v = .{ q_, k_, v_ }; + + const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; + const err_args = .{ q, k, v, opts.attn_mask }; + stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); + stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); + stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); + + //if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape())) { + // return cuda.sdpa(q, k, v, opts); + //} + + if (q.dim(.h) != k.dim(.h)) { + stdx.debug.assert(@mod(q.dim(.h), k.dim(.h)) == 0, err_template ++ "Different number of heads for keys and queries, but can't repeat keys.", err_args); + // Note: we don't try to repeat queries. + // Repeating keys is the interesting optimisation cause it reduces KV cache memory usage. + const num_rep: u63 = @intCast(@divExact(q.dim(.h), k.dim(.h))); + k, v = .{ k.repeat1d(.h, num_rep), v.repeat1d(.h, num_rep) }; + } + const attn_mask = if (opts.attn_mask) |m| m else null; + + const dims = zml.helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { + stdx.debug.panic(err_template ++ "Inputs have incompatible shapes.", err_args); + }; + const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); + const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); + k = k.mul(head_scaling.convert(k.dtype())); + std.debug.print("k.shape: {f}\n", .{k.shape()}); + + var attn_weights = q.dot(k, .hd); + std.debug.print("attn_weights.shape: {f}\n", .{attn_weights.shape()}); + // log.debug("attn_weights : {}, attn_mask : {?}", .{ attn_weights, attn_mask }); + if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); + attn_weights = attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); + + var attn = attn_weights.dot(v, .k); + return attn.transpose(q.shape()); +} diff --git a/mlir/main.zig b/mlir/main.zig index 772701d7..fabd616f 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -14,8 +14,14 @@ const Shape = zml.Shape; const EnumLiteral = @TypeOf(.enum_literal); const Platform = zml.Platform; +pub const std_options: std.Options = .{ + .log_level = .debug, + .log_scope_levels = &.{ + .{ .scope = .@"zml/async", .level = .info }, + }, +}; + var global_compilation_context: ?*CompilationContext = null; -pub const MAX_RANK: u8 = 8; pub const TensorOriginType = enum { argument, @@ -29,10 +35,14 @@ pub const TensorOrigin = union(TensorOriginType) { pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; + pub const MAX_RANK = Shape.MAX_RANK; + + pub const _Donation = union(enum) { no_buffer, input_buffer, arg: u16 }; id: usize, auto_broadcast: bool = false, _shape: zml.Shape, + _donation: _Donation = .no_buffer, tensor_origin: TensorOrigin = .{ .argument = {} }, fn mlirType(self: Tensor, mlir_ctx: *mlir.Context) *const mlir.Type { @@ -193,7 +203,16 @@ pub const Tensor = struct { //std.debug.print("{}\n", .{args_kind}); const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); - return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, &.{}); + + var batching_axes: stdx.BoundedArray([2]i8, Shape.MAX_RANK) = .{}; + for (0..lhs.rank()) |lhs_tag_index| { + const lhs_tag = lhs.shape().tag(lhs_tag_index); + if (lhs_tag == Shape.toTag(args)) continue; + if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { + batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); + } + } + return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); } pub fn dotGeneral( @@ -268,8 +287,7 @@ pub const Tensor = struct { .dot_precision = .fast, }, loc, - ); - CompilationContext.current().currentBlock().appendOwnedOperation(op); + ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } @@ -304,18 +322,58 @@ pub const Tensor = struct { return binaryOp(@src(), "add", dialects.stablehlo.add)(self, other); } + pub fn sub(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "sub", dialects.stablehlo.subtract)(self, other); + } + pub fn mul(self: Tensor, other: Tensor) Tensor { return binaryOp(@src(), "mul", dialects.stablehlo.multiply)(self, other); } + /// Returns a Tensor containing the element-wise division of the input Tensors. + pub fn div(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "div", dialects.stablehlo.divide)(self, other); + } + pub fn maximum(self: Tensor, other: Tensor) Tensor { return binaryOp(@src(), "maximum", dialects.stablehlo.maximum)(self, other); } + /// Returns a Tensor containing the element-wise cosine of the input Tensor. + pub fn cos(self: Tensor) Tensor { + const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, cosine_op.result(0)); + } + + /// Returns a Tensor containing the element-wise sine of the input Tensor. + pub fn sin(self: Tensor) Tensor { + const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, sine_op.result(0)); + } + pub fn relu(self: Tensor) Tensor { return self.maximum(.constant(self.dtype().zero())); } + /// Flattens the given axis and the next one, into one new axis. + pub fn flatten(self: Tensor, axis_: anytype) Tensor { + // TODO: move to torch.zig, this is equivalent to merge + const old_shape = self._shape; + const a = self.axis(axis_); + // stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {} on the last axis {}.", .{ self, axis }); + const new_shape = old_shape.remove(a + 1).set(a, old_shape.dim(a) * old_shape.dim(a + 1)); + + const tensor_type = mlir.rankedTensorType(new_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), new_shape.dtype())); + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), + self.value(), + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + // log.debug("flatten({d}, {d}) -> {d}", .{ self.dims(), axis_, new_shape[0 .. self.rank() - 1] }); + return _result(new_shape, reshaped_val.result(0)); + } + pub fn flattenAll(self: Tensor) Tensor { // TODO: rename to just flatten, once flatten is moved to torch return self.reshape(.{self.count()}); @@ -388,38 +446,36 @@ pub const Tensor = struct { stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); stdx.debug.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); - const ctx = CompilationContext.current(); //const loc = ctx.location(@src(), "arange({}, dtype={})", .{ args, dt }); - const loc = mlir.Location.unknown(ctx.mlir_ctx); + const loc = mlir.Location.unknown(mlirCtx()); const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; const sh = Shape.init(.{n_steps}, dt); - var op = dialects.stablehlo.iota(ctx.mlir_ctx, 0, mlir.rankedTensorType(sh.dims(), zml.mlir.Type.fromDType(ctx.mlir_ctx, sh.dtype())), loc).appendTo(CompilationContext.current().currentBlock()); + var op = dialects.stablehlo.iota(mlirCtx(), 0, mlir.rankedTensorType(sh.dims(), zml.mlir.Type.fromDType(mlirCtx(), sh.dtype())), loc).appendTo(currentBlock()); var res = _result(sh, op.result(0)); _ = &res; - //if (args.step != 1) { - // res = res.scale(args.step); - //} + if (args.step != 1) { + res = res.scale(args.step); + } - //if (args.start != 0) { - // res = res.addConstant(args.start); - //} + if (args.start != 0) { + res = res.addConstant(args.start); + } return res; } pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); - const ctx = self.getContext().mlir_ctx; - const tensor_type = mlir.rankedTensorType(output_shape.dims(), zml.mlir.Type.fromDType(ctx, output_shape.dtype())); + const tensor_type = mlir.rankedTensorType(output_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), output_shape.dtype())); //const loc = self.getContext().location(@src(), "reshape({f})", .{output_shape}); const reshape_op = dialects.stablehlo.reshape( - self.getContext().mlir_ctx, + mlirCtx(), self.value(), tensor_type, - .unknown(ctx), - ).appendTo(CompilationContext.current().currentBlock()); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(output_shape, reshape_op.result(0)); } @@ -429,14 +485,13 @@ pub const Tensor = struct { } //const loc = self.getContext().location(@src(), "convert({f},to={s})", .{ self, @tagName(to) }); - const mlir_ctx = self.getContext().mlir_ctx; - const res_type = mlir.rankedTensorType(self.dims(), zml.mlir.Type.fromDType(mlir_ctx, to)); + const res_type = mlir.rankedTensorType(self.dims(), zml.mlir.Type.fromDType(mlirCtx(), to)); const op = dialects.stablehlo.convert( - mlir_ctx, + mlirCtx(), self.value(), res_type, - .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentBlock()); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(self._shape.withDtype(to), op.result(0)); } @@ -462,7 +517,7 @@ pub const Tensor = struct { on_true.value(), on_false.value(), loc, - ).appendTo(CompilationContext.current().currentBlock()); + ).appendTo(currentBlock()); return _result(on_true._shape, op.result(0)); } @@ -507,7 +562,7 @@ pub const Tensor = struct { direction, getComparisonType(self.dtype()), loc, - ).appendTo(CompilationContext.current().currentBlock()); + ).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); } @@ -602,18 +657,38 @@ pub const Tensor = struct { .attributes = &.{ .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes_)), }, - .verify = false, + .verify = true, .location = .unknown(mlirCtx()), }).appendTo(CompilationContext.current().currentBlock()); + // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. + // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. + // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, + // to find the correct mlir.Value, but we first broadcast before creating the final + // Tensor struct. + var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (0..Tensor.MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { + broadcasting_axes.append(@intCast(i)) catch unreachable; + } + } + inline for (0..result.len) |i| { - const val = reduce_op.result(i); var reduced_shape: Shape = inputs[i].shape(); for (axes_) |a| { reduced_shape = reduced_shape.setDim(a, 1); } - result[i] = Tensor._result(result[i].shape(), val).broad(reduced_shape); + const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), reduced_shape.dtype())); + const broad_op = dialects.stablehlo.broadcast_in_dim( + mlirCtx(), + reduce_op.result(i), + broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + result[i] = Tensor._result(reduced_shape, broad_op.result(0)); } return result; @@ -665,8 +740,7 @@ pub const Tensor = struct { } const res_shape = tensors[0]._shape.set(a, concatenated_dim); - const ctx = tensors[0].getContext(); - const op = dialects.stablehlo.concatenate(ctx.mlir_ctx, buffer[0..tensors.len], a, .unknown(ctx.mlir_ctx)).appendTo(currentBlock()); + const op = dialects.stablehlo.concatenate(mlirCtx(), buffer[0..tensors.len], a, .unknown(mlirCtx())).appendTo(currentBlock()); // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); return _result(res_shape, op.result(0)); } @@ -681,6 +755,1476 @@ pub const Tensor = struct { }; } + pub const Rng = struct { + _state: Tensor, + algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, + + pub fn init() Rng { + return .{ ._state = .init(zml.Shape.init(.{2}, .u64)) }; + } + + pub fn initBuffer(platform: Platform, seed: u128) !Bufferized(Rng) { + return .{ + ._state = try zml.Buffer.fromBytes(platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), + }; + } + + ///// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. + ///// + ///// The given Rng state should not be used anymore (or you'll get the same numbers again). + ///// The output is guaranteed to be deterministic function of `self` Rng state, + ///// but it is not guaranteed to be deterministic between implementations. + //pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { + // const ctx = CompilationContext.current(); + // const loc = ctx.location(@src(), "rand.bitGen({f})", .{sh}); + // const op = dialects.stablehlo.rng_bit_generator( + // ctx.mlirCtx(), + // self.algorithm, + // self._state.value(), + // mlirx.tensorType(ctx.mlirCtx(), self._state._shape), + // mlirx.tensorType(ctx.mlirCtx(), sh), + // loc, + // ); + // return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; + //} + + //fn update(self: Rng, new_state: mlir.Value) Rng { + // return .{ + // ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), + // .algorithm = self.algorithm, + // }; + //} + + ///// Returns a Tensor of the given shape, filled with uniformly sampled floating point numbers from an interval, + ///// and a new Rng state. + ///// + ///// https://en.wikipedia.org/wiki/Continuous_uniform_distribution + //pub fn uniform( + // self: Rng, + // shape_: Shape, + // opts: struct { min: f64 = 0, max: f64 = 1 }, + //) struct { Rng, Tensor } { + // const dt = if (shape_.dtype().isFloat()) shape_.dtype() else .f32; + + // const mantissa_bit_count = @import("dtype.zig").mantissaSize(dt); + // const bit_count: usize = dt.bitSizeOf(); + // const rng_bit_count = if (mantissa_bit_count < 8) 8 else bit_count; + // const uint_dtype: DataType = switch (bit_count) { + // 8 => .u8, + // 16 => .u16, + // 32 => .u32, + // 64 => .u64, + // else => stdx.debug.panic("uniform don't support non-byte aligned dtype. Got: {f}", .{shape_}), + // }; + + // const rng, const bits = self.bitGenerator(shape_.withDtype(uint_dtype)); + + // // Erase bits outside of mantissa. + // var float_bits = bits.shiftRightLogical(scalar(rng_bit_count - mantissa_bit_count, uint_dtype)); + + // // Set exponent bits to represent e^0 (eg 127 for f32). + // float_bits = float_bits.logical(.OR, scalar(1, dt).bitCast(uint_dtype)); + + // // float_bits now uniformly represents number in [1, 2[ range. + // // Let's convert to floats, and subtract one to go to [0, 1[ range. + // var floats = float_bits.bitCast(dt).sub(scalar(1, dt)); + // floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); + + // // Convert back to integer if needed. + // return .{ rng, floats.convert(shape_.dtype()) }; + //} + + //test uniform { + // const zml = @import("zml.zig"); + // const Stats = struct { + // const Stats = @This(); + + // mean: Tensor, + // variance: Tensor, + // min: Tensor, + // max: Tensor, + + // pub fn uniformStats( + // rand: Rng, + // shape_: Shape, + // opts: struct { min: f64, max: f64 }, + // ) struct { Rng, Stats } { + // const rng, const data = rand.uniform(shape_, .{ .min = opts.min, .max = opts.max }); + // const mean_ = data.mean(0); + // const variance = data.sub(mean_.broad(data.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + // return .{ rng, .{ + // .mean = mean_, + // .variance = variance, + // .min = data.min(0), + // .max = data.max(0), + // } }; + // } + // }; + + // const platform = zml.testing.env(); + // // Compute stats over a uniform distribution on [-2, 10]. + // const rand, const stats = try zml.testing.compileAndCallWithTensors( + // platform, + // Stats.uniformStats, + // .{ Rng.shape(), zml.Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, + // .{try Rng.init(platform, 1234)}, + // ); + + // // Check the Rng state has been modified. + // try std.testing.expect(try rand._state.getValue(u128) != 1234); + + // // Check the mean and variance are close to theoritical values. + // const mean_ = try stats.mean.getValue(f32); + // try std.testing.expectApproxEqAbs(4, mean_, 0.03); + + // const variance = try stats.variance.getValue(f32); + // try std.testing.expectApproxEqAbs(12.0 * 12.0 / 12.0, variance, 0.01); + + // // Check that no value is outside of the interval + // // and we have samples close to the edges. + // const min_ = try stats.min.getValue(f32); + // try std.testing.expect(min_ >= -2); + // try std.testing.expectApproxEqAbs(-2, min_, 0.05); + + // const max_ = try stats.max.getValue(f32); + // try std.testing.expect(max_ < 10); + // try std.testing.expectApproxEqAbs(10, max_, 0.05); + //} + + ///// Returns a Tensor of the given shape, filled with floating point numbers sampled from a normal distribution. + ///// + ///// Note: this uses stablehlo.rng which is deprecated. + ///// https://github.com/openxla/stablehlo/blob/main/rfcs/20240503-opset-deprecations.md + //pub fn normal(sh: Shape, opts: struct { mean: f64 = 0, stddev: f64 = 1 }) Tensor { + // stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); + + // const ctx = CompilationContext.current(); + // const loc = ctx.location(@src(), "rand.normal({f}, mean={},stddev={})", .{ sh, opts.mean, opts.stddev }); + // const a = Tensor.constant(.{}, Data.init(sh.dtype(), opts.mean)); + // const b = Tensor.constant(.{}, Data.init(sh.dtype(), opts.stddev)); + // const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); + // const op = dialect.stablehlo.rng(ctx.mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, loc); + // return _result(sh, op.result(0)); + //} + + ///// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. + ///// + ///// Often used in ML because of the reparametrization tricks. + ///// Sampling from a gumbel distribution is equivalent to sample + ///// from a softmax distribution, but doesn't require to compute the sum of exponentials. + ///// https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks + ///// See `sampleTokens` for a practical use case. + ///// Note: we only implement the μ=0, β=1 version. + //pub fn gumbel(self: Rng, shape_: Shape) struct { Rng, Tensor } { + // const rand, const u = self.uniform( + // // Always use .f32 to have a big enough mantissa. + // shape_.withDtype(.f32), + // // We don't want 0 to be sampled otherwise `log` will return -inf. + // .{ .min = std.math.floatEps(f32), .max = 1 }, + // ); + // return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; + //} + + //test gumbel { + // const zml = @import("zml.zig"); + // const Stats = struct { + // const Stats = @This(); + + // mean: Tensor, + // variance: Tensor, + // actual_dist: Tensor, + + // pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { + // const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); + // const rng, const data = rand.gumbel(s); + // const flat = data.flattenAll(); + // const mean_ = flat.mean(0); + // const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + + // // Test out the gumbel reparametrization trick + // var x = target_dist.log().withTags(.{.d}).broad(s); + // x = x.add(data); + // const samples = x.argMax(.d).indices.squeeze(.d); + + // // count 0, 1, 2 and 3 in samples: + // // - map 0 to 1, 1 to 2**16, 2 to 2**32, 3 to N**58 + // // - sum in u64 + // // - split to [4]u16 + // const powers = blk: { + // var powers: [4]u64 = undefined; + // for (&powers, 0..) |*p, i| p.* = std.math.pow(u64, 2, i * 16); + // break :blk powers; + // }; + // const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); + // const counts = values.gatherValues(.d, samples, .{}).sum(.n).bitCast(.u16); + // const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); + // return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; + // } + // }; + + // const platform = zml.testing.env(); + // const tgt_dist = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; + // const rand, const stats = try zml.testing.compileAndCallWithTensors( + // platform, + // Stats.gumbelStats, + // .{ Rng.shape(), zml.Shape.init(.{tgt_dist.len}, .f32) }, + // .{ try Rng.init(platform, 1234), try .fromArray(platform, tgt_dist) }, + // ); + // // Check the Rng state has been modified. + // try std.testing.expect(try rand._state.getValue(i128) != 1234); + + // // Check the mean and variance are close to theoritical values. + // const mean_ = try stats.mean.getValue(f32); + // try std.testing.expectApproxEqAbs(0.5772, mean_, 0.02); + + // const variance = try stats.variance.getValue(f32); + // const pi = std.math.pi; + // try std.testing.expectApproxEqAbs(pi * pi / 6.0, variance, 0.03); + + // // Check the distribution obtained with the gumbel trick matches the target distribution. + // const actual_dist = try stats.actual_dist.getValue([4]f32); + // scoped_log.debug("tgt_dist: {d}, actual_dist: {d}", .{ tgt_dist, actual_dist }); + // for (tgt_dist, actual_dist) |tgt, actual| { + // // We normalize tgt_dist to make it a well formed distribution. + // // We didn't do it before calling gumbel, because the gumbel trick + // // doesn't require normalized distributions as input. + // try std.testing.expectApproxEqAbs(tgt / 10.0, actual, 0.05); + // } + //} + }; + + /// Tell PJRT compiler that memory should be reuse between the two tensors. + /// The compiler is already aggressively reusing tensors for intermediate results, + /// but this API allows to reuse buffer between input and output arguments + /// of a given function. + /// Note this is visible from the outside. The caller of a function with donations + /// is not allowed to reuse the donated input buffer after the call. + /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. + pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { + //var res = self; + //res._donation = origin._donation; + //switch (origin.tensor_origin) {} + _ = origin; // autofix + return self; + } + + pub const GatherOpts = struct { indices_are_sorted: bool = false }; + + /// For each coordinate in `indices`, + /// `gatherValues` extracts a single value of the given tensor. + /// + /// * axes_ is a single axis, or a tuple of axis: .b, or .{ .b, .c } + /// * indices is an integer tensor + /// * result is a tensor whose shape is similar to the input shape + /// where the gathered axes have been replaced by axes from 'indices'. + /// + /// Some example input for the base case where we work on one axis: + /// - gatherValues(f:[a]->float, .a, ind:[n]->int)[n] == f[ind[n]] + /// - gatherValues(f:[a, b], .a, ind:[n])[n, b] == f[ind[n], b] + /// - gatherValues(f: [a,b,c], .{.b}, ind: [n,m])[a, n, m, c] == f[a, ind[n, m], c] + /// + /// If an axis in common between `self` and `indices`, + /// it is treated as a "batching" axis, meaning that semantically + /// the operator is doing a gatherValues one time per dimension of this axis: + /// - gatherValues(f: [a,b,c], .{.b}, ind: [a,n])[a, n] == f[a, ind[a, n]] + /// + /// It is an error to have an axis present in `self`, `axes_` and `indices`. + /// + /// If several axes are passed, then the last axis of indices is treated as coordinates: + /// - gatherValues(f: [a,b,c], .{.b, .c}, ind: [n,2])[a, n] == f[a, ind[n][0], ind[n][1]] + /// - gatherValues(f: [a,b,c,d], .{.b, .c}, ind: [a, n,2])[a, n, d] == f[a, ind[a, n][0], ind[a, n][1], d] + /// + /// It is possible to use gatherValues without tags, but batching won't be available. + pub fn gatherValues(self: Tensor, coord_axes: anytype, indices: Tensor, opts: GatherOpts) Tensor { + // scoped_log.debug("gatherValues({}, {any}, {})", .{ self, coord_axes, indices }); + const single_coord, const coord_axes_ = _parseGatherCoord(self, coord_axes); + + stdx.debug.assert(coord_axes_.len > 0, "gatherValues expects 1 or more axes to operate one, received none. Example: `x.gatherValues(.a, indices, .{{}})`", .{}); + for (coord_axes_.constSlice(), 0..) |a, i| { + if (i > 0) { + stdx.debug.assert(a == coord_axes_.get(i - 1) + 1, "gatherValues expects 'coord_axes' to be sequential. But {any} aren't sequential in {f}", .{ coord_axes, self }); + } + } + + const LocalAxisKind = enum { batching, offset, collapsed, indices }; + var self_kind: stdx.BoundedArray(LocalAxisKind, MAX_RANK) = .{}; + var indices_batch_axes: Shape.DimsArray = .{}; + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const maybe_coord_ax = std.mem.indexOfScalar(u3, coord_axes_.constSlice(), @intCast(self_ax)); + if (indices._shape.hasTag(t)) |id_ax| { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_kind.appendAssumeCapacity(.batching); + indices_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(maybe_coord_ax == null, "gatherValues expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'coord_axes_={any}' and in 'indices={f}'", .{ self._shape._tags.get(self_ax), self, coord_axes, indices }); + } else if (maybe_coord_ax) |_| { + // for gatherValues we collapsed all gathered axes + // (contrary to gatherSlices where we collapse none) + self_kind.appendAssumeCapacity(.collapsed); + } else { + self_kind.appendAssumeCapacity(.offset); + } + } + + // When we receive several coord_axes we need an extra dimension to store + // one index per axis, which makes the coordinates of one value. + // Otherwi se stablehlo uses the "indices.rank()" default value. + const index_coord_axis = if (single_coord) + indices.rank() + else blk: { + const ax = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); + stdx.debug.assert(indices.dim(ax) == coord_axes_.len, "gatherValues with axes={any}, expects indices to be of shape [..., {}], got: {f}", .{ coord_axes, coord_axes_.len, indices }); + break :blk ax; + }; + + // compute res shape + var res_shape = Shape.init(.{}, self.dtype()); + var res_kind: stdx.BoundedArray(LocalAxisKind, MAX_RANK) = .{}; + for (self_kind.constSlice(), 0..) |kind, ax_usize| { + const ax: u3 = @intCast(ax_usize); + if (ax == coord_axes_.get(0)) { + // The first val_ax is special cause this is the place where we insert indices axes. + for (indices._shape.tags(), 0..indices.rank()) |t, id_ax| { + if (id_ax == index_coord_axis) continue; + if (std.mem.indexOfScalar(i64, indices_batch_axes.constSlice(), @intCast(id_ax))) |_| { + // batching dim are already in res + continue; + } + + res_shape = res_shape.appendDim(indices.dim(id_ax), t); + res_kind.appendAssumeCapacity(.indices); + } + } + switch (kind) { + .collapsed => continue, + else => { + res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); + res_kind.appendAssumeCapacity(kind); + }, + } + } + + // This is not a gather, but a dynamicSlice. + // Sometimes the backend recognize this pattern, but not always. + // So let us handle that. + if (indices.count() == 1) { + return self.dynamicSlice1d(coord_axes_.get(0), .{ .start = indices.flattenAll().squeeze(0), .len = 1 }).reshape(res_shape); + } + + var slice_dims: Shape.DimsArray = .{}; + for (self_kind.constSlice(), self.dims()) |k, d| { + slice_dims.appendAssumeCapacity(switch (k) { + .batching, .collapsed => 1, + .offset => d, + .indices => unreachable, + }); + } + + // scoped_log.debug("gatherValues --> {} {any}", .{ res_shape, res_kind.constSlice() }); + const gather_op = dialects.stablehlo.gather( + mlirCtx(), + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = _collectAxes(LocalAxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(LocalAxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(LocalAxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = indices_batch_axes.constSlice(), + .start_index_map = _collectAxes(LocalAxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = index_coord_axis, + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + //const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); + //stdx.debug.assert(mlir_shape.eql(res_shape), "gatherValues expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + return _result(res_shape, gather_op.result(0)); + } + + /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. + pub fn rsqrt(self: Tensor) Tensor { + const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, rsqrt_op.result(0)); + } + + /// Drops a 1-dim axis at the given index + pub fn squeeze(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); + + const new_shape = self._shape.remove(a); + // log.debug("squeeze({}, {d}={d}) -> ({})", .{ self, axis, a, new_shape }); + + return _result(new_shape, self.reshape(new_shape).value()); + } + + /// Returns a Tensor containing the sum of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. + pub fn sum(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + const result = reduce(.{self}, .{Tensor.scalar(0, self.dtype())}, &.{a}, struct { + pub fn acc(args: ReduceArgs) struct { Tensor } { + const x = args.left; + const res = args.right; + const result = res.add(x.convert(res.dtype())); + return .{result}; + } + }.acc, .{}); + return result.@"0"; + } + + inline fn wrapIndex(self: Tensor, axis_: usize, idx: i64) i64 { + return if (idx < 0) self.dim(axis_) + idx else idx; + } + + /// Returns a Tensor containing the mean of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. + pub fn mean(self: Tensor, axis_: anytype) Tensor { + return self.sum(axis_).divByConst(self.dim(axis_)); + } + + /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. + pub fn divByConst(self: Tensor, b: anytype) Tensor { + return self.div(Tensor.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. + pub inline fn scale(self: Tensor, val: anytype) Tensor { + return self.mul(Tensor.scalar(val, self.dtype())); + } + + /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. + pub fn addConstant(self: Tensor, b: anytype) Tensor { + return self.add(Tensor.scalar(b, self.dtype())); + } + + /// Returns a 0-rank Tensor with the given value. + pub fn scalar(val: anytype, dt: zml.DataType) Tensor { + const data = zml.Data.init(dt, val); + switch (dt.class()) { + .float => stdx.debug.assert(!std.math.isNan(val), "scalar(NaN) is probably due to compiling a model with an uninitialized field", .{}), + else => {}, + } + return Tensor.constant(data); + } + + /// Slices the input Tensor along a specific axis, with a start offset known at runtime. + /// Note: this doesn't support tagging, if you have tags, + /// you should use `dynamicSlice` directly. + pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { + stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {f}", .{slice_.start}); + + const a = self.axis(axis_); + const new_shape = self._shape.set(a, slice_.len); + + var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** MAX_RANK; + start_indices[a] = slice_.start.value(); + + const op = dialects.stablehlo.dynamic_slice( + mlirCtx(), + self.value(), + new_shape.dims(), + start_indices[0..self.rank()], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(new_shape, op.result(0)); + } + + pub const DynSlice = struct { start: Tensor, len: i64 }; + + pub const Slice = struct { + start: i64 = 0, + end: i64 = to_the_end, + step: i32 = 1, + singleton: bool = false, + + pub fn single(offset: i64) Slice { + return .{ .start = offset, .end = offset + 1, .singleton = true }; + } + + const to_the_end = std.math.maxInt(i64); + + pub fn format( + self: Slice, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + if (self.singleton) { + try writer.print("[{}]", .{self.start}); + } else if (self.end == to_the_end and self.step == 1) { + try writer.print("[{}..]", .{self.start}); + } else if (self.step == 1) { + try writer.print("[{}..{}]", .{ self.start, self.end }); + } else { + try writer.print("[{}..{}:{}]", .{ self.start, self.end, self.step }); + } + } + }; + + /// Slices the input Tensor over the given axis using the given parameters. + pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { + var slices = [_]Slice{.{}} ** MAX_RANK; + slices[self.axis(axis_)] = s; + return self.slice(slices[0..self.rank()]); + } + + /// Slices the input Tensor using the given parameters. + pub fn slice(self: Tensor, slices: []const Slice) Tensor { + var start_indices: [MAX_RANK]i64 = undefined; + var strides: [MAX_RANK]i64 = undefined; + var limit_indices: [MAX_RANK]i64 = undefined; + var res_shape: Shape = self._shape; + + for (slices, 0..) |s, a| { + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); + + const args: Slice = .{ + .start = self.wrapIndex(a, s.start), + .end = if (s.end == Slice.to_the_end) self.dim(a) else self.wrapIndex(a, s.end), + .step = s.step, + }; + start_indices[a] = args.start; + limit_indices[a] = args.end; + strides[a] = args.step; + res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); + } + + const result_type = mlir.rankedTensorType(res_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), res_shape.dtype())); + const slice_op = dialects.stablehlo.slice( + mlirCtx(), + self.value(), + start_indices[0..self.rank()], + limit_indices[0..self.rank()], + strides[0..self.rank()], + result_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + var res = _result(res_shape, slice_op.result(0)); + var to_remove: Shape.AxesArray = .{}; + for (slices, 0..) |s, a| { + if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); + } + return res.reshape(res_shape.removeMany(to_remove.constSlice())); + } + + /// Returns a Tensor containing the Sigmoid Linear Unit (SiLU) activation function applied to each element of the input Tensor. + /// + /// silu(x) = x σ(x) + /// https://paperswithcode.com/method/silu + pub fn silu(x: Tensor) Tensor { + return x.mul(x.sigmoid()); + } + + /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. + pub fn sigmoid(self: Tensor) Tensor { + const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + pub const logistic = sigmoid; + + /// Splits the given axis in several axes. + /// eg: `Tensor.init(.{ .a = 10, .b = 3 }).split(.a, .{.a1 = 5, .a2 = 2});` + /// The number of elements in the split shape must match the number of element + /// in the target axis. + pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { + const new_shape = self._shape.splitAxis(ax, split_shape); + + const tensor_type = mlir.rankedTensorType(new_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), new_shape.dtype())); + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), + self.value(), + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(new_shape, reshaped_val.result(0)); + } + + /// Gathers slices along the given axes with runtime indices. + /// * slice_shape represents the shape of the slices to extract, + /// it must be smaller than original shape. + /// It must use a subset of self axes. + /// If slice_shape is **not** tagged, then it must have the same rank than self. + /// * `indices` represents a set of coordinates. + /// The coordinates are read from the `.coord` axis, or last axis if `.coord` is not found. + /// The coordinate axis must have `slice_shape.rank()` dims. + /// The coordinates represent the "top-left" corner of the slice to extract. + /// * the output tensor starts with axes from `indices`. + /// * if the input tensor has tagged axes, matching `indices` axes, + /// they will be considered "batching" axes. + /// + /// Sample input/output shapes: + /// * gatherSlices([A, B, C, D], .{.b=B', .c=C'}, [N, 2]) -> [N, A, B', C', D] + /// * gatherSlices(x(a,b,c,d), .{.b=B', .c=C'}, g(n,m)) = z(n, a, b', c', d) = x(a, g(n, 0) + b', g(n, 1) + c', d) + /// + /// Note: the axis order of the result is different from gatherValues. + /// This is because gatherSlices, favorizes contiguous copy of the extracted slices, + /// while gatherValues, always copy values one by one, and as such don't have the same issues. + /// In our example the contiguous dimension .d is not sliced + /// and gatherSlices can copy data by group of C'*D elements. + pub fn gatherSlices(self: Tensor, slice_shape_: anytype, indices: Tensor, opts: GatherOpts) Tensor { + const slice_shape = if (@TypeOf(slice_shape_) == Shape) slice_shape_ else Shape.init(slice_shape_, .i32); + // scoped_log.debug("gatherSlice({}, {f}, {})", .{ self, slice_shape, indices }); + + const tagged_api = slice_shape.isFullyTagged(); + if (tagged_api) { + for (slice_shape.tags()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "gatherSlices expects `slices_shape` to only use tags from `self`. But {s} wasn't found in {f}", .{ t, self }); + } + } else { + // For untagged api, we require all slices to be specified. + // Note: we could relax this and right align the slice. + stdx.debug.assert(slice_shape.rank() == self.rank(), "gatherSlices expects `slice_shape.rank()` to match `self.rank()`. Got: gatherSlices({f}, slice={f}). To avoid specifying all axes in `slice_shape`, you can use tags.", .{ self, slice_shape }); + } + + const index_coord_axis = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); + stdx.debug.assert(indices.dim(index_coord_axis) == slice_shape.rank(), "gatherSlices({f}, slice={f}, indices) expects 'indices' to be a tensor [..., {}], got {f}", .{ self, slice_shape, slice_shape.rank(), indices }); + + // Compute result shape + var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); + var slice_dims = self._shape._dims; + var self_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var indices_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var start_index_map: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var self_offset_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); + + if (tagged_api and indices._shape.hasTag(t) != null) { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_batch_axes.appendAssumeCapacity(@intCast(self_ax)); + indices_batch_axes.appendAssumeCapacity(indices._shape.axis(t)); + slice_dims.set(self_ax, 1); + stdx.debug.assert(slice_shape.hasTag(t) == null, "gatherSlices expect axes to be either batches or slices axes. Axis {s} has been found both in `slices={f}` and `indices={f}`", .{ t, slice_shape, indices }); + } else if (maybe_slice_ax) |slice_ax| { + // Specified axes contains the start offset of the slices, + // and are collected in `start_index_map`. + const slice_dim = slice_shape.dim(slice_ax); + stdx.debug.assert(slice_dim <= self._shape.dim(self_ax), "gatherSlices expects `slice_shape` to be smaller than `self.shape()`. On axis {s}, got {f} > {f}.", .{ t, slice_shape, self._shape }); + slice_dims.set(self_ax, slice_dim); + res_shape = res_shape.appendDim(slice_dim, t); + start_index_map.appendAssumeCapacity(@intCast(self_ax)); + self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); + } else { + // non-batching, non-indexed axes + res_shape = res_shape.appendDim(self.dim(self_ax), t); + self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); + } + } + + const gather_op = dialects.stablehlo.gather( + mlirCtx(), + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = self_offset_axes.constSlice(), + .collapsed_slice_dims = &.{}, + .operand_batching_dims = self_batch_axes.constSlice(), + .start_indices_batching_dims = indices_batch_axes.constSlice(), + .start_index_map = start_index_map.constSlice(), + .index_vector_dim = index_coord_axis, + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, gather_op.result(0)); + } + + /// Merges two or more contiguous axes into one axis. + pub fn merge(self: Tensor, merges_: anytype) Tensor { + return self.reshape(self._shape.mergeAxes(merges_)); + } + + /// Slices a Tensor across many axes, with runtime known offsets. + /// + /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. + /// When using the tagged API it is allowed to not specify some axes. + /// But with the non-tagged API all slices need to be specified. + /// Examples: + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ .a = .{ .start = a_off, .len = 11}); + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ + /// .a = .{ .start = a_off, .len = 11 }, + /// .b = .{ .start = b_off, .len = 12 }, + /// }); + /// Tensor(.{ 20,30,40}).dynamicSlice(.{.{ .start = scalar(0, .i32), .len = 20 }, .{ .start = b_off, .len = 12 }, .{ .start = scalar(0, .i32), .len = 40 }}); + /// ``` + pub fn dynamicSlice(self: Tensor, slices_: anytype) Tensor { + // TODO: the untagged api is a bit verbose. Should I allow: `Tensor(.{ 20,30,40}).dynamicSlice(.{.{}, .{ .start = b_off, .len = 12 }, .{}});` ?? + // + const slices, const slices_tags = Shape.parseStruct(DynSlice, slices_); + + // TODO use slices and slices_tags for the format. + // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" + // which is kinda ugly. + + const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; + const zero = Tensor.scalar(0, idx_dtype).value(); + var offset_values = [_]*const mlir.Value{zero} ** MAX_RANK; + var res_shape = self._shape; + for (slices.constSlice(), 0..) |slice_, i| { + const offset = slice_.start; + const len = slice_.len; + if (slices_tags.len == 0) { + stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {d} and {d}", .{ self.rank(), slices.len }); + + offset_values[i] = offset.value(); + res_shape._dims.set(i, len); + + stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {d} and {d} for index {d}", .{ len, self.dim(i), i }); + } else { + const t = slices_tags.get(i); + const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {f})", .{ t, self._shape }); + + stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {d} and {d} for axis {s}", .{ len, self.dim(a), t }); + + offset_values[a] = offset.value(); + res_shape._dims.set(a, len); + } + } + const op = dialects.stablehlo.dynamic_slice(mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. + /// + /// The output dtype will be `.i32`, unless the given axis has a too big dimension, in that case we use `.i64`. + /// In most program this shouldn't matter, because typically this will be used in a comparison, + /// or explicitly converted by the user to do floating point arithmetic. + pub fn iota(sh: Shape, axis_: anytype) Tensor { + const a = sh.axis(axis_); + const dt: zml.DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; + const res_shape = sh.withDtype(dt); + + const tensor_type = mlir.rankedTensorType(res_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), res_shape.dtype())); + var op = dialects.stablehlo.iota( + mlirCtx(), + a, + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + /// Returns a Tensor containing the result of the outer product between the input Tensors. + pub fn outer(self: Tensor, other: Tensor) Tensor { + if (self.rank() + other.rank() == 1) { + return self.mul(other); + } + + const res_shape = self.shape().outer(other.shape()); + return self.broad(res_shape).mul(other.broad(res_shape)); + } + + /// Appends a 1-dim axis, with the given tag. + pub fn appendAxes(self: Tensor, t: anytype) Tensor { + // stdx.debug.assert(self.rank() < Tensor.MAX_RANK - t.len, "appendAxis expects tensor rank to be small enough in order to extend it, got {} and {} (max is {})", .{ self.rank(), t.len, Tensor.MAX_RANK }); + + return self.insertAxes(.last, t); + } + + /// Repeats a Tensor several times along the given axis. + /// + /// * repeat1d(x, axis, 4) = concat(&.{x, x, x, x}, axis); + /// * repeat1d([0, 1, 2, 3], 0, 2) = [0, 1, 2, 3, 0, 1, 2, 3] + pub fn repeat1d(self: Tensor, axis_: anytype, n_rep: u63) Tensor { + if (n_rep == 1) { + return self; + } + + const a = self.axis(axis_); + const broadshape = self._shape.insert(a + 1, .{n_rep}); + const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + + var res = self.broadcast(broadshape, repeat_dims.dims()).flatten(a); + // Restor the tag that has been lost by flatten. + res._shape._tags.set(a, self._shape.tag(a)); + + return res; + } + + /// Returns a Tensor containing the softmax function applied to each element of the input Tensor. + pub fn softmax(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + const max_val = self.max(a); + const row_mask = max_val.cmp(.GT, Tensor.scalar(-std.math.inf(f64), self.dtype())); + + const exp_diff_max = self.sub(self.max(a).broad(self._shape)).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a).broad(self._shape)); + + // If a row is full -inf return full 0 instead of full nan, + // this fix attention when mask hides a full row. + return row_mask.broad(self.shape()).select(res, Tensor.scalar(0, self.dtype())); + } + + inline fn toI64(values: anytype) []i64 { + var res: [Tensor.MAX_RANK]i64 = undefined; + for (values, 0..) |val, i| res[i] = @intCast(val); + return res[0..values.len]; + } + + fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { + var perm: stdx.BoundedArray(struct { u8, bool }, Tensor.MAX_RANK) = .{}; + // Don't rewrite on invalid inputs. + if (permutation.len > x.rank()) return false; + for (permutation) |ax| { + const squeezable = x.dim(ax) == 1; + perm.appendAssumeCapacity(.{ @intCast(ax), squeezable }); + } + + var effective_ax: u8 = 0; + for (0..perm.len) |i| { + const ax, const squeezable = perm.get(i); + if (squeezable) { + // Effectively squeeze this axis by decrementing axes coming after by 1. + for (i..perm.len) |j| { + if (perm.buffer[j][0] > ax) { + perm.buffer[j][0] -= 1; + } + } + continue; + } + + if (ax != effective_ax) return false; + effective_ax += 1; + } + + return true; + } + + /// Returns a transposed Tensor computed using the given axes. + pub fn transpose(self: Tensor, axes_: anytype) Tensor { + const axes__ = self.axes(axes_).constSlice(); + const default_perm = [MAX_RANK]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; + const no_op = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + + const permutation: []const i64 = if (axes__.len == 0) + default_perm[MAX_RANK - self.rank() ..] + else + toI64(axes__); + + stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, MAX_RANK + 2)] }); + + if (std.mem.eql(i64, permutation, no_op[0..self.rank()])) { + return self; + } + + const res_shape = self._shape.transpose(permutation); + if (transposeIsJustAReshape(self.shape(), permutation)) { + return self.reshape(res_shape); + } + + const tensor_type = mlir.rankedTensorType(res_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), res_shape.dtype())); + const op = dialects.stablehlo.transpose( + mlirCtx(), + self.value(), + tensor_type, + .{ .permutation = toI64(permutation) }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + /// Embeds a buffer with concrete values into an Mlir program. + pub fn constantTensor(bytes: []const u8, shape_: zml.Shape) Tensor { + const elem_type = zml.mlir.Type.fromDType(mlirCtx(), shape_.dtype()); + //const elem_type = zml.mlir.denseElementAttrType(shape.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{shape.dtype()}); + const constant_op = dialects.stablehlo.constant(mlirCtx(), shape_.dims(), elem_type, bytes, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(shape_, constant_op.result(0)); + } + + /// Inserts 1-dim axes at the given position, with the given tags. + /// `.{.a = 5, .b = 4}.insert(.b, .{ .c, .d }) -> .{ .a = 5, .c = 1, .d = 1, .b = 4 }` + pub fn insertAxes(self: Tensor, axis_: anytype, tags: anytype) Tensor { + const tags_ = Shape.parseTags(tags); + const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + self.rank() + else + self.axis(axis_); + + var res_shape = self._shape; + const ones = [_]i64{1} ** MAX_RANK; + res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; + res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + return self.reshape(res_shape); + } + + /// Returns a Tensor containing the maximum over a given axis. + pub fn max(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + const result = reduce(.{self}, .{Tensor.constant(self.dtype().minValue())}, &.{a}, struct { + pub fn cmp(values: ReduceArgs) struct { Tensor } { + return .{values.left.maximum(values.right.convert(values.left.dtype()))}; + } + }.cmp, .{}); + return result[0]; + } + + /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. + pub fn exp(self: Tensor) Tensor { + const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Generalized version of scatter to many inputs. + /// See `zml.Tensor.scatterSlices` for documentation on scatter. + /// + /// This allows to use the same indices to update several tensors at once, + /// and where the update function is allow to look at elements from the different tensors + /// to compute the final value. + /// + /// This sounds nice but in practice XLA doesn't support this well on GPU, + /// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. + pub fn scatter( + inputs: anytype, + index_tensors: anytype, + updates: anytype, + comptime func: anytype, + context: anytype, + opts: Tensor.ScatterOpts, + ) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const update_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].left = Tensor.init(zml.Shape.init(.{}, inputs[i].dtype())); + args[i].right = Tensor.init(zml.Shape.init(.{}, inputs[i].dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), zml.mlir.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].right.dims(), zml.mlir.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const update_block = mlir.Block.init(&block_types, &block_locs); + errdefer update_block.deinit(); + + CompilationContext.current().pushBlock(update_block); + defer CompilationContext.current().popBlock(); + + inline for (0..inputs.len) |i| { + CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, args[i].left.id, i) catch unreachable; + CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, args[i].right.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(update_block); + break :b .{ update_block, result }; + }; + + // Note: I was a bit lazy here, and I only look at tags on the first tensor. + // we probably should check all of them. + const self = zml.meta.first(Tensor, inputs); + const update = zml.meta.first(Tensor, updates); + var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); + + if (indices_per_axis.len == 0) return inputs; + + // validate coord axes: all coord_axes should exist inside self + for (indices_axes.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); + } + + // Handle scalar indices by broadcasting them to the indices with the highest rank. + const indices_shape = blk: { + var higher_rank = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |indices| { + if (indices.rank() > higher_rank.rank()) { + higher_rank = indices.shape(); + } + } + break :blk higher_rank; + }; + for (indices_per_axis.slice()) |*idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); + stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); + idx.* = idx.broad(indices_shape); + } + + // rewrite simple scatters to dynamicUpdateSlice. + if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { + return .{self.dynamicUpdateSlice(index_tensors, update)}; + } + + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. + var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); + const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); + // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; + // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var updates_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); + + const op = dialects.stablehlo.scatter( + mlirCtx(), + &input_values, + &.{indices.value()}, + &updates_values, + update_block, + .{ + .update_window_dims = _collectAxes(AxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(AxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(AxisKind, config.op_kind, .batching).constSlice(), + .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), + .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), + .index_vector_dim = indices.rank() - 1, + .indices_are_sorted = opts.indices_are_sorted, + .unique_indices = opts.indices_are_unique, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), op.result(i)); + } + + return result; + } + + const ScatterConfig = struct { + op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + indices_batch_axes: Shape.DimsArray = .{}, + scatter_to_operand_axes: Shape.DimsArray = .{}, + updates_transpose: Shape.AxesArray = .{}, + }; + + const AxisKind = enum { batching, update_window, inserted_window, window_id }; + + fn scatterConfig( + op: Shape, + update: Shape, + indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: Shape.TagsArray, + ) ScatterConfig { + var op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var indices_batch_axes: Shape.DimsArray = .{}; + var scatter_to_operand_axes: Shape.DimsArray = .{}; + var updates_transpose: Shape.AxesArray = .{}; + + const tagged_api = indices_axes.len > 0; + const indices = indices_per_axis.get(0).shape(); + + if (tagged_api) { + for (indices_axes.constSlice()) |t| { + scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); + } + for (indices.tags()) |t| { + stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); + updates_transpose.appendAssumeCapacity(update.axis(t)); + } + + for (op.tags()) |t| { + if (update.hasTag(t)) |up_ax| { + updates_transpose.appendAssumeCapacity(up_ax); + + if (indices.hasTag(t)) |id_ax| { + if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { + // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim + op_kind.appendAssumeCapacity(.inserted_window); + } else { + // tag is in op, indices and updates -> it's a batching dim + op_kind.appendAssumeCapacity(.batching); + indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); + } + } else { + op_kind.appendAssumeCapacity(.update_window); + } + } else { + op_kind.appendAssumeCapacity(.inserted_window); + } + } + + for (update.tags(), 0..) |t, up_ax| { + // Handle batch axes right away. + if (op.hasTag(t)) |self_ax| { + if (op_kind.get(self_ax) == .batching) { + up_kind.appendAssumeCapacity(.batching); + continue; + } + } + if (indices.hasTag(t) != null) { + up_kind.appendAssumeCapacity(.window_id); + } else if (op.hasTag(t)) |self_ax| { + stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); + up_kind.appendAssumeCapacity(.update_window); + } else { + // TODO: consider accepting untagged update here. + std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); + } + } + } else { + for (0..indices_per_axis.len) |i| { + op_kind.appendAssumeCapacity(.inserted_window); + scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); + up_kind.appendAssumeCapacity(.window_id); + } + for (indices_per_axis.len..op.rank()) |_| { + op_kind.appendAssumeCapacity(.update_window); + } + for (indices_per_axis.len..update.rank()) |_| { + up_kind.appendAssumeCapacity(.update_window); + } + for (0..update.rank()) |i| { + updates_transpose.appendAssumeCapacity(@intCast(i)); + } + } + + return .{ + .op_kind = op_kind, + .up_kind = up_kind, + .indices_batch_axes = indices_batch_axes, + .scatter_to_operand_axes = scatter_to_operand_axes, + .updates_transpose = updates_transpose, + }; + } + + //test scatterConfig { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // const Local = struct { + // pub fn _idx(idx_shape: anytype) Tensor { + // return Tensor.constant(idx_shape, .{ .i32 = 0 }); + // } + // }; + + // const idx = Local._idx; + // const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); + + // // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + // { + // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); + // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + // const cfg = scatterConfig(op, update, indices, coords_tags); + // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); + // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); + // } + + // // similar, but use the normalized form where .a is no longer an explicit batching axis. + // { + // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); + // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + // const cfg = scatterConfig(op, update, indices, coords_tags); + // try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); + // try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); + // } + //} + + /// Concatenate all indices tensor in one tensor. + /// + /// Is allowed to reorder stuff to simplify the job of the backend, + /// and to expand the batching dims. + fn scatterPrepareIndices( + cfg: *ScatterConfig, + op: Shape, + update: Shape, + indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: *Shape.TagsArray, + ) Tensor { + var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; + const batching = _collectAxes(AxisKind, cfg.op_kind, .batching); + for (batching.constSlice()) |batch_ax| { + const id_shape = indices_per_axis.get(0).shape(); + // batching requires tagging, so we're sure to have a tag here. + const batch_tag = op.tag(batch_ax); + indices_axes.appendAssumeCapacity(batch_tag); + const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); + indices_per_axis.appendAssumeCapacity(batch_id); + cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; + cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; + old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); + } + cfg.indices_batch_axes = .{}; + + // Reorder the axes so that in indices_per_axis is ordered like in op if possible. + // TODO: transpose updates if needed + var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; + var scatter_to_op_axes: Shape.DimsArray = .{}; + + while (old_scatter_to_op_axes.len > 0) { + const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; + const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); + const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); + + scatter_to_op_axes.appendAssumeCapacity(op_ax); + indices.appendAssumeCapacity(scatter_idx); + } + cfg.scatter_to_operand_axes = scatter_to_op_axes; + + for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { + if (i != sc_ax) { + //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); + break; + } + } + return Tensor.stack(indices.constSlice(), .last, .coord); + } + + pub const ScatterOpts = struct { + /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final offset in `self` + /// Result is undefined if the promise is violated. + indices_are_sorted: bool = false, + + /// Promise scatter that slices don't overlap. + /// Result is undefined if the promise is violated. + /// This allows for better code generation, because it means that updates can be applied in parallel. + indices_are_unique: bool = false, + + /// Function used to update previous value in `self` with values from `updates`. + /// If `update_fn` is not associative (ie the order of execution matters), + /// then you should make sure the slices don't overlap, + /// otherwise the result will depend on the runtime scheduling + /// of the operator which is backend specific. + update_fn: *const fn (ReduceArgs) struct { Tensor } = increment, + + pub fn increment(values: ReduceArgs) struct { Tensor } { + return .{values.left.add(values.right)}; + } + + pub fn override(values: ReduceArgs) struct { Tensor } { + return .{values.right}; + } + }; + + /// Update the given tensor, by copying `values` into slice by slice into `self`. + /// The slices are chosen at runtime by interpreting indices as coordinates into `self`. + /// This is a generalized version of `dynamicUpdateSlice` where more than one offset can be specified at a time. + /// + /// ### Arguments + /// + /// - Return a tensor with same shape than `self`, with updated content. + /// - `indices` is a set of Tensor (typically rank 1), representing coordinates into `self`. + /// all indices must have the same shape, but scalars are accepted. + /// - each `indices` entry contains offset along an axes into `self`. + /// Typically axes are identified by their tags, but in the absence of tags on `indices`, + /// The entry in indices will be assigned to axes of `self` from major to minor axis. + /// It is recommended to have indices referencing only major axes of `self` for better performance. + /// - `values` shape is obtained by concatenating the shape of `indices` with the shape of the slices to be extracted. + /// - `opts`: `zml.Tensor.ScatterOpts` des + /// + /// ### Sample input/output shapes with corresponding pseudo-code. + /// + /// Basic `scatterSlices` with the first two axes (.a, .b) being indexed, and full (.c, .d) slice copies: + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N]}, y[N, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],c,d] += y[n, c, d]; + /// }} + /// } + /// return z; + /// } + /// ``` + /// + /// `scatterSlices` with the first three axes (.a, .b, .c) being indexed, and a partial copy of (.c, .d). + /// Note that .c axis is present both in the indices and updates, and `updates.dim(.c) < self.dim(.c)`. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N], .c=off_c[N]}, y[N, C', D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C') |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],off_c[n]+c,d] += y[n, c, d]; + /// }} + /// } + /// return z; + /// } + /// ``` + /// + /// `scatterSlices` with the first axis .a being indexed, and where .b is used as a batching axis. + /// Note that here .b axis is present in `self`, `off_a`, and `updates`, + /// and is not mentionned in the axes of indices. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[B,N]}, y[N, B, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..B) |b| { + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[b,n],b,c,d] += y[n, b, c, d]; + /// }} + /// } + /// } + /// return z; + /// } + /// ``` + /// + /// ### Warnings + /// + /// - if `opts.update_fn` is not associative not all calls to `scatterSlices` are sound. + /// In particular if you scatter overlapping slices, with `zml.Tensor.ScatterOpts.override`, + /// then the result will depend on the execution order that you don't control. + /// - `scatterSlices` is a very expressive operator, and can lead to complicated code generation + /// that requires host<->device synchronization. + /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. + pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { + //scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); + + const UpdateType = @TypeOf(ScatterOpts.increment); + + const Custom = struct { + pub fn inc(values: ReduceArgs, custom: *const UpdateType) struct { Tensor } { + return @call(.auto, custom, .{values}); + } + }; + + return scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; + } + + /// Concatenates the input Tensors along a new axis. The Tensors must have the same shape. + /// For x, y, z of shape .{ .a = 10, .b = 11, .c = 12 }: + /// - Tensor.stack(&.{x, y, z}, .b, .layers) -> .{ .a, .layers, .b, .c } + /// - Tensor.stack(&.{x, y, z}, 1, .layers) -> .{ .a, .layers, .b, .c } + /// - Tensor.stack(&.{x, y, z}, .last, .layers) -> .{ .a, .b, .c, .layers } + pub fn stack(tensors: []const Tensor, axis_: anytype, tag: anytype) Tensor { + // Note: we could ask the compilation context for some memory instead of stack allocating + stdx.debug.assert(tensors.len <= 32, "stack only supports up to 32 tensors, got {}", .{tensors.len}); + + const shape0 = tensors[0]._shape; + const res_shape = shape0.insertTag(axis_, 1, tag); + + for (tensors[1..]) |tensor| { + stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); + } + + var reshaped: [32]Tensor = undefined; + for (tensors, 0..) |tensor, i| { + reshaped[i] = tensor.reshape(res_shape); + } + + // Be careful here: we need to resolve ax before calling concatenate, + // because we added an axis, so all + const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + shape0.rank() + else + shape0.axis(axis_); + + return Tensor.concatenate(reshaped[0..tensors.len], ax); + } + + /// Updates a part of the input Tensor using the given 'update' Tensor, with runtime known offsets. + /// + /// The offsets are specified similarly to the dynamicSlice api. + /// It's semantically equivalent to: + /// self.dynamicSlice(offsets_) := update + /// Examples: + /// ``` + /// Tensor(.{ .a = 2, .b = 5 }).dynamicUpdateSlice(.{ .a = scalar(1, .i32) }, Tensor(.{ .b = 5 })); + /// ``` + pub fn dynamicUpdateSlice(self: Tensor, offset_: anytype, update_: Tensor) Tensor { + // TODO: add updateSlice for when the offset isn't dynamic + stdx.debug.assert(self.dtype() == update_.dtype(), "dynamicUpdateSlice expects input and 'update_' tensors to be of the same type, got {} and {}", .{ self.dtype(), update_.dtype() }); + + const offset, const offset_tags = Shape.parseStruct(Tensor, offset_); + // log.debug("offset: {any}, offset_tags: {any}", .{ offset, offset_tags }); + for (offset.constSlice(), 0..) |start_idx, i| { + stdx.debug.assert(start_idx.rank() == 0, "dynamicUpdateSlice expects 'offset_' tensor ranks to be equal to 0, got {} at index {}", .{ start_idx.rank(), i }); + } + + const tagged_api = update_._shape.isFullyTagged() and self._shape.isFullyTagged() and offset_tags.len > 0; + // When using tags, we can safely insert axis with a 1-dim. + // the offset into the inserted axis will need to be specified through indices. + var update = update_; + if (tagged_api) { + // Check that all update tags are known. + for (update._shape._tags.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "dynamicUpdateSlice expects 'update_' tensor tags to be a subset of input tensor tags but {s} is missing (input shape is {f})", .{ t, self._shape }); + } + + var update_shape = self._shape; + var prev_ax: i8 = -1; + for (self._shape.tags(), 0..) |t, self_ax| { + if (update._shape.hasTag(t)) |up_ax| { + stdx.debug.assert(up_ax == prev_ax + 1, "dynamicUpdateSlice expects 'update_' and input tensor axis to have the same order, got {f} and {f}. (hint: you need to explicitly transpose 'update_')", .{ update_, self }); + + update_shape._dims.set(self_ax, update.dim(up_ax)); + prev_ax = up_ax; + } else { + update_shape._dims.set(self_ax, 1); + } + } + update = update.reshape(update_shape); + } + + stdx.debug.assert(self.rank() == update.rank(), "dynamicUpdateSlice expects input and computed update tensors to have the same rank, got {f} and {f}", .{ self, update }); + + for (self.dims(), update.dims(), 0..) |self_d, up_d, ax| { + const t = self._shape.debugTag(ax); + stdx.debug.assert(up_d <= self_d, "dynamicUpdateSlice expects 'update_' dimensions to be less than or equal to their corresponding dimension in input tensor, got {} and {} for axis .{s}", .{ up_d, self_d, t }); + + if (tagged_api and up_d < self_d) { + const axis_has_offset = std.mem.indexOfScalar(Shape.Tag, offset_tags.constSlice(), self._shape._tags.get(ax)) != null; + + stdx.debug.assert(axis_has_offset, "dynamicUpdateSlice expects 'update_' dimensions to be equal to their corresponding dimension in input tensor, got {} and {} for axis .{s} (hint: you need to provide an offset)", .{ up_d, self_d, t }); + } + } + + const idx_dtype = if (offset.len > 0) offset.get(0).dtype() else .i32; + const zero = Tensor.scalar(0, idx_dtype).value(); + var offset_values: [MAX_RANK]*const mlir.Value = undefined; + if (offset_tags.len == 0) { + // Without offset tags we need the same number of offset than rank. + stdx.debug.assert(self.rank() == offset.len, "dynamicUpdateSlice expects input tensor rank and 'offset_' length to be equal, got {} and {}", .{ self.rank(), offset.len }); + + for (offset.constSlice(), 0..) |idx, i| { + offset_values[i] = idx.value(); + } + } else { + // If an axis isn't specified, update the full slice. + // This is only allowed when using tagged sliced. + offset_values = .{zero} ** MAX_RANK; + for (offset.constSlice(), offset_tags.constSlice()) |start, t| { + const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); + offset_values[a] = start.value(); + } + } + + const op = dialects.stablehlo.dynamic_update_slice( + mlirCtx(), + self.value(), + update.value(), + offset_values[0..self.rank()], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + pub fn getContext(self: Tensor) *CompilationContext { _ = self; // autofix return global_compilation_context.?; @@ -696,6 +2240,28 @@ pub const Tensor = struct { } }; +fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, Tensor.MAX_RANK) } { + const AxesT = @TypeOf(axes_); + const axes_is_scalar = AxesT == EnumLiteral or AxesT == comptime_int or @typeInfo(AxesT) == .int; + + const coord_axes = if (axes_is_scalar) + stdx.BoundedArray(u3, Tensor.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable + else + self.axes(axes_); + + return .{ axes_is_scalar, coord_axes }; +} + +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { + var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (bounded_array.constSlice(), 0..) |v, ax| { + if (v == value) { + res.appendAssumeCapacity(@intCast(ax)); + } + } + return res; +} + const MoE = struct { experts: Mlp, router: zml.nn.Linear, @@ -839,7 +2405,6 @@ const Mnist = struct { /// just two linear layers + relu activation pub fn forward(self: Mnist, input: Tensor) Tensor { var x = input.flattenAll().convert(.f32).withTags(.{.d}); - std.debug.print("x.shape: {f}\n", .{x.shape()}); for (self.layers) |layer| { x = layer.forward(x).withTags(.{.d}); } @@ -885,7 +2450,7 @@ pub fn singleTransfer(allocator: std.mem.Allocator, shape: Shape, platform: Plat var transfer = try Transfer.init(allocator, &.{shape}, platform); errdefer transfer.deinit(allocator, platform); - const entry = .{ + const entry: TransferEntry = .{ .buffer = zml.Buffer.fromPjrtBuffers(platform, shape, &.{transfer.get(0).buffer}), .writer = transfer.get(0).writer(), }; @@ -909,7 +2474,15 @@ pub fn multiTransfer(allocator: std.mem.Allocator, shapes: anytype, platform: Pl return .{ transfer, entries }; } -fn autoLoad(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferStore4.View, platform: Platform) !Bufferized(@TypeOf(model)) { +pub fn bufferTransfer(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, name: []const u8, platform: Platform) !zml.Buffer { + const shape = buffer_store.getShape(name).?; + var transfer, const entry = try singleTransfer(allocator, shape, platform); + defer transfer.deinit(allocator, platform); + buffer_store.getReader(name).stream(entry.writer); + return entry.buffer; +} + +pub fn autoLoad(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferStore5.View, platform: Platform) !Bufferized(@TypeOf(model)) { const Model = @TypeOf(model); var result: Bufferized(Model) = undefined; @@ -921,7 +2494,7 @@ fn autoLoad(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferSt var transfer: Transfer = try .init(arena.allocator(), shapes, platform); defer transfer.deinit(arena.allocator(), platform); - const readers = try arena.allocator().alloc(BufferStore4.Reader, shapes.len); + const readers = try arena.allocator().alloc(BufferStore5.Reader, shapes.len); { var index: usize = 0; @@ -929,7 +2502,7 @@ fn autoLoad(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferSt switch (type_info) { .@"struct" => |struct_type_info| { inline for (struct_type_info.fields) |field| { - const reader = buffer_store.get(field.name).?.reader(); + const reader = buffer_store.getReader(field.name); readers[index] = reader; index += 1; } @@ -939,7 +2512,7 @@ fn autoLoad(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferSt } const LocalContext = struct { - readers: []BufferStore4.Reader, + readers: []BufferStore5.Reader, shapes: []const Shape, platform: Platform, transfer: *Transfer, @@ -1273,6 +2846,33 @@ pub fn asyncMain2() !void { defer exe.deinit(); } +pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: llama.LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + if (skip_llama3_encoding) { + // Copy so the ownership is the same in both branches. + return try allocator.dupe(u32, try encoder.encode(prompt)); + } + + const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); + + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline }); + + try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); + + return tokens.toOwnedSlice(allocator); +} + pub fn asyncMain() !void { var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; defer _ = gpa.deinit(); @@ -1301,6 +2901,18 @@ pub fn asyncMain() !void { }; defer allocator.free(model_weights_path); + const model_tokenizer_path = try std.fs.path.join(allocator, &.{ process_args[1], "tokenizer.json" }); + defer allocator.free(model_tokenizer_path); + + var tokenizer = blk: { + std.log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); + var timer = try stdx.time.Timer.start(); + defer std.log.info("Loaded tokenizer from {s} [{D}]", .{ model_tokenizer_path, timer.read() }); + + break :blk try zml.tokenizer.Tokenizer.fromFile(allocator, model_tokenizer_path); + }; + errdefer tokenizer.deinit(); + const config = blk: { var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); defer config_json_file.close() catch unreachable; @@ -1312,6 +2924,12 @@ pub fn asyncMain() !void { break :blk config_obj; }; + const options: llama.LlamaLM.Options = .{ + .sampling_strategy = null, + .max_seq_len = 256, + .qkv_type = .merged, + }; + var old_buffer_store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); defer old_buffer_store.deinit(); @@ -1324,12 +2942,160 @@ pub fn asyncMain() !void { const platform = context.autoPlatform(.{}); context.printAvailablePlatforms(platform); - var llama_model = try llama.LlamaLM.init(allocator, buffer_store.view(), config); + var llama_model = try llama.LlamaLM.init(allocator, buffer_store.view(), config, options); defer llama_model.deinit(allocator); - const llama_buffers = try llama.LlamaLM.loadBuffers(allocator, llama_model, buffer_store.view(), platform); + const tokens = Tensor.init(zml.Shape.init(.{ .s = 1 }, .u32)); + const token_index = Tensor.init(zml.Shape.init(.{}, .u32)); + const kv_cache = llama.KvCache.init(zml.Shape.init(.{ .layer = config.num_hidden_layers, .k = options.max_seq_len, .h = config.num_key_value_heads, .hd = 128 }, .bf16)); + const rng = Tensor.Rng.init(); + + const exe = try compileModel(allocator, llama.LlamaLM.forward, llama_model, .{ tokens, token_index, kv_cache, rng }, platform); + defer exe.deinit(); + + // Compile executables used when loading buffers + const merge_qkv_exe = try compileMergeQkvExe(allocator, buffer_store.view(), platform); + defer merge_qkv_exe.deinit(); + + const precompute_qkv_exe = try compilePrecomputeQkvExe(allocator, buffer_store.view(), platform); + defer precompute_qkv_exe.deinit(); + + const llama_buffers = try llama.LlamaLM.loadBuffers(allocator, llama_model, buffer_store.view(), platform, .{ .merge_qkv_exe = merge_qkv_exe, .precompute_qkv_exe = precompute_qkv_exe }); defer llama_buffers.deinit(allocator); + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + const prompt = "What is the capital of France ?"; + const tokenized_prompt = try tokenizePrompt(allocator, tokenizer, config, prompt, false); + defer allocator.free(tokenized_prompt); + + var rng_buffer = try Tensor.Rng.initBuffer(platform, 0); + defer rng_buffer._state.deinit(); + + var kv_cache_buffer = try llama.KvCache.initBuffer(kv_cache.k.shape(), platform); + defer kv_cache_buffer.layer_index.deinit(); + defer kv_cache_buffer.k.deinit(); + defer kv_cache_buffer.v.deinit(); + + var token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{0})); + defer token_index_buffer.deinit(); + + var token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{0})); + defer token_buffer.deinit(); + + var new_token_id: u32 = undefined; + for (tokenized_prompt[0..], 0..) |token_id, i| { + std.debug.print("Running iteration {} token id: {}\n", .{ i, token_id }); + token_index_buffer.deinit(); + token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{@intCast(i)})); + + token_buffer.deinit(); + token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{token_id})); + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + args.set(.{ llama_buffers, token_buffer, token_index_buffer, kv_cache_buffer, rng_buffer }); + std.log.debug("Calling exe", .{}); + exe.call(args, &results); + + token_buffer.deinit(); + kv_cache_buffer.layer_index.deinit(); + kv_cache_buffer.k.deinit(); + kv_cache_buffer.v.deinit(); + rng_buffer._state.deinit(); + results.fill(&.{ &token_buffer, &kv_cache_buffer, &rng_buffer._state }); + + const host = try token_buffer.toHostAlloc(allocator); + defer host.deinit(allocator); + + const predicted_token = host.items(u32)[0]; + const predicted_token_str = try tokenizer_decoder.decode(&.{predicted_token}); + std.debug.print("Predicted: {} {s}\n", .{ predicted_token, predicted_token_str }); + new_token_id = predicted_token; + + //const embeds_buffer = b: { + // var args = try exe.args(allocator); + // defer args.deinit(allocator); + + // var results = try exe.results(allocator); + // defer results.deinit(allocator); + + // args.set(.{ llama_buffers, token_buffer, token_index_buffer, kv_cache_buffer, rng_buffer }); + // std.log.debug("Calling exe", .{}); + // exe.call(args, &results); + + // //token_buffer.deinit(); + // kv_cache_buffer.layer_index.deinit(); + // kv_cache_buffer.k.deinit(); + // kv_cache_buffer.v.deinit(); + // rng_buffer._state.deinit(); + // var embeds_buffer: zml.Buffer = undefined; + // results.fill(&.{ &embeds_buffer, &kv_cache_buffer, &rng_buffer._state }); + + // break :b embeds_buffer; + //}; + //defer embeds_buffer.deinit(); + + //{ + // var args = try exe2.args(allocator); + // defer args.deinit(allocator); + + // var results = try exe2.results(allocator); + // defer results.deinit(allocator); + + // args.set(.{ llama_buffers, embeds_buffer, token_index_buffer, kv_cache_buffer, rng_buffer }); + // exe2.call(args, &results); + + // kv_cache_buffer.layer_index.deinit(); + // kv_cache_buffer.k.deinit(); + // kv_cache_buffer.v.deinit(); + // rng_buffer._state.deinit(); + // var hidden_buffer: zml.Buffer = undefined; + // results.fill(&.{ &hidden_buffer, &kv_cache_buffer, &rng_buffer._state }); + + // const host = try hidden_buffer.toHostAlloc(allocator); + // defer host.deinit(allocator); + // std.debug.print("Predicted: {any}\n", .{host.items(f32)}); + //} + } + + for (0..10) |i| { + std.debug.print("Running iteration {}\n", .{i}); + token_index_buffer.deinit(); + token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{@intCast(i + tokenized_prompt.len)})); + + token_buffer.deinit(); + token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{new_token_id})); + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + args.set(.{ llama_buffers, token_buffer, token_index_buffer, kv_cache_buffer, rng_buffer }); + std.log.debug("Calling exe", .{}); + exe.call(args, &results); + + token_buffer.deinit(); + kv_cache_buffer.layer_index.deinit(); + kv_cache_buffer.k.deinit(); + kv_cache_buffer.v.deinit(); + rng_buffer._state.deinit(); + results.fill(&.{ &token_buffer, &kv_cache_buffer, &rng_buffer._state }); + + const host = try token_buffer.toHostAlloc(allocator); + defer host.deinit(allocator); + + const predicted_token = host.items(u32)[0]; + const predicted_token_str = try tokenizer_decoder.decode(&.{predicted_token}); + std.debug.print("Predicted: {} {s}\n", .{ predicted_token, predicted_token_str }); + new_token_id = predicted_token; + } + //const process_args = try std.process.argsAlloc(allocator); //defer std.process.argsFree(allocator, process_args); //const pt_model = process_args[1]; @@ -1448,6 +3214,25 @@ pub fn asyncMain() !void { //try testMnist(allocator, buffer_store2, mnist, platform); } +fn compileMergeQkvExe(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, platform: Platform) !Exe { + const proj: llama.SeparateQkv = .init(buffer_store.withPrefix("model.layers.0.self_attn")); + const exe = try compile(allocator, llama.mergeQkv3, .{proj}, platform); + errdefer exe.deinit(); + + return exe; +} + +fn compilePrecomputeQkvExe(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, platform: Platform) !Exe { + const proj: llama.SeparateQkv = .init(buffer_store.withPrefix("model.layers.0.self_attn")); + const embedding = Tensor.init(buffer_store.getShape("model.embed_tokens.weight").?).withTags(.{ .voc, .d }); + const layer_norm: llama.RmsNorm = .init(buffer_store.withPrefix("model.layers.0.input_layernorm"), 1e-5); + + const exe = try compile(allocator, llama.precomputeQkv0, .{ proj, layer_norm, embedding }, platform); + errdefer exe.deinit(); + + return exe; +} + //fn testUnifiedKvCache(allocator: std.mem.Allocator, platform: Platform) !void { // const kv_cache: KvCache = .{ // .unified = Tensor.init(Shape.init(.{ .layer = 10, .h = 16, .hd = 128 }, .f32)), @@ -1511,7 +3296,7 @@ fn testLinear(linear: Linear, x: Tensor) Tensor { return linear.forward(x); } -const CompilationContext = struct { +pub const CompilationContext = struct { allocator: std.mem.Allocator, mlir_registry: *mlir.DialectRegistry, @@ -1854,7 +3639,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m return loaded_executable; } -const Exe = struct { +pub const Exe = struct { platform: Platform, exe: *pjrt.LoadedExecutable, @@ -1967,6 +3752,7 @@ const Exe = struct { var context: LocalContext = .{ .self = self, .current_index = offset }; zml.meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const zml.Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); for (0..context_.self.flat_buffers.num_devices) |device_index| { context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); } @@ -2030,6 +3816,7 @@ const Exe = struct { var context: LocalContext = .{ .self = self, .current_index = 0 }; zml.meta.visit(struct { fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { + //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); var shards: zml.Buffer.Shards = .{}; for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); @@ -2378,7 +4165,7 @@ pub const ShardWriter = struct { } }; -const Slice = struct { +const TensorSlice = struct { shape: Shape, data: []const u8, }; @@ -2397,7 +4184,7 @@ pub const BufferStore4 = struct { self.slices.deinit(self.allocator); } - pub fn add(self: *BufferStore4, key: []const u8, slice: Slice, associated_tensor_id: ?usize) !void { + pub fn add(self: *BufferStore4, key: []const u8, slice: TensorSlice, associated_tensor_id: ?usize) !void { const key_copy = try self.allocator.dupe(u8, key); errdefer self.allocator.free(key_copy); @@ -2418,7 +4205,7 @@ pub const BufferStore4 = struct { var it = buffer_store.buffers.iterator(); while (it.next()) |entry| { const descriptor_entry = tensor_descriptor.getPtr(entry.key_ptr.*).?; - const slice: Slice = .{ .shape = entry.value_ptr.shape(), .data = entry.value_ptr.bytes() }; + const slice: TensorSlice = .{ .shape = entry.value_ptr.shape(), .data = entry.value_ptr.bytes() }; try new_buffer_store.add(entry.key_ptr.*, slice, descriptor_entry.associated_tensor_id); } @@ -2441,7 +4228,7 @@ pub const BufferStore4 = struct { }; pub const Entry = struct { - slice: Slice, + slice: TensorSlice, associated_tensor_id: ?usize = null, pub fn reader(entry: *const Entry) Reader { return .{ .entry = entry }; @@ -2596,6 +4383,24 @@ pub const BufferStore5 = struct { prefix_buffer: [256]u8 = undefined, prefix_length: usize = 0, + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + pub fn parent(self: *const View) View { + const slice = self.prefix() orelse unreachable; + const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); + var buffer: [256]u8 = undefined; + @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = index + 1, + }; + } + pub fn withPrefix(self: *const View, prefix_: []const u8) View { var buffer: [256]u8 = undefined; const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; @@ -2644,7 +4449,7 @@ pub const BufferStore5 = struct { return self.maybeCreateTensorWithTags(subkey, tagz).?; } - pub fn getReader(self: View, subkey: []const u8) ?Reader { + pub fn getReader(self: View, subkey: []const u8) Reader { return self.getMaybeReader(subkey).?; } @@ -2666,5 +4471,16 @@ pub const BufferStore5 = struct { const entry_ptr = self.store.getPtrFromKey(key) orelse return null; return entry_ptr.shape; } + + pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { + var buffer: [256]u8 = undefined; + const key = if (opts.no_prefix) + subkey + else b: { + break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + }; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } }; }; diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 06bcd6fa..30317127 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -2,7 +2,7 @@ const builtin = @import("builtin"); const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { - .macos => MacOsTracer, + .macos => FakeTracer, .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, else => FakeTracer, }; From 94fe98d50fcf26c4d56f83281bd28699bde106d5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 31 Oct 2025 16:36:37 +0100 Subject: [PATCH 44/47] code freeze before cleaning and sharding addition --- mlir/main.zig | 1807 +++++++++++++++++++++++++++++++++++++++--- stdx/meta.zig | 4 + zml/meta.zig | 87 ++ zml/tools/tracer.zig | 21 + zml/zml.zig | 1 + 5 files changed, 1801 insertions(+), 119 deletions(-) diff --git a/mlir/main.zig b/mlir/main.zig index fabd616f..be96b4fb 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -14,6 +14,310 @@ const Shape = zml.Shape; const EnumLiteral = @TypeOf(.enum_literal); const Platform = zml.Platform; +const Dims = stdx.BoundedArray(i64, zml.Shape.MAX_RANK); +const log = std.log.scoped(.main); + +const safetensor = + \\{ + \\ "metadata": { + \\ "total_size": 16060522496 + \\ }, + \\ "weight_map": { + \\ "lm_head.weight": "model-00004-of-00004.safetensors", + \\ "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors", + \\ "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + \\ "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + \\ "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.norm.weight": "model-00004-of-00004.safetensors" + \\ } + \\} +; + pub const std_options: std.Options = .{ .log_level = .debug, .log_scope_levels = &.{ @@ -28,6 +332,8 @@ pub const TensorOriginType = enum { value, }; +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); + pub const TensorOrigin = union(TensorOriginType) { argument: void, value: *const mlir.Value, @@ -37,12 +343,9 @@ pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; pub const MAX_RANK = Shape.MAX_RANK; - pub const _Donation = union(enum) { no_buffer, input_buffer, arg: u16 }; - id: usize, auto_broadcast: bool = false, _shape: zml.Shape, - _donation: _Donation = .no_buffer, tensor_origin: TensorOrigin = .{ .argument = {} }, fn mlirType(self: Tensor, mlir_ctx: *mlir.Context) *const mlir.Type { @@ -57,7 +360,7 @@ pub const Tensor = struct { } fn currentBlock() *mlir.Block { - return CompilationContext.current().currentBlock(); + return CompilationContext.current().currentScope().block; } pub fn init(shape_: zml.Shape) Tensor { @@ -397,7 +700,7 @@ pub const Tensor = struct { res_shape.dims(), zml.mlir.Type.fromDType(ctx.mlir_ctx, res_shape.dtype()), ); - const broadcast_op = dialects.stablehlo.broadcast_in_dim(ctx.mlir_ctx, self.value(), axes_, result_type, .unknown(ctx.mlir_ctx)).appendTo(ctx.currentBlock()); + const broadcast_op = dialects.stablehlo.broadcast_in_dim(ctx.mlir_ctx, self.value(), axes_, result_type, .unknown(ctx.mlir_ctx)).appendTo(currentBlock()); return _result(res_shape, broadcast_op.result(0)); } @@ -627,9 +930,10 @@ pub const Tensor = struct { CompilationContext.current().pushBlock(reduce_block); defer CompilationContext.current().popBlock(); + const scope = CompilationContext.current().currentScope(); inline for (0..inits.len) |i| { - CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, args[i].left.id, i) catch unreachable; - CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, args[i].right.id, i + inits.len) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; } var result = @call(.auto, func, args ++ context); @@ -659,7 +963,7 @@ pub const Tensor = struct { }, .verify = true, .location = .unknown(mlirCtx()), - }).appendTo(CompilationContext.current().currentBlock()); + }).appendTo(currentBlock()); // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. @@ -748,13 +1052,18 @@ pub const Tensor = struct { pub fn value(self: Tensor) *const mlir.Value { return switch (self.tensor_origin) { .argument => b: { - const argument_index = self.getContext().getArgumentIndex(self.id); - break :b self.getContext().currentBlock().argument(argument_index); + const argument_index = self.getContext().currentScope().id_to_argument.get(self.id).?; + break :b self.getContext().currentScope().block.argument(argument_index); }, .value => |v| v, }; } + /// Returns the donation data of the tensor. + pub fn donation(self: Tensor) ?usize { + return CompilationContext.current().currentScope().id_to_donation.get(self.id); + } + pub const Rng = struct { _state: Tensor, algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, @@ -788,12 +1097,12 @@ pub const Tensor = struct { // return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; //} - //fn update(self: Rng, new_state: mlir.Value) Rng { - // return .{ - // ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), - // .algorithm = self.algorithm, - // }; - //} + fn update(self: Rng, new_state: mlir.Value) Rng { + return .{ + ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), + .algorithm = self.algorithm, + }; + } ///// Returns a Tensor of the given shape, filled with uniformly sampled floating point numbers from an interval, ///// and a new Rng state. @@ -1001,10 +1310,22 @@ pub const Tensor = struct { /// is not allowed to reuse the donated input buffer after the call. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { - //var res = self; - //res._donation = origin._donation; - //switch (origin.tensor_origin) {} - _ = origin; // autofix + const compilation_context = CompilationContext.current(); + const scope = compilation_context.currentScope(); + switch (origin.tensor_origin) { + .argument => { + const argument_index = scope.id_to_argument.get(origin.id).?; + + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = argument_index; + }, + .value => { + if (scope.id_to_donation.get(origin.id)) |origin_donation| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = origin_donation; + } + }, + } return self; } @@ -1716,9 +2037,10 @@ pub const Tensor = struct { CompilationContext.current().pushBlock(update_block); defer CompilationContext.current().popBlock(); + const scope = CompilationContext.current().currentScope(); inline for (0..inputs.len) |i| { - CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, args[i].left.id, i) catch unreachable; - CompilationContext.current().currentMapping().put(CompilationContext.current().allocator, args[i].right.id, i + inputs.len) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inputs.len) catch unreachable; } var result = @call(.auto, func, args ++ context); @@ -2425,7 +2747,7 @@ const Mnist = struct { pub const TransferEntry = struct { buffer: zml.Buffer, - writer: Transfer.Writer, + writer: *Transfer.Writer, }; fn TransferReturnType(comptime ShapeType: type) type { @@ -2452,7 +2774,7 @@ pub fn singleTransfer(allocator: std.mem.Allocator, shape: Shape, platform: Plat const entry: TransferEntry = .{ .buffer = zml.Buffer.fromPjrtBuffers(platform, shape, &.{transfer.get(0).buffer}), - .writer = transfer.get(0).writer(), + .writer = &transfer.get(0).writer, }; return .{ transfer, entry }; @@ -2468,7 +2790,7 @@ pub fn multiTransfer(allocator: std.mem.Allocator, shapes: anytype, platform: Pl var entries: [shapes.len]TransferEntry = undefined; inline for (shapes, 0..) |shape, index| { entries[index].buffer = zml.Buffer.fromPjrtBuffers(platform, shape, &.{transfer.get(index).buffer}); - entries[index].writer = transfer.get(index).writer(); + entries[index].writer = &transfer.get(index).writer; } return .{ transfer, entries }; @@ -2477,8 +2799,22 @@ pub fn multiTransfer(allocator: std.mem.Allocator, shapes: anytype, platform: Pl pub fn bufferTransfer(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, name: []const u8, platform: Platform) !zml.Buffer { const shape = buffer_store.getShape(name).?; var transfer, const entry = try singleTransfer(allocator, shape, platform); - defer transfer.deinit(allocator, platform); - buffer_store.getReader(name).stream(entry.writer); + defer transfer.deinit(platform); + + const tensor_desc = buffer_store.store.getPtrFromKey(name).?; + + var tensor_resource: Resource = .{ + .http = try .init(buffer_store.store.http_client, allocator, tensor_desc.resource_uri, .{ .offset = tensor_desc.offset, .size = tensor_desc.shape.byteSize() }), + }; + defer tensor_resource.deinit(); + + const buffer_reader = try allocator.alloc(u8, BUF_16_MB); + defer allocator.free(buffer_reader); + + var reader = tensor_resource.reader(buffer_reader); + + _ = try reader.interface().streamRemaining(&entry.writer.interface); + try entry.writer.interface.flush(); return entry.buffer; } @@ -2533,6 +2869,9 @@ pub fn autoLoad(allocator: std.mem.Allocator, model: anytype, buffer_store: Buff } pub fn loadBuffersFromId(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferStore5.View, platform: Platform) !Bufferized(@TypeOf(model)) { + //const id = zml.tools.tracer.tracer().frameStart("loadBuffersFromId"); + //defer zml.tools.tracer.tracer().frameEnd(id, "loadBuffersFromId"); + const Model = @TypeOf(model); var result: Bufferized(Model) = undefined; initBufferizedFrom(model, &result); @@ -2544,23 +2883,49 @@ pub fn loadBuffersFromId(allocator: std.mem.Allocator, model: anytype, buffer_st std.debug.print("Found {d} shapes\n", .{shapes.len}); var transfer: Transfer = try .init(arena.allocator(), shapes, platform); - defer transfer.deinit(arena.allocator(), platform); + defer transfer.deinit(platform); - const readers = try collectReaders(arena.allocator(), buffer_store, &model); - std.debug.print("Found {d} readers\n", .{readers.len}); + const tensor_descs = try collectTensorDesc(arena.allocator(), buffer_store, &model); + + const buffer_reader = try allocator.alloc(u8, BUF_16_MB); + defer allocator.free(buffer_reader); + + //const readers = try collectReaders(arena.allocator(), buffer_store, &model); + //std.debug.print("Found {d} readers\n", .{readers.len}); const LocalContext = struct { - readers: []BufferStore5.Reader, + tensor_descs: []TensorDesc, shapes: []const Shape, platform: Platform, transfer: *Transfer, index: usize = 0, + buffer_reader: []u8, + buffer_store_view: BufferStore5.View, + allocator: std.mem.Allocator, + }; + var context: LocalContext = .{ + .tensor_descs = tensor_descs, + .shapes = shapes, + .platform = platform, + .transfer = &transfer, + .buffer_reader = buffer_reader, + .buffer_store_view = buffer_store, + .allocator = allocator, }; - var context: LocalContext = .{ .readers = readers, .shapes = shapes, .platform = platform, .transfer = &transfer }; zml.meta.visit(struct { fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { - const writer = context_.transfer.get(context_.index).writer(); - context_.readers[context_.index].stream(writer); + const tensor_desc = context_.tensor_descs[context_.index]; + + var tensor_resource: Resource = .{ + .http = HttpResource.init(context_.buffer_store_view.store.http_client, context_.allocator, tensor_desc.resource_uri, .{ .offset = tensor_desc.offset, .size = tensor_desc.shape.byteSize() }) catch unreachable, + }; + defer tensor_resource.deinit(); + + var reader = tensor_resource.reader(context_.buffer_reader); + const entry = context_.transfer.get(context_.index); + + _ = reader.interface().streamRemaining(&entry.writer.interface) catch unreachable; + entry.writer.interface.flush() catch unreachable; buffer.* = zml.Buffer.fromPjrtBuffers(context_.platform, context_.shapes[context_.index], &.{context_.transfer.get(context_.index).buffer}); context_.index += 1; @@ -2675,9 +3040,38 @@ pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { const Transfer = struct { entries: []Entry, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + arena: std.heap.ArenaAllocator, pub const Writer = struct { - entry: *Entry, + offset: usize = 0, + interface: std.Io.Writer, + + pub fn init(buffer: []u8) Writer { + return .{ + .interface = .{ + .buffer = buffer, + .end = 0, + .vtable = &.{ + .drain = drain, + }, + }, + }; + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + _ = data; // autofix + _ = splat; // autofix + const writer: *Writer = @alignCast(@fieldParentPtr("interface", w)); + const entry: *Entry = @alignCast(@fieldParentPtr("writer", writer)); + stdx.debug.assert(writer.offset + w.end <= entry.shape.byteSize(), "Can't write more data than required", .{}); + const is_last_transfer = writer.offset + w.end >= entry.shape.byteSize(); + log.debug("Writing {} bytes", .{w.end}); + _ = entry.transfer_manager.transferData(entry.platform.pjrt_api, entry.buffer_index, w.buffer[0..w.end], @intCast(writer.offset), is_last_transfer) catch return error.WriteFailed; + const written = w.end; + writer.offset += written; + w.end = 0; + return 0; + } }; pub const Entry = struct { @@ -2686,21 +3080,21 @@ const Transfer = struct { buffer_index: usize, buffer: *pjrt.Buffer, platform: Platform, - - pub fn writer(entry: *Entry) Writer { - return .{ .entry = entry }; - } + writer: Writer, }; pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: Platform) !Transfer { const shape_specs = try allocator.alloc(pjrt.ShapeSpec, shapes.len); defer allocator.free(shape_specs); + var temp_arena = std.heap.ArenaAllocator.init(allocator); + defer temp_arena.deinit(); + var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); + errdefer arena.deinit(); for (shape_specs, shapes) |*spec, shape| { - const dims = try arena.allocator().dupe(i64, shape.dims()); + const dims = try temp_arena.allocator().dupe(i64, shape.dims()); spec.* = pjrt.ShapeSpec.init(dims, bufferTypeFromDtype(shape.dtype())); } @@ -2709,31 +3103,31 @@ const Transfer = struct { const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); errdefer transfer_manager.deinit(platform.pjrt_api); - const count = transfer_manager.bufferCount(platform.pjrt_api) catch unreachable; - for (0..count) |index| { - const buffer_size = transfer_manager.bufferSize(platform.pjrt_api, index) catch unreachable; - std.debug.print("index: {d} - size: {d}\n", .{ index, buffer_size }); - } - - const entries = try allocator.alloc(Entry, shapes.len); - errdefer allocator.free(entries); + //const count = transfer_manager.bufferCount(platform.pjrt_api) catch unreachable; + //for (0..count) |index| { + // const buffer_size = transfer_manager.bufferSize(platform.pjrt_api, index) catch unreachable; + // std.debug.print("index: {d} - size: {d}\n", .{ index, buffer_size }); + //} + const entries = try arena.allocator().alloc(Entry, shapes.len); for (entries, shapes, 0..) |*e, shape, index| { const buffer = try transfer_manager.retrieveBuffer(platform.pjrt_api, index); + const writer_buffer = try arena.allocator().alloc(u8, BUF_16_MB); e.* = .{ .shape = shape, .transfer_manager = transfer_manager, .buffer_index = index, .buffer = buffer, .platform = platform, + .writer = .init(writer_buffer), }; } - return .{ .entries = entries, .transfer_manager = transfer_manager }; + return .{ .entries = entries, .transfer_manager = transfer_manager, .arena = arena }; } - pub fn deinit(self: Transfer, allocator: std.mem.Allocator, platform: Platform) void { - allocator.free(self.entries); + pub fn deinit(self: Transfer, platform: Platform) void { + self.arena.deinit(); self.transfer_manager.deinit(platform.pjrt_api); } @@ -2812,6 +3206,163 @@ pub fn main() !void { try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); } +pub fn asyncMain3() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + // var tmp_dir = std.testing.tmpDir(.{}); + // defer tmp_dir.cleanup(); + + // const filename = "file.bin"; + // _ = try createBinFile(tmp_dir, filename, BUF_8_KB, null); + + // const file_path = try tmp_dir.dir.realpathAlloc(allocator, filename); + // defer allocator.free(file_path); + var client: std.http.Client = .{ .allocator = allocator, .write_buffer_size = 8192 }; + try client.initDefaultProxies(arena.allocator()); + defer client.deinit(); + + if (false) { + //var resource: Resource = .{ .memory = try .init(safetensor) }; + //defer resource.deinit(); + + const buffer_reader = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_reader); + + //var resource: Resource = .{ .http = try .init(&client, "http://localhost:8000/model.safetensors.index.json") }; + var resource: Resource = .{ .http = try .init(&client, allocator, "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/raw/main/model.safetensors.index.json", null) }; + defer resource.deinit(); + + //var resource: Resource = .{ .file = try .init("/Users/corendos/models/Llama-3.1-8B-Instruct/model.safetensors.index.json", buffer_reader) }; + //defer resource.deinit(); + + var reader = resource.reader(buffer_reader); + + var parsed_safetensors = try parseSafetensorsIndex(arena.allocator(), &resource, reader.interface()); + defer parsed_safetensors.deinit(); + { + //"model.layers.0.self_attn.o_proj.weight":{"dtype":"BF16","shape":[4096,4096],"data_offsets":[1411399680,1444954112]} + var tensor_resource: Resource = .{ .http = try .init( + &client, + allocator, + "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model-00001-of-00004.safetensors", + .{ .offset = 8, .size = 4096 }, + ) }; + defer tensor_resource.deinit(); + + var reader2 = tensor_resource.reader(buffer_reader); + + const buffer_writer = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_writer); + var writer = std.fs.File.stdout().writer(buffer_writer); + + const result = try reader2.interface().streamRemaining(&writer.interface); + std.log.debug("{}", .{result}); + try writer.interface.flush(); + } + } + + if (false) { + const shape = zml.Shape.init(.{ 128256, 4096 }, .bf16); + + var transfer = try Transfer.init(allocator, &.{shape}, platform); + defer transfer.deinit(platform); + + const entry = transfer.get(0); + + const start = 0; + const end = 1050673152; + var tensor_resource: Resource = .{ + .http = try .init( + &client, + allocator, + "http://localhost:8000/model-00001-of-00004.safetensors", + //"https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model-00001-of-00004.safetensors", + .{ .offset = start, .size = end - start }, + ), + }; + defer tensor_resource.deinit(); + + const buffer_reader = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_reader); + var reader = tensor_resource.reader(buffer_reader); + + const written = try reader.interface().streamRemaining(&entry.writer.interface); + try entry.writer.interface.flush(); + std.log.debug("Written {d} bytes", .{written}); + } + + if (true) { + const buffer_reader = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_reader); + + var resource: Resource = .{ .http = try .init(&client, allocator, std.Uri.parse("https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model.safetensors.index.json") catch unreachable, null) }; + defer resource.deinit(); + + var reader = resource.reader(buffer_reader); + + var resource_index = try parseSafetensorsIndex(arena.allocator(), &resource, reader.interface()); + defer resource_index.deinit(); + + var registry: TensorRegistry = try .initWithMetadata(allocator, resource_index.metadata); + defer registry.deinit(); + + var seen: std.ArrayHashMapUnmanaged(std.Uri, void, ResourceURIContext, false) = .empty; + + var it = resource_index.map.iterator(); + while (it.next()) |entry| { + if (seen.get(entry.value_ptr.*)) |_| {} else { + try seen.put(arena.allocator(), entry.value_ptr.*, {}); + + std.debug.print("subresource uri: {f}\n", .{entry.value_ptr.fmt(.all)}); + + var subresource: Resource = .{ .http = try .init(&client, allocator, entry.value_ptr.*, null) }; + defer subresource.deinit(); + + std.log.debug("Opened subresource", .{}); + + var reader2 = subresource.reader(buffer_reader); + + try parseSafetensors(®istry, entry.value_ptr.*, reader2.interface()); + } + } + + //const tensor_desc = registry.tensors.get("model.layers.0.self_attn.o_proj.weight").?; + const tensor_desc = registry.tensors.get("model.embed_tokens.weight").?; + + var tensor_resource: Resource = .{ + .http = try .init( + &client, + allocator, + tensor_desc.resource_uri, + .{ .offset = tensor_desc.offset, .size = tensor_desc.shape.byteSize() }, + ), + }; + defer tensor_resource.deinit(); + + var reader3 = tensor_resource.reader(buffer_reader); + + var transfer = try Transfer.init(allocator, &.{tensor_desc.shape}, platform); + defer transfer.deinit(platform); + + const entry = transfer.get(0); + + const written = try reader3.interface().streamRemaining(&entry.writer.interface); + try entry.writer.interface.flush(); + std.log.debug("Written {d} bytes", .{written}); + } +} + pub fn asyncMain2() !void { var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; defer _ = gpa.deinit(); @@ -2930,15 +3481,68 @@ pub fn asyncMain() !void { .qkv_type = .merged, }; - var old_buffer_store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); - defer old_buffer_store.deinit(); + var client: std.http.Client = .{ .allocator = allocator, .write_buffer_size = 8192 }; + try client.initDefaultProxies(arena.allocator()); + defer client.deinit(); - var buffer_store: BufferStore5 = try .fromBufferStore(allocator, old_buffer_store); + const buffer_reader = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_reader); + + //var resource: Resource = .{ .http = try .init(&client, allocator, std.Uri.parse("https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model.safetensors.index.json") catch unreachable, null) }; + var resource: Resource = .{ .http = try .init(&client, allocator, std.Uri.parse("http://localhost:8000/model.safetensors.index.json") catch unreachable, null) }; + defer resource.deinit(); + + var reader = resource.reader(buffer_reader); + + var resource_index = try parseSafetensorsIndex(arena.allocator(), &resource, reader.interface()); + defer resource_index.deinit(); + + var registry: TensorRegistry = try .initWithMetadata(allocator, resource_index.metadata); + defer registry.deinit(); + + //var registry: TensorRegistry = .init(allocator); + //defer registry.deinit(); + + //var resource: Resource = .{ .http = try .init(&client, allocator, std.Uri.parse("https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model.safetensors") catch unreachable, null) }; + //defer resource.deinit(); + + //var reader = resource.reader(buffer_reader); + + //try parseSafetensors(®istry, resource.uri(), reader.interface()); + + var seen: std.ArrayHashMapUnmanaged(std.Uri, void, ResourceURIContext, false) = .empty; + + var it = resource_index.map.iterator(); + while (it.next()) |entry| { + if (seen.get(entry.value_ptr.*)) |_| {} else { + try seen.put(arena.allocator(), entry.value_ptr.*, {}); + + std.debug.print("subresource uri: {f}\n", .{entry.value_ptr.fmt(.all)}); + + var subresource: Resource = .{ .http = try .init(&client, allocator, entry.value_ptr.*, null) }; + defer subresource.deinit(); + + std.log.debug("Opened subresource", .{}); + + var reader2 = subresource.reader(buffer_reader); + + try parseSafetensors(®istry, entry.value_ptr.*, reader2.interface()); + } + } + + //var old_buffer_store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); + //defer old_buffer_store.deinit(); + + //var buffer_store: BufferStore5 = try .fromBufferStore(allocator, old_buffer_store); + var buffer_store: BufferStore5 = try .init(allocator, ®istry, &client); defer buffer_store.deinit(); var context = try zml.Context.init(); defer context.deinit(); + //zml.tools.tracer.initTracer(); + //defer zml.tools.tracer.deinitTracer(); + const platform = context.autoPlatform(.{}); context.printAvailablePlatforms(platform); @@ -2978,20 +3582,21 @@ pub fn asyncMain() !void { defer kv_cache_buffer.k.deinit(); defer kv_cache_buffer.v.deinit(); - var token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{0})); - defer token_index_buffer.deinit(); + //var token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{0})); + //defer token_index_buffer.deinit(); - var token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{0})); - defer token_buffer.deinit(); + //var token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{0})); + //defer token_buffer.deinit(); var new_token_id: u32 = undefined; for (tokenized_prompt[0..], 0..) |token_id, i| { std.debug.print("Running iteration {} token id: {}\n", .{ i, token_id }); - token_index_buffer.deinit(); - token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{@intCast(i)})); + var token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{@intCast(i)})); + defer token_index_buffer.deinit(); + + var token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{token_id})); + defer token_buffer.deinit(); - token_buffer.deinit(); - token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{token_id})); var args = try exe.args(allocator); defer args.deinit(allocator); @@ -3002,11 +3607,11 @@ pub fn asyncMain() !void { std.log.debug("Calling exe", .{}); exe.call(args, &results); - token_buffer.deinit(); - kv_cache_buffer.layer_index.deinit(); - kv_cache_buffer.k.deinit(); - kv_cache_buffer.v.deinit(); - rng_buffer._state.deinit(); + //token_buffer.deinit(); + //kv_cache_buffer.layer_index.deinit(); + //kv_cache_buffer.k.deinit(); + //kv_cache_buffer.v.deinit(); + //rng_buffer._state.deinit(); results.fill(&.{ &token_buffer, &kv_cache_buffer, &rng_buffer._state }); const host = try token_buffer.toHostAlloc(allocator); @@ -3063,13 +3668,14 @@ pub fn asyncMain() !void { //} } + var token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{new_token_id})); + defer token_buffer.deinit(); + for (0..10) |i| { std.debug.print("Running iteration {}\n", .{i}); - token_index_buffer.deinit(); - token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{@intCast(i + tokenized_prompt.len)})); + var token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{@intCast(i + tokenized_prompt.len)})); + defer token_index_buffer.deinit(); - token_buffer.deinit(); - token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{new_token_id})); var args = try exe.args(allocator); defer args.deinit(allocator); @@ -3093,7 +3699,6 @@ pub fn asyncMain() !void { const predicted_token = host.items(u32)[0]; const predicted_token_str = try tokenizer_decoder.decode(&.{predicted_token}); std.debug.print("Predicted: {} {s}\n", .{ predicted_token, predicted_token_str }); - new_token_id = predicted_token; } //const process_args = try std.process.argsAlloc(allocator); @@ -3297,6 +3902,27 @@ fn testLinear(linear: Linear, x: Tensor) Tensor { } pub const CompilationContext = struct { + pub const Scope = struct { + block: *mlir.Block, + id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), + arena: std.heap.ArenaAllocator, + + pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(allocator); + return .{ + .block = block, + .id_to_argument = .empty, + .id_to_donation = .empty, + .arena = arena, + }; + } + + pub fn deinit(self: *Scope) void { + self.arena.deinit(); + } + }; + allocator: std.mem.Allocator, mlir_registry: *mlir.DialectRegistry, @@ -3305,8 +3931,7 @@ pub const CompilationContext = struct { //mlir_op_pass_manager: *mlir.OpPassManager, module: *mlir.Module, - blocks: stdx.BoundedArray(*mlir.Block, 16) = .{}, - mappings: stdx.BoundedArray(std.AutoArrayHashMapUnmanaged(usize, usize), 16) = .{}, + scopes: stdx.BoundedArray(Scope, 16) = .{}, pub fn init(allocator: std.mem.Allocator) CompilationContext { mlir.registerPasses("Transforms"); @@ -3353,28 +3978,19 @@ pub const CompilationContext = struct { self.mlir_registry.deinit(); } - pub fn currentBlock(self: *const CompilationContext) *mlir.Block { - return self.blocks.get(self.blocks.len - 1); - } - - pub fn currentMapping(self: *CompilationContext) *std.AutoArrayHashMapUnmanaged(usize, usize) { - return &self.mappings.slice()[self.blocks.len - 1]; - } - - pub fn getArgumentIndex(self: *CompilationContext, tensor_id: usize) usize { - return self.currentMapping().get(tensor_id).?; + pub fn currentScope(self: *CompilationContext) *Scope { + return &self.scopes.slice()[self.scopes.len - 1]; } pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { - self.blocks.appendAssumeCapacity(block); - self.mappings.appendAssumeCapacity(.{}); + const scope = Scope.initFromBlock(self.allocator, block); + self.scopes.appendAssumeCapacity(scope); } pub fn popBlock(self: *CompilationContext) void { - _ = self.blocks.pop(); - var maybe_popped = self.mappings.pop(); - if (maybe_popped) |*popped| { - popped.deinit(self.allocator); + var maybe_popped_scope = self.scopes.pop(); + if (maybe_popped_scope) |*popped| { + popped.deinit(); } } @@ -3397,10 +4013,10 @@ pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx. defer compilation_context.deinit(); const result = emitMlir(&compilation_context, func, args) catch unreachable; + defer result.output_info.deinit(compilation_context.allocator); + defer compilation_context.allocator.free(result.input_shapes); _ = result.func.appendTo(compilation_context.module.body()); - defer compilation_context.allocator.free(result.output_shapes); - defer compilation_context.allocator.free(result.input_shapes); var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); @@ -3408,7 +4024,7 @@ pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx. const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), platform, compilation_context.module, null) catch unreachable; const num_devices = platform.sharding().num_replicas * platform.sharding().num_partitions; - const exe = try Exe.init(allocator, platform, loaded_executable, result.input_shapes, result.output_shapes, num_devices); + const exe = try Exe.init(allocator, platform, loaded_executable, result.input_shapes, result.output_info.shapes, num_devices); errdefer exe.deinit(); return exe; @@ -3419,10 +4035,12 @@ fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { list: *std.array_list.Managed(Shape), }; var list = std.array_list.Managed(Shape).init(allocator); + errdefer list.deinit(); + var context: LocalContext = .{ .list = &list }; - zml.meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - ctx_.list.append(tensor.shape()) catch unreachable; + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.shape()); } }.cb, &context, v); @@ -3461,12 +4079,80 @@ fn collectReaders(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, return try list.toOwnedSlice(); } +fn collectTensorDesc(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, v: anytype) ![]TensorDesc { + const LocalContext = struct { + list: *std.array_list.Managed(TensorDesc), + buffer_store: BufferStore5.View, + }; + var list = std.array_list.Managed(TensorDesc).init(allocator); + var context: LocalContext = .{ .list = &list, .buffer_store = buffer_store }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + const tensor_desc = ctx_.buffer_store.store.getPtrFromId(tensor.id).?.*; + ctx_.list.append(tensor_desc) catch unreachable; + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +pub const OutputInfo = struct { + shapes: []Shape, + values: []*const mlir.Value, + donations: []?usize, + + pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + allocator.free(self.values); + allocator.free(self.donations); + } +}; + +fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + value_list: *std.array_list.Managed(*const mlir.Value), + donation_list: *std.array_list.Managed(?usize), + }; + + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); + var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); + errdefer value_list.deinit(); + var donation_list = std.array_list.Managed(?usize).init(allocator); + errdefer donation_list.deinit(); + + var context: LocalContext = .{ .shape_list = &shape_list, .value_list = &value_list, .donation_list = &donation_list }; + + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + try ctx_.value_list.append(tensor.value()); + try ctx_.donation_list.append(tensor.donation()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + .values = try value_list.toOwnedSlice(), + .donations = try donation_list.toOwnedSlice(), + }; +} + const EmitMlirResult = struct { func: *mlir.Operation, input_shapes: []const Shape, - output_shapes: []const Shape, + output_info: OutputInfo, }; +fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { + const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); + for (res, attributes) |*r, attr| { + r.* = mlir.dictionaryAttribute(mlir_ctx, attr.constSlice()); + } + return res; +} + fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); defer arena.deinit(); @@ -3489,11 +4175,12 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar }; zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - _ = ctx_.compilation_context.currentBlock().addArgument( + std.log.debug("Argument {} - shape: {f}", .{ ctx_.current_argument_id, tensor.shape() }); + _ = ctx_.compilation_context.currentScope().block.addArgument( tensor.mlirType(ctx_.compilation_context.mlir_ctx), .unknown(ctx_.compilation_context.mlir_ctx), ); - ctx_.compilation_context.currentMapping().put(ctx_.compilation_context.allocator, tensor.id, ctx_.current_argument_id) catch unreachable; + ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; ctx_.current_argument_id += 1; } }.cb, &context, &args); @@ -3501,21 +4188,32 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar const input_shapes = try collectShapes(compilation_context.allocator, &args); errdefer compilation_context.allocator.free(input_shapes); - const output_shapes = b: { + const input_attributes = try arena.allocator().alloc(AttributeList, input_shapes.len); + @memset(input_attributes, .{}); + + const output_info = b: { global_compilation_context = compilation_context; defer global_compilation_context = null; + const result = @call(.auto, func, args); - const output_shapes = try collectShapes(compilation_context.allocator, &result); - const output_values = try collectValues(arena.allocator(), &result); - _ = dialects.func.returns(compilation_context.mlir_ctx, output_values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentBlock()); - break :b output_shapes; + + const output_info = try collectOutputInfo(compilation_context.allocator, &result); + errdefer output_info.deinit(compilation_context.allocator); + + break :b output_info; }; - errdefer compilation_context.allocator.free(output_shapes); + errdefer output_info.deinit(compilation_context.allocator); + + for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { + input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); + }; + _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ .name = "main", - .block = compilation_context.currentBlock(), + .block = compilation_context.currentScope().block, .location = .unknown(compilation_context.mlir_ctx), + .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), }); compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { @@ -3529,7 +4227,7 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar return .{ .func = mlir_func, .input_shapes = input_shapes, - .output_shapes = output_shapes, + .output_info = output_info, }; } @@ -4274,7 +4972,7 @@ pub const BufferStore4 = struct { }; }; -pub const BufferStore5 = struct { +pub const BufferStoreOld = struct { key_map: std.StringHashMapUnmanaged(usize), id_map: std.AutoHashMapUnmanaged(usize, usize), entries: std.ArrayList(Entry), @@ -4287,7 +4985,7 @@ pub const BufferStore5 = struct { associated_tensor_id: ?usize = null, }; - pub fn init(allocator: std.mem.Allocator) !BufferStore5 { + pub fn init(allocator: std.mem.Allocator) !BufferStoreOld { return .{ .key_map = .empty, .id_map = .empty, @@ -4296,7 +4994,7 @@ pub const BufferStore5 = struct { }; } - pub fn deinit(self: *BufferStore5) void { + pub fn deinit(self: *BufferStoreOld) void { var it = self.key_map.iterator(); while (it.next()) |entry| { self.allocator.free(entry.key_ptr.*); @@ -4306,7 +5004,7 @@ pub const BufferStore5 = struct { self.entries.deinit(self.allocator); } - pub fn add(self: *BufferStore5, key: []const u8, shape: zml.Shape, data: []const u8) !void { + pub fn add(self: *BufferStoreOld, key: []const u8, shape: zml.Shape, data: []const u8) !void { const new_entry_index = self.entries.items.len; const new_entry = try self.entries.addOne(self.allocator); errdefer _ = self.entries.pop(); @@ -4324,7 +5022,7 @@ pub const BufferStore5 = struct { new_entry.* = .{ .shape = shape, .data = data }; } - fn bindIdToKey(self: *BufferStore5, key: []const u8, id: usize) !void { + fn bindIdToKey(self: *BufferStoreOld, key: []const u8, id: usize) !void { const index = self.key_map.get(key).?; const gop = try self.id_map.getOrPut(self.allocator, id); @@ -4336,8 +5034,8 @@ pub const BufferStore5 = struct { gop.value_ptr.* = index; } - pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore) !BufferStore5 { - var new_buffer_store: BufferStore5 = try .init(allocator); + pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore) !BufferStoreOld { + var new_buffer_store: BufferStoreOld = try .init(allocator); errdefer new_buffer_store.deinit(); var it = buffer_store.buffers.iterator(); @@ -4348,17 +5046,17 @@ pub const BufferStore5 = struct { return new_buffer_store; } - fn getPtrFromKey(self: *const BufferStore5, key: []const u8) ?*Entry { + fn getPtrFromKey(self: *const BufferStoreOld, key: []const u8) ?*Entry { const index = self.key_map.get(key) orelse return null; return &self.entries.items[index]; } - fn getPtrFromId(self: *const BufferStore5, id: usize) ?*Entry { + fn getPtrFromId(self: *const BufferStoreOld, id: usize) ?*Entry { const index = self.id_map.get(id) orelse return null; return &self.entries.items[index]; } - pub fn view(self: *BufferStore5) View { + pub fn view(self: *BufferStoreOld) View { return .{ .store = self }; } @@ -4378,7 +5076,7 @@ pub const BufferStore5 = struct { }; pub const View = struct { - store: *BufferStore5, + store: *BufferStoreOld, prefix_buffer: [256]u8 = undefined, prefix_length: usize = 0, @@ -4484,3 +5182,874 @@ pub const BufferStore5 = struct { } }; }; + +pub const BufferStore5 = struct { + registry: *TensorRegistry, + id_map: std.AutoHashMapUnmanaged(usize, *TensorDesc), + allocator: std.mem.Allocator, + // Temporary + http_client: *std.http.Client, + + pub fn init(allocator: std.mem.Allocator, registry: *TensorRegistry, http_client: *std.http.Client) !BufferStore5 { + return .{ + .registry = registry, + .id_map = .empty, + .allocator = allocator, + .http_client = http_client, + }; + } + + pub fn deinit(self: *BufferStore5) void { + self.id_map.deinit(self.allocator); + } + + fn bindIdToKey(self: *BufferStore5, key: []const u8, id: usize) !void { + const tensor_desc_ptr = self.registry.tensors.getPtr(key).?; + + const gop = try self.id_map.getOrPut(self.allocator, id); + if (gop.found_existing) { + stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.value_ptr.* }); + } + errdefer self.id_map.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = tensor_desc_ptr; + } + + fn getPtrFromKey(self: *const BufferStore5, key: []const u8) ?*TensorDesc { + const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; + return tensor_desc_ptr; + } + + fn getPtrFromId(self: *const BufferStore5, id: usize) ?*TensorDesc { + const tensor_desc_ptr = self.id_map.get(id) orelse return null; + return tensor_desc_ptr; + } + + pub fn view(self: *BufferStore5) View { + return .{ .store = self }; + } + + pub const View = struct { + store: *BufferStore5, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + pub fn parent(self: *const View) View { + const slice = self.prefix() orelse unreachable; + const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); + var buffer: [256]u8 = undefined; + @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = index + 1, + }; + } + + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + fn prefix(self: *const View) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + + const tensor = Tensor.init(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensor(self: View, subkey: []const u8) Tensor { + return self.maybeCreateTensor(subkey).?; + } + + pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + ptr.shape = ptr.shape.withTags(tagz).withSharding(.{0}); + + const tensor = Tensor.init(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) Tensor { + return self.maybeCreateTensorWithTags(subkey, tagz).?; + } + + //pub fn getReader(self: View, subkey: []const u8) Reader { + // return self.getMaybeReader(subkey).?; + //} + + //pub fn getMaybeReader(self: View, subkey: []const u8) ?Reader { + // var buffer: [256]u8 = undefined; + // const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + // const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + // return .{ .entry = entry_ptr }; + //} + + //pub fn getReaderFromId(self: View, id: usize) ?Reader { + // const entry_ptr = self.store.getPtrFromId(id) orelse return null; + // return .{ .entry = entry_ptr }; + //} + + pub fn getShape(self: View, subkey: []const u8) ?Shape { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + + pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { + var buffer: [256]u8 = undefined; + const key = if (opts.no_prefix) + subkey + else b: { + break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + }; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + }; +}; + +const KB = 1024; +const MB = 1024 * KB; + +const BUF_1_KB = 1 * KB; +const BUF_4_KB = 4 * KB; +const BUF_8_KB = 8 * KB; +const BUF_16_KB = 16 * KB; +const BUF_32_KB = 32 * KB; +const BUF_64_KB = 64 * KB; + +const BUF_1_MB = 1 * MB; +const BUF_4_MB = 4 * MB; +const BUF_8_MB = 8 * MB; +const BUF_16_MB = 16 * MB; +const BUF_32_MB = 32 * MB; +const BUF_64_MB = 64 * MB; +const BUF_128_MB = 128 * MB; +const BUF_256_MB = 256 * MB; + +pub const MemoryResource = struct { + uri: ResourceURI, + data: []const u8, + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator, data: []const u8) !MemoryResource { + return .{ + .uri = try std.Uri.parse("memory://"), + .data = data, + .allocator = allocator, + }; + } + + pub fn deinit(_: *MemoryResource) void {} +}; + +pub const FileResource = struct { + uri: ResourceURI, + uri_storage: []const u8, + file: std.fs.File, + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator, path: []const u8) !FileResource { + const uri_storage = try std.fmt.allocPrint(allocator, "file://{s}", .{path}); + errdefer allocator.free(uri_storage); + + const uri = std.Uri.parse(uri_storage); + + const file = try std.fs.openFileAbsolute(path, .{ .mode = .read_only }); + errdefer file.close(); + + return .{ + .uri = uri, + .uri_storage = uri_storage, + .file = file, + .allocator = allocator, + }; + } + + // check ownership + pub fn deinit(self: *FileResource) void { + self.allocator.free(self.uri_storage); + self.file.close(); + } +}; + +const Range = struct { + offset: usize, + size: usize, +}; + +pub const HttpResource = struct { + uri: ResourceURI, + + request: std.http.Client.Request, + arena: std.heap.ArenaAllocator, + + pub fn init(client: *std.http.Client, allocator: std.mem.Allocator, uri: std.Uri, maybe_range: ?Range) !HttpResource { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + var request = try client.request(.GET, uri, .{}); + errdefer request.deinit(); + + request.headers.authorization = .{ .override = "Bearer " }; + + const extra_headers = if (maybe_range) |range| b: { + var list: std.ArrayList(std.http.Header) = .empty; + const name = try arena.allocator().dupe(u8, "Range"); + const value = try std.fmt.allocPrint(arena.allocator(), "bytes={}-{}", .{ range.offset, range.offset + range.size - 1 }); + try list.append(arena.allocator(), .{ .name = name, .value = value }); + + break :b try list.toOwnedSlice(arena.allocator()); + } else &.{}; + request.extra_headers = extra_headers; + try request.sendBodiless(); + + const redirect_buffer = try arena.allocator().alloc(u8, 1024 * 1024); + const response = try request.receiveHead(redirect_buffer); + _ = response; + + return .{ + .uri = uri, + .request = request, + .arena = arena, + }; + } + + // check ownership + pub fn deinit(self: *HttpResource) void { + self.request.deinit(); + self.arena.deinit(); + } +}; + +pub const Resource = union(enum) { + memory: MemoryResource, + file: FileResource, + http: HttpResource, + + pub fn deinit(self: *Resource) void { + switch (self.*) { + .memory => |*m| m.deinit(), + .file => |*f| f.deinit(), + .http => |*h| h.deinit(), + } + } + + pub fn reader(self: *Resource, buffer: []u8) IoReader { + return .init(self, buffer); + } + + pub fn uri(self: *Resource) std.Uri { + return switch (self.*) { + inline else => |*r| r.uri, + }; + } +}; + +pub const IoReader = struct { + reader: union(enum) { + memory: std.io.Reader, + file: std.fs.File.Reader, + http: *std.io.Reader, + }, + resource: *Resource, + + pub fn init(resource: *Resource, buffer: []u8) IoReader { + return .{ + .reader = switch (resource.*) { + .memory => |*m| .{ .memory = std.Io.Reader.fixed(m.data) }, + .file => |*f| .{ .file = f.file.reader(buffer) }, + .http => |*h| .{ .http = h.request.reader.bodyReader(buffer, h.request.response_transfer_encoding, h.request.response_content_length) }, + }, + .resource = resource, + }; + } + + pub fn interface(self: *IoReader) *std.io.Reader { + return switch (self.reader) { + .memory => |*m| m, + .file => |*f| &f.interface, + .http => |h| h, + }; + } +}; + +pub const Metadata = union(enum) { + null: void, + int: i64, + float: f64, + bool: bool, + string: []const u8, + + array_bool: []const bool, + array_int: []const i64, + array_float: []const f64, + array_string: []const []const u8, + + pub const ItemType = enum { + int, + float, + bool, + string, + + pub fn toZigType(comptime kind: ItemType) type { + return switch (kind) { + .int => i64, + .float => f64, + .bool => bool, + .string => []const u8, + }; + } + }; + + pub fn wrap(x: anytype) Metadata { + return switch (@TypeOf(x)) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, + inline f16, f32, f64 => .{ .float = @floatCast(x) }, + bool => .{ .bool = x }, + []const u8 => .{ .string = x }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), + }; + } + + pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { + return switch (@TypeOf(any_slice[0])) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => { + const res = try allocator.alloc(i64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @intCast(val); + return .{ .array_int = res }; + }, + inline f16, f32, f64 => { + const res = try allocator.alloc(f64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @floatCast(val); + return .{ .array_float = res }; + }, + bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, + []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), + }; + } + + pub fn format( + self: Metadata, + writer: *std.Io.Writer, + ) !void { + switch (self) { + .null => _ = try writer.write("null"), + .string => |s| try writer.print("{s}", .{s}), + .bool => |b| try writer.print("{}", .{b}), + .int => |i| try writer.print("{d}", .{i}), + .float => |f| try writer.print("{d}", .{f}), + .array_bool => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.write(", "); + try writer.print("{}", .{v}); + } + try writer.writeByte(']'); + }, + .array_int => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.write(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_float => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.write(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_string => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.write(", "); + try writer.print("\"{s}\"", .{v}); + } + try writer.writeByte(']'); + }, + } + } + + pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { + switch (self) { + .string => |s| allocator.free(s), + .array_bool => |s| allocator.free(s), + .array_int => |s| allocator.free(s), + .array_float => |s| allocator.free(s), + .array_string => |s| { + for (s) |str| allocator.free(str); + allocator.free(s); + }, + else => {}, + } + } + + pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { + return switch (self) { + .null => .null, + .int => |v| .{ .int = v }, + .float => |v| .{ .float = v }, + .bool => |v| .{ .bool = v }, + .string => |s| .{ .string = try allocator.dupe(u8, s) }, + .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, + .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, + .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, + .array_string => |s| blk: { + const new_slice = try allocator.alloc([]const u8, s.len); + errdefer allocator.free(new_slice); + for (s, 0..) |str, i| { + new_slice[i] = try allocator.dupe(u8, str); + } + break :blk .{ .array_string = new_slice }; + }, + }; + } +}; + +pub const ResourceURI = std.Uri; +pub const Resources = std.ArrayHashMapUnmanaged(ResourceURI, Resource, ResourceURIContext, false); +pub const Tensors = std.StringArrayHashMapUnmanaged(TensorDesc); +pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); + +const ResourceURIContext = struct { + // todo: implement full uri + pub fn hash(_: ResourceURIContext, uri: ResourceURI) u32 { + return std.array_hash_map.hashString(uri.path.percent_encoded); + } + + pub fn eql(_: ResourceURIContext, a: ResourceURI, b: ResourceURI, _: usize) bool { + return std.mem.eql(u8, a.path.percent_encoded, b.path.percent_encoded); + } +}; + +pub const ResourceType = enum { + index, + safetensors, + unknown, +}; + +const TensorDesc = struct { + resource_uri: ResourceURI, + name: []const u8, + shape: Shape, + offset: u64, + + pub fn byteSize(self: Tensor) u64 { + return self.shape.byteSize(); + } +}; + +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, + + tensors: Tensors, + metadata: Metadatas, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .{}, + }; + } + + pub fn initWithMetadata( + allocator: std.mem.Allocator, + metadata: Metadatas, + ) !TensorRegistry { + var arena = std.heap.ArenaAllocator.init(allocator); + + return .{ + .arena = arena, + .tensors = .{}, + .metadata = blk: { + var arena_allocator = arena.allocator(); + var new_metadata: Metadatas = .{}; + + var it = metadata.iterator(); + while (it.next()) |entry| { + const key = try arena_allocator.dupe(u8, entry.key_ptr.*); + const value = try entry.value_ptr.*.clone(arena_allocator); + try new_metadata.put(arena_allocator, key, value); + } + break :blk new_metadata; + }, + }; + } + + pub fn deinit(self: *TensorRegistry) void { + const allocator = self.arena.allocator(); + self.tensors.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } +}; + +pub const ModelPathResolver = struct { + arena: std.heap.ArenaAllocator, + + pub fn init(allocator: std.mem.Allocator) ModelPathResolver { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + }; + } + + pub fn deinit(self: *ModelPathResolver) void { + self.arena.deinit(); + } + + pub fn resolve(self: *ModelPathResolver, path: []const u8) !ResourceURI { + const arena_allocator = self.arena.allocator(); + + const resource_uri: ResourceURI = ResourceURI.parse(path) catch .{ + .scheme = "file", + .path = .{ .percent_encoded = path }, + }; + + if (std.mem.eql(u8, resource_uri.scheme, FileResource.scheme)) { + const stat = std.fs.cwd().statFile(resource_uri.path.percent_encoded) catch |err| { + if (err == error.FileNotFound) { + log.warn("Path not found: {s}", .{resource_uri.path.percent_encoded}); + return error.FileNotFound; + } + return err; + }; + + if (stat.kind == .file) { + return resource_uri; + } else if (stat.kind == .directory) { + const index_path = try std.fs.path.join(arena_allocator, &[_][]const u8{ resource_uri.path.percent_encoded, "model.safetensors.index.json" }); + const model_path = try std.fs.path.join(arena_allocator, &[_][]const u8{ resource_uri.path.percent_encoded, "model.safetensors" }); + + if (std.fs.cwd().statFile(index_path)) |_| { + return .{ .scheme = "file", .path = .{ .percent_encoded = index_path } }; + } else |_| { + if (std.fs.cwd().statFile(model_path)) |_| { + return .{ .scheme = "file", .path = .{ .percent_encoded = model_path } }; + } else |_| { + return error.ModelFileNotFound; + } + } + } else { + log.err("Path is neither a file nor a directory: {s}", .{resource_uri.path.percent_encoded}); + return error.InvalidPath; + } + } else { + return resource_uri; + } + } + + pub fn resolveFromArgs(self: *ModelPathResolver, args: *std.process.ArgIterator) !ResourceURI { + _ = args.next().?; // Skip program name + + const arg = args.next() orelse { + log.err("No model path provided", .{}); + return error.NoModelPathProvided; + }; + + return self.resolve(arg); + } +}; + +const ResourceIndex = struct { + arena: std.heap.ArenaAllocator, + + map: std.StringArrayHashMapUnmanaged(ResourceURI), + metadata: Metadatas, + + pub fn init(allocator: std.mem.Allocator) ResourceIndex { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .map = .{}, + .metadata = .{}, + }; + } + + pub fn deinit(self: *ResourceIndex) void { + const allocator = self.arena.allocator(); + self.map.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } +}; + +fn parseSafetensorsIndex( + allocator: std.mem.Allocator, + resource: *Resource, + reader: *std.io.Reader, +) !ResourceIndex { + const resource_uri = resource.uri(); + const path = resource_uri.path.percent_encoded; + const basename = path[0 .. std.mem.lastIndexOfScalar(u8, path, '/').? + 1]; + + var resource_index: ResourceIndex = .init(allocator); + errdefer resource_index.deinit(); + + const arena_allocator = resource_index.arena.allocator(); + + var json_reader: std.json.Reader = .init(arena_allocator, reader); + + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + arena_allocator, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + const weight_map = index.object.get("weight_map"); + + if (weight_map) |wm| { + var it = wm.object.iterator(); + + while (it.next()) |entry| { + const weight_name = entry.key_ptr.*; + const filename = entry.value_ptr.string; + + const resource_path = try arena_allocator.alloc(u8, basename.len + filename.len); + + @memcpy(resource_path[0..basename.len], basename); + @memcpy(resource_path[basename.len..], filename); + + const sibling_uri: ResourceURI = .{ + .scheme = resource_uri.scheme, + .user = resource_uri.user, + .password = resource_uri.password, + .host = resource_uri.host, + .port = resource_uri.port, + .path = .{ .percent_encoded = resource_path }, + }; + + _ = try resource_index.map.put(arena_allocator, try arena_allocator.dupe(u8, weight_name), sibling_uri); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + if (index.object.get("__metadata__")) |metadata_val| { + resource_index.metadata = try parseMetadata(arena_allocator, metadata_val); + } + + return resource_index; +} + +fn parseSafetensors( + registry: *TensorRegistry, + resource_uri: ResourceURI, + reader: *std.io.Reader, +) !void { + var arena_allocator = registry.arena.allocator(); + + const json_header_length: usize = @intCast(try reader.takeInt(u64, .little)); + log.debug("json_header_length: {}", .{json_header_length}); + const json_data = try arena_allocator.alloc(u8, json_header_length); + defer arena_allocator.free(json_data); + + try reader.readSliceAll(json_data); + + const data_start_offset = 8 + json_header_length; + const metadata_val = try std.json.parseFromSliceLeaky(std.json.Value, arena_allocator, json_data, .{}); + + var it = metadata_val.object.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + + if (std.mem.eql(u8, key, "__metadata__")) { + registry.metadata = try parseMetadata(arena_allocator, value); + continue; + } + + const shape_field = value.object.get("shape").?.array; + + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); + continue; + } + + const offset_field = value.object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const end: u64 = @intCast(offset_field.array.items[1].integer); + const dtype = try stringToDtype(value.object.get("dtype").?.string); + + var dims: Dims = .{}; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + const shape: Shape = .init(dims.constSlice(), dtype); + const size_in_bytes = end - start; + std.debug.assert(size_in_bytes == shape.byteSize()); + + const tensor_name = try arena_allocator.dupe(u8, key); + const tensor: TensorDesc = .{ + .resource_uri = resource_uri, + .name = tensor_name, + .shape = shape, + .offset = data_start_offset + start, + }; + + try registry.tensors.put(arena_allocator, tensor_name, tensor); + } +} + +fn stringToDtype(safetensor_type: []const u8) !zml.DataType { + const map = std.StaticStringMap(zml.DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, + }); + + return map.get(safetensor_type) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); + return error.UnsupportedDataType; + }; +} + +fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { + const key = prefix.items; + return switch (val) { + .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), + .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), + .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), + .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), + .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), + .array => |v| { + if (v.items.len == 0) return; + if (validSlice(v)) |item_type| { + const data: Metadata = switch (item_type) { + .bool => blk: { + const values = try allocator.alloc(bool, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.bool; + break :blk .{ .array_bool = values }; + }, + .integer => blk: { + const values = try allocator.alloc(i64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.integer; + break :blk .{ .array_int = values }; + }, + .float => blk: { + const values = try allocator.alloc(f64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.float; + break :blk .{ .array_float = values }; + }, + inline .string, .number_string => |tag| blk: { + const values = try allocator.alloc([]const u8, v.items.len); + for (v.items, 0..) |item, i| { + values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); + } + break :blk .{ .array_string = values }; + }, + .null, .array, .object => unreachable, + }; + try metadatas.put(allocator, try allocator.dupe(u8, key), data); + } else { + for (v.items, 0..) |item, i| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + try populateMetadata(allocator, prefix, item, metadatas); + prefix.items.len = old_len; + } + } + }, + .object => |v| { + var obj_iter = v.iterator(); + while (obj_iter.next()) |entry| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.appendSliceAssumeCapacity(entry.key_ptr.*); + try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); + prefix.items.len = old_len; + } + }, + }; +} + +const StringBuilder = std.ArrayListUnmanaged(u8); + +pub fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { + var metadatas: Metadatas = .{}; + var prefix_buf: [BUF_1_KB]u8 = undefined; + var prefix = StringBuilder.initBuffer(&prefix_buf); + + try populateMetadata(allocator, &prefix, val, &metadatas); + + return metadatas; +} + +fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { + if (v.items.len == 0) return null; + + const item_type: std.meta.Tag(std.json.Value) = v.items[0]; + switch (item_type) { + .null, .array, .object => return null, + else => {}, + } + + for (v.items[1..]) |item| { + if (item != item_type) + return null; + } + + return item_type; +} diff --git a/stdx/meta.zig b/stdx/meta.zig index 0441d689..3dc8db32 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -167,6 +167,10 @@ pub fn FnResult(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } +pub fn FnError(comptime func: anytype) type { + return FnSignature(func, null).ReturnErrorSet.?; +} + pub fn Head(Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { diff --git a/zml/meta.zig b/zml/meta.zig index ee2fbc68..88516cef 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -3,6 +3,7 @@ const testing = std.testing; const stdx = @import("stdx"); const FnParam = stdx.meta.FnParam; +const FnError = stdx.meta.FnError; const asSlice = stdx.meta.asSlice; test { @@ -588,6 +589,92 @@ test visit { } } +pub fn VisitReturn(comptime cb: anytype) type { + return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| + @Type(std.builtin.Type{ .error_union = .{ .error_set = error_set, .payload = void } }) + else + void; +} + +pub fn visit2(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn(cb) { + const Callback = @TypeOf(cb); + const Ptr = @TypeOf(v); + const type_info_v = @typeInfo(Ptr); + if (type_info_v != .pointer) { + stdx.debug.compileError("zml.meta.visit2({}) is expecting a pointer/slice input, but received: {}", .{ Callback, Ptr }); + } + const ptr_info = type_info_v.pointer; + const Child = ptr_info.child; + + const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { + .pointer => |info| .{ info.child, !info.is_const }, + else => stdx.debug.compileError("zml.meta.visit2 is expecting a callback with a pointer as second argument but found {}", .{FnParam(cb, 1)}), + }; + // Abort if v doesnt' contain any K. + if (comptime !Contains(Ptr, K)) return; + + // Handle simple cases. + switch (Ptr) { + *const K, *K => return try cb(ctx, v), + *const ?K, *?K => return if (v.*) |*val| try cb(ctx, val) else {}, + []const K, []K => { + for (v) |*v_elem| try cb(ctx, v_elem); + return; + }, + else => {}, + } + + // Handle stdx.BoundedArray that contains uninitalized data. + if (@typeInfo(Child) == .@"struct" and @hasDecl(Child, "constSlice") and @hasDecl(Child, "slice")) { + return visit2(cb, ctx, if (mutating_cb) v.slice() else v.constSlice()); + } + + // Recursively visit2 fields of v. + switch (ptr_info.size) { + .one => switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + switch (field_type_info) { + .pointer => try visit2(cb, ctx, @field(v, field.name)), + .array, .optional, .@"union", .@"struct" => try visit2(cb, ctx, &@field(v, field.name)), + else => {}, + } + }, + .array => |_| for (v) |*elem| try visit2(cb, ctx, elem), + .optional => if (v.* != null) try visit2(cb, ctx, &v.*.?), + .@"union" => switch (v.*) { + inline else => |*v_field| try visit2(cb, ctx, v_field), + }, + else => stdx.debug.compileError("zml.meta.visit2({}) doesn't support fields of type: {}", .{ Callback, Child }), + }, + .slice => { + for (v) |*v_elem| { + switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + if (field_type_info == .pointer) { + try visit2(cb, ctx, @field(v_elem, field.name)); + } else { + try visit2(cb, ctx, &@field(v_elem, field.name)); + } + }, + .array => |_| for (v) |*elem| try visit2(cb, ctx, elem), + .optional => if (v.* != null) try visit2(cb, ctx, &v.*.?), + .@"union" => switch (v_elem.*) { + inline else => |*v_field| try visit2(cb, ctx, v_field), + }, + else => stdx.debug.compileError("zml.meta.visit2({}) doesn't support fields of type: {}", .{ Callback, Child }), + } + } + }, + .many, .c => stdx.debug.compileError("zml.meta.visit2({}) doesn't support [*] style pointers, got: {}", .{ Callback, Ptr }), + } +} + pub fn count(T: type, value: anytype) u32 { var counter: u32 = 0; visit(struct { diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 30317127..70d787ff 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -7,6 +7,27 @@ pub const Tracer = switch (builtin.os.tag) { else => FakeTracer, }; +var _tracer: Tracer = undefined; +var _tracer_initialized: bool = false; + +pub fn tracer() *Tracer { + if (!_tracer_initialized) { + unreachable; + } + return &_tracer; +} + +pub fn initTracer() void { + if (_tracer_initialized) return; + _tracer = Tracer.init("tracer"); + _tracer_initialized = true; +} + +pub fn deinitTracer() void { + if (!_tracer_initialized) return; + _tracer.deinit(); +} + const CudaTracer = struct { // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so diff --git a/zml/zml.zig b/zml/zml.zig index 4f8b9244..2d9f7853 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -47,6 +47,7 @@ pub const c = @import("c"); // pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { + pub const tracer = @import("tools/tracer.zig"); pub const Tracer = @import("tools/tracer.zig").Tracer; }; From 5a81f0058435993ee9a1f7a5a04fdf5e6527c113 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 31 Oct 2025 16:58:50 +0100 Subject: [PATCH 45/47] cleanup: removed all unused code If required we can look at the commit history to retrieve deleted code --- mlir/main.zig | 1436 +------------------------------------------------ 1 file changed, 2 insertions(+), 1434 deletions(-) diff --git a/mlir/main.zig b/mlir/main.zig index be96b4fb..ca976cda 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -327,13 +327,13 @@ pub const std_options: std.Options = .{ var global_compilation_context: ?*CompilationContext = null; +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); + pub const TensorOriginType = enum { argument, value, }; -const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); - pub const TensorOrigin = union(TensorOriginType) { argument: void, value: *const mlir.Value, @@ -2584,190 +2584,11 @@ pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK return res; } -const MoE = struct { - experts: Mlp, - router: zml.nn.Linear, - - pub fn init(tensor_descriptor: TensorDescriptor.View) MoE { - return .{ - .experts = .{ - .gate_up_proj = .{ - // We need to bitcast the scale cause safetensors doesn't encode f8 types correctly - .scale = tensor_descriptor.getWithTags("gate_up_proj_scales", .{ .expert, .out, .d }).?, - // We don't bitcast here because PJRT doesn't handle packed host buffers - .blocks = tensor_descriptor.getWithTags("gate_up_proj_blocks", .{ .expert, .out, .d, .d_block }), - .blocks_dtype = .f4e2m1, - .bias = tensor_descriptor.getWithTags("gate_up_proj_bias", .{ .expert, .d }), - }, - .down_proj = .{ - .blocks = tensor_descriptor.getWithTags("down_proj_blocks", .{ .expert, .out, .d, .d_block }), - .blocks_dtype = .f4e2m1, - .scale = tensor_descriptor.getWithTags("down_proj_scales", .{ .expert, .out, .d }), - .bias = tensor_descriptor.getWithTags("down_proj_bias", .{ .expert, .d }), - }, - }, - .router = .{ - .weight = tensor_descriptor.getWithTags("router.weight", .{ .dout, .d }).?, - .bias = tensor_descriptor.getWithTags("router.bias", .{.d}).?, - }, - }; - } - - pub fn loadBuffers(allocator: std.mem.Allocator, moe: MoE, buffer_store: BufferStore4.View, platform: Platform) !Bufferized(MoE) { - var transfer, const entries = try multiTransfer(allocator, .{ - moe.experts.gate_up_proj.scale.shape(), - moe.experts.gate_up_proj.blocks.shape(), - moe.experts.gate_up_proj.bias.shape(), - moe.experts.down_proj.scale.shape(), - moe.experts.down_proj.blocks.shape(), - moe.experts.down_proj.bias.shape(), - moe.router.weight.shape(), - moe.router.bias.shape(), - }, platform); - defer transfer.deinit(allocator, platform); - - const gate_up_proj_scales, const gate_up_proj_blocks, const gate_up_proj_bias, const down_proj_scales, const down_proj_blocks, const down_proj_bias, const router_weight, const router_bias = entries; - buffer_store.get("gate_up_proj_scales").?.reader().stream(gate_up_proj_scales.writer); - buffer_store.get("gate_up_proj_blocks").?.reader().stream(gate_up_proj_blocks.writer); - buffer_store.get("gate_up_proj_bias").?.reader().stream(gate_up_proj_bias.writer); - buffer_store.get("down_proj_scales").?.reader().stream(down_proj_scales.writer); - buffer_store.get("down_proj_blocks").?.reader().stream(down_proj_blocks.writer); - buffer_store.get("down_proj_bias").?.reader().stream(down_proj_bias.writer); - buffer_store.get("router.weight").?.reader().stream(router_weight.writer); - buffer_store.get("router.bias").?.reader().stream(router_bias.writer); - - return .{ - .experts = .{ - .gate_up_proj = .{ - .scale = gate_up_proj_scales, - .blocks = gate_up_proj_blocks, - .bias = gate_up_proj_bias, - }, - .down_proj = .{ - .scale = down_proj_scales, - .blocks = down_proj_blocks, - .bias = down_proj_bias, - }, - }, - .router = .{ - .weight = router_weight, - .bias = router_bias, - }, - }; - } -}; - -pub const BlockScaledLinear = struct { - blocks: Tensor, - scale: Tensor, - bias: ?Tensor = null, - blocks_dtype: zml.DataType, -}; - -pub const Mlp = struct { - gate_up_proj: BlockScaledLinear, // {.out = intermediate_size * 2, .d = hidden_size / block_size, .d_block = block_size } - down_proj: BlockScaledLinear, // {.out = hidden_size * 2, .d = intermediate_size / block_size, .d_block = block_size } - - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const dt = x.dtype(); - var gate, var up = zml.nn.splitRealImg(self.gate_up_proj.forward(x), .interleaved); - gate = .minimum(gate, .scalar(7, dt)); - up = .clamp(up, .scalar(-7, dt), .scalar(7, dt)); - - const out = gate.quickGelu().mul(up.addConstant(1)); - return zml.call(self.down_proj, .forward, .{out}); - } - - pub fn format(self: Mlp, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Mlp(gate_up_proj=.{f}, down_proj=.{f})", .{ self.gate_up_proj, self.down_proj }); - } -}; - -/// Model definition -const Mnist = struct { - layers: [2]Layer, - - const Layer = struct { - weight: Tensor, - bias: Tensor, - - pub fn init(tensor_descriptor: TensorDescriptor.View) Layer { - return .{ - .weight = tensor_descriptor.getWithTags("weight", .{ .dout, .d }).?, - .bias = tensor_descriptor.getWithTags("bias", .{.d}).?, - }; - } - - pub fn forward(self: Layer, input: Tensor) Tensor { - return self.weight.dot(input, .d).add(self.bias).relu(); - } - - pub fn loadBuffers(allocator: std.mem.Allocator, layer: Layer, buffer_store: BufferStore4.View, platform: Platform) !Bufferized(Layer) { - var transfer, const entries = try multiTransfer(allocator, .{ layer.weight.shape(), layer.bias.shape() }, platform); - defer transfer.deinit(allocator, platform); - - const weight, const bias = entries; - buffer_store.get("weight").?.reader().stream(weight.writer); - buffer_store.get("bias").?.reader().stream(bias.writer); - - return .{ - .weight = weight.buffer, - .bias = bias.buffer, - }; - } - }; - - pub fn init(tensor_descriptor: TensorDescriptor.View) Mnist { - return .{ .layers = .{ - Layer.init(tensor_descriptor.withPrefix("fc1")), - Layer.init(tensor_descriptor.withPrefix("fc2")), - } }; - } - - /// just two linear layers + relu activation - pub fn forward(self: Mnist, input: Tensor) Tensor { - var x = input.flattenAll().convert(.f32).withTags(.{.d}); - for (self.layers) |layer| { - x = layer.forward(x).withTags(.{.d}); - } - return x.argMax(0).indices.convert(.u8); - } - - pub fn loadBuffers(allocator: std.mem.Allocator, model: Mnist, buffer_store: BufferStore4.View, platform: Platform) !Bufferized(Mnist) { - return .{ - .layers = .{ - try Layer.loadBuffers(allocator, model.layers[0], buffer_store.withPrefix("fc1"), platform), - //try Layer.loadBuffers(allocator, model.layers[1], buffer_store.withPrefix("fc2"), platform), - //try autoLoad(allocator, model.layers[1], buffer_store.withPrefix("fc2"), platform), - try loadBuffersFromId(allocator, model.layers[1], buffer_store.withPrefix("fc2"), platform), - }, - }; - } -}; - pub const TransferEntry = struct { buffer: zml.Buffer, writer: *Transfer.Writer, }; -fn TransferReturnType(comptime ShapeType: type) type { - if (ShapeType == zml.Shape) { - return std.meta.Tuple(&.{ Transfer, TransferEntry }); - } - const type_info = @typeInfo(ShapeType); - return switch (type_info) { - .@"struct" => |struct_type_info| b: { - if (!struct_type_info.is_tuple) unreachable; - inline for (struct_type_info.fields) |field| { - if (field.type != zml.Shape) unreachable; - } - const types = [1]type{Transfer} ++ [_]type{TransferEntry} ** struct_type_info.fields.len; - break :b std.meta.Tuple(&types); - }, - else => unreachable, - }; -} - pub fn singleTransfer(allocator: std.mem.Allocator, shape: Shape, platform: Platform) !struct { Transfer, TransferEntry } { var transfer = try Transfer.init(allocator, &.{shape}, platform); errdefer transfer.deinit(allocator, platform); @@ -2890,9 +2711,6 @@ pub fn loadBuffersFromId(allocator: std.mem.Allocator, model: anytype, buffer_st const buffer_reader = try allocator.alloc(u8, BUF_16_MB); defer allocator.free(buffer_reader); - //const readers = try collectReaders(arena.allocator(), buffer_store, &model); - //std.debug.print("Found {d} readers\n", .{readers.len}); - const LocalContext = struct { tensor_descs: []TensorDesc, shapes: []const Shape, @@ -3103,12 +2921,6 @@ const Transfer = struct { const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); errdefer transfer_manager.deinit(platform.pjrt_api); - //const count = transfer_manager.bufferCount(platform.pjrt_api) catch unreachable; - //for (0..count) |index| { - // const buffer_size = transfer_manager.bufferSize(platform.pjrt_api, index) catch unreachable; - // std.debug.print("index: {d} - size: {d}\n", .{ index, buffer_size }); - //} - const entries = try arena.allocator().alloc(Entry, shapes.len); for (entries, shapes, 0..) |*e, shape, index| { const buffer = try transfer_manager.retrieveBuffer(platform.pjrt_api, index); @@ -3136,267 +2948,10 @@ const Transfer = struct { } }; -pub const Linear = struct { - weight: Tensor, - bias: ?Tensor = null, - tag: Shape.Tag, - - pub fn init(weight: Tensor, bias: ?Tensor, dot_tag: anytype) Linear { - return .{ - .weight = weight, - .bias = bias, - .tag = zml.Shape.toTag(dot_tag), - }; - } - - pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight.convert(x.dtype()), self.tag); - - // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); - return if (self.bias) |bias| y.add(bias.autoBroadcast()) else y; - } -}; - -const KvCacheKind = enum { - split, - unified, -}; - -const KvCache = union(KvCacheKind) { - split: struct { - k: Tensor, - v: Tensor, - }, - unified: Tensor, -}; - -fn testKvCache(kv_cache: KvCache, input: Tensor) KvCache { - return switch (kv_cache) { - .split => |split_kv_cache| .{ - .split = .{ - .k = split_kv_cache.k.add(input.autoBroadcast()), - .v = split_kv_cache.v.add(input.autoBroadcast()), - }, - }, - .unified => |unified_kv_cache| .{ .unified = unified_kv_cache.add(input.autoBroadcast()) }, - }; -} - -const RmsNorm = struct { - weight: Tensor, - eps: f32 = 1e-5, - - /// L2 normalization of input tensor along `.d` axis. - pub fn forward(self: RmsNorm, input: Tensor) Tensor { - const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); - const normalized = zml.nn.rmsNorm(x, .d, self.eps); - return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); - } -}; - -pub fn testLinear2(linear1: Linear, linear2: Linear, x: Tensor) Tensor { - const out1 = linear1.forward(x).rename(.{ .dout = .d }); - std.debug.print("out1: {f}\n", .{out1}); - const out2 = linear2.forward(out1).rename(.{ .dout = .d }); - std.debug.print("out2: {f}\n", .{out1}); - return out2; -} - pub fn main() !void { try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); } -pub fn asyncMain3() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(.{}); - context.printAvailablePlatforms(platform); - - // var tmp_dir = std.testing.tmpDir(.{}); - // defer tmp_dir.cleanup(); - - // const filename = "file.bin"; - // _ = try createBinFile(tmp_dir, filename, BUF_8_KB, null); - - // const file_path = try tmp_dir.dir.realpathAlloc(allocator, filename); - // defer allocator.free(file_path); - var client: std.http.Client = .{ .allocator = allocator, .write_buffer_size = 8192 }; - try client.initDefaultProxies(arena.allocator()); - defer client.deinit(); - - if (false) { - //var resource: Resource = .{ .memory = try .init(safetensor) }; - //defer resource.deinit(); - - const buffer_reader = try allocator.alloc(u8, BUF_4_MB); - defer allocator.free(buffer_reader); - - //var resource: Resource = .{ .http = try .init(&client, "http://localhost:8000/model.safetensors.index.json") }; - var resource: Resource = .{ .http = try .init(&client, allocator, "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/raw/main/model.safetensors.index.json", null) }; - defer resource.deinit(); - - //var resource: Resource = .{ .file = try .init("/Users/corendos/models/Llama-3.1-8B-Instruct/model.safetensors.index.json", buffer_reader) }; - //defer resource.deinit(); - - var reader = resource.reader(buffer_reader); - - var parsed_safetensors = try parseSafetensorsIndex(arena.allocator(), &resource, reader.interface()); - defer parsed_safetensors.deinit(); - { - //"model.layers.0.self_attn.o_proj.weight":{"dtype":"BF16","shape":[4096,4096],"data_offsets":[1411399680,1444954112]} - var tensor_resource: Resource = .{ .http = try .init( - &client, - allocator, - "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model-00001-of-00004.safetensors", - .{ .offset = 8, .size = 4096 }, - ) }; - defer tensor_resource.deinit(); - - var reader2 = tensor_resource.reader(buffer_reader); - - const buffer_writer = try allocator.alloc(u8, BUF_4_MB); - defer allocator.free(buffer_writer); - var writer = std.fs.File.stdout().writer(buffer_writer); - - const result = try reader2.interface().streamRemaining(&writer.interface); - std.log.debug("{}", .{result}); - try writer.interface.flush(); - } - } - - if (false) { - const shape = zml.Shape.init(.{ 128256, 4096 }, .bf16); - - var transfer = try Transfer.init(allocator, &.{shape}, platform); - defer transfer.deinit(platform); - - const entry = transfer.get(0); - - const start = 0; - const end = 1050673152; - var tensor_resource: Resource = .{ - .http = try .init( - &client, - allocator, - "http://localhost:8000/model-00001-of-00004.safetensors", - //"https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model-00001-of-00004.safetensors", - .{ .offset = start, .size = end - start }, - ), - }; - defer tensor_resource.deinit(); - - const buffer_reader = try allocator.alloc(u8, BUF_4_MB); - defer allocator.free(buffer_reader); - var reader = tensor_resource.reader(buffer_reader); - - const written = try reader.interface().streamRemaining(&entry.writer.interface); - try entry.writer.interface.flush(); - std.log.debug("Written {d} bytes", .{written}); - } - - if (true) { - const buffer_reader = try allocator.alloc(u8, BUF_4_MB); - defer allocator.free(buffer_reader); - - var resource: Resource = .{ .http = try .init(&client, allocator, std.Uri.parse("https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model.safetensors.index.json") catch unreachable, null) }; - defer resource.deinit(); - - var reader = resource.reader(buffer_reader); - - var resource_index = try parseSafetensorsIndex(arena.allocator(), &resource, reader.interface()); - defer resource_index.deinit(); - - var registry: TensorRegistry = try .initWithMetadata(allocator, resource_index.metadata); - defer registry.deinit(); - - var seen: std.ArrayHashMapUnmanaged(std.Uri, void, ResourceURIContext, false) = .empty; - - var it = resource_index.map.iterator(); - while (it.next()) |entry| { - if (seen.get(entry.value_ptr.*)) |_| {} else { - try seen.put(arena.allocator(), entry.value_ptr.*, {}); - - std.debug.print("subresource uri: {f}\n", .{entry.value_ptr.fmt(.all)}); - - var subresource: Resource = .{ .http = try .init(&client, allocator, entry.value_ptr.*, null) }; - defer subresource.deinit(); - - std.log.debug("Opened subresource", .{}); - - var reader2 = subresource.reader(buffer_reader); - - try parseSafetensors(®istry, entry.value_ptr.*, reader2.interface()); - } - } - - //const tensor_desc = registry.tensors.get("model.layers.0.self_attn.o_proj.weight").?; - const tensor_desc = registry.tensors.get("model.embed_tokens.weight").?; - - var tensor_resource: Resource = .{ - .http = try .init( - &client, - allocator, - tensor_desc.resource_uri, - .{ .offset = tensor_desc.offset, .size = tensor_desc.shape.byteSize() }, - ), - }; - defer tensor_resource.deinit(); - - var reader3 = tensor_resource.reader(buffer_reader); - - var transfer = try Transfer.init(allocator, &.{tensor_desc.shape}, platform); - defer transfer.deinit(platform); - - const entry = transfer.get(0); - - const written = try reader3.interface().streamRemaining(&entry.writer.interface); - try entry.writer.interface.flush(); - std.log.debug("Written {d} bytes", .{written}); - } -} - -pub fn asyncMain2() !void { - var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; - defer _ = gpa.deinit(); - //const allocator = std.heap.c_allocator; - const allocator = gpa.allocator(); - - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(.{}); - context.printAvailablePlatforms(platform); - - const exe = b: { - const linear1: Linear = .init( - Tensor.init(zml.Shape.init(.{ .dout = 128, .d = 128 }, .f32)), - Tensor.init(zml.Shape.init(.{ .dout = 128 }, .f32)), - .d, - ); - const linear2: Linear = .init( - Tensor.init(zml.Shape.init(.{ .dout = 64, .d = 128 }, .f32)), - Tensor.init(zml.Shape.init(.{ .dout = 64 }, .f32)), - .d, - ); - - const x = Tensor.init(zml.Shape.init(.{ .b = 16, .s = 16, .d = 128 }, .f32)); - - const exe = try compile(allocator, testLinear2, .{ linear1, linear2, x }, platform); - errdefer exe.deinit(); - - break :b exe; - }; - defer exe.deinit(); -} - pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: llama.LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { var encoder = try tokenizer.encoder(); defer encoder.deinit(); @@ -3530,10 +3085,6 @@ pub fn asyncMain() !void { } } - //var old_buffer_store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); - //defer old_buffer_store.deinit(); - - //var buffer_store: BufferStore5 = try .fromBufferStore(allocator, old_buffer_store); var buffer_store: BufferStore5 = try .init(allocator, ®istry, &client); defer buffer_store.deinit(); @@ -3582,12 +3133,6 @@ pub fn asyncMain() !void { defer kv_cache_buffer.k.deinit(); defer kv_cache_buffer.v.deinit(); - //var token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{0})); - //defer token_index_buffer.deinit(); - - //var token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{0})); - //defer token_buffer.deinit(); - var new_token_id: u32 = undefined; for (tokenized_prompt[0..], 0..) |token_id, i| { std.debug.print("Running iteration {} token id: {}\n", .{ i, token_id }); @@ -3607,11 +3152,6 @@ pub fn asyncMain() !void { std.log.debug("Calling exe", .{}); exe.call(args, &results); - //token_buffer.deinit(); - //kv_cache_buffer.layer_index.deinit(); - //kv_cache_buffer.k.deinit(); - //kv_cache_buffer.v.deinit(); - //rng_buffer._state.deinit(); results.fill(&.{ &token_buffer, &kv_cache_buffer, &rng_buffer._state }); const host = try token_buffer.toHostAlloc(allocator); @@ -3621,51 +3161,6 @@ pub fn asyncMain() !void { const predicted_token_str = try tokenizer_decoder.decode(&.{predicted_token}); std.debug.print("Predicted: {} {s}\n", .{ predicted_token, predicted_token_str }); new_token_id = predicted_token; - - //const embeds_buffer = b: { - // var args = try exe.args(allocator); - // defer args.deinit(allocator); - - // var results = try exe.results(allocator); - // defer results.deinit(allocator); - - // args.set(.{ llama_buffers, token_buffer, token_index_buffer, kv_cache_buffer, rng_buffer }); - // std.log.debug("Calling exe", .{}); - // exe.call(args, &results); - - // //token_buffer.deinit(); - // kv_cache_buffer.layer_index.deinit(); - // kv_cache_buffer.k.deinit(); - // kv_cache_buffer.v.deinit(); - // rng_buffer._state.deinit(); - // var embeds_buffer: zml.Buffer = undefined; - // results.fill(&.{ &embeds_buffer, &kv_cache_buffer, &rng_buffer._state }); - - // break :b embeds_buffer; - //}; - //defer embeds_buffer.deinit(); - - //{ - // var args = try exe2.args(allocator); - // defer args.deinit(allocator); - - // var results = try exe2.results(allocator); - // defer results.deinit(allocator); - - // args.set(.{ llama_buffers, embeds_buffer, token_index_buffer, kv_cache_buffer, rng_buffer }); - // exe2.call(args, &results); - - // kv_cache_buffer.layer_index.deinit(); - // kv_cache_buffer.k.deinit(); - // kv_cache_buffer.v.deinit(); - // rng_buffer._state.deinit(); - // var hidden_buffer: zml.Buffer = undefined; - // results.fill(&.{ &hidden_buffer, &kv_cache_buffer, &rng_buffer._state }); - - // const host = try hidden_buffer.toHostAlloc(allocator); - // defer host.deinit(allocator); - // std.debug.print("Predicted: {any}\n", .{host.items(f32)}); - //} } var token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{new_token_id})); @@ -3686,11 +3181,6 @@ pub fn asyncMain() !void { std.log.debug("Calling exe", .{}); exe.call(args, &results); - token_buffer.deinit(); - kv_cache_buffer.layer_index.deinit(); - kv_cache_buffer.k.deinit(); - kv_cache_buffer.v.deinit(); - rng_buffer._state.deinit(); results.fill(&.{ &token_buffer, &kv_cache_buffer, &rng_buffer._state }); const host = try token_buffer.toHostAlloc(allocator); @@ -3700,123 +3190,6 @@ pub fn asyncMain() !void { const predicted_token_str = try tokenizer_decoder.decode(&.{predicted_token}); std.debug.print("Predicted: {} {s}\n", .{ predicted_token, predicted_token_str }); } - - //const process_args = try std.process.argsAlloc(allocator); - //defer std.process.argsFree(allocator, process_args); - //const pt_model = process_args[1]; - //const t10kfilename = process_args[2]; - //_ = t10kfilename; // autofix - - //// Read model shapes. - //// Note this works because Mnist struct uses the same layer names as the pytorch model - //var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); - //defer buffer_store.deinit(); - - //var buffer_store2 = try BufferStore2.fromBufferStore(allocator, buffer_store, platform); - //defer buffer_store2.deinit(); - - //var tensor_descriptor = try TensorDescriptor.fromBufferStore(allocator, buffer_store); - //defer tensor_descriptor.deinit(); - - //// Compile MNIST Model - //const mnist: Mnist = .init(tensor_descriptor.view()); - //const input: Tensor = .init(zml.Shape.init(.{ 28, 28 }, .u8)); - - //var buffer_store3 = try BufferStore3.fromTensorDescriptor(allocator, tensor_descriptor, buffer_store, platform); - //defer buffer_store3.deinit(); - - //std.debug.print("{f}\n", .{mnist.layers[0].weight.shape()}); - - //const exe = try compileModel(allocator, Mnist.forward, mnist, .{input}, platform); - //defer exe.deinit(); - - //// Create model, input and output buffers. - //const input_buffer = try zml.Buffer.fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]u8{0} ** (28 * 28))); - //_ = input_buffer; // autofix - ////const mnist_buffer = try Mnist.initBuffer(buffer_store2); - - //const mnist_buffer = try bufferized(allocator, mnist, buffer_store3); - //_ = mnist_buffer; // autofix - - //var buffer_store4: BufferStore4 = try .fromBufferStore(allocator, buffer_store, tensor_descriptor); - //defer buffer_store4.deinit(); - - //const mnist_buffer2 = try Mnist.loadBuffers(allocator, mnist, buffer_store4.view(), platform); - //_ = mnist_buffer2; // autofix - - //var it = tensor_descriptor.map.iterator(); - //while (it.next()) |entry| { - // std.debug.print("key: \"{s}\" - value: {any}\n", .{ entry.key_ptr.*, entry.value_ptr.* }); - //} - - //const mnist_buffer, const arena = try bufferized(mnist, buffer_store2); - - // Create arguments (also prepare them) and results helper. - //var args = try exe.args(allocator); - //defer args.deinit(allocator); - //args.set(.{ mnist_buffer, input_buffer }); - - //var results = try exe.results(allocator); - //defer results.deinit(allocator); - - //exe.call(args, &results); - //const output = results.get(zml.Buffer); - //defer output.deinit(); - - //const linear: Linear = .{ - // .weight = Tensor.init(Shape.init(.{ .d = 128, .d_out = 128 }, .f32)), - // .bias = Tensor.init(Shape.init(.{ .d_out = 128 }, .f32)), - // .tag = Shape.toTag(.d), - //}; - - //const mlp: Mlp = .{ - // .up_proj = .{ - // .weight = Tensor.init(Shape.init(.{ .d_hidden = 128, .d = 128 }, .f32)), - // .bias = Tensor.init(Shape.init(.{ .d_hidden = 128 }, .f32)), - // }, - // .gate_proj = .{ - // .weight = Tensor.init(Shape.init(.{ .d_hidden = 128, .d = 128 }, .f32)), - // .bias = Tensor.init(Shape.init(.{ .d_hidden = 128 }, .f32)), - // }, - // .down_proj = .{ - // .weight = Tensor.init(Shape.init(.{ .d = 128, .d_hidden = 128 }, .f32)), - // .bias = Tensor.init(Shape.init(.{ .d = 128 }, .f32)), - // }, - //}; - //_ = mlp; // autofix - - //const x = Tensor.init(Shape.init(.{ .b = 16, .d = 128 }, .f32)); - - //compileModel(allocator, Mnist.forward, mnist, .{input}); - //const exe = try compile(allocator, testLinear, .{ linear, x }, platform); - //defer exe.deinit(); - - //const linear_buffers = zml.meta.MapRestrict(Tensor, zml.Buffer).map(Linear){ - // .weight = try zml.Buffer.fromBytes(platform, linear.weight.shape(), std.mem.sliceAsBytes(&[_]f32{2} ** (128 * 128))), - // .bias = try zml.Buffer.fromBytes(platform, linear.bias.?.shape(), std.mem.sliceAsBytes(&[_]f32{3} ** (128))), - //}; - //const x_buffer = try zml.Buffer.fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** (16 * 128))); - - //var inputs: [3]zml.Buffer = undefined; - //var outputs: [1]zml.Buffer = undefined; - - //serialize(&.{ linear_buffers, x_buffer }, &inputs); - //exe.call(&inputs, &outputs); - //var result: zml.Buffer = undefined; - //deserialize(&result, outputs[0..1]); - - //var host = try result.toHostAlloc(allocator); - //defer host.deinit(allocator); - - //std.debug.print("{any}\n", .{host.items(f32)}); - - //compile(allocator, testMlp, .{ mlp, x }); - - //compile(allocator, testDot, .{ Tensor.init(.init(.{ .m = 256, .n = 128 }, .f32)), Tensor.init(.init(.{ .b = 16, .n = 128, .k = 256 }, .f32)) }); - - //try testSplitKvCache(allocator, platform); - //try testUnifiedKvCache(allocator, platform); - //try testMnist(allocator, buffer_store2, mnist, platform); } fn compileMergeQkvExe(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, platform: Platform) !Exe { @@ -3838,69 +3211,6 @@ fn compilePrecomputeQkvExe(allocator: std.mem.Allocator, buffer_store: BufferSto return exe; } -//fn testUnifiedKvCache(allocator: std.mem.Allocator, platform: Platform) !void { -// const kv_cache: KvCache = .{ -// .unified = Tensor.init(Shape.init(.{ .layer = 10, .h = 16, .hd = 128 }, .f32)), -// }; -// -// const input = Tensor.init(Shape.init(.{ .hd = 128 }, .f32)); -// const exe = try compile(allocator, testKvCache, .{ kv_cache, input }, platform); -// defer exe.deinit(); -// -// var kv_cache_buffers: zml.meta.MapRestrict(Tensor, zml.Buffer).map(KvCache) = .{ -// .unified = try zml.Buffer.fromBytes(platform, kv_cache.unified.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** (10 * 16 * 128))), -// }; -// const input_buffer = try zml.Buffer.fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** 128)); -// -// var inputs: [2]zml.Buffer = undefined; -// var outputs: [1]zml.Buffer = undefined; -// -// serialize(&.{ kv_cache_buffers, input_buffer }, &inputs); -// exe.call(&inputs, &outputs); -// deserialize(&kv_cache_buffers, &outputs); -//} - -//fn testSplitKvCache(allocator: std.mem.Allocator, platform: Platform) !void { -// const kv_cache: KvCache = .{ -// .split = .{ -// .k = Tensor.init(Shape.init(.{ .layer = 10, .h = 8, .hd = 128 }, .f32)), -// .v = Tensor.init(Shape.init(.{ .layer = 10, .h = 8, .hd = 128 }, .f32)), -// }, -// }; -// -// const input = Tensor.init(Shape.init(.{ .hd = 128 }, .f32)); -// const exe = try compile(allocator, testKvCache, .{ kv_cache, input }, platform); -// defer exe.deinit(); -// -// var kv_cache_buffers: zml.meta.MapRestrict(Tensor, zml.Buffer).map(KvCache) = .{ -// .split = .{ -// .k = try zml.Buffer.fromBytes(platform, kv_cache.split.k.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** (10 * 8 * 128))), -// .v = try zml.Buffer.fromBytes(platform, kv_cache.split.k.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** (10 * 8 * 128))), -// }, -// }; -// const input_buffer = try zml.Buffer.fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{1} ** 128)); -// -// var inputs: [3]zml.Buffer = undefined; -// var outputs: [2]zml.Buffer = undefined; -// -// serialize(&.{ kv_cache_buffers, input_buffer }, &inputs); -// exe.call(&inputs, &outputs); -// deserialize(&kv_cache_buffers, &outputs); -//} - -fn testDot(a: Tensor, b: Tensor) Tensor { - _ = a.dot(b, .n); - _ = a.dot(b, .{ .n = .k }); - _ = a.dot(b, .{ .contracting = .{ .n = .k }, .batching = .{.b} }); - _ = a.dot(b, .{ .contracting = .{.n}, .batching = .{.b} }); - unreachable; - //return a.dotGeneral(b, &.{.{ 1, 1 }}, &.{}); -} - -fn testLinear(linear: Linear, x: Tensor) Tensor { - return linear.forward(x); -} - pub const CompilationContext = struct { pub const Scope = struct { block: *mlir.Block, @@ -3999,11 +3309,6 @@ pub const CompilationContext = struct { } }; -const Context = struct { - compilation_context: *CompilationContext, - current_id: usize = 0, -}; - fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), platform: Platform) !Exe { return compile(allocator, func, .{model} ++ args, platform); } @@ -4047,38 +3352,6 @@ fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { return try list.toOwnedSlice(); } -fn collectValues(allocator: std.mem.Allocator, v: anytype) ![]*const mlir.Value { - const LocalContext = struct { - list: *std.array_list.Managed(*const mlir.Value), - }; - var list = std.array_list.Managed(*const mlir.Value).init(allocator); - var context: LocalContext = .{ .list = &list }; - zml.meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - ctx_.list.append(tensor.value()) catch unreachable; - } - }.cb, &context, v); - - return try list.toOwnedSlice(); -} - -fn collectReaders(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, v: anytype) ![]BufferStore5.Reader { - const LocalContext = struct { - list: *std.array_list.Managed(BufferStore5.Reader), - buffer_store: BufferStore5.View, - }; - var list = std.array_list.Managed(BufferStore5.Reader).init(allocator); - var context: LocalContext = .{ .list = &list, .buffer_store = buffer_store }; - zml.meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - const reader = ctx_.buffer_store.getReaderFromId(tensor.id).?; - ctx_.list.append(reader) catch unreachable; - } - }.cb, &context, v); - - return try list.toOwnedSlice(); -} - fn collectTensorDesc(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, v: anytype) ![]TensorDesc { const LocalContext = struct { list: *std.array_list.Managed(TensorDesc), @@ -4552,269 +3825,6 @@ pub const Exe = struct { } }; -pub const TensorDescriptor = struct { - map: std.StringHashMapUnmanaged(Entry), - allocator: std.mem.Allocator, - - const Entry = struct { - shape: zml.Shape, - associated_tensor_id: ?usize = null, - }; - - pub const View = struct { - tensor_descriptor: *const TensorDescriptor, - - prefix_buffer: [256]u8 = undefined, - prefix_length: usize = 0, - - pub fn withPrefix(self: *const View, prefix_: []const u8) View { - var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; - - return .{ - .tensor_descriptor = self.tensor_descriptor, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - fn prefix(self: *const View) ?[]const u8 { - return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; - } - - pub fn get(self: *const View, subkey: []const u8) ?zml.Shape { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - std.debug.print("Trying to get: {s}\n", .{key}); - const entry = self.tensor_descriptor.getPtr(key) orelse return null; - return entry.shape; - } - - pub fn getWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - std.debug.print("Trying to get: {s}\n", .{key}); - const ptr = self.tensor_descriptor.getPtr(key) orelse return null; - ptr.shape = ptr.shape.withTags(tagz).withSharding(.{0}); - const tensor = Tensor.init(ptr.shape); - ptr.associated_tensor_id = tensor.id; - return tensor; - } - }; - - pub fn init(allocator: std.mem.Allocator) !TensorDescriptor { - return .{ - .map = .{}, - .allocator = allocator, - }; - } - - pub fn deinit(self: *TensorDescriptor) void { - var it = self.map.iterator(); - while (it.next()) |entry| { - self.allocator.free(entry.key_ptr.*); - } - self.map.deinit(self.allocator); - } - - pub fn add(self: *TensorDescriptor, key: []const u8, shape: zml.Shape) !void { - const key_copy = try self.allocator.dupe(u8, key); - errdefer self.allocator.free(key_copy); - - const gop = try self.map.getOrPut(self.allocator, key_copy); - if (gop.found_existing) { - return error.AlreadyExists; - } - - errdefer self.map.removeByPtr(gop.key_ptr); - - gop.value_ptr.* = .{ .shape = shape }; - } - - pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore) !TensorDescriptor { - var new_tensor_descriptor: TensorDescriptor = try .init(allocator); - errdefer new_tensor_descriptor.deinit(); - - var it = buffer_store.buffers.iterator(); - while (it.next()) |entry| { - try new_tensor_descriptor.add(entry.key_ptr.*, entry.value_ptr.shape()); - } - - return new_tensor_descriptor; - } - - pub fn withPrefix(self: *const TensorDescriptor, prefix_: []const u8) TensorDescriptor { - var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; - - return .{ - .map = self.map, - .allocator = self.allocator, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - fn prefix(self: *const TensorDescriptor) ?[]const u8 { - return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; - } - - fn getPtr(self: *const TensorDescriptor, key: []const u8) ?*Entry { - return self.map.getPtr(key); - } - - pub fn view(self: *const TensorDescriptor) View { - return .{ .tensor_descriptor = self }; - } -}; - -const BufferStore2 = struct { - buffers: std.StringHashMapUnmanaged(zml.Buffer), - allocator: std.mem.Allocator, - - prefix_buffer: [256]u8 = undefined, - prefix_length: usize = 0, - - pub fn init(allocator: std.mem.Allocator) !BufferStore2 { - return .{ - .buffers = .{}, - .allocator = allocator, - }; - } - - pub fn deinit(self: *BufferStore2) void { - var it = self.buffers.iterator(); - while (it.next()) |entry| { - entry.value_ptr.deinit(); - self.allocator.free(entry.key_ptr.*); - } - self.buffers.deinit(self.allocator); - } - - pub fn add(self: *BufferStore2, key: []const u8, buffer: zml.Buffer) !void { - const key_copy = try self.allocator.dupe(u8, key); - errdefer self.allocator.free(key_copy); - - const gop = try self.buffers.getOrPut(self.allocator, key_copy); - if (gop.found_existing) { - return error.AlreadyExists; - } - - errdefer self.buffers.removeByPtr(gop.key_ptr); - - gop.value_ptr.* = buffer; - } - - pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore, platform: Platform) !BufferStore2 { - var new_buffer_store: BufferStore2 = try .init(allocator); - errdefer new_buffer_store.deinit(); - - var it = buffer_store.buffers.iterator(); - while (it.next()) |entry| { - try new_buffer_store.add(entry.key_ptr.*, try entry.value_ptr.toDevice(platform)); - } - - return new_buffer_store; - } - - pub fn withPrefix(self: *const BufferStore2, prefix_: []const u8) BufferStore2 { - var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; - - return .{ - .buffers = self.buffers, - .allocator = self.allocator, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - fn prefix(self: *const BufferStore2) ?[]const u8 { - return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; - } - - fn get(self: *const BufferStore2, subkey: []const u8) ?zml.Buffer { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - std.debug.print("Trying to get: {s}\n", .{key}); - return self.buffers.get(key); - } -}; - -const BufferStore3 = struct { - buffers: std.ArrayList(zml.Buffer) = .{}, - id_to_buffer: std.AutoHashMapUnmanaged(usize, usize) = .{}, - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator) !BufferStore3 { - return .{ - .allocator = allocator, - }; - } - - pub fn deinit(self: *BufferStore3) void { - self.buffers.deinit(self.allocator); - self.id_to_buffer.deinit(self.allocator); - } - - pub fn fromTensorDescriptor(allocator: std.mem.Allocator, tensor_descriptor: TensorDescriptor, old_buffer_store: zml.aio.BufferStore, platform: Platform) !BufferStore3 { - var buffer_store: BufferStore3 = try .init(allocator); - errdefer for (buffer_store.buffers.items) |*item| item.deinit(); - errdefer buffer_store.deinit(); - - var it = tensor_descriptor.map.iterator(); - while (it.next()) |entry| { - var host_buffer = old_buffer_store.get(entry.key_ptr.*).?; - host_buffer._shape = entry.value_ptr.shape; - const buffer = try host_buffer.toDevice(platform); - errdefer buffer.deinit(); - - try buffer_store.add(buffer, entry.value_ptr.associated_tensor_id.?); - } - return buffer_store; - } - - pub fn add(self: *BufferStore3, buffer: zml.Buffer, id: usize) !void { - const gop = try self.id_to_buffer.getOrPut(self.allocator, id); - if (gop.found_existing) return error.AlreadyExists; - - errdefer self.id_to_buffer.removeByPtr(gop.key_ptr); - - const new_index = self.buffers.items.len; - - try self.buffers.append(self.allocator, buffer); - errdefer _ = self.buffers.pop(); - - gop.value_ptr.* = new_index; - } - - pub fn getFromId(self: *const BufferStore3, id: usize) ?zml.Buffer { - const index = self.id_to_buffer.get(id) orelse return null; - return self.buffers.items[index]; - } -}; - -pub fn bufferized(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferStore3) !Bufferized(@TypeOf(model)) { - const T = @TypeOf(model); - const BufferizedT = Bufferized(T); - var buf: BufferizedT = undefined; - - const LocalContext = struct { - buffer_store: *const BufferStore3, - }; - - var context: LocalContext = .{ .buffer_store = &buffer_store }; - - try zml.meta.mapAlloc(struct { - fn cb(context_: *LocalContext, tensor: Tensor) zml.Buffer { - const buffer = context_.buffer_store.getFromId(tensor.id).?; - return buffer; - } - }.cb, allocator, &context, model, &buf); - - return buf; -} - pub fn Bufferized(comptime T: type) type { // TODO: we should strip out the non-buffer fields. // Currently it's confusing cause the Bufferized struct contains field that are never read. @@ -4823,366 +3833,6 @@ pub fn Bufferized(comptime T: type) type { return zml.meta.MapRestrict(Tensor, zml.Buffer).map(T); } -pub const ShardWriter = struct { - tensor_shape: Shape, - shard_shape: Shape, - interface: std.Io.Writer, - out: *std.Io.Writer, - - tensor_index: usize = 0, - shard_index: usize = 0, - shard_coordinates: [2]usize, - - fn init(out: *std.Io.Writer, buffer: []u8, tensor_shape: Shape, shard_shape: Shape) ShardWriter { - return .{ - .tensor_shape = tensor_shape, - .shard_shape = shard_shape, - .interface = .{ - .buffer = buffer, - .vtable = &.{ .drain = drain }, - }, - .out = out, - }; - } - - pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self: *ShardWriter = @alignCast(@fieldParentPtr("interface", w)); - _ = self; // autofix - _ = data; // autofix - _ = splat; // autofix - } - - fn computeCoordinates(self: ShardWriter, offset: usize) [2]usize { - const y = offset / self.tensor_shape.dim(0); - const x = offset % self.tensor_shape.dim(0); - - const x_shard = @divFloor(x, self.shard_shape.dim(1)); - const y_shard = @divFloor(y, self.shard_shape.dim(0)); - - return .{ x_shard, y_shard }; - } -}; - -const TensorSlice = struct { - shape: Shape, - data: []const u8, -}; - -pub const BufferStore4 = struct { - slices: std.StringHashMapUnmanaged(Entry), - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator) !BufferStore4 { - return .{ .slices = .{}, .allocator = allocator }; - } - - pub fn deinit(self: *BufferStore4) void { - var it = self.slices.iterator(); - while (it.next()) |entry| self.allocator.free(entry.key_ptr.*); - self.slices.deinit(self.allocator); - } - - pub fn add(self: *BufferStore4, key: []const u8, slice: TensorSlice, associated_tensor_id: ?usize) !void { - const key_copy = try self.allocator.dupe(u8, key); - errdefer self.allocator.free(key_copy); - - const gop = try self.slices.getOrPut(self.allocator, key_copy); - if (gop.found_existing) { - return error.AlreadyExists; - } - - errdefer self.slices.removeByPtr(gop.key_ptr); - - gop.value_ptr.* = .{ .slice = slice, .associated_tensor_id = associated_tensor_id }; - } - - pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore, tensor_descriptor: TensorDescriptor) !BufferStore4 { - var new_buffer_store: BufferStore4 = try .init(allocator); - errdefer new_buffer_store.deinit(); - - var it = buffer_store.buffers.iterator(); - while (it.next()) |entry| { - const descriptor_entry = tensor_descriptor.getPtr(entry.key_ptr.*).?; - const slice: TensorSlice = .{ .shape = entry.value_ptr.shape(), .data = entry.value_ptr.bytes() }; - try new_buffer_store.add(entry.key_ptr.*, slice, descriptor_entry.associated_tensor_id); - } - - return new_buffer_store; - } - - pub const Reader = struct { - entry: *const Entry, - pub fn stream(self: Reader, writer: Transfer.Writer) void { - const chunk_size = 1024 * 1024; - const chunk_count = (self.entry.slice.data.len + chunk_size - 1) / chunk_size; - - for (0..chunk_count) |chunk_index| { - const start = chunk_index * chunk_size; - const end = @min((chunk_index + 1) * chunk_size, self.entry.slice.data.len); - const is_last_transfer = chunk_index == chunk_count - 1; - _ = writer.entry.transfer_manager.transferData(writer.entry.platform.pjrt_api, writer.entry.buffer_index, self.entry.slice.data[start..end], @intCast(start), is_last_transfer) catch unreachable; - } - } - }; - - pub const Entry = struct { - slice: TensorSlice, - associated_tensor_id: ?usize = null, - pub fn reader(entry: *const Entry) Reader { - return .{ .entry = entry }; - } - }; - - pub fn get(self: BufferStore4, key: []const u8) ?*Entry { - std.debug.print("Trying to get {s}\n", .{key}); - return self.slices.getPtr(key); - } - - pub fn view(self: *const BufferStore4) View { - return .{ .buffer_store = self }; - } - - pub const View = struct { - buffer_store: *const BufferStore4, - - prefix_buffer: [256]u8 = undefined, - prefix_length: usize = 0, - - pub fn withPrefix(self: *const View, prefix_: []const u8) View { - var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; - - return .{ - .buffer_store = self.buffer_store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - fn prefix(self: *const View) ?[]const u8 { - return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; - } - - pub fn get(self: *const View, subkey: []const u8) ?*Entry { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - std.debug.print("Trying to get: {s}\n", .{key}); - return self.buffer_store.get(key); - } - }; -}; - -pub const BufferStoreOld = struct { - key_map: std.StringHashMapUnmanaged(usize), - id_map: std.AutoHashMapUnmanaged(usize, usize), - entries: std.ArrayList(Entry), - allocator: std.mem.Allocator, - - pub const Entry = struct { - shape: zml.Shape, - // TODO(Corentin): Replace with reader ? - data: []const u8, - associated_tensor_id: ?usize = null, - }; - - pub fn init(allocator: std.mem.Allocator) !BufferStoreOld { - return .{ - .key_map = .empty, - .id_map = .empty, - .entries = .empty, - .allocator = allocator, - }; - } - - pub fn deinit(self: *BufferStoreOld) void { - var it = self.key_map.iterator(); - while (it.next()) |entry| { - self.allocator.free(entry.key_ptr.*); - } - self.key_map.deinit(self.allocator); - self.id_map.deinit(self.allocator); - self.entries.deinit(self.allocator); - } - - pub fn add(self: *BufferStoreOld, key: []const u8, shape: zml.Shape, data: []const u8) !void { - const new_entry_index = self.entries.items.len; - const new_entry = try self.entries.addOne(self.allocator); - errdefer _ = self.entries.pop(); - - const key_copy = try self.allocator.dupe(u8, key); - errdefer self.allocator.free(key_copy); - - const gop = try self.key_map.getOrPut(self.allocator, key_copy); - if (gop.found_existing) { - return error.AlreadyExists; - } - errdefer self.key_map.removeByPtr(gop.key_ptr); - - gop.value_ptr.* = new_entry_index; - new_entry.* = .{ .shape = shape, .data = data }; - } - - fn bindIdToKey(self: *BufferStoreOld, key: []const u8, id: usize) !void { - const index = self.key_map.get(key).?; - - const gop = try self.id_map.getOrPut(self.allocator, id); - if (gop.found_existing) { - stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.value_ptr.* }); - } - errdefer self.id_map.removeByPtr(gop.key_ptr); - - gop.value_ptr.* = index; - } - - pub fn fromBufferStore(allocator: std.mem.Allocator, buffer_store: zml.aio.BufferStore) !BufferStoreOld { - var new_buffer_store: BufferStoreOld = try .init(allocator); - errdefer new_buffer_store.deinit(); - - var it = buffer_store.buffers.iterator(); - while (it.next()) |entry| { - try new_buffer_store.add(entry.key_ptr.*, entry.value_ptr.shape(), entry.value_ptr.bytes()); - } - - return new_buffer_store; - } - - fn getPtrFromKey(self: *const BufferStoreOld, key: []const u8) ?*Entry { - const index = self.key_map.get(key) orelse return null; - return &self.entries.items[index]; - } - - fn getPtrFromId(self: *const BufferStoreOld, id: usize) ?*Entry { - const index = self.id_map.get(id) orelse return null; - return &self.entries.items[index]; - } - - pub fn view(self: *BufferStoreOld) View { - return .{ .store = self }; - } - - pub const Reader = struct { - entry: *const Entry, - pub fn stream(self: Reader, writer: Transfer.Writer) void { - const chunk_size = 1024 * 1024; - const chunk_count = (self.entry.data.len + chunk_size - 1) / chunk_size; - - for (0..chunk_count) |chunk_index| { - const start = chunk_index * chunk_size; - const end = @min((chunk_index + 1) * chunk_size, self.entry.data.len); - const is_last_transfer = chunk_index == chunk_count - 1; - _ = writer.entry.transfer_manager.transferData(writer.entry.platform.pjrt_api, writer.entry.buffer_index, self.entry.data[start..end], @intCast(start), is_last_transfer) catch unreachable; - } - } - }; - - pub const View = struct { - store: *BufferStoreOld, - - prefix_buffer: [256]u8 = undefined, - prefix_length: usize = 0, - - pub fn root(self: *const View) View { - return .{ - .store = self.store, - }; - } - - pub fn parent(self: *const View) View { - const slice = self.prefix() orelse unreachable; - const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); - var buffer: [256]u8 = undefined; - @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = index + 1, - }; - } - - pub fn withPrefix(self: *const View, prefix_: []const u8) View { - var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - fn prefix(self: *const View) ?[]const u8 { - return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; - } - - pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - - const ptr = self.store.getPtrFromKey(key) orelse return null; - - const tensor = Tensor.init(ptr.shape); - self.store.bindIdToKey(key, tensor.id) catch unreachable; - - return tensor; - } - - pub fn createTensor(self: View, subkey: []const u8) Tensor { - return self.maybeCreateTensor(subkey).?; - } - - pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - - const ptr = self.store.getPtrFromKey(key) orelse return null; - ptr.shape = ptr.shape.withTags(tagz).withSharding(.{0}); - - const tensor = Tensor.init(ptr.shape); - self.store.bindIdToKey(key, tensor.id) catch unreachable; - - return tensor; - } - - pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) Tensor { - return self.maybeCreateTensorWithTags(subkey, tagz).?; - } - - pub fn getReader(self: View, subkey: []const u8) Reader { - return self.getMaybeReader(subkey).?; - } - - pub fn getMaybeReader(self: View, subkey: []const u8) ?Reader { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - return .{ .entry = entry_ptr }; - } - - pub fn getReaderFromId(self: View, id: usize) ?Reader { - const entry_ptr = self.store.getPtrFromId(id) orelse return null; - return .{ .entry = entry_ptr }; - } - - pub fn getShape(self: View, subkey: []const u8) ?Shape { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; - } - - pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { - var buffer: [256]u8 = undefined; - const key = if (opts.no_prefix) - subkey - else b: { - break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - }; - const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; - } - }; -}; - pub const BufferStore5 = struct { registry: *TensorRegistry, id_map: std.AutoHashMapUnmanaged(usize, *TensorDesc), @@ -5301,22 +3951,6 @@ pub const BufferStore5 = struct { return self.maybeCreateTensorWithTags(subkey, tagz).?; } - //pub fn getReader(self: View, subkey: []const u8) Reader { - // return self.getMaybeReader(subkey).?; - //} - - //pub fn getMaybeReader(self: View, subkey: []const u8) ?Reader { - // var buffer: [256]u8 = undefined; - // const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - // const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - // return .{ .entry = entry_ptr }; - //} - - //pub fn getReaderFromId(self: View, id: usize) ?Reader { - // const entry_ptr = self.store.getPtrFromId(id) orelse return null; - // return .{ .entry = entry_ptr }; - //} - pub fn getShape(self: View, subkey: []const u8) ?Shape { var buffer: [256]u8 = undefined; const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; @@ -5730,72 +4364,6 @@ pub const TensorRegistry = struct { } }; -pub const ModelPathResolver = struct { - arena: std.heap.ArenaAllocator, - - pub fn init(allocator: std.mem.Allocator) ModelPathResolver { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - } - - pub fn deinit(self: *ModelPathResolver) void { - self.arena.deinit(); - } - - pub fn resolve(self: *ModelPathResolver, path: []const u8) !ResourceURI { - const arena_allocator = self.arena.allocator(); - - const resource_uri: ResourceURI = ResourceURI.parse(path) catch .{ - .scheme = "file", - .path = .{ .percent_encoded = path }, - }; - - if (std.mem.eql(u8, resource_uri.scheme, FileResource.scheme)) { - const stat = std.fs.cwd().statFile(resource_uri.path.percent_encoded) catch |err| { - if (err == error.FileNotFound) { - log.warn("Path not found: {s}", .{resource_uri.path.percent_encoded}); - return error.FileNotFound; - } - return err; - }; - - if (stat.kind == .file) { - return resource_uri; - } else if (stat.kind == .directory) { - const index_path = try std.fs.path.join(arena_allocator, &[_][]const u8{ resource_uri.path.percent_encoded, "model.safetensors.index.json" }); - const model_path = try std.fs.path.join(arena_allocator, &[_][]const u8{ resource_uri.path.percent_encoded, "model.safetensors" }); - - if (std.fs.cwd().statFile(index_path)) |_| { - return .{ .scheme = "file", .path = .{ .percent_encoded = index_path } }; - } else |_| { - if (std.fs.cwd().statFile(model_path)) |_| { - return .{ .scheme = "file", .path = .{ .percent_encoded = model_path } }; - } else |_| { - return error.ModelFileNotFound; - } - } - } else { - log.err("Path is neither a file nor a directory: {s}", .{resource_uri.path.percent_encoded}); - return error.InvalidPath; - } - } else { - return resource_uri; - } - } - - pub fn resolveFromArgs(self: *ModelPathResolver, args: *std.process.ArgIterator) !ResourceURI { - _ = args.next().?; // Skip program name - - const arg = args.next() orelse { - log.err("No model path provided", .{}); - return error.NoModelPathProvided; - }; - - return self.resolve(arg); - } -}; - const ResourceIndex = struct { arena: std.heap.ArenaAllocator, From aa822ac1ac18e273ae65ef2b79e2caf13b13d71d Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sat, 8 Nov 2025 15:23:26 +0100 Subject: [PATCH 46/47] wip --- mlir/BUILD.bazel | 1 + mlir/dialects/stablehlo/stablehlo.zig | 11 + mlir/main.zig | 579 ++++++++++++++-- mlir/mlir.zig | 14 +- mlir/writer.zig | 10 +- pjrt/pjrt.zig | 12 + zml/BUILD.bazel | 1 + zml/buffer.zig | 106 ++- zml/hostbuffer.zig | 8 +- zml/partitioning.zig | 906 ++++++++++++++++++++++++++ zml/platform.zig | 305 ++++++++- zml/shape.zig | 780 ++++++++++++++++++++-- zml/zml.zig | 1 + 13 files changed, 2596 insertions(+), 138 deletions(-) create mode 100644 zml/partitioning.zig diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index dc065330..8da9cf73 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -41,6 +41,7 @@ zig_binary( main = "main.zig", srcs = [ "llama.zig", + "writer.zig", ], visibility = ["//visibility:public"], deps = [ diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index d4f0ae72..19b8b7f8 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -866,6 +866,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result mlir.RankedTensorType.get( &.{@intCast(ol.len)}, mlir.indexType(ctx), + null, ).shaped(), ol, ), @@ -880,6 +881,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result mlir.RankedTensorType.get( &.{@intCast(ol.len)}, mlir.indexType(ctx), + null, ).shaped(), ol, ), @@ -898,6 +900,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result mlir.RankedTensorType.get( &.{@intCast(rl.len)}, mlir.indexType(ctx), + null, ).shaped(), rl, ), @@ -912,6 +915,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result mlir.RankedTensorType.get( &.{@intCast(rl.len)}, mlir.indexType(ctx), + null, ).shaped(), rl, ), @@ -951,6 +955,13 @@ pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir }, location); } +pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ + .results = .{ .flat = &.{mlir.rankedTensorType(&.{}, mlir.integerType(ctx, .u32))} }, + .location = location, + }); +} + pub const DotDimensionNumbersAttribute = opaque { const M = mlir.Methods(DotDimensionNumbersAttribute, mlir.Attribute); diff --git a/mlir/main.zig b/mlir/main.zig index ca976cda..9cba916a 100644 --- a/mlir/main.zig +++ b/mlir/main.zig @@ -13,6 +13,10 @@ const pjrt = zml.pjrt; const Shape = zml.Shape; const EnumLiteral = @TypeOf(.enum_literal); const Platform = zml.Platform; +const Mesh = zml.partitioning.Mesh; +const Sharding = zml.partitioning.Sharding; + +const RandomReader = @import("writer.zig").RandomReader; const Dims = stdx.BoundedArray(i64, zml.Shape.MAX_RANK); const log = std.log.scoped(.main); @@ -346,7 +350,9 @@ pub const Tensor = struct { id: usize, auto_broadcast: bool = false, _shape: zml.Shape, - tensor_origin: TensorOrigin = .{ .argument = {} }, + //tensor_origin: TensorOrigin = .{ .argument = {} }, + _mesh: ?Mesh = null, + _value: ?*const mlir.Value = null, fn mlirType(self: Tensor, mlir_ctx: *mlir.Context) *const mlir.Type { return mlir.rankedTensorType( @@ -597,7 +603,8 @@ pub const Tensor = struct { pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { const res: Tensor = .{ ._shape = sh, - .tensor_origin = .{ .value = val }, + ._value = val, + //.tensor_origin = .{ .value = val }, .id = Tensor.current_id.fetchAdd(1, .seq_cst), }; @@ -1050,13 +1057,11 @@ pub const Tensor = struct { } pub fn value(self: Tensor) *const mlir.Value { - return switch (self.tensor_origin) { - .argument => b: { - const argument_index = self.getContext().currentScope().id_to_argument.get(self.id).?; - break :b self.getContext().currentScope().block.argument(argument_index); - }, - .value => |v| v, - }; + if (self.getContext().currentScope().id_to_argument.get(self.id)) |argument_index| { + return self.getContext().currentScope().block.argument(argument_index); + } else if (self._value) |v| { + return v; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); } /// Returns the donation data of the tensor. @@ -1312,20 +1317,13 @@ pub const Tensor = struct { pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { const compilation_context = CompilationContext.current(); const scope = compilation_context.currentScope(); - switch (origin.tensor_origin) { - .argument => { - const argument_index = scope.id_to_argument.get(origin.id).?; - - const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; - gop.value_ptr.* = argument_index; - }, - .value => { - if (scope.id_to_donation.get(origin.id)) |origin_donation| { - const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; - gop.value_ptr.* = origin_donation; - } - }, - } + if (scope.id_to_argument.get(origin.id)) |argument_index| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = argument_index; + } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = origin_donation; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); return self; } @@ -2547,6 +2545,40 @@ pub const Tensor = struct { return _result(self._shape, op.result(0)); } + pub fn withSharding(self: Tensor, axes_: anytype) Tensor { + const partitioned_shape = self._shape.withPartitioning(axes_); + //scoped_log.debug("withSharding {}: {} -> {}", .{ axes_, self._shape, partitioned_shape }); + + const ctx = self.getContext(); + const mesh_ = ctx.currentMesh(); + + if (mesh_.isSinglePartition()) return self; + + const sharding: Sharding = .init(mesh_, partitioned_shape); + + const op = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + .{ + .call_target_name = "Sharding", + .has_side_effect = false, + .backend_config = null, + .additional_attributes = &.{mlir.NamedAttribute.named(ctx.mlir_ctx, "mhlo.sharding", ctx.getShardingAttr(sharding))}, + .api_version = .original, + }, + .unknown(ctx.mlir_ctx), + ).appendTo(currentBlock()); + + return _result(partitioned_shape, op.result(0)); + } + + pub fn withMesh(self: Tensor, mesh: Mesh) Tensor { + var t = self; + t._mesh = mesh; + return t; + } + pub fn getContext(self: Tensor) *CompilationContext { _ = self; // autofix return global_compilation_context.?; @@ -2949,7 +2981,326 @@ const Transfer = struct { }; pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain3); +} + +pub fn asyncMain2() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var generator = std.Random.DefaultPrng.init(0); + const random = generator.random(); + + const buffer_reader = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_reader); + + var reader = RandomReader.init(random, buffer_reader); + _ = &reader; // autofix + + const file = try std.fs.openFileAbsolute("/dev/random", .{}); + defer file.close(); + + var file_reader = file.reader(buffer_reader); + + const buffer_writer = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_writer); + + var writer = std.Io.Writer.Discarding.init(buffer_writer); + + const COPY_SIZE = 1 * BUF_1_GB; + + var timer = try stdx.time.Timer.start(); + try file_reader.interface.streamExact(&writer.writer, COPY_SIZE); + try writer.writer.flush(); + const elapsed = timer.read(); + + const elapsed_s: f64 = @as(f64, @floatFromInt(elapsed.ns)) / std.time.ns_per_s; + + std.log.info("Elapsed: {f} - Throughput: {}GB/s", .{ elapsed, @as(f64, COPY_SIZE / (1024 * 1024 * 1024)) / elapsed_s }); +} + +pub const Adder = struct { + weight: Tensor, + + pub fn init(allocator: std.mem.Allocator, mesh: Mesh) Adder { + _ = allocator; // autofix + var tensor = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32).withPartitioning(.{ .m = .x, .n = .y })); + tensor._mesh = mesh; + return .{ .weight = tensor }; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, model: Adder, platform: Platform) Bufferized(Adder) { + _ = allocator; // autofix + _ = model; // autofix + _ = platform; // autofix + } + + pub fn forward(self: Adder, input: Tensor) Tensor { + return self.weight.add(input);//.withSharding(.{ .m = .x, .n = .y }); + } +}; + +pub const Simple = struct { + weight: Tensor, + internal: Internal, + + pub const Internal = struct { + weight: Tensor, + + pub fn init(mesh: Mesh) Internal { + return .{ + .weight = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32)).withMesh(mesh), + }; + } + + pub fn forward(self: Internal, input: Tensor) Tensor { + return self.weight.add(input); + } + }; + + pub fn init(mesh: Mesh) Simple { + return .{ + .weight = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32)).withMesh(mesh), + .internal = .init(mesh), + }; + } + + pub fn forward(self: Simple, input: Tensor) Tensor { + const temp = self.weight.mul(input); + return zmlCall(.forward, .{ self.internal, temp }); + //return self.internal.forward(temp); + } +}; + +//pub fn Func(func_name: @TypeOf(.enum_literal), args: type) type { +// const func = ; +// return @TypeOf(func); +//} + +pub fn zmlCall(comptime func_name: @TypeOf(.enum_literal), args: anytype) stdx.meta.FnResult(@field(@TypeOf(args[0]), @tagName(func_name))) { + return zmlCallInternal(func_name, args) catch unreachable; +} + +pub fn zmlCallInternal(comptime func_name: @TypeOf(.enum_literal), args: anytype) !stdx.meta.FnResult(@field(@TypeOf(args[0]), @tagName(func_name))) { + const compilation_context = CompilationContext.current(); + + var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); + defer arena.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + compilation_context.pushBlock(block); + + const LocalContext = struct { + current_argument_id: usize = 0, + compilation_context: *CompilationContext, + }; + var context: LocalContext = .{ .compilation_context = compilation_context }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + std.log.debug("Argument {} - shape: {f}", .{ ctx_.current_argument_id, tensor.shape() }); + _ = ctx_.compilation_context.currentScope().block.addArgument( + tensor.mlirType(ctx_.compilation_context.mlir_ctx), + .unknown(ctx_.compilation_context.mlir_ctx), + ); + ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; + ctx_.current_argument_id += 1; + } + }.cb, &context, &args); + + const input_shapes = try collectShapes(arena.allocator(), &args); + const input_meshes = try collectMesh(arena.allocator(), &args); + const input_attributes = try arena.allocator().alloc(AttributeList, input_shapes.len); + @memset(input_attributes, .{}); + + const func = @field(@TypeOf(args[0]), @tagName(func_name)); + var result = @call(.auto, func, args); + const output_info = try collectOutputInfo(arena.allocator(), &result); + + const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); + @memset(output_attributes, .{}); + const mesh = compilation_context.currentMesh(); + compilation_context.addShardingAttributes(mesh, input_attributes, output_attributes, input_shapes, input_meshes, output_info); + _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ + .name = @tagName(func_name), + .block = compilation_context.currentScope().block, + .location = .unknown(compilation_context.mlir_ctx), + .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), + }).appendTo(compilation_context.module.body()); + _ = mlir_func; // autofix + + const output_types = try arena.allocator().alloc(*const mlir.Type, output_info.values.len); + for (output_info.values, output_types) |value, *t| t.* = value.type_(); + + compilation_context.popBlock(); + + const input_values = try collectValues(arena.allocator(), &args); + const call_op = dialects.func.call(compilation_context.mlir_ctx, @tagName(func_name), input_values, output_types, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + const LocalContext2 = struct { + current_argument_id: usize = 0, + call_op: *const mlir.Operation, + }; + var context2: LocalContext2 = .{ .call_op = call_op }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext2, tensor: *Tensor) void { + tensor._value = ctx_.call_op.operand(ctx_.current_argument_id); + ctx_.current_argument_id += 1; + } + }.cb, &context2, &result); + + return result; +} + +pub fn asyncMain3() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const devices = platform.getDevices(); + + for (devices, 0..) |device, i| { + const desc = device.getDescription(platform.pjrt_api); + log.info(" Device #{d}:", .{i}); + log.info(" - ID: {}", .{desc.getId(platform.pjrt_api)}); + log.info(" - Process Index: {}", .{desc.getProcessIndex(platform.pjrt_api)}); + log.info(" - Kind: {s}", .{desc.getKind(platform.pjrt_api)}); + log.info(" - Debug String: {s}", .{desc.debugString(platform.pjrt_api)}); + log.info(" - To String: {s}", .{desc.toString(platform.pjrt_api)}); + + const attributes = desc.getAttributes(platform.pjrt_api); + if (attributes.len > 0) { + log.info(" - Attributes:", .{}); + for (attributes) |attr| { + log.info(" - {s}:", .{attr.name()}); + switch (attr.kind()) { + .string => log.info(" (string): {s}", .{attr.inner.unnamed_0.string_value[0..attr.inner.value_size]}), + .int64 => { + log.info(" (int64): {}", .{attr.inner.unnamed_0.int64_value}); + if (std.mem.eql(u8, attr.name(), "memory_bandwidth")) { + const bandwidth_gbps: f32 = @as(f32, @floatFromInt(attr.inner.unnamed_0.int64_value)) / (1024.0 * 1024.0 * 1024.0); + log.info(" (Computed Bandwidth: {:.2} GB/s)", .{bandwidth_gbps}); + } + if (std.mem.eql(u8, attr.name(), "core_count")) { + log.info(" (Core Count: {})", .{attr.inner.unnamed_0.int64_value}); + } + }, + .int64list => { + const list_slice = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + log.info(" (int64list): {any}", .{list_slice}); + }, + .float => log.info(" (float): {}", .{attr.inner.unnamed_0.float_value}), + .bool => log.info(" (bool): {any}", .{attr.inner.unnamed_0.bool_value}), + } + } + } else { + log.info(" - No specific attributes.", .{}); + } + } + + const topology: Shape = .init(.{ .x = 2, .y = 2 }, .u8); + + const mesh: Mesh = .init(topology); + + const adder: Adder = .init(allocator, mesh); + + var input = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32).withPartitioning(.{ .m = .x, .n = .y })); + input._mesh = mesh; + const input_shard_shape = zml.Shape.init(.{ .m = 64, .n = 64 }, .f32); + + const exe = try compileModel(allocator, Adder.forward, adder, .{input}, mesh, platform); + defer exe.deinit(); + + var input_buffer: zml.Buffer = .{ ._shape = input.shape(), ._api = platform.pjrt_api, ._shards = .{} }; + var weight_buffer: zml.Buffer = .{ ._shape = adder.weight.shape(), ._api = platform.pjrt_api, ._shards = .{} }; + for (0..mesh.numPartitions()) |partition_index| { + const shard_data = try allocator.alloc(f32, 64 * 64); + defer allocator.free(shard_data); + const value = std.math.pow(f32, 10, @floatFromInt(partition_index)); + @memset(shard_data, value); + + const input_pjrt_buffer = b: { + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = std.mem.sliceAsBytes(shard_data).ptr, + .buffer_type = .f32, + .dims = input_shard_shape.dims(), + .byte_strides = input_shard_shape.computeByteStrides().slice(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .device = platform.getDevices()[partition_index], + }); + + if (event) |ev| ev.deinit(platform.pjrt_api); + break :b pjrt_buffer; + }; + + const weight_pjrt_buffer = b: { + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = std.mem.sliceAsBytes(shard_data).ptr, + .buffer_type = .f32, + .dims = input_shard_shape.dims(), + .byte_strides = input_shard_shape.computeByteStrides().slice(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .device = platform.getDevices()[partition_index], + }); + + if (event) |ev| ev.deinit(platform.pjrt_api); + break :b pjrt_buffer; + }; + + input_buffer._shards.appendAssumeCapacity(input_pjrt_buffer); + weight_buffer._shards.appendAssumeCapacity(weight_pjrt_buffer); + } + defer input_buffer.deinit(); + defer weight_buffer.deinit(); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + args.set(.{ weight_buffer, input_buffer }); + + exe.call(args, &results); + + var output_buffer = results.get(zml.Buffer); + output_buffer._mesh = mesh; + defer output_buffer.deinit(); + + const output_raw_data = try output_buffer.toHost(allocator); + defer allocator.free(output_raw_data); + + log.info("output: {any}", .{std.mem.bytesAsSlice(f32, output_raw_data)}); +} + +pub fn asyncMain4() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const mesh = Mesh.single(); + + const simple: Simple = .init(mesh); + + const input = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32)).withMesh(mesh); + + const exe = try compileModel(allocator, Simple.forward, simple, .{input}, mesh, platform); + defer exe.deinit(); } pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: llama.LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { @@ -3242,8 +3593,10 @@ pub const CompilationContext = struct { module: *mlir.Module, scopes: stdx.BoundedArray(Scope, 16) = .{}, + _main_mesh: Mesh, + _current_mesh: ?Mesh = null, - pub fn init(allocator: std.mem.Allocator) CompilationContext { + pub fn init(allocator: std.mem.Allocator, mesh: Mesh) CompilationContext { mlir.registerPasses("Transforms"); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { @@ -3257,9 +3610,6 @@ pub const CompilationContext = struct { module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); const pass_manager = mlir.PassManager.init(mlir_ctx); - pass_manager.enableIRPrinting(.{ - .printBeforeAll = true, - }); { var opm = pass_manager.asOpPassManager(); const passes: []const []const u8 = &.{ @@ -3278,6 +3628,7 @@ pub const CompilationContext = struct { .mlir_ctx = mlir_ctx, .mlir_pass_manager = pass_manager, .module = module, + ._main_mesh = mesh, }; } @@ -3307,29 +3658,96 @@ pub const CompilationContext = struct { pub fn current() *CompilationContext { return global_compilation_context.?; } + + pub fn pushMesh(self: *CompilationContext, mesh: Mesh) void { + if (self._current_mesh) |m| { + if (m.eql(mesh)) { + stdx.debug.panic("Mesh already pushed: {}", .{m}); + } + + stdx.debug.panic("Mesh already exists: {}", .{m}); + } + + self._current_mesh = mesh; + // log.info("Pushed mesh: {}", .{mesh}); + } + + pub fn popMesh(self: *CompilationContext) void { + if (self._current_mesh) |m| { + _ = m; // autofix + self._current_mesh = null; + // log.info("Poped mesh: {}", .{m}); + } else { + stdx.debug.panic("No mesh to pop, main mesh: {}", .{self._main_mesh}); + } + } + + pub fn currentMesh(self: *CompilationContext) Mesh { + return self._current_mesh orelse self._main_mesh; + } + + pub fn addShardingAttributes( + self: *const CompilationContext, + mesh: Mesh, + arg_attrs: []AttributeList, + res_attrs: []AttributeList, + input_shapes: []const Shape, + input_meshes: []const ?Mesh, + output_info: OutputInfo, + ) void { + if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. + + const default_layout = mlir.NamedAttribute.named(self.mlir_ctx, "mhlo.layout_mode", mlir.stringAttribute(self.mlir_ctx, "default")); + for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { + attr.appendAssumeCapacity(default_layout); + if (input_mesh) |m| { + // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); + attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); + } else { + // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); + attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + } + } + + for (res_attrs, output_info.shapes) |*attr, shape| { + attr.appendAssumeCapacity(default_layout); + attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + } + } + + pub fn getShardingAttr(self: *const CompilationContext, sharding: Sharding) *const mlir.Attribute { + var sharding_str: stdx.BoundedArray(u8, 128) = .{}; + sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; + return mlir.stringAttribute(self.mlir_ctx, sharding_str.constSlice()); + } }; -fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), platform: Platform) !Exe { - return compile(allocator, func, .{model} ++ args, platform); +fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), mesh: Mesh, platform: Platform) !Exe { + return compile(allocator, func, .{model} ++ args, mesh, platform); } -pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { - var compilation_context: CompilationContext = .init(allocator); +pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx.meta.FnArgs(func), mesh: Mesh, platform: Platform) !Exe { + var compilation_context: CompilationContext = .init(allocator, mesh); defer compilation_context.deinit(); const result = emitMlir(&compilation_context, func, args) catch unreachable; defer result.output_info.deinit(compilation_context.allocator); defer compilation_context.allocator.free(result.input_shapes); + defer compilation_context.allocator.free(result.input_meshes); _ = result.func.appendTo(compilation_context.module.body()); + compilation_context.module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, mesh.numReplicas())); + compilation_context.module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, mesh.numPartitions())); var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), platform, compilation_context.module, null) catch unreachable; + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), mesh, platform, compilation_context.module, null) catch unreachable; - const num_devices = platform.sharding().num_replicas * platform.sharding().num_partitions; - const exe = try Exe.init(allocator, platform, loaded_executable, result.input_shapes, result.output_info.shapes, num_devices); + log.warn("******** ZML generated MLIR ********", .{}); + log.warn("{f}", .{compilation_context.module.operation()}); + + const exe = try Exe.init(allocator, platform, loaded_executable, result.input_shapes, result.output_info.shapes, mesh.numDevices()); errdefer exe.deinit(); return exe; @@ -3352,6 +3770,57 @@ fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { return try list.toOwnedSlice(); } +fn collectMesh(allocator: std.mem.Allocator, v: anytype) ![]?Mesh { + const LocalContext = struct { + list: *std.array_list.Managed(?Mesh), + }; + var list = std.array_list.Managed(?Mesh).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor._mesh); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectValues(allocator: std.mem.Allocator, v: anytype) ![]*const mlir.Value { + const LocalContext = struct { + list: *std.array_list.Managed(*const mlir.Value), + }; + var list = std.array_list.Managed(*const mlir.Value).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.value()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectType(allocator: std.mem.Allocator, v: anytype) ![]*const mlir.Type { + const LocalContext = struct { + list: *std.array_list.Managed(*const mlir.Type), + }; + var list = std.array_list.Managed(*const mlir.Type).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.value().type_()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + fn collectTensorDesc(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, v: anytype) ![]TensorDesc { const LocalContext = struct { list: *std.array_list.Managed(TensorDesc), @@ -3414,7 +3883,9 @@ fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { const EmitMlirResult = struct { func: *mlir.Operation, + // TODO: Unify like output_info input_shapes: []const Shape, + input_meshes: []const ?Mesh, output_info: OutputInfo, }; @@ -3460,6 +3931,8 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar const input_shapes = try collectShapes(compilation_context.allocator, &args); errdefer compilation_context.allocator.free(input_shapes); + const input_meshes = try collectMesh(compilation_context.allocator, &args); + errdefer compilation_context.allocator.free(input_meshes); const input_attributes = try arena.allocator().alloc(AttributeList, input_shapes.len); @memset(input_attributes, .{}); @@ -3477,11 +3950,19 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar }; errdefer output_info.deinit(compilation_context.allocator); + const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); + @memset(output_attributes, .{}); + + const mesh = compilation_context.currentMesh(); + for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); }; + compilation_context.addShardingAttributes(mesh, input_attributes, output_attributes, input_shapes, input_meshes, output_info); _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + //self.add + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ .name = "main", .block = compilation_context.currentScope().block, @@ -3500,6 +3981,7 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar return .{ .func = mlir_func, .input_shapes = input_shapes, + .input_meshes = input_meshes, .output_info = output_info, }; } @@ -3526,13 +4008,11 @@ fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_are } } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, mesh: Mesh, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { //const tracer = Tracer.init("ai.zml.compilation"); //const compile_frame = tracer.frameStart("pjrt compilation"); //defer tracer.frameEnd(compile_frame, "pjrt compilation"); - const sharding = platform.sharding(); - var upb_alloc: upb.Allocator = .init(arena); const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); defer c.upb_Arena_Free(upb_arena); @@ -3543,20 +4023,24 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); - c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); - c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); - c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, mesh.numReplicas()); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, mesh.numPartitions()); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, !mesh.isSinglePartition()); c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); - c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); - c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, mesh.numReplicas()); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, mesh.numPartitions()); - const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); - for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { + const ordered_devices = try zml.platform.createDeviceMesh(arena, mesh, platform); + stdx.debug.assert(ordered_devices.len == mesh.numPartitions(), "createDeviceMesh returned {} devices, but mesh requires {}", .{ ordered_devices.len, mesh.numPartitions() }); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, mesh.numPartitions(), upb_arena); + for (ordered_devices, computation_devices[0..mesh.numPartitions()]) |device, *computation_device| { computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); - _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); + const device_id = @as(i64, @intCast(device.getDescription(platform.pjrt_api).getId(platform.pjrt_api))); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(device_id), upb_arena); } break :device_assignment_blk device_assignment; }); @@ -3973,6 +4457,7 @@ pub const BufferStore5 = struct { const KB = 1024; const MB = 1024 * KB; +const GB = 1024 * MB; const BUF_1_KB = 1 * KB; const BUF_4_KB = 4 * KB; @@ -3990,6 +4475,8 @@ const BUF_64_MB = 64 * MB; const BUF_128_MB = 128 * MB; const BUF_256_MB = 256 * MB; +const BUF_1_GB = 1 * GB; + pub const MemoryResource = struct { uri: ResourceURI, data: []const u8, diff --git a/mlir/mlir.zig b/mlir/mlir.zig index b017693d..41b7b5f8 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1156,7 +1156,7 @@ pub const Operation = opaque { sizes.appendAssumeCapacity(@intCast(segment_operands.len)); } state.addAttributes(&.{ - .named(ctx, "operandSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32)).shaped(), sizes.constSlice())), + .named(ctx, "operandSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), }); }, }; @@ -1169,7 +1169,7 @@ pub const Operation = opaque { sizes.appendAssumeCapacity(@intCast(segment_results.len)); } state.addAttributes(&.{ - .named(ctx, "resultSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32)).shaped(), sizes.constSlice())), + .named(ctx, "resultSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), }); }, }; @@ -1364,12 +1364,12 @@ pub const RankedTensorType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); - pub fn get(dimensions: []const i64, elemType: *const Type) *const RankedTensorType { + pub fn get(dimensions: []const i64, elemType: *const Type, encoding: ?*Attribute) *const RankedTensorType { return @ptrCast(c.mlirRankedTensorTypeGet( @intCast(dimensions.len), @ptrCast(dimensions), elemType.ptr(), - c.mlirAttributeGetNull(), + if (encoding) |e| e.ptr() else c.mlirAttributeGetNull(), ).ptr); } @@ -1402,7 +1402,11 @@ pub const RankedTensorType = opaque { }; pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { - return @ptrCast(RankedTensorType.get(dimensions, elem_type)); + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); +} + +pub fn rankedTensorTypeWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); } pub const DenseElementsAttribute = opaque { diff --git a/mlir/writer.zig b/mlir/writer.zig index c00d8fa6..2f76436a 100644 --- a/mlir/writer.zig +++ b/mlir/writer.zig @@ -154,10 +154,14 @@ pub const RandomReader = struct { } pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + //std.log.info("Calling stream with limit: {any}", .{limit}); const self: *RandomReader = @alignCast(@fieldParentPtr("interface", r)); - const real_limit = limit.toInt() orelse 1024; - for (0..real_limit) |_| try w.writeByte(self.random.int(u8)); - return real_limit; + const bytes = w.unusedCapacitySlice(); + const to_write = @min(bytes.len, limit.toInt() orelse bytes.len); + self.random.bytes(bytes[0..to_write]); + //@memset(bytes[0..to_write], 0xca); + + return to_write; } }; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 80a57469..f4564f22 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -610,6 +610,18 @@ pub const DeviceDescription = opaque { return @intCast(ret.process_index); } + pub fn getAttributes(self: *const DeviceDescription, api: *const Api) []const NamedValue { + const ret = api.call(.PJRT_DeviceDescription_Attributes, .{ + .device_description = self.inner(), + }) catch unreachable; + + if (ret.attributes) |attrs| { + return @ptrCast(attrs[0..ret.num_attributes]); + } + + return &.{}; + } + pub fn getKind(self: *const DeviceDescription, api: *const Api) []const u8 { const ret = api.call(.PJRT_DeviceDescription_Kind, .{ .device_description = self.inner(), diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e0722f70..6e1b0df0 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -37,6 +37,7 @@ zig_library( "nn.zig", "nn/cuda.zig", "ops.zig", + "partitioning.zig", "pjrtx.zig", "platform.zig", "posix.zig", diff --git a/zml/buffer.zig b/zml/buffer.zig index 3553bb94..36d20102 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -8,6 +8,8 @@ const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Sharding = @import("partitioning.zig").Sharding; +const Mesh = @import("partitioning.zig").Mesh; test { std.testing.refAllDecls(@This()); @@ -43,6 +45,7 @@ pub const Buffer = struct { _shape: Shape, _api: *const pjrt.Api, _shards: Shards, + _mesh: ?Mesh = null, // ? todo pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); @@ -314,14 +317,14 @@ pub const Buffer = struct { /// Copies the content of the Buffer back to host, in the given buffer, /// and return a new `HostBuffer` object with the same shape. /// The returned `HostBuffer` doesn't own the memory. - pub fn toHost(self: Buffer, output: []u8) !HostBuffer { - stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); - } + //pub fn toHost(self: Buffer, output: []u8) !HostBuffer { + // stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + // const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); + // if (maybe_event) |event| { + // try event.await_(self._api); + // } + // return HostBuffer.fromBytes(self.shape(), output); + //} /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. @@ -335,6 +338,91 @@ pub const Buffer = struct { return output; } + pub fn toHost(self: Buffer, allocator: std.mem.Allocator) ![]u8 { + const sharding: Sharding = .init(self._mesh.?, self.shape()); + stdx.debug.assert(self._shards.len == sharding.mesh.numPartitions(), "Expected {} PJRT buffers, got {}", .{ sharding.mesh.numPartitions(), self._shards.len }); + + const global_buffer = try allocator.alloc(u8, sharding.global_shape.byteSize()); + errdefer allocator.free(global_buffer); + + if (sharding.getType() == .replicated) { + if (self._shards.len > 0) { + const event = try self._shards.get(0).toHostBuffer(self._api, global_buffer); + if (event) |e| try e.await_(self._api); + } + return global_buffer; + } + + const shard_shape = sharding.shardShape(); + const temp_shard_buffer = try allocator.alloc(u8, shard_shape.byteSize()); + defer allocator.free(temp_shard_buffer); + + const element_size = sharding.global_shape.dtype().sizeOf(); + const global_element_strides = sharding.global_shape.computeElementStrides(); + const rk = sharding.global_shape.rank(); + + var devices: [MAX_NUM_SHARDS]*const pjrt.Device = undefined; + for (self._shards.constSlice(), 0..) |shard, i| { + devices[i] = try shard.getDevice(self._api); + } + + var sharding_iter = sharding.iterator(devices[0..self._shards.len]); + while (sharding_iter.next()) |device_shard| { + const shard_index: usize = @intCast(device_shard.index); + const src_buffer = self._shards.get(shard_index); + + const event = try src_buffer.toHostBuffer(self._api, temp_shard_buffer); + if (event) |e| try e.await_(self._api); + if (rk == 0) { + if (global_buffer.len > 0) { + @memcpy(global_buffer[0..element_size], temp_shard_buffer[0..element_size]); + } + continue; + } + + const last_dim_size = shard_shape.dim(rk - 1); + const contiguous_row_bytes = @as(usize, @intCast(last_dim_size)) * element_size; + const outer_dims_shape = shard_shape.remove(rk - 1); + + var iter = outer_dims_shape.iterator(); + + while (iter.next()) |item| { + const src_flat_index_start_of_row = item.flat_index * @as(usize, @intCast(last_dim_size)); + const src_byte_offset = src_flat_index_start_of_row * element_size; + + var dest_flat_index_start_of_row: i64 = 0; + + for (0..rk) |dim_idx| { + const shard_coord_in_dim = if (dim_idx < rk - 1) item.coords[dim_idx] else 0; + + const global_coord_for_dim = blk: { + const part_spec = sharding.global_shape.partition(dim_idx); + if (part_spec != .axis) break :blk shard_coord_in_dim; // Replicated axis + const mesh_axis_tag = part_spec.toTag(); + if (device_shard.indices.hasTag(mesh_axis_tag) != null) { + const coord = device_shard.indices.dim(mesh_axis_tag); + break :blk (coord * shard_shape.dim(dim_idx)) + shard_coord_in_dim; + } else { + break :blk shard_coord_in_dim; + } + }; + + dest_flat_index_start_of_row += global_coord_for_dim * global_element_strides.get(dim_idx); + } + + const dest_byte_offset = @as(usize, @intCast(dest_flat_index_start_of_row)) * element_size; + + if (contiguous_row_bytes > 0) { + @memcpy( + global_buffer[dest_byte_offset..][0..contiguous_row_bytes], + temp_shard_buffer[src_byte_offset..][0..contiguous_row_bytes], + ); + } + } + } + + return global_buffer; + } /// Frees the accelerator memory. /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. @@ -387,7 +475,7 @@ pub const Buffer = struct { fn hasShardedAxis(self: Buffer) bool { if (self._shards.len == 1) return false; - return @reduce(.Or, self._shape._sharding_info); + return self._shape.hasAtLeastOnePartitionedAxis(); } pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 117cb1e4..cce98b22 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -30,7 +30,7 @@ pub const HostBuffer = struct { pub fn empty(allocator: std.mem.Allocator, sh: Shape) error{OutOfMemory}!HostBuffer { return .{ ._shape = sh, - ._strides = sh.computeStrides().buffer, + ._strides = sh.computeByteStrides().buffer, ._data = (try allocator.alignedAlloc(u8, .@"64", sh.byteSize())).ptr, ._memory = .{ .managed = .@"64" }, }; @@ -43,7 +43,7 @@ pub const HostBuffer = struct { stdx.debug.assert(shape_.byteSize() == data_.len, "shape {} and data {} don't match", .{ shape_.byteSize(), data_.len }); return .{ ._shape = shape_, - ._strides = shape_.computeStrides().buffer, + ._strides = shape_.computeByteStrides().buffer, ._data = data_.ptr, ._memory = .unmanaged, }; @@ -172,7 +172,7 @@ pub const HostBuffer = struct { // TODO we should allow interpreting the output as @Vector(8, f32) when the tensor is f32. stdx.debug.assert(DataType.fromZigType(T) == self.dtype(), "Can't reinterpret {f} as {s}", .{ self, @typeName(T) }); stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); - const ptr: [*]const T = @alignCast(@ptrCast(self._data)); + const ptr: [*]const T = @ptrCast(@alignCast(self._data)); return ptr[0..self._shape.count()]; } @@ -225,7 +225,7 @@ pub const HostBuffer = struct { pub fn isContiguous(self: HostBuffer) bool { const _strides = self._strides; - const cont_strides = self._shape.computeStrides(); + const cont_strides = self._shape.computeByteStrides(); for (self._shape.dims(), _strides[0..self.rank()], cont_strides.constSlice()) |d, stride, cont_stride| { if (d != 1 and stride != cont_stride) return false; } diff --git a/zml/partitioning.zig b/zml/partitioning.zig new file mode 100644 index 00000000..cb2e8027 --- /dev/null +++ b/zml/partitioning.zig @@ -0,0 +1,906 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const stdx = @import("stdx"); + +const pjrtx = @import("pjrtx.zig"); +//const slice = @import("slice.zig"); +//const Shaped = slice.Shaped; +const Buffer = @import("buffer.zig").Buffer; +const Context = @import("context.zig").Context; +const DataType = @import("dtype.zig").DataType; +const Device = @import("pjrtx.zig").Device; +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; + +const log = std.log.scoped(.@"zml/partitioning"); + +test { + std.testing.refAllDecls(Mesh); + std.testing.refAllDecls(DeviceShard); + std.testing.refAllDecls(Sharding); +} + +pub const MaxMeshAxes: u8 = 3; +pub const MaxMeshSize: u8 = MaxMeshAxes * Shape.MAX_RANK; + +/// Iterates through all possible multi-dimensional indices of a topology. +/// The iteration order is row-major. +pub const TopologyIndicesIterator = struct { + index: usize = 0, + topology: Shape, + indices: Shape, + + pub fn next(self: *TopologyIndicesIterator) ?Shape { + if (self.index >= self.topology.count()) return null; + + const current_indices = self.indices; + + // Prepare for the next iteration + self.index += 1; + + // Use a standard row-major increment logic + if (self.index < self.topology.count()) { + const rank = self.topology.rank(); + var i: usize = rank; + + while (i > 0) { + i -= 1; + const dim: u4 = @intCast(i); + + const current_value = self.indices.dim(dim); + if (current_value < self.topology.dim(dim) - 1) { + self.indices = self.indices.setDim(dim, current_value + 1); + return current_indices; + } else { + self.indices = self.indices.setDim(dim, 0); + } + } + } + + return current_indices; + } +}; + +pub const Mesh = struct { + topology: Shape, + + pub fn init(topology: anytype) Mesh { + const self: Mesh = .{ + .topology = .init(topology, .u8), + }; + + if (self.rank() == 0) { + stdx.debug.panic("Mesh must have at least one tagged axis defined, got: {any}", .{topology}); + } + + if (!self.topology.isFullyTagged()) { + stdx.debug.panic("Mesh must be fully tagged, got: {any}", .{topology}); + } + + if (self.rank() > MaxMeshAxes) { + stdx.debug.panic("Mesh rank ({}) exceeds maximum allowed axes ({})", .{ self.rank(), MaxMeshAxes }); + } + + return self; + } + + pub fn single() Mesh { + const topology: Shape = .init(.{ .x = 1 }, .u8); + return .init(topology); + } + + pub fn auto(platform: Platform) Mesh { + const num_devices = platform.getDevices().len; + + if (num_devices == 0) { + stdx.debug.panic("No devices available in the platform: {}", .{platform.target}); + } + + if (num_devices > MaxMeshSize) { + stdx.debug.panic("Too many devices ({}) for a mesh, max is {}", .{ num_devices, MaxMeshSize }); + } + + const topology: Shape = .init(.{ .x = num_devices }, .u8); + + return .init(topology); + } + + pub fn reshape(self: Mesh, new_shape: anytype) Mesh { + const new_topology = self.topology.reshape(new_shape); + return .init(new_topology); + } + + pub fn flatten(self: Mesh, new_axis_name: anytype) Mesh { + const total_devices = self.numDevices(); + + var new_topology = Shape.init(.{}, .u8); + new_topology = new_topology.appendDim(total_devices, @tagName(new_axis_name)); + return Mesh.init(new_topology); + } + + test flatten { + { + // Flatten a 2D mesh + const mesh = Mesh.init(.{ .x = 2, .y = 4 }); + const flattened = mesh.flatten(.all); + const expected = Mesh.init(.{ .all = 8 }); + + try std.testing.expect(flattened.eql(expected)); + try std.testing.expectEqual(1, flattened.rank()); + try std.testing.expectEqual(8, flattened.numDevices()); + } + + { + // Flatten a 3D mesh + const mesh = Mesh.init(.{ .data = 2, .model = 3, .pipeline = 4 }); + const flattened = mesh.flatten(.devices); + const expected = Mesh.init(.{ .devices = 24 }); + try std.testing.expect(flattened.eql(expected)); + try std.testing.expectEqual(1, flattened.rank()); + try std.testing.expectEqual(24, flattened.numDevices()); + } + { + // Flattening a 1D mesh should just rename the axis + const mesh = Mesh.init(.{ .x = 16 }); + const flattened = mesh.flatten(.y); + const expected = Mesh.init(.{ .y = 16 }); + try std.testing.expect(flattened.eql(expected)); + try std.testing.expectEqual(1, expected.rank()); + } + } + + pub fn rank(self: Mesh) i64 { + return @intCast(self.topology.rank()); + } + + pub fn is1D(self: Mesh) bool { + return self.rank() == 1; + } + + pub fn is2D(self: Mesh) bool { + return self.rank() == 2; + } + + pub fn is3D(self: Mesh) bool { + return self.rank() == 3; + } + + pub fn axis(self: Mesh, ax: anytype) i64 { + return self.topology.dim(ax); + } + + pub fn hasManyPartitions(self: Mesh) bool { + return self.numPartitions() > 1; + } + + pub fn isSinglePartition(self: Mesh) bool { + return self.numPartitions() == 1; + } + + pub fn iterator(self: Mesh) TopologyIndicesIterator { + var indices = self.topology; + + for (0..indices.rank()) |dim| { + indices = indices.setDim(dim, 0); + } + + return .{ + .topology = self.topology, + .indices = indices, + }; + } + + pub fn numPartitions(self: Mesh) u8 { + return @intCast(self.topology.count()); + } + + pub fn numReplicas(_: Mesh) u8 { + return 1; + } + + pub fn numDevices(self: Mesh) u8 { + return self.numPartitions() * self.numReplicas(); + } + + pub fn eql(self: Mesh, other: Mesh) bool { + return self.topology.eql(other.topology); + } + + pub fn format( + self: Mesh, + writer: anytype, + ) !void { + try writer.print("Mesh(topology={f} rank={} numDevices={})", .{ self.topology, self.rank(), self.numDevices() }); + } +}; + +test "Mesh / 1D mesh with 1 partition" { + const mesh: Mesh = .init(.{ .x = 1 }); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / 1D mesh with 8 partitions" { + const mesh: Mesh = .init(.{ .x = 8 }); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 8); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(mesh.hasManyPartitions()); + try std.testing.expect(!mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 8); + try std.testing.expect(mesh.numDevices() == 8); +} + +test "Mesh / 2D mesh with 1 partition" { + const mesh: Mesh = .init(.{ .x = 1, .y = 1 }); + try std.testing.expect(mesh.rank() == 2); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.axis(.y) == 1); + try std.testing.expect(!mesh.is1D()); + try std.testing.expect(mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / 2D mesh with 6 partitions" { + const mesh: Mesh = .init(.{ .x = 2, .y = 3 }); + try std.testing.expect(mesh.rank() == 2); + try std.testing.expect(mesh.axis(.x) == 2); + try std.testing.expect(mesh.axis(.y) == 3); + try std.testing.expect(!mesh.is1D()); + try std.testing.expect(mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(mesh.hasManyPartitions()); + try std.testing.expect(!mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 6); + try std.testing.expect(mesh.numDevices() == 6); +} + +test "Mesh / 3D mesh with 1 partition" { + const mesh: Mesh = .init(.{ .x = 1, .y = 1, .z = 1 }); + try std.testing.expect(mesh.rank() == 3); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.axis(.y) == 1); + try std.testing.expect(mesh.axis(.z) == 1); + try std.testing.expect(!mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / 3D mesh with 64 partitions" { + const mesh_3d: Mesh = .init(.{ .x = 4, .y = 4, .z = 4 }); + try std.testing.expect(mesh_3d.rank() == 3); + try std.testing.expect(mesh_3d.axis(.x) == 4); + try std.testing.expect(mesh_3d.axis(.y) == 4); + try std.testing.expect(mesh_3d.axis(.z) == 4); + try std.testing.expect(!mesh_3d.is1D()); + try std.testing.expect(!mesh_3d.is2D()); + try std.testing.expect(mesh_3d.is3D()); + try std.testing.expect(mesh_3d.hasManyPartitions()); + try std.testing.expect(!mesh_3d.isSinglePartition()); + try std.testing.expect(mesh_3d.numPartitions() == 64); + try std.testing.expect(mesh_3d.numDevices() == 64); +} + +test "Mesh / single" { + const mesh: Mesh = .single(); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / auto with 4 devices" { + const platform = env(.{ .cpu = .{ .cpu_device_count = 4 } }); + const mesh: Mesh = .auto(platform); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 4); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(mesh.hasManyPartitions()); + try std.testing.expect(!mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 4); + try std.testing.expect(mesh.numDevices() == 4); +} + +test "Mesh / 1D mesh iterator" { + const mesh = Mesh.init(.{ .x = 4 }); + var iter = mesh.iterator(); + var count: u8 = 0; + + while (iter.next()) |indices| : (count += 1) { + const expected_shape = Shape.init(.{ .x = count }, .u8); + try std.testing.expect(indices.eqlWithTags(expected_shape)); + } + + try std.testing.expectEqual(4, count); + try std.testing.expect(iter.next() == null); +} + +test "Mesh / 2D mesh iterator" { + const mesh = Mesh.init(.{ .x = 2, .y = 3 }); + var iter = mesh.iterator(); + var count: u8 = 0; + const expected_indices = [_]Shape{ + .init(.{ .x = 0, .y = 0 }, .u8), + .init(.{ .x = 0, .y = 1 }, .u8), + .init(.{ .x = 0, .y = 2 }, .u8), + .init(.{ .x = 1, .y = 0 }, .u8), + .init(.{ .x = 1, .y = 1 }, .u8), + .init(.{ .x = 1, .y = 2 }, .u8), + }; + + while (iter.next()) |indices| : (count += 1) { + try std.testing.expect(indices.eqlWithTags(expected_indices[count])); + } + try std.testing.expectEqual(6, count); + try std.testing.expect(iter.next() == null); +} + +test "Mesh / 3D mesh iterator" { + const mesh = Mesh.init(.{ .x = 2, .y = 2, .z = 2 }); + var iter = mesh.iterator(); + var count: u8 = 0; + const expected_indices = [_]Shape{ + .init(.{ .x = 0, .y = 0, .z = 0 }, .u8), + .init(.{ .x = 0, .y = 0, .z = 1 }, .u8), + .init(.{ .x = 0, .y = 1, .z = 0 }, .u8), + .init(.{ .x = 0, .y = 1, .z = 1 }, .u8), + .init(.{ .x = 1, .y = 0, .z = 0 }, .u8), + .init(.{ .x = 1, .y = 0, .z = 1 }, .u8), + .init(.{ .x = 1, .y = 1, .z = 0 }, .u8), + .init(.{ .x = 1, .y = 1, .z = 1 }, .u8), + }; + + while (iter.next()) |indices| : (count += 1) { + try std.testing.expect(indices.eqlWithTags(expected_indices[count])); + } + + try std.testing.expectEqual(8, count); + try std.testing.expect(iter.next() == null); +} + +pub const DeviceShard = struct { + index: u64, + topology: Shape, + global_shape: Shape, + indices: Shape, + shard_shape: Shape, + device: *const Device, + + pub const SliceSpec = struct { + start_offset: usize, + byte_strides: [Shape.MAX_RANK]i64, + dims: [Shape.MAX_RANK]i64, + num_dims: u4, + }; + + pub fn specs(self: DeviceShard) SliceSpec { + const rank = self.global_shape.rank(); + + const host_byte_strides_ba = self.global_shape.computeByteStrides(); + + var shard_start_offset_bytes: i64 = 0; + for (0..rank) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + + // Check if the device's indices actually have this axis. + // If not, this dimension is effectively replicated for this device. + if (self.indices.hasTag(mesh_axis_tag) != null) { + const device_coord_on_mesh_axis = self.indices.dim(mesh_axis_tag); + const shard_dim_size = self.shard_shape.dim(i); + const start_element_in_dim = device_coord_on_mesh_axis * shard_dim_size; + shard_start_offset_bytes += start_element_in_dim * host_byte_strides_ba.get(i); + } + } + } + + var transfer_dims_buffer: [Shape.MAX_RANK]i64 = undefined; + for (0..rank) |i| { + transfer_dims_buffer[i] = self.shard_shape.dim(i); + } + + return .{ + .start_offset = @intCast(shard_start_offset_bytes), + .byte_strides = host_byte_strides_ba.buffer, + .dims = transfer_dims_buffer, + .num_dims = rank, + }; + } + + pub fn format( + self: DeviceShard, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + const slice_specs = self.specs(); + try writer.print("DeviceShard(index={d}/{d} topology={} indices={} shard_shape={} ({d}B) global_shape={} ({d}B) start_offset={d} byte_strides={any} device={})", .{ + self.index + 1, + self.topology.count(), + self.topology, + self.indices, + self.shard_shape, + self.shard_shape.byteSize(), + self.global_shape, + self.global_shape.byteSize(), + slice_specs.start_offset, + slice_specs.byte_strides[0..slice_specs.num_dims], + self.device, + }); + } +}; + +pub const DeviceShardIterator = struct { + index: usize = 0, + indices_iterator: TopologyIndicesIterator, + sharding: Sharding, + devices: []const *const pjrtx.Device, + + pub fn next(self: *DeviceShardIterator) ?DeviceShard { + const num_partitions = self.sharding.mesh.numPartitions(); // todo proxy numPartitions + + if (self.index >= num_partitions) return null; + + defer self.index += 1; + + return .{ + .index = self.index, + .topology = self.sharding.mesh.topology, + .global_shape = self.sharding.global_shape, + .indices = self.indices_iterator.next().?, + .shard_shape = self.sharding.shardShape(), + .device = self.devices[self.index], + }; + } +}; + +pub const Sharding = struct { + mesh: Mesh, + global_shape: Shape, + + pub const Type = enum { + replicated, + maximal, + manual, + }; + + pub fn init(mesh: Mesh, shape: Shape) Sharding { + // Validate that the shape is divisible by the mesh where partitioned. + for (0..shape.rank()) |dim_idx| { + const part_spec = shape.partition(dim_idx); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + + if (mesh.topology.hasTag(mesh_axis_tag) == null) { + // This partitioning is invalid for this mesh, so we skip the divisibility check. + // The dimension will effectively be replicated. + log.debug("Partitioning axis '{s}' not found in mesh {f}. Treating dimension {s} as replicated.", .{ + std.mem.span(mesh_axis_tag), + mesh, + shape.debugTag(dim_idx), + }); + continue; + } + + const global_dim = shape.dim(dim_idx); + const mesh_dim = mesh.topology.dim(mesh_axis_tag); + if (@rem(global_dim, mesh_dim) != 0) { + stdx.debug.panic( + "Tensor dimension {s} (size={d}) is not divisible by mesh axis {s} (size={d})", + .{ shape.debugTag(dim_idx), global_dim, std.mem.span(mesh_axis_tag), mesh_dim }, + ); + } + } + } + + return .{ + .mesh = mesh, + .global_shape = shape, + }; + } + + test "init / non existent mesh axis" { + const mesh: Mesh = .init(.{ .x = 4 }); + const shape: Shape = .init(.{ .m = 8, .n = 12 }, .i32); + const sharding = Sharding.init(mesh, shape.withPartitioning(.{ .m = .x, .n = .y })); + const shard_shape = sharding.shardShape(); + try std.testing.expectEqual(2, shard_shape.dim(.m)); + try std.testing.expectEqual(12, shard_shape.dim(.n)); + } + + pub fn getType(self: Sharding) Type { + var has_any_partitioned_axis = false; + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + has_any_partitioned_axis = true; + break; + } + } + } + + if (!has_any_partitioned_axis) { + return .replicated; + } + + var is_fully_partitioned = true; + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + switch (part_spec) { + .replicated, .open, .unknown => { + is_fully_partitioned = false; + break; + }, + .axis => |tag| { + if (self.mesh.topology.hasTag(tag) == null) { + is_fully_partitioned = false; + break; + } + }, + } + } + + if (is_fully_partitioned) { + return .maximal; + } + + return .manual; + } + + pub fn shardShape(self: Sharding) Shape { + var shard_: Shape = .init(.{}, self.global_shape.dtype()); + const rank = self.global_shape.rank(); + + for (0..rank) |dim| { + const part_spec = self.global_shape.partition(dim); + const global_dim_size = self.global_shape.dim(dim); + const tag = self.global_shape.tag(dim); + + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + + // If the specified mesh axis exists, partition. Otherwise, replicate. + if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + const mesh_dim = self.mesh.topology.dim(mesh_axis_tag); + const shard_dim_size = if (mesh_dim > 1) @divExact(global_dim_size, mesh_dim) else global_dim_size; + shard_ = shard_.appendDim(shard_dim_size, tag); + } else { + // Mesh axis does not exist, so this dimension is replicated. + shard_ = shard_.appendDim(global_dim_size, tag); + } + } else { + // For replicated, open, or unknown, the shard dimension is the full global dimension. + shard_ = shard_.appendDim(global_dim_size, tag); + } + } + + return shard_; + } + + pub fn iterator(self: Sharding, devices: []const *const pjrtx.Device) DeviceShardIterator { + const indices_iterator = self.mesh.iterator(); + + return .{ .indices_iterator = indices_iterator, .sharding = self, .devices = devices }; + } + + pub fn getShardingAttr(self: Sharding) std.BoundedArray(u8, 128) { + var sharding_str: std.BoundedArray(u8, 128) = .{}; + self.writeShardingRepresentation(sharding_str.writer()) catch unreachable; + return sharding_str; + } + + pub fn writeShardingRepresentation(self: Sharding, writer: anytype) !void { + if (self.getType() == .replicated) { + try writer.writeAll("{replicated}"); + return; + } + + var product_of_tensor_tiles: i64 = 1; + var has_partitioned_axis = false; + + // Calculate the product of explicitly defined tile dimensions + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + product_of_tensor_tiles *= self.mesh.topology.dim(mesh_axis_tag); + has_partitioned_axis = true; + } + } + } + + if (!has_partitioned_axis) { + try writer.writeAll("{replicated}"); + return; + } + + const is_fully_sharded_on_mesh = (product_of_tensor_tiles == self.mesh.numPartitions()); + + try writer.writeAll("{devices=["); + // The tile array must have the same rank as the tensor, for both partial and full sharding. + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + var tile_dim: i64 = 1; + if (part_spec == .axis) { + if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + tile_dim = self.mesh.topology.dim(part_spec.toTag()); + } + } + try writer.print("{d}", .{tile_dim}); + if (i < self.global_shape.rank() - 1) { + try writer.writeByte(','); + } + } + + try writer.writeAll("]<=["); + try writer.print("{}", .{self.mesh.numDevices()}); + //const mesh_dims = self.mesh.topology.dims(); + //for (mesh_dims, 0..) |dim, i| { + // try writer.print("{d}", .{dim}); + // if (i < mesh_dims.len - 1) { + // try writer.writeByte(','); + // } + //} + + if (is_fully_sharded_on_mesh) { + try writer.writeAll("]}"); + } else { + try writer.writeAll("] last_tile_dim_replicate}"); + } + } + + // pub fn writeShardingRepresentation(self: Sharding, writer: anytype) !void { + // if (self.getType() == .replicated) { + // try writer.writeAll("{replicated}"); + // return; + // } + + // var product_of_tensor_tiles: i64 = 1; + // var num_partitioned_dims: u32 = 0; + + // // Calculate the product of explicitly defined tile dimensions + // for (0..self.global_shape.rank()) |i| { + // const part_spec = self.global_shape.partition(i); + // if (part_spec == .axis) { + // const mesh_axis_tag = part_spec.toTag(); + // if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + // product_of_tensor_tiles *= self.mesh.topology.dim(mesh_axis_tag); + // num_partitioned_dims += 1; + // } + // } + // } + + // const is_fully_sharded_on_mesh = (product_of_tensor_tiles == self.mesh.numPartitions()); + + // if (is_fully_sharded_on_mesh) { + // // This is the easy case. The tensor is partitioned across all mesh devices. + // // The tile array must have the same rank as the tensor. + // try writer.writeAll("{devices=["); + // for (0..self.global_shape.rank()) |i| { + // const part_spec = self.global_shape.partition(i); + // var tile_dim: i64 = 1; + // if (part_spec == .axis) { + // if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + // tile_dim = self.mesh.topology.dim(part_spec.toTag()); + // } + // } + // try writer.print("{d}", .{tile_dim}); + // if (i < self.global_shape.rank() - 1) { + // try writer.writeByte(','); + // } + // } + // try writer.print("]<=[{d}]}}", .{self.mesh.numPartitions()}); + // } else { + // // This is partial sharding. This is more complex. + // // The `devices` array lists the tiling only for the *partitioned* dimensions. + // // The remaining devices are handled by `last_tile_dim_replicate`. + // try writer.writeAll("{devices=["); + // var first = true; + // for (0..self.global_shape.rank()) |i| { + // const part_spec = self.global_shape.partition(i); + // if (part_spec == .axis) { + // const mesh_axis_tag = part_spec.toTag(); + // if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + // if (!first) try writer.writeByte(','); + // try writer.print("{d}", .{self.mesh.topology.dim(mesh_axis_tag)}); + // first = false; + // } + // } + // } + // try writer.print("]<=[{d}] last_tile_dim_replicate}}", .{self.mesh.numPartitions()}); + // } + // } + + test "Sharding MLIR representation" { + // Case 1: Fully Replicated + const mesh_rep = Mesh.init(.{ .x = 4 }); + const shape_rep = Shape.init(.{ .m = 8, .k = 8 }, .i32); + const sharding_rep = Sharding.init(mesh_rep, shape_rep); + var attr_rep_ba = sharding_rep.getShardingAttr(); + try std.testing.expectEqualStrings("{replicated}", attr_rep_ba.constSlice()); + + // Case 2: Partially partitioned + const mesh_part = Mesh.init(.{ .x = 2, .y = 4 }); + const shape_part = Shape.init(.{ .m = 8, .k = 12 }, .i32).withPartitioning(.{ .m = .x }); + const sharding_part = Sharding.init(mesh_part, shape_part); + var attr_part_ba = sharding_part.getShardingAttr(); + // dim 'm' is partitioned on 'x' (size 2), dim 'k' is replicated (size 1) + try std.testing.expectEqualStrings("{devices=[2,1]<=[8]}", attr_part_ba.constSlice()); + + // Case 3: Fully partitioned + const mesh_full = Mesh.init(.{ .x = 2, .y = 4 }); + const shape_full = Shape.init(.{ .m = 8, .k = 12 }, .i32).withPartitioning(.{ .m = .x, .k = .y }); + const sharding_full = Sharding.init(mesh_full, shape_full); + var attr_full_ba = sharding_full.getShardingAttr(); + // dim 'm' partitioned on 'x' (size 2), dim 'k' partitioned on 'y' (size 4) + try std.testing.expectEqualStrings("{devices=[2,4]<=[8]}", attr_full_ba.constSlice()); + + // Case 4: More complex 3D case with a replicated dimension in the middle + const mesh_3d = Mesh.init(.{ .x = 2, .y = 2, .z = 3 }); + const shape_3d = Shape.init(.{ .b = 4, .h = 8, .w = 6 }, .i32).withPartitioning(.{ .b = .x, .w = .z }); + const sharding_3d = Sharding.init(mesh_3d, shape_3d); + var attr_3d_ba = sharding_3d.getShardingAttr(); + // b -> x (size 2), h -> replicated (size 1), w -> z (size 3) + try std.testing.expectEqualStrings("{devices=[2,1,3]<=[12]}", attr_3d_ba.constSlice()); + } + + pub fn format( + self: Sharding, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + try writer.print("Sharding(global_shape={} mesh={})", .{ self.global_shape, self.mesh }); + } +}; + +//fn testShardingCase(allocator: std.mem.Allocator, mesh: Mesh, shape: Shape, partition_spec: anytype, verbose: bool) !void { +// _platform = null; // Reset the global platform +// const platform = env(.{ .cpu = .{ .cpu_device_count = @intCast(mesh.numDevices()) } }); +// +// const shape_partitioned = shape.withPartitioning(partition_spec); +// const sharding = Sharding.init(mesh, shape_partitioned); +// std.debug.print("\n--- Testing Case (verbose={any}) --- {}\n", .{ verbose, sharding }); +// +// const original_data = try slice.arange(allocator, shape_partitioned, .{}); +// defer allocator.free(original_data); +// if (verbose) { +// //std.debug.print("Original full slice of data: {any}\n\n", .{Shaped(i32, shape_partitioned, original_data)}); +// } +// +// // Scatter +// var pjrt_buffers = std.ArrayList(*pjrtx.Buffer).init(allocator); +// defer pjrt_buffers.deinit(); +// +// var sharding_iter = sharding.iterator(platform.getDevices()); +// while (sharding_iter.next()) |device_shard| { +// if (verbose) std.debug.print("{}\n", .{device_shard}); +// const specs = device_shard.specs(); +// const args = pjrtx.Client.BufferFromHostBufferArgs{ +// .data = original_data[specs.start_offset..].ptr, +// .buffer_type = bufferTypeFromDtype(device_shard.shard_shape.dtype()), +// .dims = specs.dims[0..specs.num_dims], +// .byte_strides = specs.byte_strides[0..specs.num_dims], +// .host_buffer_semantics = .ImmutableUntilTransferCompletes, +// .device = device_shard.device, +// }; +// const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); +// if (event) |e| try e.await_(platform.pjrt_api); +// try pjrt_buffers.append(pjrt_buffer); +// +// // Optional: verify the transferred data for debugging +// if (verbose) { +// const shard_shape = device_shard.shard_shape; +// const fetched_data = try allocator.alloc(u8, shard_shape.byteSize()); +// defer allocator.free(fetched_data); +// const fetch_event = try pjrt_buffer.toHostBuffer(platform.pjrt_api, fetched_data, .{}); +// if (fetch_event) |e| try e.await_(platform.pjrt_api); +// //std.debug.print("Fetched shard data: {any}\n\n", .{Shaped(i32, shard_shape, fetched_data)}); +// } +// } +// +// // Gather +// if (verbose) std.debug.print("--- Reassembling from shards ---\n", .{}); +// const sharded_buffer = Buffer.fromPjrtBuffers(platform, sharding, pjrt_buffers.items); +// defer sharded_buffer.deinit(); // This will deinit all the underlying PJRT buffers. +// +// const reassembled_data = try sharded_buffer.toHost(allocator); +// defer allocator.free(reassembled_data); +// +// //if (verbose) std.debug.print("Reassembled full slice of data: {any}\n\n", .{Shaped(i32, sharding.global_shape, reassembled_data)}); +// +// // Verify +// try std.testing.expectEqualSlices(u8, original_data, reassembled_data); +// std.debug.print("✅ Verification successful for: {}\n\n", .{sharding}); +//} + +//test "Sharding All Cases" { +// const allocator = std.testing.allocator; +// const verbose = true; +// +// // Fully replicated +// try testShardingCase(allocator, .init(.{ .x = 4 }), .init(.{ .m = 8, .k = 8 }, .i32), .{}, verbose); +// // 1D sharding on a 1D mesh +// try testShardingCase(allocator, .init(.{ .x = 8 }), .init(.{ .m = 16, .k = 16 }, .i32), .{ .k = .x }, verbose); +// // 2D sharding on a 2D mesh +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 4 }), .init(.{ .m = 8, .k = 12 }, .i32), .{ .m = .x, .k = .y }, verbose); +// // 3D tensor on a 2D mesh, partitioned on non-contiguous dimensions +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .b = 4, .h = 8, .w = 6 }, .i32), .{ .b = .x, .w = .y }, verbose); +// // 3D sharding on a 3D mesh +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 4, .z = 3 }), .init(.{ .m = 4, .k = 8, .j = 6 }, .i32), .{ .m = .x, .k = .y, .j = .z }, verbose); +// // 2D tensor, 2D mesh, but only partitioned on one axis +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 3 }), .init(.{ .m = 10, .k = 6 }, .i32), .{ .k = .x }, verbose); +// // Replicated on a 2D mesh +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .m = 4, .k = 4 }, .i32), .{}, verbose); +// // 3D tensor, 2D mesh, one replicated dimension in the middle +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .batch = 4, .seq_len = 10, .features = 8 }, .i32), .{ .batch = .x, .features = .y }, verbose); +// // 2D mesh where one dimension is 1 +// try testShardingCase(allocator, .init(.{ .x = 1, .y = 4 }), .init(.{ .m = 8, .k = 8 }, .i32), .{ .m = .x, .k = .y }, verbose); +// // Partially partitioned, non-divisible on replicated axis +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 3 }), .init(.{ .m = 8, .k = 10 }, .i32), .{ .m = .x }, verbose); +// // Single device mesh +// try testShardingCase(allocator, .init(.{ .x = 1 }), .init(.{ .m = 8, .k = 10 }, .i32), .{ .m = .x }, verbose); +// // NEW: 3D tensor, 2D mesh, partitioned on first two axes. Good test for new reassemble logic. +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .b = 4, .h = 8, .w = 6 }, .i32), .{ .b = .x, .h = .y }, verbose); +//} + +// todo: temp +pub fn bufferTypeFromDtype(dt: DataType) pjrtx.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrtx.BufferType, @tagName(tag)), + }; +} + +pub fn dtypeFromBufferType(pjrt_type: pjrtx.BufferType) DataType { + return switch (pjrt_type) { + .invalid => @panic("Found an invalid pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; +} + +// todo: temp (zig deps and compilation story...) +var _platform: ?Platform = null; + +pub fn env(opts: Platform.CreateOptions) Platform { + if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); + if (_platform == null) { + var ctx = Context.init() catch unreachable; + _platform = ctx.autoPlatform(opts).withCompilationOptions(.{ + .xla_dump_to = "/tmp/zml/tests-partitioning/", + .sharding_enabled = true, + }); + } + + return _platform.?; +} diff --git a/zml/platform.zig b/zml/platform.zig index 0bd8a2a5..b59d49a5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,6 +3,8 @@ const std = @import("std"); const runtimes = @import("runtimes"); pub const Target = runtimes.Platform; const stdx = @import("stdx"); +const Mesh = @import("partitioning.zig").Mesh; +const Shape = @import("shape.zig").Shape; const pjrt = @import("pjrtx.zig"); @@ -10,6 +12,305 @@ const log = std.log.scoped(.zml); pub const available_targets = std.enums.values(Target); +/// Represents the physical arrangement of hardware devices. +/// It contains the devices ordered according to their physical topology and the shape of that topology. +pub const PhysicalMesh = struct { + allocator: std.mem.Allocator, + devices: []const *const pjrt.Device, + shape: Shape, + + pub fn deinit(self: *PhysicalMesh) void { + self.allocator.free(self.devices); + self.* = undefined; + } +}; + +// Optimal orderings for TPU collectives, from JAX. +const tpu_v5_lite_2x2_ring_order = [_]u8{ 0, 1, 3, 2 }; +const tpu_v5_lite_2x2x2_iota_order = [_]u8{ 0, 4, 2, 6, 1, 5, 3, 7 }; +// Same ring order for v5e tray, v6e tray, and v7x 2x4 tray +const tpu_tray_ring_order_8 = [_]u8{ 0, 1, 2, 3, 7, 6, 5, 4 }; + +/// Sorts devices primarily by their coordinates (lexicographically, reversed) and then by core_on_chip. +/// This matches JAX's z,y,x,core sorting for creating a physical mesh. +/// This function is allocation-free. +fn sortDevicesForPhysicalMesh(context: *const pjrt.Api, lhs_dev: *const pjrt.Device, rhs_dev: *const pjrt.Device) bool { + const api: *const pjrt.Api = context; + + const lhs_desc = lhs_dev.getDescription(api); + const rhs_desc = rhs_dev.getDescription(api); + + var lhs_coords: ?[]const i64 = null; + var lhs_core: i64 = 0; + for (lhs_desc.getAttributes(api)) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + lhs_coords = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + lhs_core = attr.inner.unnamed_0.int64_value; + } + } + + var rhs_coords: ?[]const i64 = null; + var rhs_core: i64 = 0; + for (rhs_desc.getAttributes(api)) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + rhs_coords = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + rhs_core = attr.inner.unnamed_0.int64_value; + } + } + + // TPUs are expected to have coordinates. + // If not, we can't sort them topologically, so we treat them as equal and let the original order prevail. + if (lhs_coords == null or rhs_coords == null) { + return false; + } + + const lhs_coords_slice = lhs_coords.?; + const rhs_coords_slice = rhs_coords.?; + + stdx.debug.assert(lhs_coords_slice.len == rhs_coords_slice.len, "Devices have different coordinate ranks", .{}); + var i = lhs_coords_slice.len; + while (i > 0) { + i -= 1; + if (lhs_coords_slice[i] != rhs_coords_slice[i]) { + return lhs_coords_slice[i] < rhs_coords_slice[i]; + } + } + + return lhs_core < rhs_core; +} + +/// Sorts devices topologically for consistent ordering across platforms. +fn getTopologicallySortedDevices( + allocator: std.mem.Allocator, + devices: []const *const pjrt.Device, + api: *const pjrt.Api, +) ![]*const pjrt.Device { + const sorted_devices = try allocator.dupe(*const pjrt.Device, devices); + std.sort.pdq(*const pjrt.Device, sorted_devices, api, sortDevicesForPhysicalMesh); + return sorted_devices; +} + +/// Creates a device mesh that arranges devices for optimal collective performance, +/// similar to `jax.experimental.mesh_utils.create_device_mesh`. +pub fn createDeviceMesh( + allocator: std.mem.Allocator, + logical_mesh: Mesh, + platform: Platform, +) ![]const *const pjrt.Device { + const api = platform.pjrt_api; + const devices = platform.getDevices(); + + const num_logical_devices: u64 = logical_mesh.topology.count(); + stdx.debug.assert(num_logical_devices <= devices.len, "Number of devices {} must be >= the product of logical_mesh_shape {f}", .{ devices.len, logical_mesh.topology }); + const active_devices = devices[0..@intCast(num_logical_devices)]; + + if (active_devices.len == 0) return &.{}; + + // Always sort devices topologically first to ensure a consistent base order + // for mapping logical devices to physical devices across all platforms. + const sequential_devices = try getTopologicallySortedDevices(allocator, active_devices, api); + + // For TPUs, we might apply an additional reordering for performance. + // For other platforms, we will use the topologically sorted list directly. + switch (platform.target) { + .tpu => { + // Keep the TPU-specific reordering logic, but operate on the sorted list. + const first_device_kind = active_devices[0].getDescription(api).getKind(api); + + if (std.mem.eql(u8, first_device_kind, "TPU v5 lite") or std.mem.eql(u8, first_device_kind, "TPU v6 lite")) { + if (try createTpuV5LiteMesh(allocator, logical_mesh, sequential_devices, api)) |ordered_devices| { + allocator.free(sequential_devices); // Free the intermediate sorted list + return ordered_devices; + } + } else if (std.mem.eql(u8, first_device_kind, "TPU7x")) { + if (try createTpu7xMesh(allocator, logical_mesh, sequential_devices, api)) |ordered_devices| { + allocator.free(sequential_devices); // Free the intermediate sorted list + return ordered_devices; + } + } + + // Fallback for other TPU kinds (e.g., v2, v3, v4, v5p). + log.warn("Using generic TPU physical mesh creation for {s}. For optimal performance, a specific handler may be needed.", .{first_device_kind}); + var physical_mesh = try getPhysicalTpuMesh(allocator, sequential_devices, api); + defer physical_mesh.deinit(); + return try createDeviceMeshForNdTorus(allocator, physical_mesh, logical_mesh); + }, + .cuda, .rocm, .cpu, .neuron => { + // For other platforms, the topologically sorted list is the final list. + return sequential_devices; + }, + } +} + +/// Creates a device mesh for TPU v5 lite and v6 lite, applying optimal orderings. +fn createTpuV5LiteMesh( + allocator: std.mem.Allocator, + logical_mesh: Mesh, + devices: []const *const pjrt.Device, + api: *const pjrt.Api, +) !?[]const *const pjrt.Device { + const sequential_devices = try allocator.dupe(*const pjrt.Device, devices); + defer allocator.free(sequential_devices); + std.sort.pdq(*const pjrt.Device, sequential_devices, api, sortDevicesForPhysicalMesh); + + const reordered_devices = try allocator.alloc(*const pjrt.Device, devices.len); + + const order: ?[]const u8 = switch (devices.len) { + 4 => &tpu_v5_lite_2x2_ring_order, + 8 => &tpu_v5_lite_2x2x2_iota_order, // JAX uses iota order for v5e 2x2x2 cubes + else => null, + }; + + if (order) |o| { + for (o, 0..) |old_idx, new_idx| { + reordered_devices[new_idx] = sequential_devices[old_idx]; + } + // Reshaping is conceptual; the flat ordered array is what matters for XLA's device_assignment. + _ = logical_mesh; + return reordered_devices; + } + + allocator.free(reordered_devices); + return null; +} + +/// Creates a device mesh for TPU v7x. +fn createTpu7xMesh( + allocator: std.mem.Allocator, + logical_mesh: Mesh, + devices: []const *const pjrt.Device, + api: *const pjrt.Api, +) !?[]const *const pjrt.Device { + const sequential_devices = try allocator.dupe(*const pjrt.Device, devices); + defer allocator.free(sequential_devices); + std.sort.pdq(*const pjrt.Device, sequential_devices, api, sortDevicesForPhysicalMesh); + + if (devices.len == 8) { + const reordered_devices = try allocator.alloc(*const pjrt.Device, devices.len); + for (tpu_tray_ring_order_8, 0..) |old_idx, new_idx| { + reordered_devices[new_idx] = sequential_devices[old_idx]; + } + _ = logical_mesh; + return reordered_devices; + } + + return null; +} + +/// Constructs a PhysicalMesh representing the N-D torus of a TPU slice. +fn getPhysicalTpuMesh(allocator: std.mem.Allocator, devices: []const *const pjrt.Device, api: *const pjrt.Api) !PhysicalMesh { + stdx.debug.assert(devices.len > 0, "Cannot create physical mesh from zero devices", .{}); + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + _ = arena; // autofix + + var min_coords: [3]i64 = .{ std.math.maxInt(i64), std.math.maxInt(i64), std.math.maxInt(i64) }; + var max_coords: [3]i64 = .{ std.math.minInt(i64), std.math.minInt(i64), std.math.minInt(i64) }; + var min_core: i64 = std.math.maxInt(i64); + var max_core: i64 = std.math.minInt(i64); + + for (devices) |device| { + const desc = device.getDescription(api); + const attrs = desc.getAttributes(api); + for (attrs) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + const coords = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + stdx.debug.assert(coords.len == 3, "Expected 3D coordinates for TPU device, got {any}", .{coords}); + for (0..3) |i| { + min_coords[i] = @min(min_coords[i], coords[i]); + max_coords[i] = @max(max_coords[i], coords[i]); + } + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + min_core = @min(min_core, attr.inner.unnamed_0.int64_value); + max_core = @max(max_core, attr.inner.unnamed_0.int64_value); + } + } + } + + const dims: [3]u64 = .{ + @intCast(max_coords[0] - min_coords[0] + 1), + @intCast(max_coords[1] - min_coords[1] + 1), + @intCast(max_coords[2] - min_coords[2] + 1), + }; + const cores_per_chip: u64 = @intCast(max_core - min_core + 1); + stdx.debug.assert(devices.len == dims[0] * dims[1] * dims[2] * cores_per_chip, "Device list does not form a contiguous cuboid.", .{}); + + var physical_shape: Shape = undefined; + var physical_dims: Shape.DimsArray = .{}; + if (cores_per_chip > 1) { + try physical_dims.appendSlice(&.{ @intCast(dims[0]), @intCast(dims[1]), @intCast(cores_per_chip) }); + physical_shape = Shape.init(physical_dims.constSlice(), .u8).withTags(.{ .x, .y, .c }); + } else { + try physical_dims.appendSlice(&.{ @intCast(dims[0]), @intCast(dims[1]), @intCast(dims[2]) }); + physical_shape = Shape.init(physical_dims.constSlice(), .u8).withTags(.{ .x, .y, .z }); + } + + const ordered_devices = try allocator.alloc(*const pjrt.Device, devices.len); + // @memset(ordered_devices, null); + + for (devices) |device| { + const desc = device.getDescription(api); + const attrs = desc.getAttributes(api); + var coords: [3]i64 = undefined; + var core_on_chip: i64 = 0; + for (attrs) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + const coords_slice = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + @memcpy(coords[0..coords_slice.len], coords_slice); + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + core_on_chip = attr.inner.unnamed_0.int64_value; + } + } + const x = @as(usize, @intCast(coords[0] - min_coords[0])); + const y = @as(usize, @intCast(coords[1] - min_coords[1])); + const z = @as(usize, @intCast(coords[2] - min_coords[2])); + const c = @as(usize, @intCast(core_on_chip - min_core)); + + const index = if (cores_per_chip > 1) (x * @as(usize, @intCast(dims[1])) * @as(usize, @intCast(cores_per_chip))) + (y * @as(usize, @intCast(cores_per_chip))) + c else (x * @as(usize, @intCast(dims[1])) * @as(usize, @intCast(dims[2]))) + (y * @as(usize, @intCast(dims[2]))) + z; + stdx.debug.assert(index < ordered_devices.len, "Index out of bounds", .{}); + // stdx.debug.assert(ordered_devices[index] == null, "Duplicate device position in physical mesh", .{}); + ordered_devices[index] = device; + } + + return PhysicalMesh{ + .allocator = allocator, + .devices = ordered_devices, + .shape = physical_shape, + }; +} + +/// Maps a logical mesh onto a physical N-D torus mesh. +fn createDeviceMeshForNdTorus( + allocator: std.mem.Allocator, + physical_mesh: PhysicalMesh, + logical_mesh: Mesh, +) ![]const *const pjrt.Device { + // This is a complex combinatorial problem. For now, we implement a simplified greedy approach + // that assumes logical axes map to single physical axes. A full implementation would explore + // combinations and splitting physical axes. + // TODO: Implement the full JAX axis assignment algorithm. + log.warn("Using simplified logical to physical mesh mapping. Performance may not be optimal.", .{}); + + var physical_axes = physical_mesh.shape._dims; + var logical_axes = logical_mesh.topology._dims; + std.sort.pdq(i64, physical_axes.slice(), {}, std.sort.desc(i64)); + std.sort.pdq(i64, logical_axes.slice(), {}, std.sort.desc(i64)); + + if (!std.mem.eql(i64, physical_axes.constSlice(), logical_axes.constSlice())) { + log.err("Physical mesh shape {any} cannot be mapped to logical mesh shape {any} with the current simplified algorithm.", .{ + physical_mesh.shape, + logical_mesh.topology, + }); + return error.MeshMappingNotSupported; + } + + // Since shapes are identical after sorting, a simple copy is sufficient for this simplified version. + return allocator.dupe(*const pjrt.Device, physical_mesh.devices); +} + pub const CompilationOptions = struct { xla_dump_to: ?[]const u8 = null, xla_dump_fusion_visualization: bool = false, @@ -113,7 +414,7 @@ const _CreateOptions = struct { /// "Best-Fit with Coalescing" algorithm bfc: Options, /// use cudaMallocAsync - @"async": Options, + async: Options, /// use raw cuMalloc platform, @@ -129,7 +430,7 @@ const _CreateOptions = struct { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); }, - .bfc, .@"async" => |opt| { + .bfc, .async => |opt| { values.appendAssumeCapacity(pjrt.NamedValue.from("allocator", self.allocator)); values.appendAssumeCapacity(pjrt.NamedValue.from("preallocate", opt.preallocate)); if (opt.memory_fraction > 0) { diff --git a/zml/shape.zig b/zml/shape.zig index ac86f314..b2d61975 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -11,28 +11,159 @@ const EnumLiteral = @TypeOf(.enum_literal); const log = std.log.scoped(.shape); test { - std.testing.refAllDecls(Shape); + testing.refAllDecls(Shape); } /// Represent the shape of a tensor. pub const Shape = struct { pub const MAX_RANK: u8 = 8; + pub const PartitionSpec = union(enum) { + axis: Tag, + open, + replicated, + unknown, + + pub fn init(comptime value: anytype) PartitionSpec { + const T = @TypeOf(value); + + if (T == PartitionSpec) { + return value; + } + + if (comptime T == EnumLiteral) { + const name = @tagName(value); + inline for (std.meta.fields(PartitionSpec)) |field| { + if (field.type == void and std.mem.eql(u8, field.name, name)) { + return @field(PartitionSpec, field.name); + } + } + } + + if (comptime isTagConvertible(T)) { + const tag_ = Shape.toTag(value); + + return .{ .axis = tag_ }; + } + + stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); + } + + test "PartitionSpec.init" { + const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; + try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); + + const spec_axis_open: PartitionSpec = .open; + try testing.expect(spec_axis_open.eql(.init(.open))); + } + + pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { + if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { + return false; + } + + switch (self) { + .axis => |t1| { + const t2 = other.axis; + return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); + }, + else => return true, + } + } + + test "PartitionSpec.eql" { + const spec_axis: PartitionSpec = .init(.a); + const spec_axis_b: PartitionSpec = .init(.b); + const spec_open: PartitionSpec = .open; + const spec_replicated: PartitionSpec = .replicated; + const spec_unknown: PartitionSpec = .unknown; + + try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); + try testing.expect(!spec_axis.eql(spec_axis_b)); + try testing.expect(!spec_axis.eql(.open)); + try testing.expect(!spec_axis.eql(.replicated)); + try testing.expect(!spec_axis.eql(.unknown)); + + try testing.expect(spec_open.eql(.open)); + try testing.expect(!spec_open.eql(spec_axis)); + try testing.expect(!spec_open.eql(.replicated)); + try testing.expect(!spec_open.eql(.unknown)); + + try testing.expect(spec_replicated.eql(.replicated)); + try testing.expect(!spec_replicated.eql(spec_axis)); + try testing.expect(!spec_replicated.eql(.open)); + try testing.expect(!spec_replicated.eql(.unknown)); + + try testing.expect(spec_unknown.eql(.unknown)); + try testing.expect(!spec_unknown.eql(spec_axis)); + try testing.expect(!spec_unknown.eql(.open)); + try testing.expect(!spec_unknown.eql(.replicated)); + } + + pub fn isClosed(self: PartitionSpec) bool { + return switch (self) { + .axis => true, + .open => false, + .replicated => true, + .unknown => false, + }; + } + + test isClosed { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.isClosed()); + + const spec_open: PartitionSpec = .open; + try testing.expect(!spec_open.isClosed()); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.isClosed()); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(!spec_unknown.isClosed()); + } + + pub fn toTag(self: PartitionSpec) Tag { + return switch (self) { + .axis => |t| return t, + .open => TagOpen, + .replicated => TagReplicated, + .unknown => TagUnknown, + }; + } + + test "PartitionSpec.toTag" { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); + + const spec_open: PartitionSpec = .open; + try testing.expect(spec_open.toTag() == TagOpen); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.toTag() == TagReplicated); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(spec_unknown.toTag() == TagUnknown); + } + }; + pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; const TagLast = "last".ptr; + const TagReplicated = "replicated".ptr; + const TagOpen = "open".ptr; pub const DimsArray = stdx.BoundedArray(i64, MAX_RANK); pub const TagsArray = stdx.BoundedArray(Tag, MAX_RANK); pub const AxesArray = stdx.BoundedArray(u3, MAX_RANK); - pub const ShardingInfo = @Vector(MAX_RANK, bool); + pub const PartitionArray = stdx.BoundedArray(PartitionSpec, MAX_RANK); const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** MAX_RANK }; _dtype: DataType, _dims: DimsArray = .{}, _tags: TagsArray = UnknownTags, - _sharding_info: ShardingInfo = @splat(false), + _partitioning: PartitionArray = .{}, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -129,6 +260,8 @@ pub const Shape = struct { pub fn init(dimz: anytype, dt: DataType) Shape { var res: Shape = .{ ._dtype = dt }; res._dims, res._tags = parseDimensions(dimz); + res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); + return res; } @@ -145,6 +278,7 @@ pub const Shape = struct { }; res._tags.append(TagUnknown) catch unreachable; } + res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -153,17 +287,17 @@ pub const Shape = struct { } pub fn rank(self: Shape) u4 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return @intCast(self._dims.len); } pub fn dim(self: Shape, ax: anytype) i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.get(self.axis(ax)); } pub fn dims(self: *const Shape) []const i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.constSlice(); } @@ -180,7 +314,7 @@ pub const Shape = struct { }; } - pub fn toTag(v: anytype) Tag { + fn toTag(v: anytype) Tag { const T = @TypeOf(v); return switch (T) { EnumLiteral => @tagName(v).ptr, @@ -190,12 +324,12 @@ pub const Shape = struct { }; } - inline fn ensureDimsAndTagsAreSync(self: Shape) void { - stdx.debug.assert(self._dims.len == self._tags.len, "Tags and dims have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + inline fn ensureAttributesAreSync(self: Shape) void { + stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); } pub fn tag(self: Shape, ax: anytype) Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.get(self.axis(ax)); } @@ -215,7 +349,7 @@ pub const Shape = struct { } pub fn tags(self: *const Shape) []const Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.constSlice(); } @@ -255,7 +389,7 @@ pub const Shape = struct { } pub fn axis(self: Shape, axis_: anytype) u3 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axis_); if (comptime stdx.meta.isInteger(T)) { @@ -270,7 +404,7 @@ pub const Shape = struct { } pub fn axes(self: Shape, axes_: anytype) AxesArray { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axes_); @@ -300,7 +434,7 @@ pub const Shape = struct { fn axisFromInt(self: Shape, a: isize) u3 { const rk: i8 = self.rank(); if (a < -rk or a > rk) { - stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); + stdx.debug.panic("Tensor {f} doesn't have dimension: {}", .{ self, a }); } return if (a < 0) @intCast(a + rk) @@ -400,13 +534,16 @@ pub const Shape = struct { if (need_comma) try writer.writeByte(','); const t = self.tag(i); if (t != TagUnknown) { - try writer.print("{s}={d}", .{ t, d }); + try writer.print("{s}={}", .{ t, d }); } else { - try writer.print("{d}", .{d}); + try writer.print("{}", .{d}); } - if (self._sharding_info[i]) { - try writer.writeByte('!'); + + const part = self._partitioning.get(i); + if (part.toTag() != TagUnknown) { + try writer.print("/{s}", .{part.toTag()}); } + need_comma = true; } if (need_comma) try writer.writeByte(','); @@ -439,13 +576,14 @@ pub const Shape = struct { pub fn reshape(self: Shape, new_shape_: anytype) Shape { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); + new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); new_shape.inferMissingAxis(self.count()); - stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); + stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {d} to {d}", .{ self.dims(), new_shape.dims() }); return new_shape; } fn inferMissingAxis(self: *Shape, n_: usize) void { - stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); + stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {}", .{self.*}); const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; // We can't use `self.count()` yet cause we have negative dims. @@ -513,6 +651,11 @@ pub const Shape = struct { var res = self; res._dims.insertSlice(ax, dims_.constSlice()) catch unreachable; res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + for (0..dims_.len) |_| { + res._partitioning.insert(ax, .unknown) catch unreachable; + } + return res; } @@ -523,7 +666,7 @@ pub const Shape = struct { } pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { - stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); + stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {}, it's already at max rank.", .{self}); const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) self.rank() @@ -533,6 +676,8 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; + res._partitioning.insert(ax, .unknown) catch unreachable; + return res; } @@ -541,6 +686,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); + res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); return res; } @@ -562,6 +708,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); + res._partitioning.appendAssumeCapacity(.unknown); return res; } @@ -570,14 +717,15 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); + _ = res._partitioning.orderedRemove(a); return res; } pub const drop = remove; test remove { - try std.testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); - try std.testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); } pub fn removeMany(self: Shape, axes_: anytype) Shape { @@ -595,20 +743,22 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; + sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; + sh._partitioning.len = rk - to_remove.len; return sh; } test removeMany { - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{12}, Shape.init(.{ 10, 11, 12 }, .f32).removeMany(.{ 0, 1 }).dims(), ); - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{ 10, 11 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).removeMany(.{ -1, -2 }).dims(), @@ -619,10 +769,16 @@ pub const Shape = struct { std.debug.assert(self.rank() == permutations.len); const permutations_ = self.axes(permutations); var res = self; + const original_dims = self._dims; + const original_tags = self._tags; + const original_parts = self._partitioning; + for (permutations_.constSlice(), 0..) |permutation, i| { - res._dims.set(i, self.dim(permutation)); - res._tags.set(i, self.tag(permutation)); + res._dims.set(i, original_dims.get(permutation)); + res._tags.set(i, original_tags.get(permutation)); + res._partitioning.set(i, original_parts.get(permutation)); } + return res; } @@ -638,6 +794,10 @@ pub const Shape = struct { Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32).transpose(.{ 0, 2, 1, 3 }), ), ); + + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); } /// Tag each ax of this shape with tags from a tuple. @@ -659,7 +819,7 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, tagz }); + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {any}", .{ self, tagz }); inline for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -698,7 +858,7 @@ pub const Shape = struct { var res = self; if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {any}", .{ self, tagz }); + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {}, got {any}", .{ self, tagz }); for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -706,7 +866,7 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {}", .{ self, tagz }); + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {}, got {}", .{ self, tagz }); inline for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -746,16 +906,233 @@ pub const Shape = struct { return res; } - pub fn withSharding(self: Shape, axes_: anytype) Shape { + pub fn withPartitioning(self: Shape, specs: anytype) Shape { + const T = @TypeOf(specs); + + var res = self.withDefaultPartitioning(); // todo add test for this new change + + if (stdx.meta.isStruct(T)) { + inline for (std.meta.fields(T)) |field| { + const partition_axis = @field(specs, field.name); + const axis_ = res.axisFromTagMaybe(toTag(field)); + + if (axis_) |ax| { + res._partitioning.set(ax, PartitionSpec.init(partition_axis)); + } else { + stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); + } + } + } else { + stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + } + + // Check that no mesh axis is used to partition multiple tensor dimensions. + var used_mesh_axes: stdx.BoundedArray(Tag, MAX_RANK) = .{ .len = 0 }; + + for (res._partitioning.constSlice()) |spec| { + const axis_tag = spec.toTag(); + + if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { + stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); + } + + switch (spec) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), + } + } + + return res; + } + + test withPartitioning { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); + } + + pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { + const T = @TypeOf(mapping); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); + var res = self; - // Reset sharding. - res._sharding_info = @splat(false); - for (self.axes(axes_).constSlice()) |ax| { - res._sharding_info[ax] = true; + for (res._partitioning.slice()) |*spec| { + switch (spec.*) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => |*old_axis_tag| { + inline for (std.meta.fields(T)) |field| { + const logical_axis_in_map = toTag(field); + + if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { + const physical_axis_from_map = @field(mapping, field.name); + spec.* = PartitionSpec.init(physical_axis_from_map); + break; + } + } + }, + } } return res; } + test mapPartitioningAxes { + const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ + .batch = .data, + .seq = .replicated, + .features = .model, + }); + + const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); + + try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); + try testing.expect(physical_shape.partition(.features).eql(.init(.y))); + try testing.expect(physical_shape.partition(.seq).eql(.replicated)); + + const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); + try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); + try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); + } + + pub fn withDefaultPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withDefaultPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + } + + pub fn withReplicatedPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withReplicatedPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withReplicatedPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); + } + + pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec == .axis) return true; + } + return false; + } + + test hasAtLeastOnePartitionedAxis { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + } + + pub fn isFullyPartitioned(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + switch (spec) { + .replicated, .open, .unknown => return false, + .axis => {}, + } + } + return true; + } + + test isFullyPartitioned { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); + try testing.expect(shape.isFullyPartitioned()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyPartitioned()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.isFullyPartitioned()); + } + + pub fn isFullyReplicated(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec != .replicated and spec != .unknown) { + return false; + } + } + return true; + } + + test isFullyReplicated { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyReplicated()); + } + + pub fn partition(self: Shape, ax: anytype) PartitionSpec { + return self._partitioning.get(self.axis(ax)); + } + + test partition { + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); + try testing.expectEqual(.open, shape.partition(.b)); + try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); + } + + pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { + for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { + if (dim_part.eql(part)) return true; + } + return false; + } + + test containsPartitionSpec { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.open)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(shape.containsPartitionSpec(.unknown)); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.open)); + } + /// Renames some of the tags in this shape. /// Shape.init(.{ .a = 10, .b = 20 }).rename(.{ .b = .batch }); // .{ .a = 10, .batch = 20 }; pub fn rename(self: Shape, renames: anytype) Shape { @@ -764,7 +1141,7 @@ pub const Shape = struct { var res = self; inline for (std.meta.fields(T)) |field| { const new_field = @field(renames, field.name); - stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); + stdx.debug.assert(self.hasTag(new_field) == null, "{}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); res._tags.set(self.axis(field), toTag(new_field)); } return res; @@ -784,7 +1161,103 @@ pub const Shape = struct { } } - pub fn computeStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + /// Computes the strides of the shape in units of elements. + /// For a shape {d0, d1, d2}, the strides would be {d1*d2, d2, 1}. + pub fn computeElementStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + return self.computeStrides(1); + } + + test computeElementStrides { + { + const s = Shape.init(.{ 4, 1024 }, .bf16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 2, 3, 4 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12, 4, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .i16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 6, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{10}, .u8); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{1}, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1, 5 }, .i32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 5, 5, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ .n = 8, .c = 3, .h = 224, .w = 224 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 3 * 224 * 224, 224 * 224, 224, 1 }, strides.constSlice()); + } + } + + /// Computes the strides of the shape in units of bytes. + /// For a shape {d0, d1, d2} of type T, the strides would be + /// {(d1*d2)*sizeof(T), d2*sizeof(T), 1*sizeof(T)}. + pub fn computeByteStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + return self.computeStrides(self.dtype().sizeOf()); + } + + test computeByteStrides { + { + const s = Shape.init(.{ 2, 3, 4 }, .f16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12 * 2, 4 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1024 }, .bf16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .c64); // 8 bytes (f32 + f32) + const strides = s.computeByteStrides(); + try std.testing.expectEqual(@sizeOf(std.math.Complex(f32)), 8); + try std.testing.expectEqualSlices(i64, &.{ 6 * 8, 1 * 8 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 128, 512 }, .i8); // 1 byte + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 512, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 1, 4096 }, .u32); // 4 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 4096 * 4, 1 * 4 }, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); // 8 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + } + + fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, MAX_RANK) { const rk = self.rank(); var strides: stdx.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; if (rk == 0) return strides; @@ -795,7 +1268,7 @@ pub const Shape = struct { // and the element size in bytes. var d: V = @bitCast(self._dims.buffer); d = @select(i64, rank_mask, d, @as(V, @splat(1))); - d = std.simd.shiftElementsLeft(d, 1, self.dtype().sizeOf()); + d = std.simd.shiftElementsLeft(d, 1, element_size); d = std.simd.prefixScan(.Mul, -1, d); strides.buffer = @bitCast(d); @@ -854,6 +1327,12 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; + _ = new_shape._partitioning.orderedRemove(ax); + + for (0..dims_.len) |_| { + new_shape._partitioning.insert(ax, .unknown) catch unreachable; + } + new_shape.inferMissingAxis(self.count()); return new_shape; } @@ -898,22 +1377,43 @@ pub const Shape = struct { /// Merge the given axes into one axis. /// eg: `Shape.init(.{.a1 = 5, .a2 = 2, .b = 3}).merge(.{ .a = .{ .a1, .a2 }); -> .{ .a = 10, .b = 3 }` pub fn mergeAxis(self: Shape, axis_: anytype, axes_: anytype) Shape { - const axes__ = self.axes(axes_); - - const first_axis = axes__.get(0); - const last_axis = axes__.get(axes__.len - 1); + const axes_to_merge = self.axes(axes_); + stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); var new_dim: i64 = 1; - for (axes__.constSlice(), first_axis..) |ax, counter| { - new_dim *= self.dim(ax); - stdx.debug.assert(ax == counter, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); + // Merging partitioned axes is complex. If ANY axis being merged is partitioned, + // the resulting sharding is ambiguous. A propagation pass must resolve this. + // Marking it 'open' is the correct signal. If all are replicated/unknown, the + // result is also replicated/unknown. + // rule: .replicated wins over .unknown. .open wins over everything. + var resulting_partition_spec: PartitionSpec = .unknown; + + for (0..axes_to_merge.len) |i| { + const current_axis = axes_to_merge.get(i); + new_dim *= self.dim(current_axis); + + if (i > 0) { + stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {} along non-contiguous axes {any}", .{ self, axes_ }); + } + + switch (self.partition(current_axis)) { + .axis, .open => resulting_partition_spec = .open, + .replicated => if (resulting_partition_spec != .open) { + resulting_partition_spec = .replicated; + }, + .unknown => {}, + } } + const first_axis_to_merge = axes_to_merge.get(0); + const num_axes_to_merge = axes_to_merge.len; + var new_shape = self; - new_shape._dims.set(first_axis, new_dim); - new_shape._dims.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.dims()[last_axis + 1 ..]) catch unreachable; - new_shape._tags.set(first_axis, if (comptime isTagConvertible(@TypeOf(axis_))) toTag(axis_) else TagUnknown); - new_shape._tags.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.tags()[last_axis + 1 ..]) catch unreachable; + + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); + new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); + new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); + return new_shape; } @@ -945,6 +1445,28 @@ pub const Shape = struct { ); } + test "mergeAxis partitioning" { + // Merging a partitioned axis with a replicated one -> open + const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); + const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged1.partition(.ab).eql(.open)); + try testing.expect(merged1.dim(.ab) == 10); + try testing.expect(merged1.rank() == 2); + + // Merging two replicated axes -> replicated + const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); + const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged2.partition(.ab).eql(.replicated)); + try testing.expect(merged2.partition(.c).eql(.init(.y))); + + // Merging two unknown axes -> unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown + const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged3.partition(.ab).eql(.unknown)); + } + pub fn mergeAxes(self: Shape, axes_: anytype) Shape { const T = @TypeOf(axes_); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); @@ -990,10 +1512,10 @@ pub const Shape = struct { return res; } - pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, MAX_RANK), TagsArray } { + pub fn parseStruct(T: type, v: anytype) struct { std.BoundedArray(T, MAX_RANK), TagsArray } { const V = @TypeOf(v); - var vals_: stdx.BoundedArray(T, MAX_RANK) = .{}; + var vals_: std.BoundedArray(T, MAX_RANK) = .{}; var tags_: TagsArray = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { @@ -1028,10 +1550,10 @@ pub const Shape = struct { } /// Parses a struct literal into a list of options for each axes. - pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, MAX_RANK) { + pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) std.BoundedArray(T, MAX_RANK) { const V = @TypeOf(options); - var res: stdx.BoundedArray(T, MAX_RANK) = .{}; + var res: std.BoundedArray(T, MAX_RANK) = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); for (options) |d| { @@ -1063,17 +1585,17 @@ pub const Shape = struct { test "comptimeShape" { comptime { const s = Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } // comptime only the shape { const s = comptime Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } } @@ -1083,7 +1605,7 @@ pub const Shape = struct { for (0..other.rank()) |ax| { if (other.tag(ax) != Shape.TagUnknown) { if (self.hasTag(other.tag(ax))) |batching_ax| { - stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({f}, {f})", .{ self, other }); + stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({}, {})", .{ self, other }); batching_axes += 1; } } @@ -1093,14 +1615,134 @@ pub const Shape = struct { return res_shape; } - pub fn concatenate(shapes: []const Shape, ax: anytype) Shape { - // TODO: Check that shapes are compatible - var new_shape = shapes[0]; - var new_dim: i64 = 0; - for (shapes) |s| { - new_dim += s.dim(ax); - } - new_shape = new_shape.setDim(ax, new_dim); - return new_shape; + pub fn iterator(self: Shape) MultiDimIterator { + return .{ + .shape = self, + .current_coords = .{0} ** Shape.MAX_RANK, + .flat_index = 0, + }; } }; + +/// Iterates over all coordinates of a shape, yielding the multi-dimensional +/// coordinates and the corresponding flat index for each element. +pub const MultiDimIterator = struct { + shape: Shape, + current_coords: [Shape.MAX_RANK]i64, + flat_index: usize, + + pub const Item = struct { + coords: [Shape.MAX_RANK]i64, + rank: u4, + flat_index: usize, + }; + + pub fn next(self: *MultiDimIterator) ?Item { + if (self.flat_index >= self.shape.count()) { + return null; + } + + const rank = self.shape.rank(); + const item = Item{ + .coords = self.current_coords, + .rank = rank, + .flat_index = self.flat_index, + }; + + self.flat_index += 1; + + var i: usize = rank; + while (i > 0) { + i -= 1; + self.current_coords[i] += 1; + if (self.current_coords[i] < self.shape.dim(i)) { + return item; + } + self.current_coords[i] = 0; + } + + return item; + } +}; + +test "MultiDimIterator / Scalar" { + const shape = Shape.scalar(.f32); + var iter = shape.iterator(); + const item = iter.next().?; + + try std.testing.expectEqual(0, item.rank); + try std.testing.expectEqual(0, item.flat_index); + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / Empty" { + const shape = Shape.init(.{ .x = 5, .y = 0, .z = 2 }, .f32); + var iter = shape.iterator(); + + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / 1D shape" { + const shape = Shape.init(.{5}, .f32); + var iter = shape.iterator(); + var count: i64 = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.coords[0]); + } + try std.testing.expectEqual(5, count); +} + +test "MultiDimIterator / 2D shape" { + const shape = Shape.init(.{ .rows = 2, .cols = 3 }, .f32); + var iter = shape.iterator(); + var items = std.ArrayList(MultiDimIterator.Item).init(std.testing.allocator); + defer items.deinit(); + + while (iter.next()) |item| { + try items.append(item); + } + + try std.testing.expectEqual(6, items.items.len); + try std.testing.expectEqualSlices(i64, &.{ 0, 0 }, items.items[0].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 1 }, items.items[1].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 2 }, items.items[2].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 0 }, items.items[3].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 1 }, items.items[4].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 2 }, items.items[5].coords[0..2]); + try std.testing.expectEqual(5, items.items[5].flat_index); +} + +test "MultiDimIterator / 3D shape" { + const shape = Shape.init(.{ .d = 2, .h = 2, .w = 2 }, .f32); + var iter = shape.iterator(); + const expected_coords = [_][3]i64{ + .{ 0, 0, 0 }, + .{ 0, 0, 1 }, + .{ 0, 1, 0 }, + .{ 0, 1, 1 }, + .{ 1, 0, 0 }, + .{ 1, 0, 1 }, + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + }; + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.flat_index); + try std.testing.expectEqualSlices(i64, &expected_coords[count], item.coords[0..3]); + } + + try std.testing.expectEqual(8, count); +} + +test "MultiDimIterator / Shape with dimension of size 1" { + const shape = Shape.init(.{ .a = 2, .b = 1, .c = 3 }, .f32); + var iter = shape.iterator(); + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(0, item.coords[1]); + } + try std.testing.expectEqual(6, count); +} diff --git a/zml/zml.zig b/zml/zml.zig index 2d9f7853..027fa92f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -44,6 +44,7 @@ pub const testing = @import("testing.zig"); pub const torch = @import("torch.zig"); pub const upb = @import("upb"); pub const c = @import("c"); +pub const partitioning = @import("partitioning.zig"); // pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { From db760496c3dd34be7a886e6258f23624cdf86aec Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 19 Nov 2025 09:46:26 -0800 Subject: [PATCH 47/47] wip: Loader + Resource --- examples/mnist/mnist.zig | 138 ++++++++++++++++++++++----------------- zml/BUILD.bazel | 2 + zml/io.zig | 59 +++++++++++++++++ zml/io/file.zig | 107 ++++++++++++++++++++++++++++++ zml/zml.zig | 1 + 5 files changed, 248 insertions(+), 59 deletions(-) create mode 100644 zml/io.zig create mode 100644 zml/io/file.zig diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 5c85f5b7..fb1479b3 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -41,81 +41,101 @@ pub fn main() !void { } pub fn asyncMain() !void { - const allocator = std.heap.c_allocator; + const allocator = std.heap.smp_allocator; - // // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); + var file_loader: zml.io.file.Loader = .init(); + defer file_loader.deinit(); - // log.info("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); + var loader: zml.io.Loader = .init(allocator); + defer loader.deinit(); - // // Auto-select platform - const platform = context.autoPlatform(.{}); - context.printAvailablePlatforms(platform); + loader.register(.file, &file_loader); - // Parse program args - const process_args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, process_args); - const pt_model = process_args[1]; - const t10kfilename = process_args[2]; + const resource = try loader.open(allocator, try std.Uri.parse("file:///Users/corendos/models/Llama-3.2-1B-Instruct/config.json?start=1&end=100")); + defer resource.deinit(); - // Memory arena dedicated to model shapes and weights - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); + var read_buffer: [4096]u8 = undefined; + var reader = resource.reader(&read_buffer); - // Read model shapes. - // Note this works because Mnist struct uses the same layer names as the pytorch model - var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); - defer buffer_store.deinit(); + var write_buffer: [4096]u8 = undefined; + var writer = std.fs.File.stdout().writer(&write_buffer); - const mnist_model = try zml.aio.populateModel(Mnist, allocator, buffer_store); - log.info("Reading model shapes from PyTorch file {s}...", .{pt_model}); + _ = try reader.interface().streamRemaining(&writer.interface); + try writer.interface.flush(); - // Start compiling - log.info("Compiling model to MLIR....", .{}); - var start_time = try std.time.Timer.start(); - var compilation = try asynk.asyncc(zml.compile, .{ allocator, Mnist.forward, .{}, .{zml.Shape.init(.{ 28, 28 }, .u8)}, buffer_store, platform }); + //// // Create ZML context + //var context = try zml.Context.init(); + //defer context.deinit(); - // While compiling, start loading weights on the platform - var model_weights = try zml.aio.loadModelBuffers(Mnist, mnist_model, buffer_store, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); + //// log.info("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); - // Wait for end of compilation and end of weights loading. - const compiled_mnist = try compilation.awaitt(); - log.info("✅ Compiled model in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + //// // Auto-select platform + //const platform = context.autoPlatform(.{}); + //context.printAvailablePlatforms(platform); - const mnist = compiled_mnist.prepare(model_weights); - defer mnist.deinit(); - log.info("✅ Weights transferred in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + //// Parse program args + //const process_args = try std.process.argsAlloc(allocator); + //defer std.process.argsFree(allocator, process_args); + //const pt_model = process_args[1]; + //const t10kfilename = process_args[2]; - log.info("Starting inference...", .{}); + //// Memory arena dedicated to model shapes and weights + //var arena_state = std.heap.ArenaAllocator.init(allocator); + //defer arena_state.deinit(); + //const arena = arena_state.allocator(); - // Load a random digit image from the dataset. - const dataset = try asynk.File.open(t10kfilename, .{ .mode = .read_only }); - defer dataset.close() catch unreachable; - var rng = std.Random.Xoshiro256.init(@intCast(std.time.timestamp())); + //// Read model shapes. + //// Note this works because Mnist struct uses the same layer names as the pytorch model + //var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); + //defer buffer_store.deinit(); - // inference - can be looped - { - const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); - var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample), .{}); - defer input.deinit(); + //const mnist_model = try zml.aio.populateModel(Mnist, allocator, buffer_store); + //log.info("Reading model shapes from PyTorch file {s}...", .{pt_model}); - printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); - defer result.deinit(); + //// Start compiling + //log.info("Compiling model to MLIR....", .{}); + //var start_time = try std.time.Timer.start(); + //var compilation = try asynk.asyncc(zml.compile, .{ allocator, Mnist.forward, .{}, .{zml.Shape.init(.{ 28, 28 }, .u8)}, buffer_store, platform }); - log.info( - \\✅ RECOGNIZED DIGIT: - \\ +-------------+ - \\{s} - \\ +-------------+ - \\ - , .{digits[try result.getValue(u8)]}); - } + //// While compiling, start loading weights on the platform + //var model_weights = try zml.aio.loadModelBuffers(Mnist, mnist_model, buffer_store, arena, platform); + //defer zml.aio.unloadBuffers(&model_weights); + + //// Wait for end of compilation and end of weights loading. + //const compiled_mnist = try compilation.awaitt(); + //log.info("✅ Compiled model in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + + //const mnist = compiled_mnist.prepare(model_weights); + //defer mnist.deinit(); + //log.info("✅ Weights transferred in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + + //log.info("Starting inference...", .{}); + + //// Load a random digit image from the dataset. + //const dataset = try asynk.File.open(t10kfilename, .{ .mode = .read_only }); + //defer dataset.close() catch unreachable; + //var rng = std.Random.Xoshiro256.init(@intCast(std.time.timestamp())); + + //// inference - can be looped + //{ + // const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); + // var sample: [28 * 28]u8 align(16) = undefined; + // _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); + // var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample), .{}); + // defer input.deinit(); + + // printDigit(sample); + // var result: zml.Buffer = mnist.call(.{input}); + // defer result.deinit(); + + // log.info( + // \\✅ RECOGNIZED DIGIT: + // \\ +-------------+ + // \\{s} + // \\ +-------------+ + // \\ + // , .{digits[try result.getValue(u8)]}); + //} } fn printDigit(digit: [28 * 28]u8) void { diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 6e1b0df0..9f61f308 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -24,6 +24,8 @@ zig_library( "aio/json.zig", "aio/safetensors.zig", "aio/tinyllama.zig", + "io.zig", + "io/file.zig", "buffer.zig", "context.zig", "dtype.zig", diff --git a/zml/io.zig b/zml/io.zig new file mode 100644 index 00000000..26227210 --- /dev/null +++ b/zml/io.zig @@ -0,0 +1,59 @@ +const std = @import("std"); +pub const file = @import("io/file.zig"); + +pub const ResourceKind = enum { file }; + +pub const Loader = struct { + file: ?*file.Loader = null, + + pub fn init(allocator: std.mem.Allocator) Loader { + _ = allocator; // autofix + return .{}; + } + + pub fn deinit(self: Loader) void { + _ = self; // autofix + } + + pub fn open(self: Loader, allocator: std.mem.Allocator, uri: std.Uri) !Resource { + inline for (std.meta.tags(ResourceKind)) |tag| { + if (std.mem.eql(u8, uri.scheme, @tagName(tag))) { + const maybe_loader = @field(self, @tagName(tag)); + if (maybe_loader) |loader| { + return @unionInit(Resource, @tagName(tag), try loader.open(allocator, uri)); + } + } + } + return error.NoLoaderFound; + } + + pub fn register(self: *Loader, kind: ResourceKind, loader: anytype) void { + @field(self, @tagName(kind)) = loader; + } +}; + +pub const Resource = union(ResourceKind) { + file: file.Resource, + + pub fn deinit(self: Resource) void { + switch (self) { + inline else => |v| v.deinit(), + } + } + + pub fn reader(self: Resource, buffer: []u8) Reader { + return switch (self) { + .file => |v| .{ .file = v.reader(buffer) }, + }; + } +}; + +pub const Reader = union(ResourceKind) { + file: file.Reader, + + pub fn interface(self: *Reader) *std.io.Reader { + return switch (self.*) { + .file => |*v| &v.interface, + }; + } +}; diff --git a/zml/io/file.zig b/zml/io/file.zig new file mode 100644 index 00000000..8c215cbe --- /dev/null +++ b/zml/io/file.zig @@ -0,0 +1,107 @@ +const std = @import("std"); + +pub const Reader = struct { + reader: std.fs.File.Reader, + remaining: std.Io.Limit, + interface: std.Io.Reader, + + pub fn init(reader: std.fs.File.Reader, limit: std.Io.Limit) Reader { + return .{ + .reader = reader, + .remaining = limit, + .interface = .{ + .vtable = &.{ + .stream = stream, + .discard = discard, + }, + .buffer = &.{}, + .seek = 0, + .end = 0, + }, + }; + } + + fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *Reader = @fieldParentPtr("interface", r); + const combined_limit = limit.min(self.remaining); + const n = try self.reader.interface.stream(w, combined_limit); + self.remaining = self.remaining.subtract(n).?; + return n; + } + + fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { + const self: *Reader = @fieldParentPtr("interface", r); + const combined_limit = limit.min(self.remaining); + const n = try self.reader.interface.discard(combined_limit); + self.remaining = self.remaining.subtract(n).?; + return n; + } +}; + +pub const Loader = struct { + pub fn init() Loader { + return .{}; + } + + pub fn deinit(self: Loader) void { + _ = self; // autofix + } + + pub fn open(self: Loader, allocator: std.mem.Allocator, uri: std.Uri) !Resource { + _ = self; // autofix + const resource: Resource = try .init(allocator, uri); + + return resource; + } +}; + +pub const Resource = struct { + file: std.fs.File, + size: usize, + start: usize, + + pub fn init(allocator: std.mem.Allocator, uri: std.Uri) !Resource { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const raw_query = if (uri.query) |query| try query.toRawMaybeAlloc(arena.allocator()) else ""; + var it = std.mem.tokenizeScalar(u8, raw_query, '&'); + var start: ?usize = null; + var end: ?usize = null; + while (it.next()) |component| { + var it2 = std.mem.tokenizeScalar(u8, component, '='); + const key = it2.next().?; + const value = it2.next() orelse ""; + if (std.mem.eql(u8, key, "start")) { + start = try std.fmt.parseInt(usize, value, 10); + } else if (std.mem.eql(u8, key, "end")) { + end = try std.fmt.parseInt(usize, value, 10); + } + } + + std.debug.print("start: {any} - end: {any}\n", .{ start, end }); + + const path = try uri.path.toRawMaybeAlloc(arena.allocator()); + + const file = try std.fs.openFileAbsolute(path, .{}); + + const size = end.? - start.?; + + return .{ + .file = file, + .size = size, + .start = start.?, + }; + } + + pub fn deinit(self: Resource) void { + self.file.close(); + } + + pub fn reader(self: Resource, buffer: []u8) Reader { + var r = std.fs.File.Reader.init(self.file, buffer); + r.seekTo(self.start) catch unreachable; + const r2 = Reader.init(r, .limited(self.size)); + return r2; + } +}; diff --git a/zml/zml.zig b/zml/zml.zig index 027fa92f..db6da357 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -45,6 +45,7 @@ pub const torch = @import("torch.zig"); pub const upb = @import("upb"); pub const c = @import("c"); pub const partitioning = @import("partitioning.zig"); +pub const io = @import("io.zig"); // pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct {