diff --git a/BUILD.bazel b/BUILD.bazel index aa1656d4..344438f7 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -10,9 +10,13 @@ zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ - "//async", - "//examples/llama", + # "//async", + # "//examples/llama", "//stdx", "//zml", + "//mlir:main", + # "//async", + # "//stdx", + # "//zml", ], ) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 5c85f5b7..fb1479b3 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -41,81 +41,101 @@ pub fn main() !void { } pub fn asyncMain() !void { - const allocator = std.heap.c_allocator; + const allocator = std.heap.smp_allocator; - // // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); + var file_loader: zml.io.file.Loader = .init(); + defer file_loader.deinit(); - // log.info("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); + var loader: zml.io.Loader = .init(allocator); + defer loader.deinit(); - // // Auto-select platform - const platform = context.autoPlatform(.{}); - context.printAvailablePlatforms(platform); + loader.register(.file, &file_loader); - // Parse program args - const process_args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, process_args); - const pt_model = process_args[1]; - const t10kfilename = process_args[2]; + const resource = try loader.open(allocator, try std.Uri.parse("file:///Users/corendos/models/Llama-3.2-1B-Instruct/config.json?start=1&end=100")); + defer resource.deinit(); - // Memory arena dedicated to model shapes and weights - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); + var read_buffer: [4096]u8 = undefined; + var reader = resource.reader(&read_buffer); - // Read model shapes. - // Note this works because Mnist struct uses the same layer names as the pytorch model - var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); - defer buffer_store.deinit(); + var write_buffer: [4096]u8 = undefined; + var writer = std.fs.File.stdout().writer(&write_buffer); - const mnist_model = try zml.aio.populateModel(Mnist, allocator, buffer_store); - log.info("Reading model shapes from PyTorch file {s}...", .{pt_model}); + _ = try reader.interface().streamRemaining(&writer.interface); + try writer.interface.flush(); - // Start compiling - log.info("Compiling model to MLIR....", .{}); - var start_time = try std.time.Timer.start(); - var compilation = try asynk.asyncc(zml.compile, .{ allocator, Mnist.forward, .{}, .{zml.Shape.init(.{ 28, 28 }, .u8)}, buffer_store, platform }); + //// // Create ZML context + //var context = try zml.Context.init(); + //defer context.deinit(); - // While compiling, start loading weights on the platform - var model_weights = try zml.aio.loadModelBuffers(Mnist, mnist_model, buffer_store, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); + //// log.info("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); - // Wait for end of compilation and end of weights loading. - const compiled_mnist = try compilation.awaitt(); - log.info("✅ Compiled model in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + //// // Auto-select platform + //const platform = context.autoPlatform(.{}); + //context.printAvailablePlatforms(platform); - const mnist = compiled_mnist.prepare(model_weights); - defer mnist.deinit(); - log.info("✅ Weights transferred in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + //// Parse program args + //const process_args = try std.process.argsAlloc(allocator); + //defer std.process.argsFree(allocator, process_args); + //const pt_model = process_args[1]; + //const t10kfilename = process_args[2]; - log.info("Starting inference...", .{}); + //// Memory arena dedicated to model shapes and weights + //var arena_state = std.heap.ArenaAllocator.init(allocator); + //defer arena_state.deinit(); + //const arena = arena_state.allocator(); - // Load a random digit image from the dataset. - const dataset = try asynk.File.open(t10kfilename, .{ .mode = .read_only }); - defer dataset.close() catch unreachable; - var rng = std.Random.Xoshiro256.init(@intCast(std.time.timestamp())); + //// Read model shapes. + //// Note this works because Mnist struct uses the same layer names as the pytorch model + //var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); + //defer buffer_store.deinit(); - // inference - can be looped - { - const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); - var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample), .{}); - defer input.deinit(); + //const mnist_model = try zml.aio.populateModel(Mnist, allocator, buffer_store); + //log.info("Reading model shapes from PyTorch file {s}...", .{pt_model}); - printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); - defer result.deinit(); + //// Start compiling + //log.info("Compiling model to MLIR....", .{}); + //var start_time = try std.time.Timer.start(); + //var compilation = try asynk.asyncc(zml.compile, .{ allocator, Mnist.forward, .{}, .{zml.Shape.init(.{ 28, 28 }, .u8)}, buffer_store, platform }); - log.info( - \\✅ RECOGNIZED DIGIT: - \\ +-------------+ - \\{s} - \\ +-------------+ - \\ - , .{digits[try result.getValue(u8)]}); - } + //// While compiling, start loading weights on the platform + //var model_weights = try zml.aio.loadModelBuffers(Mnist, mnist_model, buffer_store, arena, platform); + //defer zml.aio.unloadBuffers(&model_weights); + + //// Wait for end of compilation and end of weights loading. + //const compiled_mnist = try compilation.awaitt(); + //log.info("✅ Compiled model in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + + //const mnist = compiled_mnist.prepare(model_weights); + //defer mnist.deinit(); + //log.info("✅ Weights transferred in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + + //log.info("Starting inference...", .{}); + + //// Load a random digit image from the dataset. + //const dataset = try asynk.File.open(t10kfilename, .{ .mode = .read_only }); + //defer dataset.close() catch unreachable; + //var rng = std.Random.Xoshiro256.init(@intCast(std.time.timestamp())); + + //// inference - can be looped + //{ + // const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); + // var sample: [28 * 28]u8 align(16) = undefined; + // _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); + // var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample), .{}); + // defer input.deinit(); + + // printDigit(sample); + // var result: zml.Buffer = mnist.call(.{input}); + // defer result.deinit(); + + // log.info( + // \\✅ RECOGNIZED DIGIT: + // \\ +-------------+ + // \\{s} + // \\ +-------------+ + // \\ + // , .{digits[try result.getValue(u8)]}); + //} } fn printDigit(digit: [28 * 28]u8) void { diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 87982a7e..8da9cf73 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") cc_library( name = "c", @@ -8,8 +8,7 @@ cc_library( deps = [ "@llvm-project//mlir:CAPIArith", "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:CAPIMath", - "@llvm-project//mlir:CAPISCF", + "@llvm-project//mlir:CAPIMemRef", "@llvm-project//mlir:CAPITransforms", ], ) @@ -28,3 +27,47 @@ zig_test( name = "test", deps = [":mlir"], ) + +zig_binary( + name = "main", + #args = [ + # "$(location @mnist//:mnist.safetensors)", + # "$(location @mnist//:t10k-images.idx3-ubyte)", + #], + #data = [ + # "@mnist//:mnist.safetensors", + # "@mnist//:t10k-images.idx3-ubyte", + #], + main = "main.zig", + srcs = [ + "llama.zig", + "writer.zig", + ], + visibility = ["//visibility:public"], + deps = [ + ":mlir", + "//zml", + "//mlir/dialects", + # ":c", + "//stdx", + "//async", + ], + copts = ["-freference-trace"] +) + +zig_library( + name = "writer", + main = "writer.zig", + visibility = ["//visibility:public"], + deps = [ + ":c", + "//zml", + "//stdx", + "//async", + ], +) + +zig_test( + name = "test_writer", + deps = [":writer"], +) diff --git a/mlir/context.zig b/mlir/context.zig new file mode 100644 index 00000000..e69de29b diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 101f67ff..8b07057c 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -3,17 +3,17 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "dialects", srcs = [ - "arith.zig", + # "arith.zig", "func.zig", - "math.zig", - "scf.zig", - "tensor.zig", + # "math.zig", + # "scf.zig", + # "tensor.zig", ], import_name = "mlir/dialects", main = "dialects.zig", visibility = ["//visibility:public"], deps = [ - "//mlir", + "//mlir:mlir", "//mlir/dialects/stablehlo", "//stdx", ], diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig index 39cbc2a1..4e8e0ed9 100644 --- a/mlir/dialects/arith.zig +++ b/mlir/dialects/arith.zig @@ -1,59 +1,61 @@ const std = @import("std"); -const mlir = @import("mlir"); +const mlir = @import("mlir2"); -pub fn constant(ctx: mlir.Context, value: mlir.Attribute, location: mlir.Location) mlir.Operation { +pub fn constant(ctx: *mlir.Context, value: *const mlir.Attribute, location: *const mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, "arith.constant", .{ - .attributes = &.{.{ "value", value }}, + .attributes = &.{ + .named(ctx, "value", value), + }, .result_type_inference = true, .location = location, }); } -fn binary_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation { +fn binaryOp(comptime op_name: []const u8) type { return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { + pub fn call(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, + .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .location = location, }); } - }.call; + }; } -fn cast_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Type, mlir.Location) mlir.Operation { +fn castOp(comptime op_name: []const u8) type { return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, new_type: mlir.Type, location: mlir.Location) mlir.Operation { + pub fn call(ctx: mlir.Context, value: *const mlir.Value, new_type: *const mlir.Type, location: *const mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .results = &.{new_type}, + .operands = .{ .float = &.{value} }, + .results = .{ .flat = &.{new_type} }, .location = location, }); } - }.call; + }; } -pub const addi = binary_fn("arith.addi"); -pub const addf = binary_fn("arith.addf"); -pub const subi = binary_fn("arith.subi"); -pub const subf = binary_fn("arith.subf"); -pub const muli = binary_fn("arith.muli"); -pub const mulf = binary_fn("arith.mulf"); -pub const divsi = binary_fn("arith.divsi"); -pub const divui = binary_fn("arith.divui"); -pub const divf = binary_fn("arith.divf"); -pub const maxnumf = binary_fn("arith.maxnumf"); -pub const maxnumi = binary_fn("arith.maxnumi"); -pub const extsi = cast_fn("arith.extsi"); -pub const extui = cast_fn("arith.extui"); -pub const extf = cast_fn("arith.extf"); -pub const trunci = cast_fn("arith.trunci"); -pub const truncf = cast_fn("arith.truncf"); -pub const fptosi = cast_fn("arith.fptosi"); -pub const fptoui = cast_fn("arith.fptoui"); -pub const sitofp = cast_fn("arith.sitofp"); -pub const uitofp = cast_fn("arith.uitofp"); +pub const addi = binaryOp("arith.addi").call; +pub const addf = binaryOp("arith.addf").call; +pub const subi = binaryOp("arith.subi").call; +pub const subf = binaryOp("arith.subf").call; +pub const muli = binaryOp("arith.muli").call; +pub const mulf = binaryOp("arith.mulf").call; +pub const divsi = binaryOp("arith.divsi").call; +pub const divui = binaryOp("arith.divui").call; +pub const divf = binaryOp("arith.divf").call; +pub const maxnumf = binaryOp("arith.maxnumf").call; +pub const maxnumi = binaryOp("arith.maxnumi").call; +pub const extsi = castOp("arith.extsi").call; +pub const extui = castOp("arith.extui").call; +pub const extf = castOp("arith.extf").call; +pub const trunci = castOp("arith.trunci").call; +pub const truncf = castOp("arith.truncf").call; +pub const fptosi = castOp("arith.fptosi").call; +pub const fptoui = castOp("arith.fptoui").call; +pub const sitofp = castOp("arith.sitofp").call; +pub const uitofp = castOp("arith.uitofp").call; pub const CmpIPredicate = enum { eq, @@ -68,12 +70,12 @@ pub const CmpIPredicate = enum { uge, }; -pub fn cmpi(ctx: mlir.Context, predicate: CmpIPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn cmpi(ctx: mlir.Context, predicate: CmpIPredicate, lhs: *const mlir.Value, rhs: *const mlir.Value, location: mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, "arith.cmpi", .{ - .operands = &.{ lhs, rhs }, + .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .{ "predicate", .int(ctx, .i64, @intFromEnum(predicate)) }, + .{ "predicate", mlir.integerAttribute(ctx, .i64, @intFromEnum(predicate)) }, }, .location = location, }); @@ -98,12 +100,12 @@ pub const CmpFPredicate = enum { true, }; -pub fn cmpf(ctx: mlir.Context, predicate: CmpFPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn cmpf(ctx: mlir.Context, predicate: CmpFPredicate, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *const mlir.Operation { return mlir.Operation.make(ctx, "arith.cmpf", .{ - .operands = &.{ lhs, rhs }, + .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .{ "predicate", .int(ctx, .i64, @intFromEnum(predicate)) }, + .named(ctx, "predicate", mlir.integerAttribute(ctx, .i64, @intFromEnum(predicate))), }, .location = location, }); diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index 47f74d8e..3649d443 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,12 +1,13 @@ const std = @import("std"); -pub const arith = @import("arith.zig"); -pub const func = @import("func.zig"); -pub const math = @import("math.zig"); -pub const scf = @import("scf.zig"); -pub const tensor = @import("tensor.zig"); pub const stablehlo = @import("mlir/dialects/stablehlo"); +pub const func = @import("func.zig"); + +// pub const arith = @import("arith.zig"); +// pub const math = @import("math.zig"); +// pub const scf = @import("scf.zig"); +// pub const tensor = @import("tensor.zig"); test { std.testing.refAllDecls(@This()); } diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 868ff3bc..b9f8b7f5 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -3,50 +3,91 @@ const std = @import("std"); const mlir = @import("mlir"); const stdx = @import("stdx"); -pub fn func( - ctx: mlir.Context, - args: struct { - sym_name: []const u8, - args: []const mlir.Type, - arg_attrs: []const mlir.Attribute = &.{}, - results: []const mlir.Type, - res_attrs: []const mlir.Attribute = &.{}, - block: mlir.Block, - location: mlir.Location, - }, -) mlir.Operation { - var attrs_tuple_buffer = stdx.BoundedArray(mlir.AttrTuple, 4){}; - attrs_tuple_buffer.appendAssumeCapacity(.{ "sym_name", .string(ctx, args.sym_name) }); - attrs_tuple_buffer.appendAssumeCapacity(.{ "function_type", .type_(.function(ctx, args.args, args.results)) }); - if (args.arg_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "arg_attrs", .array(ctx, args.arg_attrs) }); - } +const FuncOpArgs = struct { + pub const Visibility = enum { + public, + private, + }; - if (args.res_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "res_attrs", .array(ctx, args.res_attrs) }); + name: []const u8, + args: ?[]const *const mlir.Type = null, + args_attributes: ?[]const *const mlir.Attribute = null, + results: ?[]const *const mlir.Type = null, + results_attributes: ?[]const *const mlir.Attribute = null, + block: *mlir.Block, + location: *const mlir.Location, + no_inline: bool = false, + visibility: Visibility = .public, +}; + +pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .{}; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .{}; + + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .{}; + attr_tuples_buffer.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), + .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), + .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( + ctx, + args.args orelse args: { + for (0..args.block.numArguments()) |i| { + args_buffer.appendAssumeCapacity(args.block.argument(i).type_()); + } + break :args args_buffer.constSlice(); + }, + args.results orelse results: { + const terminator = args.block.terminator() orelse { + @panic("block has no terminator"); + }; + for (0..terminator.numOperands()) |i| { + results_buffer.appendAssumeCapacity(terminator.operand(i).type_()); + } + break :results results_buffer.constSlice(); + }, + ))), + }); + + if (args.args_attributes) |args_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); + } + if (args.results_attributes) |results_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, results_attributes))); + } + if (args.no_inline) { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); } return mlir.Operation.make(ctx, "func.func", .{ .blocks = &.{args.block}, - .attributes = attrs_tuple_buffer.constSlice(), + .attributes = attr_tuples_buffer.constSlice(), .location = args.location, }); } -pub fn call(ctx: mlir.Context, name: [:0]const u8, values: []const mlir.Value, results: []const mlir.Type, loc: mlir.Location) mlir.Operation { +pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Value, results: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.call", .{ - .variadic_operands = &.{values}, - .results = results, - .verify = true, - .attributes = &.{.{ "callee", .symbol(ctx, name) }}, - .location = loc, + .operands = .{ .variadic = &.{values} }, + .results = .{ .flat = results }, + .attributes = &.{ + .named(ctx, "callee", mlir.flatSymbolRefAttribute(ctx, name)), + }, + .location = location, }); } -pub fn return_(ctx: mlir.Context, values: []const mlir.Value, loc: mlir.Location) mlir.Operation { +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.return", .{ - .operands = values, + .operands = .{ .flat = &.{value} }, + .location = location, .verify = false, - .location = loc, + }); +} + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "func.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, }); } diff --git a/mlir/dialects/math.zig b/mlir/dialects/math.zig deleted file mode 100644 index 62ef622c..00000000 --- a/mlir/dialects/math.zig +++ /dev/null @@ -1,36 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -const namespace = "math"; - -fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{value}, - .results = &.{value.getType()}, - .location = location, - }); - } - }; -} - -fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{ lhs, rhs }, - .results = &.{}, - .location = location, - }); - } - }; -} - -pub const ipowi = binary_fn("ipowi").call; -pub const fpowi = binary_fn("fpowi").call; -pub const tanh = unary_fn("tanh").call; -pub const sqrt = unary_fn("sqrt").call; -pub const exp = unary_fn("exp").call; -pub const exp2 = unary_fn("exp2").call; -pub const log = unary_fn("log").call; diff --git a/mlir/dialects/scf.zig b/mlir/dialects/scf.zig deleted file mode 100644 index 61fcb189..00000000 --- a/mlir/dialects/scf.zig +++ /dev/null @@ -1,61 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -pub fn ForBody(ExtraArgs: type) type { - return fn (mlir.Context, mlir.Block, ExtraArgs) mlir.Operation; -} - -pub const ForRange = struct { - start: mlir.Value, - end: mlir.Value, - step: mlir.Value, -}; - -pub fn @"for"( - ExtraArgs: type, - ctx: mlir.Context, - range: ForRange, - init_values: []const mlir.Value, - body: ForBody(ExtraArgs), - extra_args: ExtraArgs, - loc: mlir.Location, -) mlir.Operation { - const n_args = init_values.len; - var init_types_buf: [32]mlir.Type = undefined; - var locs_buf: [32]mlir.Location = undefined; - - // The first block argument is the for loop induction variable, - // followed then by all the loop-carried variables. - const init_types = init_types_buf[0 .. n_args + 1]; - const locs = locs_buf[0 .. n_args + 1]; - init_types[0] = range.start.getType(); - locs[0] = loc; - for (1.., init_values) |i, val| { - init_types[i] = val.getType(); - locs[i] = loc; - } - - const block = mlir.Block.init(init_types, locs) catch unreachable; - const yield_op = @call(.auto, body, .{ ctx, block, extra_args }); - std.debug.assert(std.mem.eql(u8, "scf.yield", yield_op.name().str())); - block.appendOperationRecursive(yield_op, .open); - - const for_op = mlir.Operation.make(ctx, "scf.for", .{ - .variadic_operands = &.{ &.{ range.start, range.end, range.step }, init_values }, - .results = init_types[1..], - .blocks = &.{block}, - .location = loc, - .verify = false, - }); - return for_op; -} - -pub fn yield(ctx: mlir.Context, res: []const mlir.Value, loc: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "scf.yield", .{ - .variadic_operands = &.{res}, - .results = &.{}, - .location = loc, - .verify = false, - }); -} diff --git a/mlir/dialects/stablehlo/BUILD.bazel b/mlir/dialects/stablehlo/BUILD.bazel index 45e69239..0596357a 100644 --- a/mlir/dialects/stablehlo/BUILD.bazel +++ b/mlir/dialects/stablehlo/BUILD.bazel @@ -6,8 +6,8 @@ zig_library( main = "stablehlo.zig", visibility = ["//mlir/dialects:__subpackages__"], deps = [ - "//mlir", "//mlir:c", + "//mlir:mlir", "//stdx", "@stablehlo//:stablehlo_dialect_capi", ], diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 0c60fd01..19b8b7f8 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -2,112 +2,106 @@ const std = @import("std"); const c = @import("c"); const mlir = @import("mlir"); +const stringRef = mlir.stringRef; const stdx = @import("stdx"); -pub const abs = functors.unary_fn("stablehlo.abs").call; -pub const cosine = functors.unary_fn("stablehlo.cosine").call; -pub const sine = functors.unary_fn("stablehlo.sine").call; -pub const exponential = functors.unary_fn("stablehlo.exponential").call; -pub const exponential_minus_one = functors.unary_fn("stablehlo.exponential_minus_one").call; -pub const floor = functors.unary_fn("stablehlo.floor").call; -pub const log = functors.unary_fn("stablehlo.log").call; -pub const log_plus_one = functors.unary_fn("stablehlo.log_plus_one").call; -pub const not = functors.unary_fn("stablehlo.not").call; -pub const negate = functors.unary_fn("stablehlo.negate").call; -pub const sqrt = functors.unary_fn("stablehlo.sqrt").call; -pub const tanh = functors.unary_fn("stablehlo.tanh").call; -pub const cbrt = functors.unary_fn("stablehlo.cbrt").call; -pub const ceil = functors.unary_fn("stablehlo.ceil").call; -pub const rsqrt = functors.unary_fn("stablehlo.rsqrt").call; -pub const count_leading_zeros = functors.unary_fn("stablehlo.count_leading_zeros").call; -pub const is_finite = functors.unary_fn("stablehlo.is_finite").call; -pub const logistic = functors.unary_fn("stablehlo.logistic").call; -pub const popcnt = functors.unary_fn("stablehlo.popcnt").call; -pub const sign = functors.unary_fn("stablehlo.sign").call; -pub const real = functors.unary_fn("stablehlo.real").call; -pub const imag = functors.unary_fn("stablehlo.imag").call; +pub const abs = unaryOp("stablehlo.abs").call; +pub const cosine = unaryOp("stablehlo.cosine").call; +pub const sine = unaryOp("stablehlo.sine").call; +pub const exponential = unaryOp("stablehlo.exponential").call; +pub const exponential_minus_one = unaryOp("stablehlo.exponential_minus_one").call; +pub const floor = unaryOp("stablehlo.floor").call; +pub const log = unaryOp("stablehlo.log").call; +pub const log_plus_one = unaryOp("stablehlo.log_plus_one").call; +pub const not = unaryOp("stablehlo.not").call; +pub const negate = unaryOp("stablehlo.negate").call; +pub const sqrt = unaryOp("stablehlo.sqrt").call; +pub const tanh = unaryOp("stablehlo.tanh").call; +pub const cbrt = unaryOp("stablehlo.cbrt").call; +pub const ceil = unaryOp("stablehlo.ceil").call; +pub const rsqrt = unaryOp("stablehlo.rsqrt").call; +pub const count_leading_zeros = unaryOp("stablehlo.count_leading_zeros").call; +pub const is_finite = unaryOp("stablehlo.is_finite").call; +pub const logistic = unaryOp("stablehlo.logistic").call; +pub const popcnt = unaryOp("stablehlo.popcnt").call; +pub const sign = unaryOp("stablehlo.sign").call; +pub const real = unaryOp("stablehlo.real").call; +pub const imag = unaryOp("stablehlo.imag").call; +pub const round_nearest_afz = unaryOp("stablehlo.round_nearest_afz").call; +pub const round_nearest_even = unaryOp("stablehlo.round_nearest_even").call; +pub const tuple = unaryOp("stablehlo.tuple").call; -pub const add = functors.binary_fn("stablehlo.add").call; -pub const multiply = functors.binary_fn("stablehlo.multiply").call; -pub const divide = functors.binary_fn("stablehlo.divide").call; -pub const subtract = functors.binary_fn("stablehlo.subtract").call; -pub const or_ = functors.binary_fn("stablehlo.or").call; -pub const xor = functors.binary_fn("stablehlo.xor").call; -pub const and_ = functors.binary_fn("stablehlo.and").call; -pub const atan2 = functors.binary_fn("stablehlo.atan2").call; -pub const maximum = functors.binary_fn("stablehlo.maximum").call; -pub const minimum = functors.binary_fn("stablehlo.minimum").call; -pub const power = functors.binary_fn("stablehlo.power").call; -pub const remainder = functors.binary_fn("stablehlo.remainder").call; -pub const shift_left = functors.binary_fn("stablehlo.shift_left").call; -pub const shift_right_arithmetic = functors.binary_fn("stablehlo.shift_right_arithmetic").call; -pub const shift_right_logical = functors.binary_fn("stablehlo.shift_right_logical").call; -pub const complex = functors.binary_fn("stablehlo.complex").call; +pub const add = binaryOp("stablehlo.add").call; +pub const multiply = binaryOp("stablehlo.multiply").call; +pub const divide = binaryOp("stablehlo.divide").call; +pub const subtract = binaryOp("stablehlo.subtract").call; +pub const or_ = binaryOp("stablehlo.or").call; +pub const xor = binaryOp("stablehlo.xor").call; +pub const and_ = binaryOp("stablehlo.and").call; +pub const atan2 = binaryOp("stablehlo.atan2").call; +pub const maximum = binaryOp("stablehlo.maximum").call; +pub const minimum = binaryOp("stablehlo.minimum").call; +pub const power = binaryOp("stablehlo.power").call; +pub const remainder = binaryOp("stablehlo.remainder").call; +pub const shift_left = binaryOp("stablehlo.shift_left").call; +pub const shift_right_arithmetic = binaryOp("stablehlo.shift_right_arithmetic").call; +pub const shift_right_logical = binaryOp("stablehlo.shift_right_logical").call; +pub const complex = binaryOp("stablehlo.complex").call; -const functors = struct { - fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); - } - }; - } +pub const bitcast_convert = castOp("stablehlo.bitcast_convert").call; +pub const convert = castOp("stablehlo.convert").call; +pub const reshape = castOp("stablehlo.reshape").call; - pub fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .location = location, - }); - } - }; - } -}; - -pub fn return_(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{&.{value}}, - .verify = false, - .location = location, - }); +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; } -pub fn returns_(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{values}, - .verify = false, - .location = location, - }); +fn unaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .location = location, + }); + } + }; } -pub fn bitcast_convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.bitcast_convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); +pub fn binaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; } -pub fn cholesky(ctx: mlir.Context, value: mlir.Value, lower: bool, location: mlir.Location) mlir.Operation { +pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.cholesky", .{ - .operands = &.{value}, + .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .{ "lower", .i1FromBool(ctx, lower) }, + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower))), }, .location = location, }); } -pub fn clamp(ctx: mlir.Context, min: mlir.Value, value: mlir.Value, max: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Value, max: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.clamp", .{ - .operands = &.{ min, value, max }, + .operands = .{ .flat = &.{ min, value, max } }, .result_type_inference = true, .location = location, }); @@ -119,18 +113,17 @@ pub const DotPrecision = union(enum) { highest, algorithm: DotAlgorithm, - pub fn precisionAttr(self: DotPrecision, ctx: mlir.Context) mlir.Attribute { - const precision = PrecisionAttribute.init(ctx, switch (self) { + pub fn precision(self: DotPrecision) PrecisionAttribute.Precision { + return switch (self) { .fast => .DEFAULT, .high => .HIGH, .highest => .HIGHEST, // When we specify the dot algorithm, we should not specify the precision. .algorithm => .DEFAULT, - }); - return precision.asAttr(); + }; } - pub fn algorithmAttr(self: DotPrecision, ctx: mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { + pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { return switch (self) { .algorithm => |algo| algo.asAttr(ctx, operand_type), else => null, @@ -149,159 +142,172 @@ pub const DotAlgorithm = struct { // bf16_6x: each input is decomposed to 3 bf16 components, then 6 dot operations are done on those components, and the result is accumulated in f32. // not sure where this is available. pub const bf16_6x: DotAlgorithm = .{ - .operand = .bf16, .accumulation = .f32, .component_count = 1, .num_primitive_operations = 6, .allow_imprecise_accumulation = false, }; - - pub fn asAttr(self: DotAlgorithm, ctx: mlir.Context, tensor_type: mlir.RankedTensorType) mlir.Attribute { - const elem_type = tensor_type.getElementType(); - - return mlir.Attribute.wrap(c.stablehloDotAlgorithmGet( - ctx._inner, - elem_type._inner, - elem_type._inner, - self.accumulation.asType(ctx)._inner, - self.component_count, - self.component_count, - self.num_primitive_operations, - self.allow_imprecise_accumulation, - )); - } }; +pub fn dotAlgorithmAttribute(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { + return @ptrCast(c.stablehloDotAlgorithmGet( + ctx.ptr(), + element_type.ptr(), + element_type.ptr(), + mlir.floatType(ctx, dot_algorithm.accumulation).ptr(), + dot_algorithm.component_count, + dot_algorithm.component_count, + dot_algorithm.num_primitive_operations, + dot_algorithm.allow_imprecise_accumulation, + ).ptr); +} + /// General matrix multiplication "a la Einstein sum" /// Note: stablehlo doesn't do type inference for dot_general pub fn dot_general( - ctx: mlir.Context, - lhs: mlir.Value, - rhs: mlir.Value, - result_type: mlir.Type, - location: mlir.Location, + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, opts: struct { lhs_batching_dimensions: []const i64, rhs_batching_dimensions: []const i64, lhs_contracting_dimensions: []const i64, rhs_contracting_dimensions: []const i64, - precision: DotPrecision, + dot_precision: DotPrecision, }, -) mlir.Operation { - const precisions: [2]mlir.Attribute = @splat(opts.precision.precisionAttr(ctx)); - const attributes = [3]mlir.AttrTuple{ - .{ - "dot_dimension_numbers", DotDimensionNumbersAttribute.init(ctx, .{ + location: *const mlir.Location, +) *mlir.Operation { + const precisions: [2]*const mlir.Attribute = @splat(precisionAttribute(ctx, opts.dot_precision.precision())); + const attributes = [3]mlir.NamedAttribute{ + .named( + ctx, + "dot_dimension_numbers", + dotDimensionNumbersAttribute(ctx, .{ .lhs_batching_dimensions = opts.lhs_batching_dimensions, .rhs_batching_dimensions = opts.rhs_batching_dimensions, .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, - }).asAttr(), - }, - .{ "precision_config", .array(ctx, &precisions) }, + }), + ), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &precisions)), // keep algorithm as the last attribute so we can omit it when it's not set. - .{ "algorithm", opts.precision.algorithmAttr(ctx, lhs.getType().as(mlir.RankedTensorType).?) orelse undefined }, + .named(ctx, "algorithm", switch (opts.dot_precision) { + .algorithm => |v| dotAlgorithmAttribute(ctx, v, lhs.type_()), + else => undefined, + }), }; - const n_attributes = if (opts.precision == .algorithm) attributes.len else attributes.len - 1; + const n_attributes = if (opts.dot_precision == .algorithm) attributes.len else attributes.len - 1; return mlir.Operation.make(ctx, "stablehlo.dot_general", .{ - .operands = &.{ lhs, rhs }, - .results = &.{result_type}, + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, .attributes = attributes[0..n_attributes], .location = location, }); } pub fn constant( - ctx: mlir.Context, + ctx: *mlir.Context, dims: []const i64, - elem_type: mlir.DenseElementsAttributeTypes, + elem_type: *const mlir.Type, raw_bytes: []const u8, - location: mlir.Location, -) mlir.Operation { + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.constant", .{ - .operands = &.{}, - .results = &.{.tensor(dims, elem_type.mlirType(ctx))}, - .attributes = &.{.{ "value", .denseElementsFromBytes(ctx, dims, elem_type, raw_bytes) }}, + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, + .attributes = &.{ + .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.get(dims, elem_type).shaped(), raw_bytes)), + }, .location = location, }); } -pub fn convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn broadcast_in_dim(ctx: mlir.Context, operand: mlir.Value, dims: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn broadcast_in_dim( + ctx: *mlir.Context, + operand: *const mlir.Value, + dims: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ - .operands = &.{operand}, - .results = &.{result_type}, + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "broadcast_dimensions", .dense(ctx, .i64, dims) }, + .named(ctx, "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims)), }, .location = location, }); } -pub fn transpose(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location, opts: struct { permutation: []const i64 }) mlir.Operation { +pub fn transpose( + ctx: *mlir.Context, + value: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { permutation: []const i64 }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.transpose", .{ - .operands = &.{value}, - .results = &.{result_type}, + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "permutation", .dense(ctx, .i64, opts.permutation) }, + .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation)), }, .location = location, }); } -pub fn slice(ctx: mlir.Context, operand: mlir.Value, start_indices: []const i64, limit_indices: []const i64, strides: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + start_indices: []const i64, + limit_indices: []const i64, + strides: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.slice", .{ - .operands = &.{operand}, - .results = &.{result_type}, + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "start_indices", .dense(ctx, .i64, start_indices) }, - .{ "limit_indices", .dense(ctx, .i64, limit_indices) }, - .{ "strides", .dense(ctx, .i64, strides) }, + .named(ctx, "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices)), + .named(ctx, "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices)), + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), }, .location = location, }); } -pub fn concatenate(ctx: mlir.Context, inputs: []const mlir.Value, dimension: i64, location: mlir.Location) mlir.Operation { +pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimension: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ - .operands = inputs, + .operands = .{ .flat = inputs }, .result_type_inference = true, .attributes = &.{ - .{ "dimension", .int(ctx, .i64, dimension) }, + .named(ctx, "dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); } -pub fn reshape(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.reshape", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn select(ctx: mlir.Context, condition: mlir.Value, then: mlir.Value, else_: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn select( + ctx: *mlir.Context, + condition: *const mlir.Value, + then: *const mlir.Value, + else_: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.select", .{ - .operands = &.{ condition, then, else_ }, - .results = &.{then.getType()}, + .operands = .{ .flat = &.{ condition, then, else_ } }, + .results = .{ .flat = &.{then.type_()} }, .location = location, }); } pub fn gather( - ctx: mlir.Context, - value: mlir.Value, - indices: mlir.Value, + ctx: *mlir.Context, + value: *const mlir.Value, + indices: *const mlir.Value, slice_sizes: []const i64, - location: mlir.Location, args: struct { offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -311,15 +317,16 @@ pub fn gather( index_vector_dim: i64, indices_are_sorted: bool = false, }, -) mlir.Operation { + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make( ctx, "stablehlo.gather", .{ - .operands = &.{ value, indices }, + .operands = .{ .flat = &.{ value, indices } }, .result_type_inference = true, .attributes = &.{ - .{ "dimension_numbers", GatherDimensionNumbersAttribute.init( + .named(ctx, "dimension_numbers", gatherDimensionNumbersAttribute( ctx, args.offset_dims, args.collapsed_slice_dims, @@ -327,20 +334,20 @@ pub fn gather( args.start_indices_batching_dims, args.start_index_map, args.index_vector_dim, - ).asAttr() }, - .{ "slice_sizes", .dense(ctx, .i64, slice_sizes) }, - .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, + )), + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, slice_sizes)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), }, .location = location, }, ); } -fn elementTypeOrSelf(typ: mlir.Type) mlir.Type { - return if (typ.as(mlir.ShapedType)) |shaped| { - return shaped.elementType(); - } else typ; -} +// fn elementTypeOrSelf(typ: *const mlir.Type) *const mlir.Type { +// return if (typ.as(mlir.ShapedType)) |shaped| { +// return shaped.elementType(); +// } else typ; +// } pub const ScatterArgs = struct { update_window_dims: []const i64, @@ -352,9 +359,9 @@ pub const ScatterArgs = struct { indices_are_sorted: bool = false, unique_indices: bool = false, - pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: mlir.Context) mlir.Attribute { - return .{ ._inner = c.stablehloScatterDimensionNumbersGet( - ctx._inner, + pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { + return @ptrCast(c.stablehloScatterDimensionNumbersGet( + ctx.ptr(), @intCast(self.update_window_dims.len), self.update_window_dims.ptr, @intCast(self.inserted_window_dims.len), @@ -366,29 +373,29 @@ pub const ScatterArgs = struct { @intCast(self.scatter_dims_to_operand_dims.len), self.scatter_dims_to_operand_dims.ptr, self.index_vector_dim, - ) }; + ).ptr); } }; pub fn scatter( - ctx: mlir.Context, - inputs: []const mlir.Value, - scatter_indices: []const mlir.Value, - updates: []const mlir.Value, - update_block: mlir.Block, + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + scatter_indices: []const *const mlir.Value, + updates: []const *const mlir.Value, + update_block: *mlir.Block, args: ScatterArgs, - location: mlir.Location, -) mlir.Operation { + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make( ctx, "stablehlo.scatter", .{ - .variadic_operands = &.{ inputs, scatter_indices, updates }, + .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, .blocks = &.{update_block}, .attributes = &.{ - .{ "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx) }, - .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, - .{ "unique_indices", .boolean(ctx, args.unique_indices) }, + .named(ctx, "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), + .named(ctx, "unique_indices", mlir.boolAttribute(ctx, args.unique_indices)), }, .result_type_inference = true, .location = location, @@ -396,163 +403,159 @@ pub fn scatter( ); } -pub fn iota(ctx: mlir.Context, dimension: i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.iota", .{ - .operands = &.{}, - .results = &.{result_type}, + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "iota_dimension", .int(ctx, .i64, dimension) }, + .named(ctx, "iota_dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); } -pub fn reverse(ctx: mlir.Context, operand: mlir.Value, dimensions: []const i64, location: mlir.Location) mlir.Operation { - const result_type = operand.getType(); +pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []const i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.reverse", .{ - .operands = &.{operand}, - .results = &.{result_type}, + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{operand.type_()} }, .attributes = &.{ - .{ "dimensions", .dense(ctx, .i64, dimensions) }, + .named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions)), }, .location = location, }); } -pub fn compare(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, comparison_direction: ComparisonDirection, compare_type: CompareType, location: mlir.Location) mlir.Operation { +pub fn compare( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + comparison_direction: ComparisonDirection.Direction, + compare_type: CompareType.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.compare", .{ - .operands = &.{ lhs, rhs }, + .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .{ "comparison_direction", comparison_direction.asAttr() }, - .{ "compare_type", compare_type.asAttr() }, + .named(ctx, "comparison_direction", comparisonDirection(ctx, comparison_direction)), + .named(ctx, "compare_type", compareType(ctx, compare_type)), }, .location = location, }); } -pub fn reduce( - ctx: mlir.Context, - inputs: []const mlir.Value, - init_values: []const mlir.Value, - dimensions: []const i64, - blkctx: anytype, - blkfn: fn (anytype, mlir.Context, []const mlir.Value, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; +// pub fn reduce( +// ctx: *mlir.Context, +// inputs: []const *const mlir.Value, +// init_values: []const *const mlir.Value, +// dimensions: []const i64, +// blkctx: anytype, +// blkfn: fn (anytype, mlir.Context, []const *const mlir.Value, []const *const mlir.Value) mlir.Operation, +// location: *const mlir.Location, +// ) *mlir.Operation { +// const MaxBlockArguments = 32; - const block_n_args = inputs.len + init_values.len; - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0..block_n_args]; - var reduce_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type: mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); - reduce_elem_types[i] = arg_type; - reduce_elem_types[inputs.len + i] = arg_type; - } - var block = mlir.Block.open(reduce_elem_types[0..block_n_args], locations) catch unreachable; - { - defer block.close(); +// const block_n_args = inputs.len + init_values.len; +// const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0..block_n_args]; +// var reduce_elem_types: [MaxBlockArguments]*const mlir.Type = undefined; +// for (inputs, 0..) |input, i| { +// const arg_type: *const mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); +// reduce_elem_types[i] = arg_type; +// reduce_elem_types[inputs.len + i] = arg_type; +// } +// var block = mlir.Block.open(reduce_elem_types[0..block_n_args], locations) catch unreachable; +// { +// defer block.close(); - var block_inputs: [MaxBlockArguments / 2]mlir.Value = undefined; - var block_accs: [MaxBlockArguments / 2]mlir.Value = undefined; - for (0..inputs.len) |i| { - block_inputs[i] = block.argument(i); - block_accs[i] = block.argument(inputs.len + i); - } - _ = blkfn(blkctx, ctx, block_inputs[0..inputs.len], block_accs[0..init_values.len]); - } +// var block_inputs: [MaxBlockArguments / 2]*const mlir.Value = undefined; +// var block_accs: [MaxBlockArguments / 2]*const mlir.Value = undefined; +// for (0..inputs.len) |i| { +// block_inputs[i] = block.argument(i); +// block_accs[i] = block.argument(inputs.len + i); +// } +// _ = blkfn(blkctx, ctx, block_inputs[0..inputs.len], block_accs[0..init_values.len]); +// } - return mlir.Operation.make(ctx, "stablehlo.reduce", .{ - .variadic_operands = &.{ inputs, init_values }, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimensions", .dense(ctx, .i64, dimensions) }, - }, - .location = location, - }); -} +// return mlir.Operation.make(ctx, "stablehlo.reduce", .{ +// .operands = .{ .variadic = &.{ inputs, init_values } }, +// .result_type_inference = true, +// .block = block, +// .attributes = &.{ +// .{ "dimensions", .dense(ctx, .i64, dimensions) }, +// }, +// .location = location, +// }); +// } -pub fn sort( - ctx: mlir.Context, - inputs: []const mlir.Value, - dimension: i64, - is_stable: bool, - blkctx: anytype, - compfn: fn (anytype, mlir.Context, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; +// pub fn sort( +// ctx: *mlir.Context, +// inputs: []const *const mlir.Value, +// dimension: i64, +// is_stable: bool, +// blkctx: anytype, +// compfn: fn (anytype, mlir.Context, []const *const mlir.Value) mlir.Operation, +// location: *const mlir.Location, +// ) *mlir.Operation { +// const MaxBlockArguments = 32; - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0 .. inputs.len * 2]; - var sort_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type: mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); - sort_elem_types[i * 2] = arg_type; - sort_elem_types[i * 2 + 1] = arg_type; - } - var block = mlir.Block.init(sort_elem_types[0 .. inputs.len * 2], locations) catch unreachable; +// const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0 .. inputs.len * 2]; +// var sort_elem_types: [MaxBlockArguments]*const mlir.Type = undefined; +// for (inputs, 0..) |input, i| { +// const arg_type: *const mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); +// sort_elem_types[i * 2] = arg_type; +// sort_elem_types[i * 2 + 1] = arg_type; +// } +// var block = mlir.Block.init(sort_elem_types[0 .. inputs.len * 2], locations) catch unreachable; - var block_inputs: [MaxBlockArguments]mlir.Value = undefined; - for (0..inputs.len * 2) |i| { - block_inputs[i] = block.argument(i); - } - _ = compfn(blkctx, ctx, block_inputs[0 .. inputs.len * 2]); +// var block_inputs: [MaxBlockArguments]*const mlir.Value = undefined; +// for (0..inputs.len * 2) |i| { +// block_inputs[i] = block.argument(i); +// } +// _ = compfn(blkctx, ctx, block_inputs[0 .. inputs.len * 2]); - return mlir.Operation.make(ctx, "stablehlo.sort", .{ - .variadic_operands = &.{inputs}, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimension", .int(ctx, .i64, dimension) }, - .{ "is_stable", .boolean(ctx, is_stable) }, - }, - .location = location, - }); -} +// return mlir.Operation.make(ctx, "stablehlo.sort", .{ +// .operands = .{ .variadic = &.{inputs} }, +// .result_type_inference = true, +// .block = block, +// .attributes = &.{ +// .{ "dimension", .int(ctx, .i64, dimension) }, +// .{ "is_stable", .boolean(ctx, is_stable) }, +// }, +// .location = location, +// }); +// } -pub fn dynamic_slice(ctx: mlir.Context, operand: mlir.Value, new_dims: []const i64, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { +pub fn dynamic_slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + new_dims: []const i64, + start_indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.dynamic_slice", .{ - .variadic_operands = &.{ &.{operand}, start_indices }, + .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, .result_type_inference = true, .attributes = &.{ - .{ "slice_sizes", .dense(ctx, .i64, new_dims) }, + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, new_dims)), }, .location = location, }); } -pub fn round_nearest_afz(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_afz", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - -pub fn round_nearest_even(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_even", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - pub const PadOpts = struct { low: []const i64, high: []const i64, interior: []const i64, }; -pub fn pad(ctx: mlir.Context, value: mlir.Value, padding_value: mlir.Value, opts: PadOpts, location: mlir.Location) mlir.Operation { +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, location: *const mlir.Location, opts: PadOpts) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.pad", .{ - .operands = &.{ value, padding_value }, + .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, .attributes = &.{ - .{ "edge_padding_low", .dense(ctx, .i64, opts.low) }, - .{ "edge_padding_high", .dense(ctx, .i64, opts.high) }, - .{ "interior_padding", .dense(ctx, .i64, opts.interior) }, + .named(ctx, "edge_padding_low", mlir.denseArrayAttribute(ctx, .i64, opts.low)), + .named(ctx, "edge_padding_high", mlir.denseArrayAttribute(ctx, .i64, opts.high)), + .named(ctx, "interior_padding", mlir.denseArrayAttribute(ctx, .i64, opts.interior)), }, .location = location, }); @@ -562,18 +565,18 @@ pub const TriangularSolveOpts = struct { left_side: bool, lower: bool, unit_diagonal: bool, - transpose_a: Transpose.Type, + transpose_a: TransposeAttribute.Type, }; -pub fn triangular_solve(ctx: mlir.Context, value: mlir.Value, other: mlir.Value, location: mlir.Location, opts: TriangularSolveOpts) mlir.Operation { +pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, opts: TriangularSolveOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ - .operands = &.{ value, other }, + .operands = .{ .flat = &.{ value, other } }, .result_type_inference = true, .attributes = &.{ - .{ "left_side", .i1FromBool(ctx, opts.left_side) }, - .{ "lower", .i1FromBool(ctx, opts.lower) }, - .{ "unit_diagonal", .i1FromBool(ctx, opts.unit_diagonal) }, - .{ "transpose_a", Transpose.init(ctx, opts.transpose_a).asAttr() }, + .named(ctx, "left_side", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.left_side))), + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.lower))), + .named(ctx, "unit_diagonal", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.unit_diagonal))), + .named(ctx, "transpose_a", transposeAttribute(ctx, opts.transpose_a)), }, .location = location, }); @@ -584,74 +587,132 @@ pub const FftOpts = struct { length: []const i64, }; -pub fn fft(ctx: mlir.Context, value: mlir.Value, location: mlir.Location, opts: FftOpts) mlir.Operation { +pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, opts: FftOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.fft", .{ - .operands = &.{value}, + .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .{ "fft_type", FftType.init(ctx, opts.kind).asAttr() }, - .{ "fft_length", .dense(ctx, .i64, opts.length) }, + .named(ctx, "fft_type", fftType(ctx, opts.kind)), + .named(ctx, "fft_length", mlir.denseArrayAttribute(ctx, .i64, opts.length)), }, .location = location, }); } -pub fn rng(ctx: mlir.Context, a: mlir.Value, b: mlir.Value, shape: mlir.Value, rng_distribution: RngDistribution.Type, location: mlir.Location) mlir.Operation { +pub const RngDistribution = opaque { + const M = mlir.Methods(RngDistribution, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + UNIFORM, + NORMAL, + }; + + pub fn get(ctx: *mlir.Context, value_: Type) *const RngDistribution { + return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + } + + pub fn value(self: *const RngDistribution) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngDistributionAttrGetValue(self.ptr())), + ) orelse unreachable; + } +}; + +pub fn rngDistribution(ctx: *mlir.Context, value: RngDistribution.Type) *const mlir.Attribute { + return @ptrCast(RngDistribution.get(ctx, value)); +} + +pub const RngAlgorithm = opaque { + const M = mlir.Methods(RngAlgorithm, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + DEFAULT, + THREE_FRY, + PHILOX, + }; + + pub fn get(ctx: *mlir.Context, value_: Type) *const RngAlgorithm { + return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + } + + pub fn value(self: *const RngAlgorithm) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngAlgorithmAttrGetValue(self.ptr())), + ) orelse unreachable; + } +}; + +pub fn rngAlgorithm(ctx: *mlir.Context, value: RngAlgorithm.Type) *const mlir.Attribute { + return @ptrCast(RngAlgorithm.get(ctx, value)); +} + +pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng", .{ - .operands = &.{ a, b, shape }, + .operands = .{ .flat = &.{ a, b, shape } }, .result_type_inference = true, .attributes = &.{ - .{ "rng_distribution", RngDistribution.init(ctx, rng_distribution).asAttr() }, + .named(ctx, "rng_distribution", rngDistribution(ctx, rng_distribution)), }, .location = location, }); } -pub fn rng_bit_generator(ctx: mlir.Context, rng_algorithm: RngAlgorithm.Type, initial_state: mlir.Value, res_state_type: mlir.Type, res_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn rng_bit_generator( + ctx: *mlir.Context, + rng_algorithm: RngAlgorithm.Type, + initial_state: *const mlir.Value, + res_state_type: *const mlir.Type, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ - .operands = &.{initial_state}, + .operands = .{ .flat = &.{initial_state} }, .results = &.{ res_state_type, res_type }, .attributes = &.{ - .{ "rng_algorithm", RngAlgorithm.init(ctx, rng_algorithm).asAttr() }, + .named(ctx, "rng_algorithm", rngAlgorithm(ctx, rng_algorithm)), }, .location = location, }); } -pub fn reduce_precision(ctx: mlir.Context, value: mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: mlir.Location) mlir.Operation { +pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ - .operands = &.{value}, + .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .{ "exponent_bits", .int(ctx, .i32, exponent_bits) }, - .{ "mantissa_bits", .int(ctx, .i32, mantissa_bits) }, + .named(ctx, "exponent_bits", mlir.integerAttribute(ctx, .i32, exponent_bits)), + .named(ctx, "mantissa_bits", mlir.integerAttribute(ctx, .i32, mantissa_bits)), }, .location = location, }); } -pub fn dynamic_update_slice(ctx: mlir.Context, operand: mlir.Value, update: mlir.Value, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { +pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, update: *const mlir.Value, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ - .variadic_operands = &.{ &.{operand}, &.{update}, start_indices }, + .operands = .{ .variadic = &.{ &.{operand}, &.{update}, start_indices } }, .result_type_inference = true, .location = location, }); } -pub fn tuple(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.tuple", .{ - .operands = values, - .result_type_inference = true, - .location = location, - }); -} - -pub fn get_tuple_element(ctx: mlir.Context, tuple_value: mlir.Value, index: i64, location: mlir.Location) mlir.Operation { +pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ - .operands = &.{tuple_value}, + .operands = .{ .flat = &.{tuple_value} }, .result_type_inference = true, .attributes = &.{ - .{ "index", .int(ctx, .i32, index) }, + .named(ctx, "index", mlir.integerAttribute(ctx, .i32, index)), }, .location = location, }); @@ -679,46 +740,44 @@ pub const ConvolutionOpts = struct { }; pub fn convolution( - ctx: mlir.Context, - lhs: mlir.Value, - rhs: mlir.Value, + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, opts: ConvolutionOpts, - res_type: mlir.Type, - location: mlir.Location, -) mlir.Operation { - var max_precisions: [2]mlir.Attribute = undefined; + location: *const mlir.Location, +) *mlir.Operation { + var max_precisions: [2]*const mlir.Attribute = undefined; for (opts.precision_config, 0..) |p, i| { - max_precisions[i] = PrecisionAttribute.init(ctx, p).asAttr(); + max_precisions[i] = precisionAttribute(ctx, p); } var window_reversal: [3]i32 = undefined; for (opts.window_reversal, 0..) |w, i| { window_reversal[i] = @intCast(@intFromBool(w)); } return mlir.Operation.make(ctx, "stablehlo.convolution", .{ - .operands = &.{ lhs, rhs }, - .results = &.{res_type}, + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, .attributes = &.{ - .{ "window_strides", .dense(ctx, .i64, opts.window_strides) }, - .{ "padding", .denseElements(ctx, opts.pad_shape, .i64, opts.pad_value) }, - .{ "lhs_dilation", .dense(ctx, .i64, opts.lhs_dilation) }, - .{ "rhs_dilation", .dense(ctx, .i64, opts.rhs_dilation) }, - .{ "window_reversal", .dense(ctx, .bool, window_reversal[0..opts.window_reversal.len]) }, - .{ - "dimension_numbers", ConvDimensionNumbersAttribute.init(ctx, .{ - .input_batch_dimension = opts.input_batch_dimension, - .input_feature_dimension = opts.input_feature_dimension, - .input_spatial_dimensions = opts.input_spatial_dimensions, - .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, - .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, - .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, - .output_batch_dimension = opts.output_batch_dimension, - .output_feature_dimension = opts.output_feature_dimension, - .output_spatial_dimensions = opts.output_spatial_dimensions, - }).asAttr(), - }, - .{ "feature_group_count", .int(ctx, .i64, opts.feature_group_count) }, - .{ "batch_group_count", .int(ctx, .i64, opts.batch_group_count) }, - .{ "precision_config", .array(ctx, &max_precisions) }, + .named(ctx, "window_strides", mlir.denseArrayAttribute(ctx, .i64, opts.window_strides)), + .named(ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(opts.pad_shape, mlir.integerType(ctx, .i64)).shaped(), opts.pad_value)), + .named(ctx, "lhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.lhs_dilation)), + .named(ctx, "rhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.rhs_dilation)), + .named(ctx, "window_reversal", mlir.denseArrayAttribute(ctx, .bool, window_reversal[0..opts.window_reversal.len])), + .named(ctx, "dimension_numbers", convDimensionNumbersAttribute(ctx, .{ + .input_batch_dimension = opts.input_batch_dimension, + .input_feature_dimension = opts.input_feature_dimension, + .input_spatial_dimensions = opts.input_spatial_dimensions, + .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, + .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, + .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, + .output_batch_dimension = opts.output_batch_dimension, + .output_feature_dimension = opts.output_feature_dimension, + .output_spatial_dimensions = opts.output_spatial_dimensions, + })), + .named(ctx, "feature_group_count", mlir.integerAttribute(ctx, .i64, opts.feature_group_count)), + .named(ctx, "batch_group_count", mlir.integerAttribute(ctx, .i64, opts.batch_group_count)), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &max_precisions)), }, .location = location, }); @@ -732,261 +791,350 @@ pub const CustomCallOpts = struct { typed_ffi = 4, }; - call_target_name: [:0]const u8, - has_side_effect: bool, - backend_config: ?mlir.Attribute, + pub const BackendConfig = union(ApiVersion) { + original: []const u8, + status_returning: []const u8, + status_returning_unified: []const u8, + typed_ffi: *const mlir.DictionaryAttribute, + }; + + call_target_name: []const u8, + has_side_effect: ?bool = null, + api_version: ?ApiVersion = null, + backend_config: ?BackendConfig = null, operand_layouts: ?[]const []const usize = null, result_layouts: ?[]const []const usize = null, - output_operand_aliases: []const i64 = &.{}, - additional_attributes: []const mlir.AttrTuple = &.{}, - api_version: ApiVersion, + output_operand_aliases: ?[]const i64 = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, }; -pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { +pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, opts: CustomCallOpts, location: *const mlir.Location) *mlir.Operation { const MAX_OPERANDS = 64; const MAX_RESULTS = 16; - - const backend_config = opts.backend_config orelse mlir.Attribute.string(ctx, ""); - if (@intFromEnum(opts.api_version) < @intFromEnum(CustomCallOpts.ApiVersion.typed_ffi)) { - stdx.debug.assert( - backend_config.isA(mlir.StringAttribute), - "API version < 4 requires a string as backend_config, got {}", - .{backend_config}, - ); - } else { - stdx.debug.assert( - backend_config.isA(mlir.DictionaryAttribute), - "API version >= 4 requires a dictionary as backend_config, got {}", - .{backend_config}, - ); - } - - var attrs: stdx.BoundedArray(mlir.AttrTuple, 32) = .{}; - attrs.appendSliceAssumeCapacity(&[_]mlir.AttrTuple{ - .{ "api_version", .int(ctx, .i32, @intFromEnum(opts.api_version)) }, - .{ "call_target_name", .string(ctx, opts.call_target_name) }, - .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, - .{ "backend_config", backend_config }, - }); - - { - var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - for (opts.output_operand_aliases) |alias| { - output_operand_aliases.appendAssumeCapacity( - OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), - ); - } - attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); - } - - const MINOR_TO_MAJOR = blk: { - const MAX_RANK = 8; - var ret: [MAX_RANK]usize = undefined; - for (0..MAX_RANK) |i| { - ret[i] = @intCast(MAX_RANK - i - 1); + const MINOR_TO_MAJOR = comptime blk: { + var ret: [mlir.ShapedType.MAX_RANK]usize = undefined; + for (0..ret.len) |i| { + ret[i] = @intCast(mlir.ShapedType.MAX_RANK - i - 1); } break :blk ret; }; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .{}; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "call_target_name", mlir.stringAttribute(ctx, opts.call_target_name)), + }); + attrs.appendSliceAssumeCapacity(opts.additional_attributes); + + if (opts.has_side_effect) |has_side_effects| { + attrs.appendAssumeCapacity( + .named(ctx, "has_side_effect", mlir.boolAttribute(ctx, has_side_effects)), + ); + } + + const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; + if (opts.api_version orelse backend_config_api_version) |v| { + attrs.appendAssumeCapacity( + .named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v))), + ); + } + if (opts.backend_config) |backend_config| { + attrs.appendAssumeCapacity( + .named(ctx, "backend_config", switch (backend_config) { + .typed_ffi => |v| @ptrCast(v), + inline else => |v| mlir.stringAttribute(ctx, v), + }), + ); + } + + if (opts.output_operand_aliases) |output_operand_aliases| { + var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; + for (output_operand_aliases) |alias| { + buffer.appendAssumeCapacity( + outputOperandAliasAttribute(ctx, .{ .operand_index = alias }), + ); + } + attrs.appendAssumeCapacity( + .named(ctx, "output_operand_aliases", mlir.arrayAttribute(ctx, buffer.constSlice())), + ); + } + + var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .{}; if (opts.operand_layouts) |layouts| { - var operand_layouts: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; for (layouts) |ol| { - operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + ol, + ), + ); } - attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); } else { - const operand_layouts = blk: { - var ret: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; - for (inputs) |input| { - const ranked_type = input.getType().as(mlir.RankedTensorType).?; - const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; - ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); - } - break :blk ret; - }; - attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); + for (inputs) |input| { + const shaped_type = input.type_().isA(mlir.ShapedType).?; + const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + ol, + ), + ); + } } + attrs.appendAssumeCapacity( + .named(ctx, "operand_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + ); + layouts_buffer.clear(); if (opts.result_layouts) |layouts| { - var result_layouts: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; for (layouts) |rl| { - result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + rl, + ), + ); } - attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); } else { - const result_layouts = blk: { - var ret: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - for (res_types) |t| { - const ranked_t = t.as(mlir.RankedTensorType).?; - const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; - ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); - } - break :blk ret; - }; - attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); + for (result_types) |result_type| { + const shaped_type = result_type.isA(mlir.ShapedType).?; + const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + rl, + ), + ); + } } - - attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); + attrs.appendAssumeCapacity( + .named(ctx, "result_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + ); return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ - .operands = inputs, - .results = res_types, + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, .attributes = attrs.constSlice(), .location = location, }); } -pub const DotDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, +pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{value}, .{ + .call_target_name = "CreateBuffer", + .api_version = .typed_ffi, + }, location); +} - pub const is_a_fn = c.stablehloAttributeIsADotDimensionNumbers; - const Self = DotDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ + .call_target_name = "Pin", + .api_version = .typed_ffi, + }, location); +} - pub fn init(ctx: mlir.Context, args: struct { +pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ + .call_target_name = "Unpin", + .api_version = .typed_ffi, + }, location); +} + +pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ + .results = .{ .flat = &.{mlir.rankedTensorType(&.{}, mlir.integerType(ctx, .u32))} }, + .location = location, + }); +} + +pub const DotDimensionNumbersAttribute = opaque { + const M = mlir.Methods(DotDimensionNumbersAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { lhs_batching_dimensions: []const i64, rhs_batching_dimensions: []const i64, lhs_contracting_dimensions: []const i64, rhs_contracting_dimensions: []const i64, - }) Self { - return .{ - ._inner = c.stablehloDotDimensionNumbersGet( - ctx._inner, - @intCast(args.lhs_batching_dimensions.len), - args.lhs_batching_dimensions.ptr, - @intCast(args.rhs_batching_dimensions.len), - args.rhs_batching_dimensions.ptr, - @intCast(args.lhs_contracting_dimensions.len), - args.lhs_contracting_dimensions.ptr, - @intCast(args.rhs_contracting_dimensions.len), - args.rhs_contracting_dimensions.ptr, - ), - }; + }; + + pub fn init(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttribute { + return @ptrCast(c.stablehloDotDimensionNumbersGet( + ctx.ptr(), + @intCast(args.lhs_batching_dimensions.len), + args.lhs_batching_dimensions.ptr, + @intCast(args.rhs_batching_dimensions.len), + args.rhs_batching_dimensions.ptr, + @intCast(args.lhs_contracting_dimensions.len), + args.lhs_contracting_dimensions.ptr, + @intCast(args.rhs_contracting_dimensions.len), + args.rhs_contracting_dimensions.ptr, + ).ptr); } - pub fn getLhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self._inner)); + pub fn getLhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self.ptr())); } - pub fn getLhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self._inner, @intCast(pos)); + pub fn getLhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self._inner)); + pub fn getRhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self.ptr())); } - pub fn getRhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self._inner, @intCast(pos)); + pub fn getRhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getLhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self._inner)); + pub fn getLhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self.ptr())); } - pub fn getLhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self._inner, @intCast(pos)); + pub fn getLhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self._inner)); + pub fn getRhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self.ptr())); } - pub fn getRhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self._inner, @intCast(pos)); + pub fn getRhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } }; -pub const GatherDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, +pub fn dotDimensionNumbersAttribute(ctx: *mlir.Context, args: DotDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(DotDimensionNumbersAttribute.init(ctx, args)); +} - pub const is_a_fn = c.stablehloAttributeIsAGatherDimensionNumbers; - const Self = GatherDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const GatherDimensionNumbersAttribute = opaque { + const M = mlir.Methods(GatherDimensionNumbersAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.eql(c.mlirAttributePrint); pub fn init( - ctx: mlir.Context, + ctx: *mlir.Context, offset_dims: []const i64, collapsed_slice_dims: []const i64, operand_batching_dims: []const i64, start_indices_batching_dims: []const i64, start_index_map: []const i64, index_vector_dim: i64, - ) Self { - return .{ - ._inner = c.stablehloGatherDimensionNumbersGet( - ctx._inner, - @intCast(offset_dims.len), - offset_dims.ptr, - @intCast(collapsed_slice_dims.len), - collapsed_slice_dims.ptr, - @intCast(operand_batching_dims.len), - operand_batching_dims.ptr, - @intCast(start_indices_batching_dims.len), - start_indices_batching_dims.ptr, - @intCast(start_index_map.len), - start_index_map.ptr, - index_vector_dim, - ), - }; + ) *const GatherDimensionNumbersAttribute { + return @ptrCast(c.stablehloGatherDimensionNumbersGet( + ctx.ptr(), + @intCast(offset_dims.len), + offset_dims.ptr, + @intCast(collapsed_slice_dims.len), + collapsed_slice_dims.ptr, + @intCast(operand_batching_dims.len), + operand_batching_dims.ptr, + @intCast(start_indices_batching_dims.len), + start_indices_batching_dims.ptr, + @intCast(start_index_map.len), + start_index_map.ptr, + index_vector_dim, + ).ptr); } - pub fn getOffsetDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self._inner)); + pub fn getOffsetDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self.ptr())); } - pub fn getOffsetDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self._inner, @intCast(pos)); + pub fn getOffsetDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self.ptr(), @intCast(pos)); } - pub fn getCollapsedSliceDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self._inner)); + pub fn getCollapsedSliceDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self.ptr())); } - pub fn getCollapsedSliceDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self._inner, @intCast(pos)); + pub fn getCollapsedSliceDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self._inner)); + pub fn getStartIndexMapSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self.ptr())); } - pub fn getOperandBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self._inner)); + pub fn getOperandBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self.ptr())); } - pub fn getOperandBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self._inner, @intCast(pos)); + pub fn getOperandBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndicesBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self._inner)); + pub fn getStartIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self.ptr())); } - pub fn getStartIndicesBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self._inner, @intCast(pos)); + pub fn getStartIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self._inner, @intCast(pos)); + pub fn getStartIndexMapElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self.ptr(), @intCast(pos)); } - pub fn getIndexVectorDim(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self._inner)); + pub fn getIndexVectorDim(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self.ptr())); } }; -pub const ConvDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, +pub fn gatherDimensionNumbersAttribute( + ctx: *mlir.Context, + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, +) *const mlir.Attribute { + return @ptrCast(GatherDimensionNumbersAttribute.init( + ctx, + offset_dims, + collapsed_slice_dims, + operand_batching_dims, + start_indices_batching_dims, + start_index_map, + index_vector_dim, + )); +} - pub const is_a_fn = c.stablehloAttributeIsAConvDimensionNumbers; - const Self = ConvDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const ConvDimensionNumbersAttribute = opaque { + const M = mlir.Methods(ConvDimensionNumbersAttribute, mlir.Attribute); - pub fn init(ctx: mlir.Context, args: struct { + pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { input_batch_dimension: i64, input_feature_dimension: i64, input_spatial_dimensions: []const i64, @@ -996,106 +1144,116 @@ pub const ConvDimensionNumbersAttribute = struct { output_batch_dimension: i64, output_feature_dimension: i64, output_spatial_dimensions: []const i64, - }) Self { - return .{ - ._inner = c.stablehloConvDimensionNumbersGet( - ctx._inner, - args.input_batch_dimension, - args.input_feature_dimension, - @intCast(args.input_spatial_dimensions.len), - args.input_spatial_dimensions.ptr, - args.kernel_input_feature_dimension, - args.kernel_output_feature_dimension, - @intCast(args.kernel_spatial_dimensions.len), - args.kernel_spatial_dimensions.ptr, - args.output_batch_dimension, - args.output_feature_dimension, - @intCast(args.output_spatial_dimensions.len), - args.output_spatial_dimensions.ptr, - ), - }; + }; + + pub fn init(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttribute { + return @ptrCast(c.stablehloConvDimensionNumbersGet( + ctx.ptr(), + args.input_batch_dimension, + args.input_feature_dimension, + @intCast(args.input_spatial_dimensions.len), + args.input_spatial_dimensions.ptr, + args.kernel_input_feature_dimension, + args.kernel_output_feature_dimension, + @intCast(args.kernel_spatial_dimensions.len), + args.kernel_spatial_dimensions.ptr, + args.output_batch_dimension, + args.output_feature_dimension, + @intCast(args.output_spatial_dimensions.len), + args.output_spatial_dimensions.ptr, + ).ptr); } - pub fn getInputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputBatchDimension(self._inner); + pub fn getInputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetInputBatchDimension(self.ptr()); } - pub fn getInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self._inner); + pub fn getInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self.ptr()); } - pub fn getInputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self._inner)); + pub fn getInputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self.ptr())); } - pub fn getInputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self._inner, @intCast(pos)); + pub fn getInputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getKernelInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self._inner); + pub fn getKernelInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self.ptr()); } - pub fn getKernelOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self._inner); + pub fn getKernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self.ptr()); } - pub fn getKernelSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self._inner)); + pub fn getKernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self.ptr())); } - pub fn getKernelSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self._inner, @intCast(pos)); + pub fn getKernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getOutputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self._inner); + pub fn getOutputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self.ptr()); } - pub fn getOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self._inner); + pub fn getOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self.ptr()); } - pub fn getOutputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self._inner)); + pub fn getOutputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self.ptr())); } - pub fn getOutputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self._inner, @intCast(pos)); + pub fn getOutputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } }; -pub const OutputOperandAliasAttribute = struct { - _inner: c.MlirAttribute, +pub fn convDimensionNumbersAttribute(ctx: *mlir.Context, args: ConvDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(ConvDimensionNumbersAttribute.init(ctx, args)); +} - pub const is_a_fn = c.stablehloAttributeIsAOutputOperandAlias; - pub const asAttr = mlir.Attribute.fromAny(OutputOperandAliasAttribute); - pub const eql = mlir.Attribute.eqlAny(OutputOperandAliasAttribute); +pub const OutputOperandAliasAttribute = opaque { + const M = mlir.Methods(OutputOperandAliasAttribute, c.MlirAttribute); - pub fn init( - ctx: mlir.Context, - output_tuple_indices: []const i64, + pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { + output_tuple_indices: []const i64 = &.{}, operand_index: i64, - operand_tuple_indices: []const i64, - ) OutputOperandAliasAttribute { - return .{ ._inner = c.stablehloOutputOperandAliasGet( - ctx._inner, - @intCast(output_tuple_indices.len), - output_tuple_indices.ptr, - @intCast(operand_index), - @intCast(operand_tuple_indices.len), - operand_tuple_indices.ptr, - ) }; + operand_tuple_indices: []const i64 = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttribute { + return @ptrCast(c.stablehloOutputOperandAliasGet( + ctx.ptr(), + @intCast(args.output_tuple_indices.len), + @ptrCast(args.output_tuple_indices), + @intCast(args.operand_index), + @intCast(args.operand_tuple_indices.len), + @ptrCast(args.operand_tuple_indices), + ).ptr); } }; -pub const PrecisionAttribute = struct { - _inner: c.MlirAttribute, +pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(OutputOperandAliasAttribute.get(ctx, args)); +} - pub const is_a_fn = c.stablehloAttributeIsAPrecisionAttr; - const Self = PrecisionAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const PrecisionAttribute = opaque { + const M = mlir.Methods(PrecisionAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Precision = enum { DEFAULT, @@ -1103,23 +1261,27 @@ pub const PrecisionAttribute = struct { HIGHEST, }; - pub fn init(ctx: mlir.Context, value: Precision) Self { - return .{ ._inner = c.stablehloPrecisionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Precision) *const PrecisionAttribute { + return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Precision { - const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self._inner)); + pub fn getValue(self: *const PrecisionAttribute) Precision { + const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self.ptr())); return std.meta.stringToEnum(Precision, value) orelse unreachable; } }; -pub const ComparisonDirection = struct { - _inner: c.MlirAttribute, +pub fn precisionAttribute(ctx: *mlir.Context, precision: PrecisionAttribute.Precision) *const mlir.Attribute { + return @ptrCast(PrecisionAttribute.init(ctx, precision)); +} - pub const is_a_fn = c.stablehloAttributeIsAComparisonDirectionAttr; - const Self = ComparisonDirection; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const ComparisonDirection = opaque { + const M = mlir.Methods(ComparisonDirection, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Direction = enum { EQ, @@ -1130,23 +1292,27 @@ pub const ComparisonDirection = struct { LT, }; - pub fn init(ctx: mlir.Context, value: Direction) Self { - return .{ ._inner = c.stablehloComparisonDirectionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Direction) *const ComparisonDirection { + return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Direction { - const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self._inner)); + pub fn getValue(self: *const ComparisonDirection) Direction { + const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); return std.meta.stringToEnum(Direction, value) orelse unreachable; } }; -pub const CompareType = struct { - _inner: c.MlirAttribute, +pub fn comparisonDirection(ctx: *mlir.Context, value: ComparisonDirection.Direction) *const mlir.Attribute { + return @ptrCast(ComparisonDirection.init(ctx, value)); +} - pub const is_a_fn = c.stablehloAttributeIsAComparisonTypeAttr; - const Self = CompareType; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const CompareType = opaque { + const M = mlir.Methods(CompareType, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { SIGNED, @@ -1155,23 +1321,27 @@ pub const CompareType = struct { TOTALORDER, }; - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloComparisonTypeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Type) *const CompareType { + return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self._inner)); + pub fn getValue(self: *const CompareType) Type { + const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub const Transpose = struct { - _inner: c.MlirAttribute, +pub fn compareType(ctx: *mlir.Context, value: CompareType.Type) *const mlir.Attribute { + return @ptrCast(CompareType.init(ctx, value)); +} - pub const is_a_fn = c.stablehloAttributeIsATransposeAttr; - const Self = Transpose; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const TransposeAttribute = opaque { + const M = mlir.Methods(TransposeAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsATransposeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { NO_TRANSPOSE, @@ -1179,23 +1349,27 @@ pub const Transpose = struct { ADJOINT, }; - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloTransposeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Type) *const TransposeAttribute { + return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self._inner)); + pub fn getValue(self: *const TransposeAttribute) Type { + const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub const FftType = struct { - _inner: c.MlirAttribute, +pub fn transposeAttribute(ctx: *mlir.Context, value: TransposeAttribute.Type) *const mlir.Attribute { + return @ptrCast(TransposeAttribute.init(ctx, value)); +} - pub const is_a_fn = c.stablehloAttributeIsAFftTypeAttr; - const Self = FftType; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const FftType = opaque { + const M = mlir.Methods(FftType, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { FFT, @@ -1204,122 +1378,45 @@ pub const FftType = struct { IRFFT, }; - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloFftTypeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Type) *const FftType { + return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self._inner)); + pub fn getValue(self: *const FftType) Type { + const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub const RngDistribution = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsARngDistributionAttr; - const Self = RngDistribution; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - UNIFORM, - NORMAL, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloRngDistributionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngDistributionAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const RngAlgorithm = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsARngAlgorithmAttr; - const Self = RngAlgorithm; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - DEFAULT, - THREE_FRY, - PHILOX, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloRngAlgorithmAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngAlgorithmAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub fn stablehloVersionFromCompatibilityRequirement(requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { - const state = struct { - var buf: [32]u8 = undefined; - - fn call(req: c.MlirStablehloCompatibilityRequirement) []u8 { - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - c.stablehloVersionFromCompatibilityRequirement(req, (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); - - return buf[0..stream.pos]; - } - }; - - return state.call(requirement); +pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { + return @ptrCast(FftType.init(ctx, value)); } -pub fn stablehloGetSmallerVersion(version1: []const u8, version2: []const u8) []const u8 { +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: mlir.StringCallbackCtx = .{ .writer = &writer }; + _ = @call(.auto, streamFn, args ++ .{ mlir.stringCallback, &sctx }); + return writer.buffered(); +} + +pub fn versionFromCompatibilityRequirement(buffer: []u8, requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { + return stringFromStream(buffer, c.stablehloVersionFromCompatibilityRequirement, .{requirement}); +} + +pub fn smallerVersion(version1: []const u8, version2: []const u8) []const u8 { var buf: [32]u8 = undefined; - - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - _ = c.stablehloGetSmallerVersion(mlir.stringRef(version1), mlir.stringRef(version2), (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); - - return if (std.mem.eql(u8, buf[0..stream.pos], version1)) version1 else version2; + const result = stringFromStream(&buf, c.stablehloGetSmallerVersion, .{ stringRef(version1), stringRef(version2) }); + return if (std.mem.eql(u8, result, version1)) version1 else version2; } -pub fn getCurrentVersion() []const u8 { +pub fn currentVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; var str: []const u8 = undefined; var once = std.once(call); fn call() void { - var stream = std.io.fixedBufferStream(&buf); - var writer_ = stream.writer(); - const ContextWriter = @TypeOf(writer_); - - c.stablehloGetCurrentVersion((struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const writer: *ContextWriter = @ptrCast(@alignCast(userdata)); - _ = writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &writer_); - - str = buf[0..stream.pos]; + str = stringFromStream(&buf, c.stablehloGetCurrentVersion, .{}); } }; @@ -1327,25 +1424,14 @@ pub fn getCurrentVersion() []const u8 { return state.str; } -pub fn getMinimumVersion() []const u8 { +pub fn minimumVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; var str: []const u8 = undefined; var once = std.once(call); fn call() void { - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - c.stablehloGetMinimumVersion((struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); - - str = buf[0..stream.pos]; + str = stringFromStream(&buf, c.stablehloGetMinimumVersion, .{}); } }; @@ -1353,14 +1439,51 @@ pub fn getMinimumVersion() []const u8 { return state.str; } -pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: anytype) !void { - var context = .{ .writer = writer }; - const WriterContext = @TypeOf(context); +pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromStringRef( + stringRef(bytecode), + stringRef(target_version), + mlir.stringCallback, + &sctx, + ); + if (sctx.err) |err| { + return err; + } + if (result.value == 0) { + return error.InvalidMlirBytecodeVersion; + } +} - try mlir.successOr(c.stablehloSerializePortableArtifactFromStringRef(mlir.stringRef(bytecode), mlir.stringRef(target_version), (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context), error.InvalidMlirBytecodeVersion); +pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromModule( + module.ptr(), + stringRef(target_version), + mlir.stringCallback, + &sctx, + true, + ); + if (sctx.err) |err| { + return err; + } + if (c.mlirLogicalResultIsFailure(result)) { + return error.InvalidMlirBytecodeVersion; + } +} + +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .flat = &.{&.{value}} }, + .location = location, + .verify = false, + }); +} + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .variadic = &.{values} }, + .verify = false, + .location = location, + }); } diff --git a/mlir/llama.zig b/mlir/llama.zig new file mode 100644 index 00000000..c009827b --- /dev/null +++ b/mlir/llama.zig @@ -0,0 +1,905 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = @import("stdx"); + +const Tensor = @import("main.zig").Tensor; +const TensorDescriptor = @import("main.zig").TensorDescriptor; +const BufferStore4 = @import("main.zig").BufferStore4; +const BufferStore5 = @import("main.zig").BufferStore5; +const CompilationContext = @import("main.zig").CompilationContext; +const Exe = @import("main.zig").Exe; +const Bufferized = @import("main.zig").Bufferized; +const loadBuffersFromId = @import("main.zig").loadBuffersFromId; +const multiTransfer = @import("main.zig").multiTransfer; +const singleTransfer = @import("main.zig").singleTransfer; +const structTransfer = @import("main.zig").structTransfer; +const bufferTransfer = @import("main.zig").bufferTransfer; +const autoLoad = @import("main.zig").autoLoad; +const initBufferizedFrom = @import("main.zig").initBufferizedFrom; +const compile = @import("main.zig").compile; + +pub const LlamaLM = struct { + pub const Config = struct { + bos_token_id: u32, + eos_token_id: stdx.json.Union(union(enum) { + int: u32, + ints: []u32, + }), + num_hidden_layers: usize, + num_attention_heads: usize, + num_key_value_heads: usize, + rope_theta: f32, + max_position_embeddings: usize, + rms_norm_eps: f32, + hf_rope_impl: bool = true, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, + }; + + pub const Options = struct { + sampling_strategy: ?zml.nn.SamplingStrategy, + max_seq_len: usize, + qkv_type: QkvKind, + }; + + pub const LoadingOptions = struct { + merge_qkv_exe: ?Exe = null, + precompute_qkv_exe: ?Exe = null, + }; + + lm_head: ?Linear, + model: Llama, + + // Options controlling generation + gen_opts: zml.nn.SamplingStrategy = .{}, + config: Config, + + pub fn init(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, config: LlamaLM.Config, options: Options) !LlamaLM { + const lm_head: ?Linear = b: { + const lm_head_weight = buffer_store.withPrefix("lm_head").maybeCreateTensorWithTags("weight", .{ .dout, .d }) orelse break :b null; + break :b .{ .weight = lm_head_weight, .tag = zml.Shape.toTag(.d) }; + }; + + return .{ + .lm_head = lm_head, + .model = try .init(allocator, buffer_store.withPrefix("model"), config, options), + .gen_opts = options.sampling_strategy orelse .{}, + .config = config, + }; + } + + pub fn deinit(self: LlamaLM, allocator: std.mem.Allocator) void { + allocator.free(self.model.layers); + } + + pub fn loadBuffers(allocator: std.mem.Allocator, llama: LlamaLM, buffer_store: BufferStore5.View, platform: zml.Platform, options: LoadingOptions) !LlamaLMBufferized { + return .{ + .lm_head = if (llama.lm_head != null) try loadBuffersFromId(allocator, llama.lm_head.?, buffer_store.withPrefix("lm_head"), platform) else null, + .model = try Llama.loadBuffers(allocator, llama.model, buffer_store.withPrefix("model"), platform, options), + }; + } + + /// Predicts the token at `token_index` position. + /// Returns: + /// - updated `tokens`, + /// - updated KV cache + /// - a Rng state to allow for probabilistic generation + pub fn forward( + self: LlamaLM, + tokens_: Tensor, + token_index: Tensor, + kv_cache: KvCache, + rng: Tensor.Rng, + ) struct { Tensor, KvCache, Tensor.Rng } { + stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); + const tokens = tokens_.withPartialTags(.{.s}); + const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache); + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + } + + pub fn sampleTokens( + self: LlamaLM, + lm_head_: ?Linear, + out_: Tensor, + rng: Tensor.Rng, + opts: zml.nn.SamplingStrategy, + ) struct { Tensor, Tensor.Rng } { + const out = out_.withPartialTags(.{ .s, .d }); + + var logits = blk: { + if (lm_head_) |lm_head| { + break :blk lm_head.forward(out).rename(.{ .dout = .d }); + } else { + break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .d); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = sampleTokens2(logits, opts, rng); + return .{ next_tokens, new_rng }; + } +}; + +pub const LlamaLMBufferized = struct { + lm_head: ?Bufferized(Linear), + model: Bufferized(Llama), + + pub fn deinit(self: LlamaLMBufferized, allocator: std.mem.Allocator) void { + allocator.free(self.model.layers); + } +}; + +pub const Linear = struct { + weight: Tensor, + bias: ?Tensor = null, + tag: zml.Shape.Tag, + + pub fn initWithTag(buffer_store: BufferStore5.View, tag: anytype) Linear { + return .{ + .weight = buffer_store.createTensorWithTags("weight", .{ .dout, tag }), + .bias = buffer_store.maybeCreateTensorWithTags("bias", .{.dout}), + .tag = zml.Shape.toTag(tag), + }; + } + + pub fn forward(self: Linear, x: Tensor) Tensor { + var y = x.dot(self.weight.convert(x.dtype()), self.tag); + + // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); + return if (self.bias) |bias| y.add(bias.autoBroadcast()) else y; + } +}; + +pub const TokenEmbedding = struct { + weight: Tensor, + + pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { + stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); + stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); + return self.weight.gatherValues(0, idx, .{}); + } +}; + +pub const Llama = struct { + embed_tokens: TokenEmbedding, + norm: RmsNorm, + layers: []TransformerLayer, + + pub fn init(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, config: LlamaLM.Config, options: LlamaLM.Options) !Llama { + const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + //const custom_options = if (i != 0) options else b: { + // var custom_options = options; + // custom_options.qkv_type = .precomputed; + // break :b custom_options; + //}; + const custom_options = options; + var buffer: [16]u8 = undefined; + const number_prefix = try std.fmt.bufPrint(&buffer, "layers.{d}", .{i}); + layer.* = .init(buffer_store.withPrefix(number_prefix), config, custom_options); + } + + return .{ + .embed_tokens = .{ .weight = buffer_store.createTensorWithTags("embed_tokens.weight", .{ .x, .y }) }, + .norm = .init(buffer_store.withPrefix("norm"), config.rms_norm_eps), + .layers = layers, + }; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, model: Llama, buffer_store: BufferStore5.View, platform: zml.Platform, options: LlamaLM.LoadingOptions) !Bufferized(Llama) { + std.debug.print("Loading Llama\n", .{}); + const layers = try allocator.alloc(Bufferized(TransformerLayer), model.layers.len); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + var buffer: [16]u8 = undefined; + const number_prefix = try std.fmt.bufPrint(&buffer, "layers.{d}", .{i}); + std.debug.print("Loading layer {d}\n", .{i}); + layer.* = try TransformerLayer.loadBuffers(allocator, model.layers[i], buffer_store.withPrefix(number_prefix), platform, options); + } + + return .{ + .embed_tokens = try loadBuffersFromId(allocator, model.embed_tokens, buffer_store.withPrefix("embed_tokens"), platform), + .layers = layers, + .norm = try loadBuffersFromId(allocator, model.norm, buffer_store.withPrefix("norm"), platform), + }; + } + + /// Forward one token, using KV cache for previous tokens. + /// Returns result and updated KV cache. + pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { + const embeds = embed(self.embed_tokens, tokens); + var hidden = embeds; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + std.debug.print("hidden.shape: {f}\n", .{hidden.shape()}); + hidden, updated_kv_cache = layer.forward(hidden, tokens, token_index, updated_kv_cache.atLayer(i)); + } + const output = self.norm.forward(hidden); + + return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; + } + + pub fn embed(embed_tokens_: TokenEmbedding, tokens_: Tensor) Tensor { + return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); + } +}; + +pub const RmsNorm = struct { + weight: Tensor, + eps: f32 = 1e-5, + + pub fn init(buffer_store: BufferStore5.View, eps: f32) RmsNorm { + return .{ + .weight = buffer_store.createTensorWithTags("weight", .{.d}), + .eps = eps, + }; + } + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: Tensor) Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + const normalized = rmsNorm(x, .d, self.eps); + return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); + } +}; + +pub const TransformerLayer = struct { + input_layernorm: RmsNorm, + self_attn: SelfAttn, + post_attention_layernorm: RmsNorm, + mlp: Mlp, + + pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config, options: LlamaLM.Options) TransformerLayer { + return .{ + .input_layernorm = .init(buffer_store.withPrefix("input_layernorm"), config.rms_norm_eps), + .self_attn = SelfAttn.init(buffer_store.withPrefix("self_attn"), config, options), + .post_attention_layernorm = .init(buffer_store.withPrefix("post_attention_layernorm"), config.rms_norm_eps), + .mlp = .init(buffer_store.withPrefix("mlp")), + }; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, layer: TransformerLayer, buffer_store: BufferStore5.View, platform: zml.Platform, options: LlamaLM.LoadingOptions) !Bufferized(TransformerLayer) { + return .{ + .input_layernorm = try loadBuffersFromId(allocator, layer.input_layernorm, buffer_store.withPrefix("input_layernorm"), platform), + .self_attn = try SelfAttn.loadBuffers(allocator, layer.self_attn, buffer_store.withPrefix("self_attn"), platform, options), + .post_attention_layernorm = try loadBuffersFromId(allocator, layer.post_attention_layernorm, buffer_store.withPrefix("post_attention_layernorm"), platform), + .mlp = try loadBuffersFromId(allocator, layer.mlp, buffer_store.withPrefix("mlp"), platform), + }; + } + + pub fn forward( + self: TransformerLayer, + x0: Tensor, + tokens: Tensor, + token_index: Tensor, + kv_cache: KvCache, + ) struct { Tensor, KvCache } { + // Self Attention + //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); + //stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); + + const params: SelfAttn.ForwardParams = switch (self.self_attn.proj) { + .merged => .{ .merged = self.input_layernorm.forward(x0) }, + .separate => .{ .separate = self.input_layernorm.forward(x0) }, + .precomputed => .{ .precomputed = .{ .x = x0, .tokens = tokens } }, + }; + const delta0, const updated_kv_cache = self.self_attn.forward(params, token_index, kv_cache); + const x1 = x0.add(delta0); + + // Fully Connected + const x1_normalized = self.post_attention_layernorm.forward(x1); + const x2 = self.mlp.forward(x1_normalized).add(x1).rename(.{ .dout = .d }); + + return .{ x2.reuseBuffer(x0), updated_kv_cache }; + } +}; + +const QkvKind = enum { + merged, + separate, + precomputed, +}; + +pub const SeparateQkv = struct { + q_proj: Linear, + k_proj: Linear, + v_proj: Linear, + + pub fn init(buffer_store: BufferStore5.View) SeparateQkv { + const q_proj = Linear.initWithTag(buffer_store.withPrefix("q_proj"), .d); + const k_proj = Linear.initWithTag(buffer_store.withPrefix("k_proj"), .d); + const v_proj = Linear.initWithTag(buffer_store.withPrefix("v_proj"), .d); + return .{ .q_proj = q_proj, .k_proj = k_proj, .v_proj = v_proj }; + } +}; + +pub const SelfAttn = struct { + pub const Proj = union(QkvKind) { + merged: Linear, + separate: SeparateQkv, + precomputed: Tensor, + }; + proj: Proj, + + o_proj: Linear, + num_heads: i64, + num_kv_heads: i64, + rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = 10_000, + }, + + pub fn init(buffer_store: BufferStore5.View, config: LlamaLM.Config, options: LlamaLM.Options) SelfAttn { + const proj: Proj = switch (options.qkv_type) { + .separate => .{ .separate = SeparateQkv.init(buffer_store) }, + .merged => b: { + const q_weight_shape = buffer_store.getShape("q_proj.weight").?; + const k_weight_shape = buffer_store.getShape("k_proj.weight").?; + const v_weight_shape = buffer_store.getShape("v_proj.weight").?; + const qkv_weight_shape = zml.Shape.concatenate(&.{ q_weight_shape, k_weight_shape, v_weight_shape }, 0); + break :b .{ .merged = .{ .weight = Tensor.init(qkv_weight_shape).withTags(.{ .dout, .d }), .tag = zml.Shape.toTag(.d) } }; + }, + .precomputed => b: { + const q_weight_shape = buffer_store.getShape("q_proj.weight").?; + const k_weight_shape = buffer_store.getShape("k_proj.weight").?; + const v_weight_shape = buffer_store.getShape("v_proj.weight").?; + const embedding_shape = buffer_store.getShapeOpts("model.embed_tokens.weight", .{ .no_prefix = true }).?; + const qkv_weight_shape = zml.Shape.concatenate(&.{ q_weight_shape, k_weight_shape, v_weight_shape }, 0); + const precomputed_qkv_weight_shape = zml.Shape.init(.{ embedding_shape.dim(0), qkv_weight_shape.dim(0) }, qkv_weight_shape.dtype()); + + break :b .{ .precomputed = Tensor.init(precomputed_qkv_weight_shape).withTags(.{ .voc, .d }) }; + }, + }; + + return .{ + .proj = proj, + .o_proj = .initWithTag(buffer_store.withPrefix("o_proj"), .d), + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }, + }; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, self_attn: SelfAttn, buffer_store: BufferStore5.View, platform: zml.Platform, options: LlamaLM.LoadingOptions) !Bufferized(SelfAttn) { + return switch (self_attn.proj) { + .separate => |separate| .{ + .proj = .{ + .separate = .{ + .q_proj = try loadBuffersFromId(allocator, separate.q_proj, buffer_store.withPrefix("q_proj"), platform), + .k_proj = try loadBuffersFromId(allocator, separate.k_proj, buffer_store.withPrefix("k_proj"), platform), + .v_proj = try loadBuffersFromId(allocator, separate.v_proj, buffer_store.withPrefix("v_proj"), platform), + }, + }, + .o_proj = try loadBuffersFromId(allocator, self_attn.o_proj, buffer_store.withPrefix("o_proj"), platform), + }, + .merged => b: { + const proj: SeparateQkv = .init(buffer_store); + + var args = try options.merge_qkv_exe.?.args(allocator); + defer args.deinit(allocator); + + var results = try options.merge_qkv_exe.?.results(allocator); + defer results.deinit(allocator); + + const proj_buffers = try loadBuffersFromId(allocator, proj, buffer_store, platform); + + args.set(.{proj_buffers}); + options.merge_qkv_exe.?.call(args, &results); + + var qkv_proj: Bufferized(Linear) = undefined; + initBufferizedFrom(self_attn.proj.merged, &qkv_proj); + + results.fill(&qkv_proj); + errdefer qkv_proj.weight.deinit(); + errdefer if (qkv_proj.bias != null) qkv_proj.bias.?.deinit(); + + const o_proj = try loadBuffersFromId(allocator, self_attn.o_proj, buffer_store.withPrefix("o_proj"), platform); + + break :b .{ + .proj = .{ + .merged = qkv_proj, + }, + .o_proj = o_proj, + }; + }, + .precomputed => b: { + const proj: SeparateQkv = .init(buffer_store); + + var args = try options.precompute_qkv_exe.?.args(allocator); + defer args.deinit(allocator); + + var results = try options.precompute_qkv_exe.?.results(allocator); + defer results.deinit(allocator); + + const proj_buffers = try loadBuffersFromId(allocator, proj, buffer_store, platform); + const embedding_buffer = try bufferTransfer(allocator, buffer_store.root(), "model.embed_tokens.weight", platform); + defer embedding_buffer.deinit(); + + const input_layernorm_buffer = try bufferTransfer(allocator, buffer_store.parent(), "input_layernorm.weight", platform); + defer input_layernorm_buffer.deinit(); + + args.set(.{ proj_buffers, input_layernorm_buffer, embedding_buffer }); + options.precompute_qkv_exe.?.call(args, &results); + + var precomputed_qkv = results.get(zml.Buffer); + errdefer precomputed_qkv.deinit(); + + const o_proj = try loadBuffersFromId(allocator, self_attn.o_proj, buffer_store.withPrefix("o_proj"), platform); + + break :b .{ + .proj = .{ + .precomputed = precomputed_qkv, + }, + .o_proj = o_proj, + }; + }, + }; + } + + const ForwardParams = union(QkvKind) { + merged: Tensor, + separate: Tensor, + precomputed: struct { x: Tensor, tokens: Tensor }, + }; + + /// Self Attention. + /// - If token_index is set, x is assumed to be the representation of one new token, + /// and kv_cache will be read for the previous tokens. + /// - If token_index is not set, x is assumed to be the representation of all tokens + /// since the beginning of the sequence, and kv_cache won't be read. + /// In both case, kv_cache will be updated with the computed key and value. + /// x: {.b, .s, .d } -> .{.b, .s, .d} + pub fn forward( + self: SelfAttn, + params: ForwardParams, + token_index: Tensor, + kv_cache: KvCache, + ) struct { Tensor, KvCache } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + std.debug.assert(@as(QkvKind, params) == @as(QkvKind, self.proj)); + var q, var k, var v = switch (self.proj) { + .separate => b: { + const q = self.proj.separate.q_proj.forward(params.separate).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); + const k = self.proj.separate.k_proj.forward(params.separate).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + const v = self.proj.separate.v_proj.forward(params.separate).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + break :b .{ q, k, v }; + }, + .merged => b: { + const qkv = self.proj.merged.forward(params.merged).splitAxis(-1, .{ .h = self.num_heads + num_kv_heads + num_kv_heads, .hd = .auto }); + const q = qkv.slice1d(.h, .{ .start = 0, .end = self.num_heads }); + const k = qkv.slice1d(.h, .{ .start = self.num_heads, .end = self.num_heads + num_kv_heads }); + const v = qkv.slice1d(.h, .{ .start = self.num_heads + num_kv_heads, .end = self.num_heads + num_kv_heads + num_kv_heads }); + break :b .{ q, k, v }; + }, + .precomputed => b: { + std.debug.print("shape: {f}\n", .{self.proj.precomputed.shape()}); + std.debug.print("tokens.shape: {f}\n", .{params.precomputed.tokens.shape()}); + const qkv = self.proj.precomputed.gatherValues(0, params.precomputed.tokens, .{}).splitAxis(-1, .{ .h = self.num_heads + num_kv_heads + num_kv_heads, .hd = .auto }); + const q = qkv.slice1d(.h, .{ .start = 0, .end = self.num_heads }); + const k = qkv.slice1d(.h, .{ .start = self.num_heads, .end = self.num_heads + num_kv_heads }); + const v = qkv.slice1d(.h, .{ .start = self.num_heads + num_kv_heads, .end = self.num_heads + num_kv_heads + num_kv_heads }); + break :b .{ q, k, v }; + }, + }; + + // Generate the attention mask. + const seq_len = kv_cache.k.dim(.k); + var attn_mask = causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); + + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); + + // In self-attention, .s axis is used both for keys and queries. + const pos_index = b: { + const temp = Tensor.arange(.{ .end = q.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = q.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + q = rope(q, pos_index, self.rope_opts); + k = rope(k, pos_index, self.rope_opts); + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + const dtype = q.dtype(); + const new_kv_cache = kv_cache.update(k, v, token_index); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + const attn_output = sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ self.o_proj.forward(attn), new_kv_cache }; + } +}; + +fn mergeQkv(q: Tensor, k: Tensor, v: Tensor) Tensor { + return Tensor.concatenate(&.{ q, k, v }, 0); +} + +pub fn mergeQkv2(q: Linear, k: Linear, v: Linear) Linear { + return .{ + .weight = Tensor.concatenate(&.{ q.weight, k.weight, v.weight }, 0), + .bias = if (q.bias != null) Tensor.concatenate(&.{ q.bias.?, k.bias.?, v.bias.? }, 0) else null, + .tag = q.tag, + }; +} + +pub fn mergeQkv3(proj: SeparateQkv) Linear { + return .{ + .weight = Tensor.concatenate(&.{ proj.q_proj.weight, proj.k_proj.weight, proj.v_proj.weight }, 0), + .bias = if (proj.q_proj.bias != null) Tensor.concatenate(&.{ proj.q_proj.bias.?, proj.k_proj.bias.?, proj.v_proj.bias.? }, 0) else null, + .tag = proj.q_proj.tag, + }; +} + +pub fn precomputeQkv0(proj: SeparateQkv, layer_norm: RmsNorm, embedding: Tensor) Tensor { + const hidden = layer_norm.forward(embedding); + const qkv_proj: Linear = .{ + .weight = Tensor.concatenate(&.{ proj.q_proj.weight, proj.k_proj.weight, proj.v_proj.weight }, 0), + .bias = if (proj.q_proj.bias != null) Tensor.concatenate(&.{ proj.q_proj.bias.?, proj.k_proj.bias.?, proj.v_proj.bias.? }, 0) else null, + .tag = proj.q_proj.tag, + }; + + return qkv_proj.forward(hidden).rename(.{ .dout = .d }); +} + +const Mlp = struct { + up_proj: Linear, // (dim -> hidden_dim) + gate_proj: Linear, // (dim -> hidden_dim) + down_proj: Linear, // (hidden_dim -> dim) + + pub fn init(buffer_store: BufferStore5.View) Mlp { + return .{ + .up_proj = .initWithTag(buffer_store.withPrefix("up_proj"), .d), + .gate_proj = .initWithTag(buffer_store.withPrefix("gate_proj"), .d), + .down_proj = .initWithTag(buffer_store.withPrefix("down_proj"), .d), + }; + } + + pub fn forward(self: Mlp, x: Tensor) Tensor { + const proj = self.up_proj.forward(x); + var output = self.gate_proj.forward(x); + output = output.silu().mul(proj).rename(.{ .dout = .d }); + return self.down_proj.forward(output); + } +}; + +pub const KvCache = struct { + k: Tensor, + v: Tensor, + layer_index: Tensor, + + pub fn init(kv_shape: zml.Shape) KvCache { + // The KV-cache is initialized with ones to detect reads of uninitialized memory. + return .{ + .k = Tensor.init(kv_shape), + .v = Tensor.init(kv_shape), + .layer_index = Tensor.init(zml.Shape.init(.{}, .u32)), + }; + } + + pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !Bufferized(KvCache) { + return .{ + .k = try zml.Buffer.constant(platform, kv_shape, 0), + .v = try zml.Buffer.constant(platform, kv_shape, 0), + .layer_index = try zml.Buffer.scalar(platform, 0, .u32), + }; + } + + pub fn keys(self: KvCache) Tensor { + return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache) Tensor { + return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + std.debug.print("new_k: {f}\n", .{new_k.shape()}); + std.debug.print("self: {f}\n", .{self.k.shape()}); + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .k = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .k = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } +}; + +pub fn sampleTokens2(activations: Tensor, opts: zml.nn.SamplingStrategy, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { + _ = opts; // autofix + //if (opts.topk <= 1) { + if (true) { + const next_tokens = activations.argMax(.voc).indices.squeeze(.voc); + return .{ next_tokens, rng }; + } + + //const topk = activations.topK(opts.topk, .voc, .{}); + //// After the topk, we don't have .voc values, anymore, only topk. + //var x = topk.values.rename(.{ .voc = .topk }); + //if (opts.temperature != 1.0) { + // x = x.scale(1 / opts.temperature); + //} + + //// Gumbel reparametrization trick: + //// Adding gumbel noise and taking the argmax is equivalent + //// to sampling from the categorical distribution produced by the softmax. + //// https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks + //const next_rng, const gumbel_noise = rng.gumbel(x.shape()); + //x = x.add(gumbel_noise); + //const topk_idx = x.argMax(.topk).indices; + + //// topk_idx is indices into topk.values ! so in the range [0, topk] + //// Convert for the original indices from the full [0, voc] range. + //const next_tokens = topk.indices.gatherValues(.voc, topk_idx.squeeze(.topk), .{}); + //// log.debug("sampleTokens({}) -> {} -> {} -> {}", .{ activations, topk.indices, topk_idx, next_tokens }); + //return .{ next_tokens, next_rng }; +} + +pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x.axis(axis); + // upcast to improve precision + const xf32 = x.convert(.f32); + const mean = xf32.mul(xf32).mean(ax); + const rsqrt = Tensor.rsqrt(mean.addConstant(eps)).convert(x.dtype()); + return x.mul(rsqrt.broad(x.shape())); +} + +/// Return causal attention masks for the given shape. +/// The last dimensions are +pub fn causalAttnMask( + attn_shape_: anytype, + dtype: zml.DataType, + attn_window_len: ?u32, +) Tensor { + const attn_shape = zml.Shape.init(attn_shape_, dtype); + stdx.debug.assert(attn_shape.rank() == 2, "causalAttnMask({f}) shape need to be exactly 2 axes", .{attn_shape}); + const qlen = attn_shape.dim(-2); + const q_idx = Tensor.iota(attn_shape, -2); + const klen = attn_shape.dim(-1); + const k_idx = Tensor.iota(attn_shape, -1); + + // all elements > main diagonal must be 0 + // (q_idx - window_len < k_idx <= q_idx) + var mask = k_idx.cmp(.LE, q_idx); + if (attn_window_len) |window_len| { + if (qlen >= window_len or klen >= window_len) { + const window_mask = q_idx.cmp(.LT, k_idx.addConstant(window_len)); + mask = mask.logical(.AND, window_mask); + } + } + + if (dtype.isFloat()) { + const zeros = Tensor.constant(dtype.zero()).broad(mask.shape()); + const minus_inf = Tensor.constant(dtype.minValue()).broad(mask.shape()); + mask = Tensor.select(mask, zeros, minus_inf); + } else { + mask = mask.convert(dtype); + } + + return mask; +} + +/// Rotary position embedding modify queries and keys tensor before compute Q * K in self attention. +/// This biases a token to look at token near him. +/// The nice thing with rope is that you can cache the modified queries and keys directly. +/// See: https://paperswithcode.com/method/rope +/// +/// Expected shapes of tensor: +/// - x: .{ .s, .hd } where .s is the sequence length and .hd the head dimension +/// - pos_idx: optional tensor which indicates which positions are needed. +/// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. +pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: zml.nn.RopeOpts) Tensor { + stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); + + const idx = if (pos_idx) |idx| blk: { + stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={f}, idx={f})", .{ x, idx }); + break :blk idx; + } else blk: { + stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={f})", .{x}); + break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); + }; + const x_real, const x_imag = splitRealImg(x, opts.layout); + + // compute sin and cos in f32 before downcasting to x type. + const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); + const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); + const cos = inv_freq_pos.cos().convert(x.dtype()).broad(x_real.shape()); + const sin = inv_freq_pos.sin().convert(x.dtype()).broad(x_real.shape()); + + // apply rotation + const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); + const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); + + // flatten last dimensions + return mergeRealImg(y_real, y_imag, opts.layout); +} + +pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: zml.nn.RopeOpts.Layout) Tensor { + return switch (layout) { + .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), + .interleaved => Tensor.concatenate(&.{ + x_real.appendAxes(.{.interleaved_real_img}), + x_imag.appendAxes(.{.interleaved_real_img}), + }, -1).flatten(-2), + }; +} + +/// {exp( - n * ln(10_000) / N ) | n in [0..N] } +pub fn invFreq(N: i64, opts: zml.nn.RopeOpts) Tensor { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + const N_half: usize = @intCast(@divExact(N, 2)); + const inv_freq = arena.allocator().alloc(f32, N_half) catch @panic("OOM"); + _invFreq(opts, inv_freq); + return Tensor.constantTensor(std.mem.sliceAsBytes(inv_freq), zml.Shape.init(.{@divExact(N, 2)}, .f32)); +} + +fn _invFreq(opts: zml.nn.RopeOpts, inv_freq: []f32) void { + const N = inv_freq.len; + // Default frequencies + for (0.., inv_freq) |n, *f| { + f.* = @exp(-@log(opts.freq_base) * stdx.math.divFloat(f32, n, N)); + } + + switch (opts.scaling) { + .default => {}, + .custom => { + stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); + @memcpy(inv_freq, opts.scaling.custom); + }, + .llama3 => |s| { + // https://arxiv.org/pdf/2309.16039 + // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. + // In Llama3 they used a higher base freq and also downscaled low frequencies. + std.debug.assert(s.low_freq_factor < s.high_freq_factor); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.high_freq_factor * (2 * std.math.pi) / M; + const f_low = s.low_freq_factor * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Linear interpolation for middle freq + const lerp: f64 = (inv_freq[n] - f_low) / (f_high - f_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, + } +} + +pub fn splitRealImg(x: Tensor, layout: zml.nn.RopeOpts.Layout) [2]Tensor { + const n = x.dim(-1); + + return switch (layout) { + .sequential => .{ + x.slice1d(-1, .{ .end = @divExact(n, 2) }), + x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), + }, + .interleaved => .{ + x.slice1d(-1, .{ .start = 0, .step = 2 }), + x.slice1d(-1, .{ .start = 1, .step = 2 }), + }, + }; +} + +pub const SdpaOpts = struct { + attn_mask: ?Tensor = null, + scale: ?Tensor = null, + allow_cudnn: bool = true, + // TODO: put a callback instead of all this field, + // so that +}; + +/// Scaled dot product attention. +/// +/// **Shapes**: +/// - q, result: .{ .h, .q, .hd } +/// - k, v: .{ .h, .k, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +/// +/// .h is allowed to differ from queries and keys as long as the key heads +/// can be repeated to match query heads. +pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { + var q, var k, var v = .{ q_, k_, v_ }; + + const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; + const err_args = .{ q, k, v, opts.attn_mask }; + stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); + stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); + stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); + + //if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape())) { + // return cuda.sdpa(q, k, v, opts); + //} + + if (q.dim(.h) != k.dim(.h)) { + stdx.debug.assert(@mod(q.dim(.h), k.dim(.h)) == 0, err_template ++ "Different number of heads for keys and queries, but can't repeat keys.", err_args); + // Note: we don't try to repeat queries. + // Repeating keys is the interesting optimisation cause it reduces KV cache memory usage. + const num_rep: u63 = @intCast(@divExact(q.dim(.h), k.dim(.h))); + k, v = .{ k.repeat1d(.h, num_rep), v.repeat1d(.h, num_rep) }; + } + const attn_mask = if (opts.attn_mask) |m| m else null; + + const dims = zml.helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { + stdx.debug.panic(err_template ++ "Inputs have incompatible shapes.", err_args); + }; + const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); + const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); + k = k.mul(head_scaling.convert(k.dtype())); + std.debug.print("k.shape: {f}\n", .{k.shape()}); + + var attn_weights = q.dot(k, .hd); + std.debug.print("attn_weights.shape: {f}\n", .{attn_weights.shape()}); + // log.debug("attn_weights : {}, attn_mask : {?}", .{ attn_weights, attn_mask }); + if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); + attn_weights = attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); + + var attn = attn_weights.dot(v, .k); + return attn.transpose(q.shape()); +} diff --git a/mlir/main.zig b/mlir/main.zig new file mode 100644 index 00000000..9cba916a --- /dev/null +++ b/mlir/main.zig @@ -0,0 +1,5110 @@ +const std = @import("std"); + +const llama = @import("llama.zig"); + +const c = zml.c; +const asynk = @import("async"); +const dialects = @import("mlir/dialects"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); +const upb = zml.upb; +const zml = @import("zml"); +const pjrt = zml.pjrt; +const Shape = zml.Shape; +const EnumLiteral = @TypeOf(.enum_literal); +const Platform = zml.Platform; +const Mesh = zml.partitioning.Mesh; +const Sharding = zml.partitioning.Sharding; + +const RandomReader = @import("writer.zig").RandomReader; + +const Dims = stdx.BoundedArray(i64, zml.Shape.MAX_RANK); +const log = std.log.scoped(.main); + +const safetensor = + \\{ + \\ "metadata": { + \\ "total_size": 16060522496 + \\ }, + \\ "weight_map": { + \\ "lm_head.weight": "model-00004-of-00004.safetensors", + \\ "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors", + \\ "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + \\ "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + \\ "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + \\ "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + \\ "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + \\ "model.norm.weight": "model-00004-of-00004.safetensors" + \\ } + \\} +; + +pub const std_options: std.Options = .{ + .log_level = .debug, + .log_scope_levels = &.{ + .{ .scope = .@"zml/async", .level = .info }, + }, +}; + +var global_compilation_context: ?*CompilationContext = null; + +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); + +pub const TensorOriginType = enum { + argument, + value, +}; + +pub const TensorOrigin = union(TensorOriginType) { + argument: void, + value: *const mlir.Value, +}; + +pub const Tensor = struct { + var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; + pub const MAX_RANK = Shape.MAX_RANK; + + id: usize, + auto_broadcast: bool = false, + _shape: zml.Shape, + //tensor_origin: TensorOrigin = .{ .argument = {} }, + _mesh: ?Mesh = null, + _value: ?*const mlir.Value = null, + + fn mlirType(self: Tensor, mlir_ctx: *mlir.Context) *const mlir.Type { + return mlir.rankedTensorType( + self.dims(), + zml.mlir.Type.fromDType(mlir_ctx, self.dtype()), + ); + } + + fn mlirCtx() *mlir.Context { + return CompilationContext.current().mlir_ctx; + } + + fn currentBlock() *mlir.Block { + return CompilationContext.current().currentScope().block; + } + + pub fn init(shape_: zml.Shape) Tensor { + return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; + } + + /// Returns the shape of a Tensor. + pub fn shape(self: Tensor) Shape { + return self._shape; + } + + /// Returns the datatype of a Tensor. + pub fn dtype(self: Tensor) zml.DataType { + return self._shape.dtype(); + } + + /// Returns the rank of a Tensor. + pub fn rank(self: Tensor) u4 { + return self._shape.rank(); + } + + /// Returns the number of element of a Tensor. + pub fn count(self: Tensor) usize { + return self._shape.count(); + } + + /// Returns the size in bytes of a Tensor. + pub fn byteSize(self: Tensor) usize { + return self._shape.byteSize(); + } + + /// Returns the dimension of axis 'axis_'. + /// + /// 'axis_' can be an integer or a tag. + pub fn dim(self: Tensor, axis_: anytype) i64 { + return self._shape.dim(axis_); + } + + /// Returns the dimensions of a Tensor as a slice. + pub fn dims(self: *const Tensor) []const i64 { + return self._shape.dims(); + } + + /// Returns the index of axis 'axis_'. + /// + /// 'axis_' can be an integer or a tag. + pub fn axis(self: Tensor, axis_: anytype) u3 { + return self._shape.axis(axis_); + } + + /// Returns the indices of each of the given axes. + /// + /// 'axis_' can be an integer or a tag. + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, Tensor.MAX_RANK) { + return self._shape.axes(axes_); + } + + /// Returns a Tensor tagged with the tags in 'tagz'. + pub fn withTags(self: Tensor, tagz: anytype) Tensor { + var res = self; + res._shape = self._shape.withTags(tagz); + return res; + } + + /// Returns a Tensor tagged partially with the tags in 'tagz'. + /// + /// If 'tagz' is of length n, the n last dimensions of the Tensor will be tagged. + pub fn withPartialTags(self: Tensor, tagz: anytype) Tensor { + var res = self; + res._shape = self._shape.withPartialTags(tagz); + return res; + } + + /// Returns a Tensor with new tag names. + pub fn rename(self: Tensor, renames: anytype) Tensor { + var res = self; + res._shape = self._shape.rename(renames); + return res; + } + + pub fn renameAxis(self: Tensor, ax: i8, name: EnumLiteral) Tensor { + var res = self; + res._shape._tags.set(self.axis(ax), @tagName(name).ptr); + return res; + } + + pub fn autoBroadcast(self: Tensor) Tensor { + var ret = self; + ret.auto_broadcast = true; + return ret; + } + + pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { + //stdx.debug.assert(lhs.rank() >= 1 and rhs.rank() >= 1, "Can't matmul({f}, {f}) ! The two tensors need to have at least rank 1.", .{ lhs, rhs }); + + const contracting = [_][2]i8{.{ -1, if (rhs.rank() >= 2) rhs.rank() - 2 else 0 }}; + if (lhs.rank() == 1 or rhs.rank() <= 2) { + // When lhs is a vector or rhs is small the torch semantics match the dot_general semantics and life is easy. + return lhs.dotGeneral(rhs, &contracting, &.{}); + } + + //stdx.debug.assert(lhs.rank() == 2, "Can't matmul({f}, {f}) ! One of the two tensors need to have a rank less than 2.", .{ lhs, rhs }); + + // Pytorch treats the extra dimensions of rhs has batching dimensions, + // and implicitly broadcast lhs along those. + // We make this broadcasting explicit. + var left_shape = rhs.shape(); + left_shape._dims.set(left_shape.axis(-2), lhs.dim(-2)); + left_shape._tags.set(left_shape.axis(-2), lhs.shape().tag(-2)); + left_shape._dims.set(left_shape.axis(-1), lhs.dim(-1)); + left_shape._tags.set(left_shape.axis(-1), lhs.shape().tag(-1)); + const lhs_broad = lhs.broadcastLeft(left_shape); + + const n_batching_axes = rhs.rank() - lhs.rank(); + var batching: [MAX_RANK][2]i8 = undefined; + for (0..n_batching_axes) |i| { + batching[i] = .{ @intCast(i), @intCast(i) }; + } + return lhs_broad.dotGeneral(rhs, &contracting, batching[0..n_batching_axes]); + } + + const ArgsKind = enum { + simple, + contracting_only, + full, + }; + + fn isFullArgsKind(comptime T: type) bool { + return std.meta.fieldIndex(T, "contracting") != null and std.meta.fieldIndex(T, "batching") != null; + } + + fn getArgsKind(comptime T: type) ArgsKind { + const type_info = @typeInfo(T); + return switch (type_info) { + .enum_literal => .simple, + .@"struct" => if (isFullArgsKind(T)) .full else .contracting_only, + else => unreachable, + }; + } + + pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { + //const args_kind = getArgsKind(@TypeOf(args)); + //std.debug.print("{}\n", .{args_kind}); + const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); + const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); + + var batching_axes: stdx.BoundedArray([2]i8, Shape.MAX_RANK) = .{}; + for (0..lhs.rank()) |lhs_tag_index| { + const lhs_tag = lhs.shape().tag(lhs_tag_index); + if (lhs_tag == Shape.toTag(args)) continue; + if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { + batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); + } + } + return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); + } + + pub fn dotGeneral( + lhs: Tensor, + rhs: Tensor, + contracting_axes: []const [2]i8, + batching_axes: []const [2]i8, + ) Tensor { + stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); + + const Axes = stdx.BoundedArray(i64, MAX_RANK); + + var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + // Validate batching axes + var lhs_batching_axes: Axes = .{}; + var rhs_batching_axes: Axes = .{}; + for (batching_axes) |b_axes| { + const l, const r = b_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + var t = lhs._shape.tag(l); + if (t == Shape.TagUnknown) t = rhs._shape.tag(r); + res_shape = res_shape.appendDim(lhs._shape.dim(l), t); + lhs_batching_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_batching_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + // Validate contracting axes + var lhs_contracting_axes: Axes = .{}; + var rhs_contracting_axes: Axes = .{}; + for (contracting_axes) |c_axes| { + const l, const r = c_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + // Result shape is obtained by concatenating batching dimensions, (already done) + // then dimensions from lhs axes that aren't contracting nor batching, + // then dimensions from rhs axes that aren't contracting nor batching. + for (0..lhs.rank()) |l| { + if (std.mem.indexOfScalar(i64, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, lhs_batching_axes.constSlice(), @intCast(l))) |_| { + continue; + } + res_shape = res_shape.appendDim(lhs._shape.dim(l), lhs._shape.tag(l)); + } + for (0..rhs.rank()) |r| { + if (std.mem.indexOfScalar(i64, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, rhs_batching_axes.constSlice(), @intCast(r))) |_| { + continue; + } + res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); + } + + const mlir_ctx = lhs.getContext().mlir_ctx; + //const loc = lhs.getContext().location(@src(), "dot({f},{f},contracting={any},batching={any}", .{ lhs, rhs, contracting_axes, batching_axes }); + const loc = mlir.Location.unknown(mlirCtx()); + const op = dialects.stablehlo.dot_general( + mlir_ctx, + lhs.value(), + rhs.value(), + mlir.rankedTensorType(res_shape.dims(), zml.mlir.Type.fromDType(mlir_ctx, res_shape.dtype())), + .{ + .lhs_batching_dimensions = lhs_batching_axes.constSlice(), + .rhs_batching_dimensions = rhs_batching_axes.constSlice(), + .lhs_contracting_dimensions = lhs_contracting_axes.constSlice(), + .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), + .dot_precision = .fast, + }, + loc, + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { + const res: Tensor = .{ + ._shape = sh, + ._value = val, + //.tensor_origin = .{ .value = val }, + .id = Tensor.current_id.fetchAdd(1, .seq_cst), + }; + + //if (builtin.mode == .Debug) { + // // Check that the MLIR value actually have the same shape. + // const other = fromMlirValue(val); + // stdx.debug.internalAssert(sh.eql(other._shape), "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); + //} + + return res; + } + + pub fn constant(val: zml.DataType.Value) Tensor { + const op = dialects.stablehlo.constant( + mlirCtx(), + &.{}, + zml.mlir.Type.fromDType(mlirCtx(), val.dtype()), + val.constSlice(), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(.init(&.{}, val.dtype()), op.result(0)).autoBroadcast(); + } + + pub fn add(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "add", dialects.stablehlo.add)(self, other); + } + + pub fn sub(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "sub", dialects.stablehlo.subtract)(self, other); + } + + pub fn mul(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "mul", dialects.stablehlo.multiply)(self, other); + } + + /// Returns a Tensor containing the element-wise division of the input Tensors. + pub fn div(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "div", dialects.stablehlo.divide)(self, other); + } + + pub fn maximum(self: Tensor, other: Tensor) Tensor { + return binaryOp(@src(), "maximum", dialects.stablehlo.maximum)(self, other); + } + + /// Returns a Tensor containing the element-wise cosine of the input Tensor. + pub fn cos(self: Tensor) Tensor { + const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, cosine_op.result(0)); + } + + /// Returns a Tensor containing the element-wise sine of the input Tensor. + pub fn sin(self: Tensor) Tensor { + const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, sine_op.result(0)); + } + + pub fn relu(self: Tensor) Tensor { + return self.maximum(.constant(self.dtype().zero())); + } + + /// Flattens the given axis and the next one, into one new axis. + pub fn flatten(self: Tensor, axis_: anytype) Tensor { + // TODO: move to torch.zig, this is equivalent to merge + const old_shape = self._shape; + const a = self.axis(axis_); + // stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {} on the last axis {}.", .{ self, axis }); + const new_shape = old_shape.remove(a + 1).set(a, old_shape.dim(a) * old_shape.dim(a + 1)); + + const tensor_type = mlir.rankedTensorType(new_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), new_shape.dtype())); + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), + self.value(), + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + // log.debug("flatten({d}, {d}) -> {d}", .{ self.dims(), axis_, new_shape[0 .. self.rank() - 1] }); + return _result(new_shape, reshaped_val.result(0)); + } + + pub fn flattenAll(self: Tensor) Tensor { + // TODO: rename to just flatten, once flatten is moved to torch + return self.reshape(.{self.count()}); + } + + pub fn broadcast(self: Tensor, output_shape: Shape, axes_: []const i64) Tensor { + stdx.debug.assert(axes_.len == self.rank(), "broadcast expects axes_ to map all axes from self to axes of the output shape, got broadcast({f}, {f}, {any})", .{ self, output_shape, axes_ }); + for (0.., axes_) |self_ax, other_ax| { + const d = self.dim(self_ax); + stdx.debug.assert(d == 1 or d == output_shape.dim(other_ax), "broadcast expects shape axes to either be 1-sized or to match the target size. got broadcast({f}, {f}, {any}), error on self axis {d} mapping to other axis {d}", .{ self, output_shape, axes_, self_ax, other_ax }); + } + + const res_shape = output_shape.withDtype(self.dtype()); + if (std.mem.eql(i64, self.dims(), output_shape.dims())) { + // No broadcast needed. We don't emit a new stablehlo value + // but we propagate output_shape tags. + return _result(res_shape, self.value()); + } + const ctx = self.getContext(); + const result_type = mlir.rankedTensorType( + res_shape.dims(), + zml.mlir.Type.fromDType(ctx.mlir_ctx, res_shape.dtype()), + ); + const broadcast_op = dialects.stablehlo.broadcast_in_dim(ctx.mlir_ctx, self.value(), axes_, result_type, .unknown(ctx.mlir_ctx)).appendTo(currentBlock()); + return _result(res_shape, broadcast_op.result(0)); + } + + pub fn broad(self: Tensor, other: Shape) Tensor { + // TODO: broad is too restrictive because sometime you only want to specify one specific axis + // Note: if you code below, make sure to update Shape.canBroadcastTo. + stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self, other }); + + // Already the right shape + if (std.mem.eql(i64, self.dims(), other.dims())) return self; + + // Non ambiguous broadcasting + // TODO: broad is error prone because of this: + // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } + if (self._shape.rank() == 0 or self._shape.rank() == other.rank()) { + const all_axes = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + return self.broadcast(other, all_axes[0..self.rank()]); + } + + // check that each axis of self maps to an axis of other + var axes_: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (self._shape.tags()) |t| { + axes_.appendAssumeCapacity(@intCast(other.axis(t))); + } + return self.broadcast(other, axes_.constSlice()); + } + + pub fn broadcastLeft(self: Tensor, output_shape: Shape) Tensor { + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastLeft expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); + + const a = output_shape.rank() - self.rank(); + if (self.rank() == output_shape.rank() and std.mem.eql(i64, self.dims(), output_shape.dims())) { + return self; + } + + return self.broadcast(output_shape, Shape.range(output_shape.rank(), output_shape.dtype()).dims()[a..]); + } + + pub const ArangeArgs = struct { + start: i64 = 0, + end: i64, + step: i64 = 1, + }; + + pub fn arange(args: ArangeArgs, dt: zml.DataType) Tensor { + stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); + stdx.debug.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); + + //const loc = ctx.location(@src(), "arange({}, dtype={})", .{ args, dt }); + const loc = mlir.Location.unknown(mlirCtx()); + + const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; + const sh = Shape.init(.{n_steps}, dt); + var op = dialects.stablehlo.iota(mlirCtx(), 0, mlir.rankedTensorType(sh.dims(), zml.mlir.Type.fromDType(mlirCtx(), sh.dtype())), loc).appendTo(currentBlock()); + var res = _result(sh, op.result(0)); + _ = &res; + + if (args.step != 1) { + res = res.scale(args.step); + } + + if (args.start != 0) { + res = res.addConstant(args.start); + } + + return res; + } + + pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { + const output_shape = self._shape.reshape(output_shape_); + const tensor_type = mlir.rankedTensorType(output_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), output_shape.dtype())); + //const loc = self.getContext().location(@src(), "reshape({f})", .{output_shape}); + const reshape_op = dialects.stablehlo.reshape( + mlirCtx(), + self.value(), + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(output_shape, reshape_op.result(0)); + } + + pub fn convert(self: Tensor, to: zml.DataType) Tensor { + if (to == self.dtype()) { + return self; + } + //const loc = self.getContext().location(@src(), "convert({f},to={s})", .{ self, @tagName(to) }); + + const res_type = mlir.rankedTensorType(self.dims(), zml.mlir.Type.fromDType(mlirCtx(), to)); + const op = dialects.stablehlo.convert( + mlirCtx(), + self.value(), + res_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(self._shape.withDtype(to), op.result(0)); + } + + pub fn select(bool_tensor: Tensor, on_true: Tensor, on_false: Tensor) Tensor { + stdx.debug.assert(bool_tensor.dtype() == .bool, "select expects input tensor type to be a boolean, got {}", .{bool_tensor.dtype()}); + stdx.debug.assert(on_true.dtype() == on_false.dtype(), "select expects 'on_true' and 'on_false' tensor types to be equal, got {} and {}", .{ on_true.dtype(), on_false.dtype() }); + + if (bool_tensor.rank() != 0 and on_true.rank() == 0) { + return bool_tensor.select(on_true.broad(bool_tensor.shape()), on_false); + } + if (bool_tensor.rank() != 0 and on_false.rank() == 0) { + return bool_tensor.select(on_true, on_false.broad(bool_tensor.shape())); + } + + stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); + stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); + + //const loc = bool_tensor.mlirCtx().location(@src()); + const loc = mlir.Location.unknown(mlirCtx()); + const op = dialects.stablehlo.select( + mlirCtx(), + bool_tensor.value(), + on_true.value(), + on_false.value(), + loc, + ).appendTo(currentBlock()); + + return _result(on_true._shape, op.result(0)); + } + + pub const ArgMaxRes = struct { + values: Tensor, + indices: Tensor, + }; + + pub const LogicalOp = enum { OR, XOR, AND }; + pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { + return switch (logical_op) { + .OR => binaryOp(@src(), "or", dialects.stablehlo.or_)(self, other), + .XOR => binaryOp(@src(), "xor", dialects.stablehlo.xor)(self, other), + .AND => binaryOp(@src(), "and", dialects.stablehlo.and_)(self, other), + }; + } + + fn getComparisonType(dt: zml.DataType) dialects.stablehlo.CompareType.Type { + return switch (dt) { + .i4, .i8, .i16, .i32, .i64 => .SIGNED, + .bool, .u4, .u8, .u16, .u32, .u64 => .UNSIGNED, + .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .FLOAT, + .c64, .c128 => @panic("Can't compare complex numbers"), + }; + } + + pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); + + if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); + if (self.rank() != 0 and other.rank() == 0) return self.cmp(direction, other.broadcast(self._shape, &.{})); + + stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); + + //const loc = self.getContext().location(@src(), "cmp(.{s})", .{@tagName(direction)}); + const loc = mlir.Location.unknown(mlirCtx()); + const op = dialects.stablehlo.compare( + mlirCtx(), + self.value(), + other.value(), + direction, + getComparisonType(self.dtype()), + loc, + ).appendTo(currentBlock()); + + return _result(self._shape.withDtype(.bool), op.result(0)); + } + + pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { + const a = x.axis(axis_); + const dt: zml.DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; + + const values, const indices = reduce( + .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, + .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, + &.{a}, + struct { + fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { + const left_gt_right = values.left.cmp(.GT, values.right); + const is_nan = values.left.cmp(.NE, values.left); + const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); + + // If values.left == values.right: keep the smallest idx. + const is_same = values.left.cmp(.EQ, values.right); + const is_first = indices.left.cmp(.LT, indices.right); + const is_same_but_first = is_same.logical(.AND, is_first); + const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + const max_idx = keep_left_idx.select(indices.left, indices.right); + + return .{ max_val, max_idx }; + } + }.cmp, + .{}, + ); + return .{ .values = values, .indices = indices }; + } + + pub const ReduceArgs = struct { + left: Tensor, + right: Tensor, + }; + + fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), zml.mlir.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), zml.mlir.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes_)), + }, + .verify = true, + .location = .unknown(mlirCtx()), + }).appendTo(currentBlock()); + + // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. + // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. + // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, + // to find the correct mlir.Value, but we first broadcast before creating the final + // Tensor struct. + var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (0..Tensor.MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { + broadcasting_axes.append(@intCast(i)) catch unreachable; + } + } + + inline for (0..result.len) |i| { + var reduced_shape: Shape = inputs[i].shape(); + for (axes_) |a| { + reduced_shape = reduced_shape.setDim(a, 1); + } + + const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), reduced_shape.dtype())); + const broad_op = dialects.stablehlo.broadcast_in_dim( + mlirCtx(), + reduce_op.result(i), + broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + result[i] = Tensor._result(reduced_shape, broad_op.result(0)); + } + + return result; + } + + fn binaryOp( + src: std.builtin.SourceLocation, + op_name: []const u8, + comptime op_fn: fn (*mlir.Context, *const mlir.Value, *const mlir.Value, *const mlir.Location) *mlir.Operation, + ) fn (Tensor, Tensor) Tensor { + _ = op_name; // autofix + _ = src; // autofix + return struct { + pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { + const other_ = if (other.auto_broadcast) other.broad(self.shape()) else other; + //stdx.debug.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {f} and {f}", .{ op_name, self, other }); + + //if (self.rank() == 0 and other.rank() != 0) { + // return binaryOpHelper(self.broad(other._shape), other); + //} + + //if (self.rank() != 0 and other.rank() == 0) { + // return binaryOpHelper(self, other.broad(self._shape)); + //} + + //stdx.debug.assert(self._shape.eql(other._shape), "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); + + const ctx = self.getContext(); + //const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other }); + const ret = op_fn(mlirCtx(), self.value(), other_.value(), .unknown(ctx.mlir_ctx)).appendTo(currentBlock()); + return _result(self._shape, ret.result(0)); + } + }.binaryOpHelper; + } + + pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { + if (tensors.len == 1) return tensors[0]; + stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); + var buffer: [32]*const mlir.Value = undefined; + std.debug.assert(tensors.len <= buffer.len); + std.debug.assert(tensors.len > 0); + const a = tensors[0].axis(axis_); + // TODO(Corendos): Check that tensor axes match. + + var concatenated_dim: i64 = 0; + for (tensors, 0..) |t, i| { + buffer[i] = t.value(); + concatenated_dim += t.dim(a); + } + + const res_shape = tensors[0]._shape.set(a, concatenated_dim); + const op = dialects.stablehlo.concatenate(mlirCtx(), buffer[0..tensors.len], a, .unknown(mlirCtx())).appendTo(currentBlock()); + // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); + return _result(res_shape, op.result(0)); + } + + pub fn value(self: Tensor) *const mlir.Value { + if (self.getContext().currentScope().id_to_argument.get(self.id)) |argument_index| { + return self.getContext().currentScope().block.argument(argument_index); + } else if (self._value) |v| { + return v; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + } + + /// Returns the donation data of the tensor. + pub fn donation(self: Tensor) ?usize { + return CompilationContext.current().currentScope().id_to_donation.get(self.id); + } + + pub const Rng = struct { + _state: Tensor, + algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, + + pub fn init() Rng { + return .{ ._state = .init(zml.Shape.init(.{2}, .u64)) }; + } + + pub fn initBuffer(platform: Platform, seed: u128) !Bufferized(Rng) { + return .{ + ._state = try zml.Buffer.fromBytes(platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), + }; + } + + ///// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. + ///// + ///// The given Rng state should not be used anymore (or you'll get the same numbers again). + ///// The output is guaranteed to be deterministic function of `self` Rng state, + ///// but it is not guaranteed to be deterministic between implementations. + //pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { + // const ctx = CompilationContext.current(); + // const loc = ctx.location(@src(), "rand.bitGen({f})", .{sh}); + // const op = dialects.stablehlo.rng_bit_generator( + // ctx.mlirCtx(), + // self.algorithm, + // self._state.value(), + // mlirx.tensorType(ctx.mlirCtx(), self._state._shape), + // mlirx.tensorType(ctx.mlirCtx(), sh), + // loc, + // ); + // return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; + //} + + fn update(self: Rng, new_state: mlir.Value) Rng { + return .{ + ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), + .algorithm = self.algorithm, + }; + } + + ///// Returns a Tensor of the given shape, filled with uniformly sampled floating point numbers from an interval, + ///// and a new Rng state. + ///// + ///// https://en.wikipedia.org/wiki/Continuous_uniform_distribution + //pub fn uniform( + // self: Rng, + // shape_: Shape, + // opts: struct { min: f64 = 0, max: f64 = 1 }, + //) struct { Rng, Tensor } { + // const dt = if (shape_.dtype().isFloat()) shape_.dtype() else .f32; + + // const mantissa_bit_count = @import("dtype.zig").mantissaSize(dt); + // const bit_count: usize = dt.bitSizeOf(); + // const rng_bit_count = if (mantissa_bit_count < 8) 8 else bit_count; + // const uint_dtype: DataType = switch (bit_count) { + // 8 => .u8, + // 16 => .u16, + // 32 => .u32, + // 64 => .u64, + // else => stdx.debug.panic("uniform don't support non-byte aligned dtype. Got: {f}", .{shape_}), + // }; + + // const rng, const bits = self.bitGenerator(shape_.withDtype(uint_dtype)); + + // // Erase bits outside of mantissa. + // var float_bits = bits.shiftRightLogical(scalar(rng_bit_count - mantissa_bit_count, uint_dtype)); + + // // Set exponent bits to represent e^0 (eg 127 for f32). + // float_bits = float_bits.logical(.OR, scalar(1, dt).bitCast(uint_dtype)); + + // // float_bits now uniformly represents number in [1, 2[ range. + // // Let's convert to floats, and subtract one to go to [0, 1[ range. + // var floats = float_bits.bitCast(dt).sub(scalar(1, dt)); + // floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); + + // // Convert back to integer if needed. + // return .{ rng, floats.convert(shape_.dtype()) }; + //} + + //test uniform { + // const zml = @import("zml.zig"); + // const Stats = struct { + // const Stats = @This(); + + // mean: Tensor, + // variance: Tensor, + // min: Tensor, + // max: Tensor, + + // pub fn uniformStats( + // rand: Rng, + // shape_: Shape, + // opts: struct { min: f64, max: f64 }, + // ) struct { Rng, Stats } { + // const rng, const data = rand.uniform(shape_, .{ .min = opts.min, .max = opts.max }); + // const mean_ = data.mean(0); + // const variance = data.sub(mean_.broad(data.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + // return .{ rng, .{ + // .mean = mean_, + // .variance = variance, + // .min = data.min(0), + // .max = data.max(0), + // } }; + // } + // }; + + // const platform = zml.testing.env(); + // // Compute stats over a uniform distribution on [-2, 10]. + // const rand, const stats = try zml.testing.compileAndCallWithTensors( + // platform, + // Stats.uniformStats, + // .{ Rng.shape(), zml.Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, + // .{try Rng.init(platform, 1234)}, + // ); + + // // Check the Rng state has been modified. + // try std.testing.expect(try rand._state.getValue(u128) != 1234); + + // // Check the mean and variance are close to theoritical values. + // const mean_ = try stats.mean.getValue(f32); + // try std.testing.expectApproxEqAbs(4, mean_, 0.03); + + // const variance = try stats.variance.getValue(f32); + // try std.testing.expectApproxEqAbs(12.0 * 12.0 / 12.0, variance, 0.01); + + // // Check that no value is outside of the interval + // // and we have samples close to the edges. + // const min_ = try stats.min.getValue(f32); + // try std.testing.expect(min_ >= -2); + // try std.testing.expectApproxEqAbs(-2, min_, 0.05); + + // const max_ = try stats.max.getValue(f32); + // try std.testing.expect(max_ < 10); + // try std.testing.expectApproxEqAbs(10, max_, 0.05); + //} + + ///// Returns a Tensor of the given shape, filled with floating point numbers sampled from a normal distribution. + ///// + ///// Note: this uses stablehlo.rng which is deprecated. + ///// https://github.com/openxla/stablehlo/blob/main/rfcs/20240503-opset-deprecations.md + //pub fn normal(sh: Shape, opts: struct { mean: f64 = 0, stddev: f64 = 1 }) Tensor { + // stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); + + // const ctx = CompilationContext.current(); + // const loc = ctx.location(@src(), "rand.normal({f}, mean={},stddev={})", .{ sh, opts.mean, opts.stddev }); + // const a = Tensor.constant(.{}, Data.init(sh.dtype(), opts.mean)); + // const b = Tensor.constant(.{}, Data.init(sh.dtype(), opts.stddev)); + // const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); + // const op = dialect.stablehlo.rng(ctx.mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, loc); + // return _result(sh, op.result(0)); + //} + + ///// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. + ///// + ///// Often used in ML because of the reparametrization tricks. + ///// Sampling from a gumbel distribution is equivalent to sample + ///// from a softmax distribution, but doesn't require to compute the sum of exponentials. + ///// https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks + ///// See `sampleTokens` for a practical use case. + ///// Note: we only implement the μ=0, β=1 version. + //pub fn gumbel(self: Rng, shape_: Shape) struct { Rng, Tensor } { + // const rand, const u = self.uniform( + // // Always use .f32 to have a big enough mantissa. + // shape_.withDtype(.f32), + // // We don't want 0 to be sampled otherwise `log` will return -inf. + // .{ .min = std.math.floatEps(f32), .max = 1 }, + // ); + // return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; + //} + + //test gumbel { + // const zml = @import("zml.zig"); + // const Stats = struct { + // const Stats = @This(); + + // mean: Tensor, + // variance: Tensor, + // actual_dist: Tensor, + + // pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { + // const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); + // const rng, const data = rand.gumbel(s); + // const flat = data.flattenAll(); + // const mean_ = flat.mean(0); + // const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + + // // Test out the gumbel reparametrization trick + // var x = target_dist.log().withTags(.{.d}).broad(s); + // x = x.add(data); + // const samples = x.argMax(.d).indices.squeeze(.d); + + // // count 0, 1, 2 and 3 in samples: + // // - map 0 to 1, 1 to 2**16, 2 to 2**32, 3 to N**58 + // // - sum in u64 + // // - split to [4]u16 + // const powers = blk: { + // var powers: [4]u64 = undefined; + // for (&powers, 0..) |*p, i| p.* = std.math.pow(u64, 2, i * 16); + // break :blk powers; + // }; + // const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); + // const counts = values.gatherValues(.d, samples, .{}).sum(.n).bitCast(.u16); + // const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); + // return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; + // } + // }; + + // const platform = zml.testing.env(); + // const tgt_dist = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; + // const rand, const stats = try zml.testing.compileAndCallWithTensors( + // platform, + // Stats.gumbelStats, + // .{ Rng.shape(), zml.Shape.init(.{tgt_dist.len}, .f32) }, + // .{ try Rng.init(platform, 1234), try .fromArray(platform, tgt_dist) }, + // ); + // // Check the Rng state has been modified. + // try std.testing.expect(try rand._state.getValue(i128) != 1234); + + // // Check the mean and variance are close to theoritical values. + // const mean_ = try stats.mean.getValue(f32); + // try std.testing.expectApproxEqAbs(0.5772, mean_, 0.02); + + // const variance = try stats.variance.getValue(f32); + // const pi = std.math.pi; + // try std.testing.expectApproxEqAbs(pi * pi / 6.0, variance, 0.03); + + // // Check the distribution obtained with the gumbel trick matches the target distribution. + // const actual_dist = try stats.actual_dist.getValue([4]f32); + // scoped_log.debug("tgt_dist: {d}, actual_dist: {d}", .{ tgt_dist, actual_dist }); + // for (tgt_dist, actual_dist) |tgt, actual| { + // // We normalize tgt_dist to make it a well formed distribution. + // // We didn't do it before calling gumbel, because the gumbel trick + // // doesn't require normalized distributions as input. + // try std.testing.expectApproxEqAbs(tgt / 10.0, actual, 0.05); + // } + //} + }; + + /// Tell PJRT compiler that memory should be reuse between the two tensors. + /// The compiler is already aggressively reusing tensors for intermediate results, + /// but this API allows to reuse buffer between input and output arguments + /// of a given function. + /// Note this is visible from the outside. The caller of a function with donations + /// is not allowed to reuse the donated input buffer after the call. + /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. + pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { + const compilation_context = CompilationContext.current(); + const scope = compilation_context.currentScope(); + if (scope.id_to_argument.get(origin.id)) |argument_index| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = argument_index; + } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = origin_donation; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + return self; + } + + pub const GatherOpts = struct { indices_are_sorted: bool = false }; + + /// For each coordinate in `indices`, + /// `gatherValues` extracts a single value of the given tensor. + /// + /// * axes_ is a single axis, or a tuple of axis: .b, or .{ .b, .c } + /// * indices is an integer tensor + /// * result is a tensor whose shape is similar to the input shape + /// where the gathered axes have been replaced by axes from 'indices'. + /// + /// Some example input for the base case where we work on one axis: + /// - gatherValues(f:[a]->float, .a, ind:[n]->int)[n] == f[ind[n]] + /// - gatherValues(f:[a, b], .a, ind:[n])[n, b] == f[ind[n], b] + /// - gatherValues(f: [a,b,c], .{.b}, ind: [n,m])[a, n, m, c] == f[a, ind[n, m], c] + /// + /// If an axis in common between `self` and `indices`, + /// it is treated as a "batching" axis, meaning that semantically + /// the operator is doing a gatherValues one time per dimension of this axis: + /// - gatherValues(f: [a,b,c], .{.b}, ind: [a,n])[a, n] == f[a, ind[a, n]] + /// + /// It is an error to have an axis present in `self`, `axes_` and `indices`. + /// + /// If several axes are passed, then the last axis of indices is treated as coordinates: + /// - gatherValues(f: [a,b,c], .{.b, .c}, ind: [n,2])[a, n] == f[a, ind[n][0], ind[n][1]] + /// - gatherValues(f: [a,b,c,d], .{.b, .c}, ind: [a, n,2])[a, n, d] == f[a, ind[a, n][0], ind[a, n][1], d] + /// + /// It is possible to use gatherValues without tags, but batching won't be available. + pub fn gatherValues(self: Tensor, coord_axes: anytype, indices: Tensor, opts: GatherOpts) Tensor { + // scoped_log.debug("gatherValues({}, {any}, {})", .{ self, coord_axes, indices }); + const single_coord, const coord_axes_ = _parseGatherCoord(self, coord_axes); + + stdx.debug.assert(coord_axes_.len > 0, "gatherValues expects 1 or more axes to operate one, received none. Example: `x.gatherValues(.a, indices, .{{}})`", .{}); + for (coord_axes_.constSlice(), 0..) |a, i| { + if (i > 0) { + stdx.debug.assert(a == coord_axes_.get(i - 1) + 1, "gatherValues expects 'coord_axes' to be sequential. But {any} aren't sequential in {f}", .{ coord_axes, self }); + } + } + + const LocalAxisKind = enum { batching, offset, collapsed, indices }; + var self_kind: stdx.BoundedArray(LocalAxisKind, MAX_RANK) = .{}; + var indices_batch_axes: Shape.DimsArray = .{}; + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const maybe_coord_ax = std.mem.indexOfScalar(u3, coord_axes_.constSlice(), @intCast(self_ax)); + if (indices._shape.hasTag(t)) |id_ax| { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_kind.appendAssumeCapacity(.batching); + indices_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(maybe_coord_ax == null, "gatherValues expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'coord_axes_={any}' and in 'indices={f}'", .{ self._shape._tags.get(self_ax), self, coord_axes, indices }); + } else if (maybe_coord_ax) |_| { + // for gatherValues we collapsed all gathered axes + // (contrary to gatherSlices where we collapse none) + self_kind.appendAssumeCapacity(.collapsed); + } else { + self_kind.appendAssumeCapacity(.offset); + } + } + + // When we receive several coord_axes we need an extra dimension to store + // one index per axis, which makes the coordinates of one value. + // Otherwi se stablehlo uses the "indices.rank()" default value. + const index_coord_axis = if (single_coord) + indices.rank() + else blk: { + const ax = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); + stdx.debug.assert(indices.dim(ax) == coord_axes_.len, "gatherValues with axes={any}, expects indices to be of shape [..., {}], got: {f}", .{ coord_axes, coord_axes_.len, indices }); + break :blk ax; + }; + + // compute res shape + var res_shape = Shape.init(.{}, self.dtype()); + var res_kind: stdx.BoundedArray(LocalAxisKind, MAX_RANK) = .{}; + for (self_kind.constSlice(), 0..) |kind, ax_usize| { + const ax: u3 = @intCast(ax_usize); + if (ax == coord_axes_.get(0)) { + // The first val_ax is special cause this is the place where we insert indices axes. + for (indices._shape.tags(), 0..indices.rank()) |t, id_ax| { + if (id_ax == index_coord_axis) continue; + if (std.mem.indexOfScalar(i64, indices_batch_axes.constSlice(), @intCast(id_ax))) |_| { + // batching dim are already in res + continue; + } + + res_shape = res_shape.appendDim(indices.dim(id_ax), t); + res_kind.appendAssumeCapacity(.indices); + } + } + switch (kind) { + .collapsed => continue, + else => { + res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); + res_kind.appendAssumeCapacity(kind); + }, + } + } + + // This is not a gather, but a dynamicSlice. + // Sometimes the backend recognize this pattern, but not always. + // So let us handle that. + if (indices.count() == 1) { + return self.dynamicSlice1d(coord_axes_.get(0), .{ .start = indices.flattenAll().squeeze(0), .len = 1 }).reshape(res_shape); + } + + var slice_dims: Shape.DimsArray = .{}; + for (self_kind.constSlice(), self.dims()) |k, d| { + slice_dims.appendAssumeCapacity(switch (k) { + .batching, .collapsed => 1, + .offset => d, + .indices => unreachable, + }); + } + + // scoped_log.debug("gatherValues --> {} {any}", .{ res_shape, res_kind.constSlice() }); + const gather_op = dialects.stablehlo.gather( + mlirCtx(), + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = _collectAxes(LocalAxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(LocalAxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(LocalAxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = indices_batch_axes.constSlice(), + .start_index_map = _collectAxes(LocalAxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = index_coord_axis, + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + //const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); + //stdx.debug.assert(mlir_shape.eql(res_shape), "gatherValues expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + return _result(res_shape, gather_op.result(0)); + } + + /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. + pub fn rsqrt(self: Tensor) Tensor { + const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, rsqrt_op.result(0)); + } + + /// Drops a 1-dim axis at the given index + pub fn squeeze(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); + + const new_shape = self._shape.remove(a); + // log.debug("squeeze({}, {d}={d}) -> ({})", .{ self, axis, a, new_shape }); + + return _result(new_shape, self.reshape(new_shape).value()); + } + + /// Returns a Tensor containing the sum of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. + pub fn sum(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + const result = reduce(.{self}, .{Tensor.scalar(0, self.dtype())}, &.{a}, struct { + pub fn acc(args: ReduceArgs) struct { Tensor } { + const x = args.left; + const res = args.right; + const result = res.add(x.convert(res.dtype())); + return .{result}; + } + }.acc, .{}); + return result.@"0"; + } + + inline fn wrapIndex(self: Tensor, axis_: usize, idx: i64) i64 { + return if (idx < 0) self.dim(axis_) + idx else idx; + } + + /// Returns a Tensor containing the mean of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. + pub fn mean(self: Tensor, axis_: anytype) Tensor { + return self.sum(axis_).divByConst(self.dim(axis_)); + } + + /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. + pub fn divByConst(self: Tensor, b: anytype) Tensor { + return self.div(Tensor.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. + pub inline fn scale(self: Tensor, val: anytype) Tensor { + return self.mul(Tensor.scalar(val, self.dtype())); + } + + /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. + pub fn addConstant(self: Tensor, b: anytype) Tensor { + return self.add(Tensor.scalar(b, self.dtype())); + } + + /// Returns a 0-rank Tensor with the given value. + pub fn scalar(val: anytype, dt: zml.DataType) Tensor { + const data = zml.Data.init(dt, val); + switch (dt.class()) { + .float => stdx.debug.assert(!std.math.isNan(val), "scalar(NaN) is probably due to compiling a model with an uninitialized field", .{}), + else => {}, + } + return Tensor.constant(data); + } + + /// Slices the input Tensor along a specific axis, with a start offset known at runtime. + /// Note: this doesn't support tagging, if you have tags, + /// you should use `dynamicSlice` directly. + pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { + stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {f}", .{slice_.start}); + + const a = self.axis(axis_); + const new_shape = self._shape.set(a, slice_.len); + + var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** MAX_RANK; + start_indices[a] = slice_.start.value(); + + const op = dialects.stablehlo.dynamic_slice( + mlirCtx(), + self.value(), + new_shape.dims(), + start_indices[0..self.rank()], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(new_shape, op.result(0)); + } + + pub const DynSlice = struct { start: Tensor, len: i64 }; + + pub const Slice = struct { + start: i64 = 0, + end: i64 = to_the_end, + step: i32 = 1, + singleton: bool = false, + + pub fn single(offset: i64) Slice { + return .{ .start = offset, .end = offset + 1, .singleton = true }; + } + + const to_the_end = std.math.maxInt(i64); + + pub fn format( + self: Slice, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + if (self.singleton) { + try writer.print("[{}]", .{self.start}); + } else if (self.end == to_the_end and self.step == 1) { + try writer.print("[{}..]", .{self.start}); + } else if (self.step == 1) { + try writer.print("[{}..{}]", .{ self.start, self.end }); + } else { + try writer.print("[{}..{}:{}]", .{ self.start, self.end, self.step }); + } + } + }; + + /// Slices the input Tensor over the given axis using the given parameters. + pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { + var slices = [_]Slice{.{}} ** MAX_RANK; + slices[self.axis(axis_)] = s; + return self.slice(slices[0..self.rank()]); + } + + /// Slices the input Tensor using the given parameters. + pub fn slice(self: Tensor, slices: []const Slice) Tensor { + var start_indices: [MAX_RANK]i64 = undefined; + var strides: [MAX_RANK]i64 = undefined; + var limit_indices: [MAX_RANK]i64 = undefined; + var res_shape: Shape = self._shape; + + for (slices, 0..) |s, a| { + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); + + const args: Slice = .{ + .start = self.wrapIndex(a, s.start), + .end = if (s.end == Slice.to_the_end) self.dim(a) else self.wrapIndex(a, s.end), + .step = s.step, + }; + start_indices[a] = args.start; + limit_indices[a] = args.end; + strides[a] = args.step; + res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); + } + + const result_type = mlir.rankedTensorType(res_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), res_shape.dtype())); + const slice_op = dialects.stablehlo.slice( + mlirCtx(), + self.value(), + start_indices[0..self.rank()], + limit_indices[0..self.rank()], + strides[0..self.rank()], + result_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + var res = _result(res_shape, slice_op.result(0)); + var to_remove: Shape.AxesArray = .{}; + for (slices, 0..) |s, a| { + if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); + } + return res.reshape(res_shape.removeMany(to_remove.constSlice())); + } + + /// Returns a Tensor containing the Sigmoid Linear Unit (SiLU) activation function applied to each element of the input Tensor. + /// + /// silu(x) = x σ(x) + /// https://paperswithcode.com/method/silu + pub fn silu(x: Tensor) Tensor { + return x.mul(x.sigmoid()); + } + + /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. + pub fn sigmoid(self: Tensor) Tensor { + const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + pub const logistic = sigmoid; + + /// Splits the given axis in several axes. + /// eg: `Tensor.init(.{ .a = 10, .b = 3 }).split(.a, .{.a1 = 5, .a2 = 2});` + /// The number of elements in the split shape must match the number of element + /// in the target axis. + pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { + const new_shape = self._shape.splitAxis(ax, split_shape); + + const tensor_type = mlir.rankedTensorType(new_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), new_shape.dtype())); + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), + self.value(), + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(new_shape, reshaped_val.result(0)); + } + + /// Gathers slices along the given axes with runtime indices. + /// * slice_shape represents the shape of the slices to extract, + /// it must be smaller than original shape. + /// It must use a subset of self axes. + /// If slice_shape is **not** tagged, then it must have the same rank than self. + /// * `indices` represents a set of coordinates. + /// The coordinates are read from the `.coord` axis, or last axis if `.coord` is not found. + /// The coordinate axis must have `slice_shape.rank()` dims. + /// The coordinates represent the "top-left" corner of the slice to extract. + /// * the output tensor starts with axes from `indices`. + /// * if the input tensor has tagged axes, matching `indices` axes, + /// they will be considered "batching" axes. + /// + /// Sample input/output shapes: + /// * gatherSlices([A, B, C, D], .{.b=B', .c=C'}, [N, 2]) -> [N, A, B', C', D] + /// * gatherSlices(x(a,b,c,d), .{.b=B', .c=C'}, g(n,m)) = z(n, a, b', c', d) = x(a, g(n, 0) + b', g(n, 1) + c', d) + /// + /// Note: the axis order of the result is different from gatherValues. + /// This is because gatherSlices, favorizes contiguous copy of the extracted slices, + /// while gatherValues, always copy values one by one, and as such don't have the same issues. + /// In our example the contiguous dimension .d is not sliced + /// and gatherSlices can copy data by group of C'*D elements. + pub fn gatherSlices(self: Tensor, slice_shape_: anytype, indices: Tensor, opts: GatherOpts) Tensor { + const slice_shape = if (@TypeOf(slice_shape_) == Shape) slice_shape_ else Shape.init(slice_shape_, .i32); + // scoped_log.debug("gatherSlice({}, {f}, {})", .{ self, slice_shape, indices }); + + const tagged_api = slice_shape.isFullyTagged(); + if (tagged_api) { + for (slice_shape.tags()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "gatherSlices expects `slices_shape` to only use tags from `self`. But {s} wasn't found in {f}", .{ t, self }); + } + } else { + // For untagged api, we require all slices to be specified. + // Note: we could relax this and right align the slice. + stdx.debug.assert(slice_shape.rank() == self.rank(), "gatherSlices expects `slice_shape.rank()` to match `self.rank()`. Got: gatherSlices({f}, slice={f}). To avoid specifying all axes in `slice_shape`, you can use tags.", .{ self, slice_shape }); + } + + const index_coord_axis = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); + stdx.debug.assert(indices.dim(index_coord_axis) == slice_shape.rank(), "gatherSlices({f}, slice={f}, indices) expects 'indices' to be a tensor [..., {}], got {f}", .{ self, slice_shape, slice_shape.rank(), indices }); + + // Compute result shape + var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); + var slice_dims = self._shape._dims; + var self_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var indices_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var start_index_map: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var self_offset_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); + + if (tagged_api and indices._shape.hasTag(t) != null) { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_batch_axes.appendAssumeCapacity(@intCast(self_ax)); + indices_batch_axes.appendAssumeCapacity(indices._shape.axis(t)); + slice_dims.set(self_ax, 1); + stdx.debug.assert(slice_shape.hasTag(t) == null, "gatherSlices expect axes to be either batches or slices axes. Axis {s} has been found both in `slices={f}` and `indices={f}`", .{ t, slice_shape, indices }); + } else if (maybe_slice_ax) |slice_ax| { + // Specified axes contains the start offset of the slices, + // and are collected in `start_index_map`. + const slice_dim = slice_shape.dim(slice_ax); + stdx.debug.assert(slice_dim <= self._shape.dim(self_ax), "gatherSlices expects `slice_shape` to be smaller than `self.shape()`. On axis {s}, got {f} > {f}.", .{ t, slice_shape, self._shape }); + slice_dims.set(self_ax, slice_dim); + res_shape = res_shape.appendDim(slice_dim, t); + start_index_map.appendAssumeCapacity(@intCast(self_ax)); + self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); + } else { + // non-batching, non-indexed axes + res_shape = res_shape.appendDim(self.dim(self_ax), t); + self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); + } + } + + const gather_op = dialects.stablehlo.gather( + mlirCtx(), + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = self_offset_axes.constSlice(), + .collapsed_slice_dims = &.{}, + .operand_batching_dims = self_batch_axes.constSlice(), + .start_indices_batching_dims = indices_batch_axes.constSlice(), + .start_index_map = start_index_map.constSlice(), + .index_vector_dim = index_coord_axis, + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, gather_op.result(0)); + } + + /// Merges two or more contiguous axes into one axis. + pub fn merge(self: Tensor, merges_: anytype) Tensor { + return self.reshape(self._shape.mergeAxes(merges_)); + } + + /// Slices a Tensor across many axes, with runtime known offsets. + /// + /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. + /// When using the tagged API it is allowed to not specify some axes. + /// But with the non-tagged API all slices need to be specified. + /// Examples: + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ .a = .{ .start = a_off, .len = 11}); + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ + /// .a = .{ .start = a_off, .len = 11 }, + /// .b = .{ .start = b_off, .len = 12 }, + /// }); + /// Tensor(.{ 20,30,40}).dynamicSlice(.{.{ .start = scalar(0, .i32), .len = 20 }, .{ .start = b_off, .len = 12 }, .{ .start = scalar(0, .i32), .len = 40 }}); + /// ``` + pub fn dynamicSlice(self: Tensor, slices_: anytype) Tensor { + // TODO: the untagged api is a bit verbose. Should I allow: `Tensor(.{ 20,30,40}).dynamicSlice(.{.{}, .{ .start = b_off, .len = 12 }, .{}});` ?? + // + const slices, const slices_tags = Shape.parseStruct(DynSlice, slices_); + + // TODO use slices and slices_tags for the format. + // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" + // which is kinda ugly. + + const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; + const zero = Tensor.scalar(0, idx_dtype).value(); + var offset_values = [_]*const mlir.Value{zero} ** MAX_RANK; + var res_shape = self._shape; + for (slices.constSlice(), 0..) |slice_, i| { + const offset = slice_.start; + const len = slice_.len; + if (slices_tags.len == 0) { + stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {d} and {d}", .{ self.rank(), slices.len }); + + offset_values[i] = offset.value(); + res_shape._dims.set(i, len); + + stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {d} and {d} for index {d}", .{ len, self.dim(i), i }); + } else { + const t = slices_tags.get(i); + const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {f})", .{ t, self._shape }); + + stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {d} and {d} for axis {s}", .{ len, self.dim(a), t }); + + offset_values[a] = offset.value(); + res_shape._dims.set(a, len); + } + } + const op = dialects.stablehlo.dynamic_slice(mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. + /// + /// The output dtype will be `.i32`, unless the given axis has a too big dimension, in that case we use `.i64`. + /// In most program this shouldn't matter, because typically this will be used in a comparison, + /// or explicitly converted by the user to do floating point arithmetic. + pub fn iota(sh: Shape, axis_: anytype) Tensor { + const a = sh.axis(axis_); + const dt: zml.DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; + const res_shape = sh.withDtype(dt); + + const tensor_type = mlir.rankedTensorType(res_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), res_shape.dtype())); + var op = dialects.stablehlo.iota( + mlirCtx(), + a, + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + /// Returns a Tensor containing the result of the outer product between the input Tensors. + pub fn outer(self: Tensor, other: Tensor) Tensor { + if (self.rank() + other.rank() == 1) { + return self.mul(other); + } + + const res_shape = self.shape().outer(other.shape()); + return self.broad(res_shape).mul(other.broad(res_shape)); + } + + /// Appends a 1-dim axis, with the given tag. + pub fn appendAxes(self: Tensor, t: anytype) Tensor { + // stdx.debug.assert(self.rank() < Tensor.MAX_RANK - t.len, "appendAxis expects tensor rank to be small enough in order to extend it, got {} and {} (max is {})", .{ self.rank(), t.len, Tensor.MAX_RANK }); + + return self.insertAxes(.last, t); + } + + /// Repeats a Tensor several times along the given axis. + /// + /// * repeat1d(x, axis, 4) = concat(&.{x, x, x, x}, axis); + /// * repeat1d([0, 1, 2, 3], 0, 2) = [0, 1, 2, 3, 0, 1, 2, 3] + pub fn repeat1d(self: Tensor, axis_: anytype, n_rep: u63) Tensor { + if (n_rep == 1) { + return self; + } + + const a = self.axis(axis_); + const broadshape = self._shape.insert(a + 1, .{n_rep}); + const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + + var res = self.broadcast(broadshape, repeat_dims.dims()).flatten(a); + // Restor the tag that has been lost by flatten. + res._shape._tags.set(a, self._shape.tag(a)); + + return res; + } + + /// Returns a Tensor containing the softmax function applied to each element of the input Tensor. + pub fn softmax(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + const max_val = self.max(a); + const row_mask = max_val.cmp(.GT, Tensor.scalar(-std.math.inf(f64), self.dtype())); + + const exp_diff_max = self.sub(self.max(a).broad(self._shape)).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a).broad(self._shape)); + + // If a row is full -inf return full 0 instead of full nan, + // this fix attention when mask hides a full row. + return row_mask.broad(self.shape()).select(res, Tensor.scalar(0, self.dtype())); + } + + inline fn toI64(values: anytype) []i64 { + var res: [Tensor.MAX_RANK]i64 = undefined; + for (values, 0..) |val, i| res[i] = @intCast(val); + return res[0..values.len]; + } + + fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { + var perm: stdx.BoundedArray(struct { u8, bool }, Tensor.MAX_RANK) = .{}; + // Don't rewrite on invalid inputs. + if (permutation.len > x.rank()) return false; + for (permutation) |ax| { + const squeezable = x.dim(ax) == 1; + perm.appendAssumeCapacity(.{ @intCast(ax), squeezable }); + } + + var effective_ax: u8 = 0; + for (0..perm.len) |i| { + const ax, const squeezable = perm.get(i); + if (squeezable) { + // Effectively squeeze this axis by decrementing axes coming after by 1. + for (i..perm.len) |j| { + if (perm.buffer[j][0] > ax) { + perm.buffer[j][0] -= 1; + } + } + continue; + } + + if (ax != effective_ax) return false; + effective_ax += 1; + } + + return true; + } + + /// Returns a transposed Tensor computed using the given axes. + pub fn transpose(self: Tensor, axes_: anytype) Tensor { + const axes__ = self.axes(axes_).constSlice(); + const default_perm = [MAX_RANK]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; + const no_op = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + + const permutation: []const i64 = if (axes__.len == 0) + default_perm[MAX_RANK - self.rank() ..] + else + toI64(axes__); + + stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, MAX_RANK + 2)] }); + + if (std.mem.eql(i64, permutation, no_op[0..self.rank()])) { + return self; + } + + const res_shape = self._shape.transpose(permutation); + if (transposeIsJustAReshape(self.shape(), permutation)) { + return self.reshape(res_shape); + } + + const tensor_type = mlir.rankedTensorType(res_shape.dims(), zml.mlir.Type.fromDType(mlirCtx(), res_shape.dtype())); + const op = dialects.stablehlo.transpose( + mlirCtx(), + self.value(), + tensor_type, + .{ .permutation = toI64(permutation) }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + /// Embeds a buffer with concrete values into an Mlir program. + pub fn constantTensor(bytes: []const u8, shape_: zml.Shape) Tensor { + const elem_type = zml.mlir.Type.fromDType(mlirCtx(), shape_.dtype()); + //const elem_type = zml.mlir.denseElementAttrType(shape.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{shape.dtype()}); + const constant_op = dialects.stablehlo.constant(mlirCtx(), shape_.dims(), elem_type, bytes, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(shape_, constant_op.result(0)); + } + + /// Inserts 1-dim axes at the given position, with the given tags. + /// `.{.a = 5, .b = 4}.insert(.b, .{ .c, .d }) -> .{ .a = 5, .c = 1, .d = 1, .b = 4 }` + pub fn insertAxes(self: Tensor, axis_: anytype, tags: anytype) Tensor { + const tags_ = Shape.parseTags(tags); + const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + self.rank() + else + self.axis(axis_); + + var res_shape = self._shape; + const ones = [_]i64{1} ** MAX_RANK; + res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; + res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + return self.reshape(res_shape); + } + + /// Returns a Tensor containing the maximum over a given axis. + pub fn max(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + const result = reduce(.{self}, .{Tensor.constant(self.dtype().minValue())}, &.{a}, struct { + pub fn cmp(values: ReduceArgs) struct { Tensor } { + return .{values.left.maximum(values.right.convert(values.left.dtype()))}; + } + }.cmp, .{}); + return result[0]; + } + + /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. + pub fn exp(self: Tensor) Tensor { + const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Generalized version of scatter to many inputs. + /// See `zml.Tensor.scatterSlices` for documentation on scatter. + /// + /// This allows to use the same indices to update several tensors at once, + /// and where the update function is allow to look at elements from the different tensors + /// to compute the final value. + /// + /// This sounds nice but in practice XLA doesn't support this well on GPU, + /// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. + pub fn scatter( + inputs: anytype, + index_tensors: anytype, + updates: anytype, + comptime func: anytype, + context: anytype, + opts: Tensor.ScatterOpts, + ) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const update_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].left = Tensor.init(zml.Shape.init(.{}, inputs[i].dtype())); + args[i].right = Tensor.init(zml.Shape.init(.{}, inputs[i].dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), zml.mlir.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].right.dims(), zml.mlir.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const update_block = mlir.Block.init(&block_types, &block_locs); + errdefer update_block.deinit(); + + CompilationContext.current().pushBlock(update_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(update_block); + break :b .{ update_block, result }; + }; + + // Note: I was a bit lazy here, and I only look at tags on the first tensor. + // we probably should check all of them. + const self = zml.meta.first(Tensor, inputs); + const update = zml.meta.first(Tensor, updates); + var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); + + if (indices_per_axis.len == 0) return inputs; + + // validate coord axes: all coord_axes should exist inside self + for (indices_axes.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); + } + + // Handle scalar indices by broadcasting them to the indices with the highest rank. + const indices_shape = blk: { + var higher_rank = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |indices| { + if (indices.rank() > higher_rank.rank()) { + higher_rank = indices.shape(); + } + } + break :blk higher_rank; + }; + for (indices_per_axis.slice()) |*idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); + stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); + idx.* = idx.broad(indices_shape); + } + + // rewrite simple scatters to dynamicUpdateSlice. + if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { + return .{self.dynamicUpdateSlice(index_tensors, update)}; + } + + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. + var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); + const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); + // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; + // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var updates_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); + + const op = dialects.stablehlo.scatter( + mlirCtx(), + &input_values, + &.{indices.value()}, + &updates_values, + update_block, + .{ + .update_window_dims = _collectAxes(AxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(AxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(AxisKind, config.op_kind, .batching).constSlice(), + .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), + .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), + .index_vector_dim = indices.rank() - 1, + .indices_are_sorted = opts.indices_are_sorted, + .unique_indices = opts.indices_are_unique, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), op.result(i)); + } + + return result; + } + + const ScatterConfig = struct { + op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + indices_batch_axes: Shape.DimsArray = .{}, + scatter_to_operand_axes: Shape.DimsArray = .{}, + updates_transpose: Shape.AxesArray = .{}, + }; + + const AxisKind = enum { batching, update_window, inserted_window, window_id }; + + fn scatterConfig( + op: Shape, + update: Shape, + indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: Shape.TagsArray, + ) ScatterConfig { + var op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var indices_batch_axes: Shape.DimsArray = .{}; + var scatter_to_operand_axes: Shape.DimsArray = .{}; + var updates_transpose: Shape.AxesArray = .{}; + + const tagged_api = indices_axes.len > 0; + const indices = indices_per_axis.get(0).shape(); + + if (tagged_api) { + for (indices_axes.constSlice()) |t| { + scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); + } + for (indices.tags()) |t| { + stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); + updates_transpose.appendAssumeCapacity(update.axis(t)); + } + + for (op.tags()) |t| { + if (update.hasTag(t)) |up_ax| { + updates_transpose.appendAssumeCapacity(up_ax); + + if (indices.hasTag(t)) |id_ax| { + if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { + // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim + op_kind.appendAssumeCapacity(.inserted_window); + } else { + // tag is in op, indices and updates -> it's a batching dim + op_kind.appendAssumeCapacity(.batching); + indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); + } + } else { + op_kind.appendAssumeCapacity(.update_window); + } + } else { + op_kind.appendAssumeCapacity(.inserted_window); + } + } + + for (update.tags(), 0..) |t, up_ax| { + // Handle batch axes right away. + if (op.hasTag(t)) |self_ax| { + if (op_kind.get(self_ax) == .batching) { + up_kind.appendAssumeCapacity(.batching); + continue; + } + } + if (indices.hasTag(t) != null) { + up_kind.appendAssumeCapacity(.window_id); + } else if (op.hasTag(t)) |self_ax| { + stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); + up_kind.appendAssumeCapacity(.update_window); + } else { + // TODO: consider accepting untagged update here. + std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); + } + } + } else { + for (0..indices_per_axis.len) |i| { + op_kind.appendAssumeCapacity(.inserted_window); + scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); + up_kind.appendAssumeCapacity(.window_id); + } + for (indices_per_axis.len..op.rank()) |_| { + op_kind.appendAssumeCapacity(.update_window); + } + for (indices_per_axis.len..update.rank()) |_| { + up_kind.appendAssumeCapacity(.update_window); + } + for (0..update.rank()) |i| { + updates_transpose.appendAssumeCapacity(@intCast(i)); + } + } + + return .{ + .op_kind = op_kind, + .up_kind = up_kind, + .indices_batch_axes = indices_batch_axes, + .scatter_to_operand_axes = scatter_to_operand_axes, + .updates_transpose = updates_transpose, + }; + } + + //test scatterConfig { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // const Local = struct { + // pub fn _idx(idx_shape: anytype) Tensor { + // return Tensor.constant(idx_shape, .{ .i32 = 0 }); + // } + // }; + + // const idx = Local._idx; + // const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); + + // // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + // { + // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); + // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + // const cfg = scatterConfig(op, update, indices, coords_tags); + // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); + // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); + // } + + // // similar, but use the normalized form where .a is no longer an explicit batching axis. + // { + // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); + // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + // const cfg = scatterConfig(op, update, indices, coords_tags); + // try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); + // try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); + // } + //} + + /// Concatenate all indices tensor in one tensor. + /// + /// Is allowed to reorder stuff to simplify the job of the backend, + /// and to expand the batching dims. + fn scatterPrepareIndices( + cfg: *ScatterConfig, + op: Shape, + update: Shape, + indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: *Shape.TagsArray, + ) Tensor { + var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; + const batching = _collectAxes(AxisKind, cfg.op_kind, .batching); + for (batching.constSlice()) |batch_ax| { + const id_shape = indices_per_axis.get(0).shape(); + // batching requires tagging, so we're sure to have a tag here. + const batch_tag = op.tag(batch_ax); + indices_axes.appendAssumeCapacity(batch_tag); + const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); + indices_per_axis.appendAssumeCapacity(batch_id); + cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; + cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; + old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); + } + cfg.indices_batch_axes = .{}; + + // Reorder the axes so that in indices_per_axis is ordered like in op if possible. + // TODO: transpose updates if needed + var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; + var scatter_to_op_axes: Shape.DimsArray = .{}; + + while (old_scatter_to_op_axes.len > 0) { + const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; + const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); + const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); + + scatter_to_op_axes.appendAssumeCapacity(op_ax); + indices.appendAssumeCapacity(scatter_idx); + } + cfg.scatter_to_operand_axes = scatter_to_op_axes; + + for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { + if (i != sc_ax) { + //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); + break; + } + } + return Tensor.stack(indices.constSlice(), .last, .coord); + } + + pub const ScatterOpts = struct { + /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final offset in `self` + /// Result is undefined if the promise is violated. + indices_are_sorted: bool = false, + + /// Promise scatter that slices don't overlap. + /// Result is undefined if the promise is violated. + /// This allows for better code generation, because it means that updates can be applied in parallel. + indices_are_unique: bool = false, + + /// Function used to update previous value in `self` with values from `updates`. + /// If `update_fn` is not associative (ie the order of execution matters), + /// then you should make sure the slices don't overlap, + /// otherwise the result will depend on the runtime scheduling + /// of the operator which is backend specific. + update_fn: *const fn (ReduceArgs) struct { Tensor } = increment, + + pub fn increment(values: ReduceArgs) struct { Tensor } { + return .{values.left.add(values.right)}; + } + + pub fn override(values: ReduceArgs) struct { Tensor } { + return .{values.right}; + } + }; + + /// Update the given tensor, by copying `values` into slice by slice into `self`. + /// The slices are chosen at runtime by interpreting indices as coordinates into `self`. + /// This is a generalized version of `dynamicUpdateSlice` where more than one offset can be specified at a time. + /// + /// ### Arguments + /// + /// - Return a tensor with same shape than `self`, with updated content. + /// - `indices` is a set of Tensor (typically rank 1), representing coordinates into `self`. + /// all indices must have the same shape, but scalars are accepted. + /// - each `indices` entry contains offset along an axes into `self`. + /// Typically axes are identified by their tags, but in the absence of tags on `indices`, + /// The entry in indices will be assigned to axes of `self` from major to minor axis. + /// It is recommended to have indices referencing only major axes of `self` for better performance. + /// - `values` shape is obtained by concatenating the shape of `indices` with the shape of the slices to be extracted. + /// - `opts`: `zml.Tensor.ScatterOpts` des + /// + /// ### Sample input/output shapes with corresponding pseudo-code. + /// + /// Basic `scatterSlices` with the first two axes (.a, .b) being indexed, and full (.c, .d) slice copies: + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N]}, y[N, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],c,d] += y[n, c, d]; + /// }} + /// } + /// return z; + /// } + /// ``` + /// + /// `scatterSlices` with the first three axes (.a, .b, .c) being indexed, and a partial copy of (.c, .d). + /// Note that .c axis is present both in the indices and updates, and `updates.dim(.c) < self.dim(.c)`. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N], .c=off_c[N]}, y[N, C', D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C') |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],off_c[n]+c,d] += y[n, c, d]; + /// }} + /// } + /// return z; + /// } + /// ``` + /// + /// `scatterSlices` with the first axis .a being indexed, and where .b is used as a batching axis. + /// Note that here .b axis is present in `self`, `off_a`, and `updates`, + /// and is not mentionned in the axes of indices. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[B,N]}, y[N, B, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..B) |b| { + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[b,n],b,c,d] += y[n, b, c, d]; + /// }} + /// } + /// } + /// return z; + /// } + /// ``` + /// + /// ### Warnings + /// + /// - if `opts.update_fn` is not associative not all calls to `scatterSlices` are sound. + /// In particular if you scatter overlapping slices, with `zml.Tensor.ScatterOpts.override`, + /// then the result will depend on the execution order that you don't control. + /// - `scatterSlices` is a very expressive operator, and can lead to complicated code generation + /// that requires host<->device synchronization. + /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. + pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { + //scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); + + const UpdateType = @TypeOf(ScatterOpts.increment); + + const Custom = struct { + pub fn inc(values: ReduceArgs, custom: *const UpdateType) struct { Tensor } { + return @call(.auto, custom, .{values}); + } + }; + + return scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; + } + + /// Concatenates the input Tensors along a new axis. The Tensors must have the same shape. + /// For x, y, z of shape .{ .a = 10, .b = 11, .c = 12 }: + /// - Tensor.stack(&.{x, y, z}, .b, .layers) -> .{ .a, .layers, .b, .c } + /// - Tensor.stack(&.{x, y, z}, 1, .layers) -> .{ .a, .layers, .b, .c } + /// - Tensor.stack(&.{x, y, z}, .last, .layers) -> .{ .a, .b, .c, .layers } + pub fn stack(tensors: []const Tensor, axis_: anytype, tag: anytype) Tensor { + // Note: we could ask the compilation context for some memory instead of stack allocating + stdx.debug.assert(tensors.len <= 32, "stack only supports up to 32 tensors, got {}", .{tensors.len}); + + const shape0 = tensors[0]._shape; + const res_shape = shape0.insertTag(axis_, 1, tag); + + for (tensors[1..]) |tensor| { + stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); + } + + var reshaped: [32]Tensor = undefined; + for (tensors, 0..) |tensor, i| { + reshaped[i] = tensor.reshape(res_shape); + } + + // Be careful here: we need to resolve ax before calling concatenate, + // because we added an axis, so all + const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + shape0.rank() + else + shape0.axis(axis_); + + return Tensor.concatenate(reshaped[0..tensors.len], ax); + } + + /// Updates a part of the input Tensor using the given 'update' Tensor, with runtime known offsets. + /// + /// The offsets are specified similarly to the dynamicSlice api. + /// It's semantically equivalent to: + /// self.dynamicSlice(offsets_) := update + /// Examples: + /// ``` + /// Tensor(.{ .a = 2, .b = 5 }).dynamicUpdateSlice(.{ .a = scalar(1, .i32) }, Tensor(.{ .b = 5 })); + /// ``` + pub fn dynamicUpdateSlice(self: Tensor, offset_: anytype, update_: Tensor) Tensor { + // TODO: add updateSlice for when the offset isn't dynamic + stdx.debug.assert(self.dtype() == update_.dtype(), "dynamicUpdateSlice expects input and 'update_' tensors to be of the same type, got {} and {}", .{ self.dtype(), update_.dtype() }); + + const offset, const offset_tags = Shape.parseStruct(Tensor, offset_); + // log.debug("offset: {any}, offset_tags: {any}", .{ offset, offset_tags }); + for (offset.constSlice(), 0..) |start_idx, i| { + stdx.debug.assert(start_idx.rank() == 0, "dynamicUpdateSlice expects 'offset_' tensor ranks to be equal to 0, got {} at index {}", .{ start_idx.rank(), i }); + } + + const tagged_api = update_._shape.isFullyTagged() and self._shape.isFullyTagged() and offset_tags.len > 0; + // When using tags, we can safely insert axis with a 1-dim. + // the offset into the inserted axis will need to be specified through indices. + var update = update_; + if (tagged_api) { + // Check that all update tags are known. + for (update._shape._tags.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "dynamicUpdateSlice expects 'update_' tensor tags to be a subset of input tensor tags but {s} is missing (input shape is {f})", .{ t, self._shape }); + } + + var update_shape = self._shape; + var prev_ax: i8 = -1; + for (self._shape.tags(), 0..) |t, self_ax| { + if (update._shape.hasTag(t)) |up_ax| { + stdx.debug.assert(up_ax == prev_ax + 1, "dynamicUpdateSlice expects 'update_' and input tensor axis to have the same order, got {f} and {f}. (hint: you need to explicitly transpose 'update_')", .{ update_, self }); + + update_shape._dims.set(self_ax, update.dim(up_ax)); + prev_ax = up_ax; + } else { + update_shape._dims.set(self_ax, 1); + } + } + update = update.reshape(update_shape); + } + + stdx.debug.assert(self.rank() == update.rank(), "dynamicUpdateSlice expects input and computed update tensors to have the same rank, got {f} and {f}", .{ self, update }); + + for (self.dims(), update.dims(), 0..) |self_d, up_d, ax| { + const t = self._shape.debugTag(ax); + stdx.debug.assert(up_d <= self_d, "dynamicUpdateSlice expects 'update_' dimensions to be less than or equal to their corresponding dimension in input tensor, got {} and {} for axis .{s}", .{ up_d, self_d, t }); + + if (tagged_api and up_d < self_d) { + const axis_has_offset = std.mem.indexOfScalar(Shape.Tag, offset_tags.constSlice(), self._shape._tags.get(ax)) != null; + + stdx.debug.assert(axis_has_offset, "dynamicUpdateSlice expects 'update_' dimensions to be equal to their corresponding dimension in input tensor, got {} and {} for axis .{s} (hint: you need to provide an offset)", .{ up_d, self_d, t }); + } + } + + const idx_dtype = if (offset.len > 0) offset.get(0).dtype() else .i32; + const zero = Tensor.scalar(0, idx_dtype).value(); + var offset_values: [MAX_RANK]*const mlir.Value = undefined; + if (offset_tags.len == 0) { + // Without offset tags we need the same number of offset than rank. + stdx.debug.assert(self.rank() == offset.len, "dynamicUpdateSlice expects input tensor rank and 'offset_' length to be equal, got {} and {}", .{ self.rank(), offset.len }); + + for (offset.constSlice(), 0..) |idx, i| { + offset_values[i] = idx.value(); + } + } else { + // If an axis isn't specified, update the full slice. + // This is only allowed when using tagged sliced. + offset_values = .{zero} ** MAX_RANK; + for (offset.constSlice(), offset_tags.constSlice()) |start, t| { + const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); + offset_values[a] = start.value(); + } + } + + const op = dialects.stablehlo.dynamic_update_slice( + mlirCtx(), + self.value(), + update.value(), + offset_values[0..self.rank()], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + pub fn withSharding(self: Tensor, axes_: anytype) Tensor { + const partitioned_shape = self._shape.withPartitioning(axes_); + //scoped_log.debug("withSharding {}: {} -> {}", .{ axes_, self._shape, partitioned_shape }); + + const ctx = self.getContext(); + const mesh_ = ctx.currentMesh(); + + if (mesh_.isSinglePartition()) return self; + + const sharding: Sharding = .init(mesh_, partitioned_shape); + + const op = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + .{ + .call_target_name = "Sharding", + .has_side_effect = false, + .backend_config = null, + .additional_attributes = &.{mlir.NamedAttribute.named(ctx.mlir_ctx, "mhlo.sharding", ctx.getShardingAttr(sharding))}, + .api_version = .original, + }, + .unknown(ctx.mlir_ctx), + ).appendTo(currentBlock()); + + return _result(partitioned_shape, op.result(0)); + } + + pub fn withMesh(self: Tensor, mesh: Mesh) Tensor { + var t = self; + t._mesh = mesh; + return t; + } + + pub fn getContext(self: Tensor) *CompilationContext { + _ = self; // autofix + return global_compilation_context.?; + } + + pub fn format( + self: Tensor, + writer: *std.Io.Writer, + ) !void { + // TODO(0.15.0) handle format + // const bare_fmt = fmt.len == 1 and fmt[0] == '_'; + try writer.print("{s}({f})", .{ @typeName(Tensor), self._shape }); + } +}; + +fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, Tensor.MAX_RANK) } { + const AxesT = @TypeOf(axes_); + const axes_is_scalar = AxesT == EnumLiteral or AxesT == comptime_int or @typeInfo(AxesT) == .int; + + const coord_axes = if (axes_is_scalar) + stdx.BoundedArray(u3, Tensor.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable + else + self.axes(axes_); + + return .{ axes_is_scalar, coord_axes }; +} + +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { + var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (bounded_array.constSlice(), 0..) |v, ax| { + if (v == value) { + res.appendAssumeCapacity(@intCast(ax)); + } + } + return res; +} + +pub const TransferEntry = struct { + buffer: zml.Buffer, + writer: *Transfer.Writer, +}; + +pub fn singleTransfer(allocator: std.mem.Allocator, shape: Shape, platform: Platform) !struct { Transfer, TransferEntry } { + var transfer = try Transfer.init(allocator, &.{shape}, platform); + errdefer transfer.deinit(allocator, platform); + + const entry: TransferEntry = .{ + .buffer = zml.Buffer.fromPjrtBuffers(platform, shape, &.{transfer.get(0).buffer}), + .writer = &transfer.get(0).writer, + }; + + return .{ transfer, entry }; +} + +pub fn multiTransfer(allocator: std.mem.Allocator, shapes: anytype, platform: Platform) !struct { Transfer, [shapes.len]TransferEntry } { + var shapes_array: [shapes.len]zml.Shape = undefined; + inline for (shapes, 0..) |shape, index| shapes_array[index] = shape; + + var transfer = try Transfer.init(allocator, &shapes_array, platform); + errdefer transfer.deinit(allocator, platform); + + var entries: [shapes.len]TransferEntry = undefined; + inline for (shapes, 0..) |shape, index| { + entries[index].buffer = zml.Buffer.fromPjrtBuffers(platform, shape, &.{transfer.get(index).buffer}); + entries[index].writer = &transfer.get(index).writer; + } + + return .{ transfer, entries }; +} + +pub fn bufferTransfer(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, name: []const u8, platform: Platform) !zml.Buffer { + const shape = buffer_store.getShape(name).?; + var transfer, const entry = try singleTransfer(allocator, shape, platform); + defer transfer.deinit(platform); + + const tensor_desc = buffer_store.store.getPtrFromKey(name).?; + + var tensor_resource: Resource = .{ + .http = try .init(buffer_store.store.http_client, allocator, tensor_desc.resource_uri, .{ .offset = tensor_desc.offset, .size = tensor_desc.shape.byteSize() }), + }; + defer tensor_resource.deinit(); + + const buffer_reader = try allocator.alloc(u8, BUF_16_MB); + defer allocator.free(buffer_reader); + + var reader = tensor_resource.reader(buffer_reader); + + _ = try reader.interface().streamRemaining(&entry.writer.interface); + try entry.writer.interface.flush(); + return entry.buffer; +} + +pub fn autoLoad(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferStore5.View, platform: Platform) !Bufferized(@TypeOf(model)) { + const Model = @TypeOf(model); + var result: Bufferized(Model) = undefined; + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const shapes = try collectShapes(arena.allocator(), &model); + + var transfer: Transfer = try .init(arena.allocator(), shapes, platform); + defer transfer.deinit(arena.allocator(), platform); + + const readers = try arena.allocator().alloc(BufferStore5.Reader, shapes.len); + + { + var index: usize = 0; + const type_info = @typeInfo(Model); + switch (type_info) { + .@"struct" => |struct_type_info| { + inline for (struct_type_info.fields) |field| { + const reader = buffer_store.getReader(field.name); + readers[index] = reader; + index += 1; + } + }, + else => unreachable, + } + } + + const LocalContext = struct { + readers: []BufferStore5.Reader, + shapes: []const Shape, + platform: Platform, + transfer: *Transfer, + index: usize = 0, + }; + var context: LocalContext = .{ .readers = readers, .shapes = shapes, .platform = platform, .transfer = &transfer }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { + const writer = context_.transfer.get(context_.index).writer(); + context_.readers[context_.index].stream(writer); + + buffer.* = zml.Buffer.fromPjrtBuffers(context_.platform, context_.shapes[context_.index], &.{context_.transfer.get(context_.index).buffer}); + context_.index += 1; + } + }.cb, &context, &result); + + return result; +} + +pub fn loadBuffersFromId(allocator: std.mem.Allocator, model: anytype, buffer_store: BufferStore5.View, platform: Platform) !Bufferized(@TypeOf(model)) { + //const id = zml.tools.tracer.tracer().frameStart("loadBuffersFromId"); + //defer zml.tools.tracer.tracer().frameEnd(id, "loadBuffersFromId"); + + const Model = @TypeOf(model); + var result: Bufferized(Model) = undefined; + initBufferizedFrom(model, &result); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const shapes = try collectShapes(arena.allocator(), &model); + std.debug.print("Found {d} shapes\n", .{shapes.len}); + + var transfer: Transfer = try .init(arena.allocator(), shapes, platform); + defer transfer.deinit(platform); + + const tensor_descs = try collectTensorDesc(arena.allocator(), buffer_store, &model); + + const buffer_reader = try allocator.alloc(u8, BUF_16_MB); + defer allocator.free(buffer_reader); + + const LocalContext = struct { + tensor_descs: []TensorDesc, + shapes: []const Shape, + platform: Platform, + transfer: *Transfer, + index: usize = 0, + buffer_reader: []u8, + buffer_store_view: BufferStore5.View, + allocator: std.mem.Allocator, + }; + var context: LocalContext = .{ + .tensor_descs = tensor_descs, + .shapes = shapes, + .platform = platform, + .transfer = &transfer, + .buffer_reader = buffer_reader, + .buffer_store_view = buffer_store, + .allocator = allocator, + }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { + const tensor_desc = context_.tensor_descs[context_.index]; + + var tensor_resource: Resource = .{ + .http = HttpResource.init(context_.buffer_store_view.store.http_client, context_.allocator, tensor_desc.resource_uri, .{ .offset = tensor_desc.offset, .size = tensor_desc.shape.byteSize() }) catch unreachable, + }; + defer tensor_resource.deinit(); + + var reader = tensor_resource.reader(context_.buffer_reader); + const entry = context_.transfer.get(context_.index); + + _ = reader.interface().streamRemaining(&entry.writer.interface) catch unreachable; + entry.writer.interface.flush() catch unreachable; + + buffer.* = zml.Buffer.fromPjrtBuffers(context_.platform, context_.shapes[context_.index], &.{context_.transfer.get(context_.index).buffer}); + context_.index += 1; + } + }.cb, &context, &result); + + return result; +} + +pub fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model))) void { + const Model = @TypeOf(model); + const type_info = @typeInfo(Bufferized(Model)); + switch (type_info) { + .@"struct" => |struct_type_info| { + if (Bufferized(Model) == zml.Buffer) return; + inline for (struct_type_info.fields) |field| { + initBufferizedFrom(@field(model, field.name), &@field(bufferized_, field.name)); + } + }, + .@"union" => { + switch (model) { + inline else => |v, tag| { + bufferized_.* = @unionInit(Bufferized(Model), @tagName(tag), undefined); + initBufferizedFrom(v, @field(bufferized_, @tagName(tag))); + }, + } + }, + .optional => { + if (model == null) { + bufferized_.* = null; + } else { + bufferized_.* = undefined; + initBufferizedFrom(model.?, &bufferized_.*.?); + } + }, + .void, .int, .@"enum", .bool, .enum_literal, .float, .pointer, .vector => {}, + else => unreachable, + } +} + +pub fn structTransfer(allocator: std.mem.Allocator, model: anytype, platform: Platform) !struct { Transfer, Transferized(@TypeOf(model)) } { + const Model = @TypeOf(model); + var result: Transferized(Model) = undefined; + initTransferizedFrom(model, &result); + + const shapes = try collectShapes(allocator, &model); + defer allocator.free(shapes); + std.debug.print("Found {d} shapes\n", .{shapes.len}); + + var transfer: Transfer = try .init(allocator, shapes, platform); + errdefer transfer.deinit(allocator, platform); + + const LocalContext = struct { + shapes: []const Shape, + platform: Platform, + transfer: *Transfer, + index: usize = 0, + }; + var context: LocalContext = .{ .shapes = shapes, .platform = platform, .transfer = &transfer }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, transfer_entry: *TransferEntry) void { + transfer_entry.buffer = zml.Buffer.fromPjrtBuffers(context_.platform, context_.shapes[context_.index], &.{context_.transfer.get(context_.index).buffer}); + transfer_entry.writer = context_.transfer.get(context_.index).writer(); + context_.index += 1; + } + }.cb, &context, &result); + + return .{ transfer, result }; +} + +pub fn Transferized(comptime T: type) type { + return zml.meta.MapRestrict(Tensor, TransferEntry).map(T); +} + +fn initTransferizedFrom(model: anytype, transferized: *Transferized(@TypeOf(model))) void { + const Model = @TypeOf(model); + const type_info = @typeInfo(Transferized(Model)); + switch (type_info) { + .@"struct" => |struct_type_info| { + if (Transferized(Model) == TransferEntry) return; + inline for (struct_type_info.fields) |field| { + initTransferizedFrom(@field(model, field.name), &@field(transferized, field.name)); + } + }, + .@"union" => { + switch (model) { + inline else => |v, tag| { + transferized.* = @unionInit(Bufferized(Model), @tagName(tag), undefined); + initTransferizedFrom(v, @field(transferized, @tagName(tag))); + }, + } + }, + .optional => { + if (model == null) { + transferized.* = null; + } else { + transferized.* = undefined; + initTransferizedFrom(model.?, &transferized.*.?); + } + }, + .void, .int, .@"enum", .bool, .enum_literal, .float, .pointer, .vector => {}, + else => unreachable, + } +} + +pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), + }; +} + +const Transfer = struct { + entries: []Entry, + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + arena: std.heap.ArenaAllocator, + + pub const Writer = struct { + offset: usize = 0, + interface: std.Io.Writer, + + pub fn init(buffer: []u8) Writer { + return .{ + .interface = .{ + .buffer = buffer, + .end = 0, + .vtable = &.{ + .drain = drain, + }, + }, + }; + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + _ = data; // autofix + _ = splat; // autofix + const writer: *Writer = @alignCast(@fieldParentPtr("interface", w)); + const entry: *Entry = @alignCast(@fieldParentPtr("writer", writer)); + stdx.debug.assert(writer.offset + w.end <= entry.shape.byteSize(), "Can't write more data than required", .{}); + const is_last_transfer = writer.offset + w.end >= entry.shape.byteSize(); + log.debug("Writing {} bytes", .{w.end}); + _ = entry.transfer_manager.transferData(entry.platform.pjrt_api, entry.buffer_index, w.buffer[0..w.end], @intCast(writer.offset), is_last_transfer) catch return error.WriteFailed; + const written = w.end; + writer.offset += written; + w.end = 0; + return 0; + } + }; + + pub const Entry = struct { + shape: Shape, + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + buffer_index: usize, + buffer: *pjrt.Buffer, + platform: Platform, + writer: Writer, + }; + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: Platform) !Transfer { + const shape_specs = try allocator.alloc(pjrt.ShapeSpec, shapes.len); + defer allocator.free(shape_specs); + + var temp_arena = std.heap.ArenaAllocator.init(allocator); + defer temp_arena.deinit(); + + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + for (shape_specs, shapes) |*spec, shape| { + const dims = try temp_arena.allocator().dupe(i64, shape.dims()); + spec.* = pjrt.ShapeSpec.init(dims, bufferTypeFromDtype(shape.dtype())); + } + + const memory = platform.pjrt_client.memoryByKind(platform.pjrt_api, .unpinned_host).?; + + const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); + errdefer transfer_manager.deinit(platform.pjrt_api); + + const entries = try arena.allocator().alloc(Entry, shapes.len); + for (entries, shapes, 0..) |*e, shape, index| { + const buffer = try transfer_manager.retrieveBuffer(platform.pjrt_api, index); + const writer_buffer = try arena.allocator().alloc(u8, BUF_16_MB); + e.* = .{ + .shape = shape, + .transfer_manager = transfer_manager, + .buffer_index = index, + .buffer = buffer, + .platform = platform, + .writer = .init(writer_buffer), + }; + } + + return .{ .entries = entries, .transfer_manager = transfer_manager, .arena = arena }; + } + + pub fn deinit(self: Transfer, platform: Platform) void { + self.arena.deinit(); + self.transfer_manager.deinit(platform.pjrt_api); + } + + pub fn get(self: *const Transfer, index: usize) *Entry { + return &self.entries[index]; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain3); +} + +pub fn asyncMain2() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var generator = std.Random.DefaultPrng.init(0); + const random = generator.random(); + + const buffer_reader = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_reader); + + var reader = RandomReader.init(random, buffer_reader); + _ = &reader; // autofix + + const file = try std.fs.openFileAbsolute("/dev/random", .{}); + defer file.close(); + + var file_reader = file.reader(buffer_reader); + + const buffer_writer = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_writer); + + var writer = std.Io.Writer.Discarding.init(buffer_writer); + + const COPY_SIZE = 1 * BUF_1_GB; + + var timer = try stdx.time.Timer.start(); + try file_reader.interface.streamExact(&writer.writer, COPY_SIZE); + try writer.writer.flush(); + const elapsed = timer.read(); + + const elapsed_s: f64 = @as(f64, @floatFromInt(elapsed.ns)) / std.time.ns_per_s; + + std.log.info("Elapsed: {f} - Throughput: {}GB/s", .{ elapsed, @as(f64, COPY_SIZE / (1024 * 1024 * 1024)) / elapsed_s }); +} + +pub const Adder = struct { + weight: Tensor, + + pub fn init(allocator: std.mem.Allocator, mesh: Mesh) Adder { + _ = allocator; // autofix + var tensor = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32).withPartitioning(.{ .m = .x, .n = .y })); + tensor._mesh = mesh; + return .{ .weight = tensor }; + } + + pub fn loadBuffers(allocator: std.mem.Allocator, model: Adder, platform: Platform) Bufferized(Adder) { + _ = allocator; // autofix + _ = model; // autofix + _ = platform; // autofix + } + + pub fn forward(self: Adder, input: Tensor) Tensor { + return self.weight.add(input);//.withSharding(.{ .m = .x, .n = .y }); + } +}; + +pub const Simple = struct { + weight: Tensor, + internal: Internal, + + pub const Internal = struct { + weight: Tensor, + + pub fn init(mesh: Mesh) Internal { + return .{ + .weight = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32)).withMesh(mesh), + }; + } + + pub fn forward(self: Internal, input: Tensor) Tensor { + return self.weight.add(input); + } + }; + + pub fn init(mesh: Mesh) Simple { + return .{ + .weight = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32)).withMesh(mesh), + .internal = .init(mesh), + }; + } + + pub fn forward(self: Simple, input: Tensor) Tensor { + const temp = self.weight.mul(input); + return zmlCall(.forward, .{ self.internal, temp }); + //return self.internal.forward(temp); + } +}; + +//pub fn Func(func_name: @TypeOf(.enum_literal), args: type) type { +// const func = ; +// return @TypeOf(func); +//} + +pub fn zmlCall(comptime func_name: @TypeOf(.enum_literal), args: anytype) stdx.meta.FnResult(@field(@TypeOf(args[0]), @tagName(func_name))) { + return zmlCallInternal(func_name, args) catch unreachable; +} + +pub fn zmlCallInternal(comptime func_name: @TypeOf(.enum_literal), args: anytype) !stdx.meta.FnResult(@field(@TypeOf(args[0]), @tagName(func_name))) { + const compilation_context = CompilationContext.current(); + + var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); + defer arena.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + compilation_context.pushBlock(block); + + const LocalContext = struct { + current_argument_id: usize = 0, + compilation_context: *CompilationContext, + }; + var context: LocalContext = .{ .compilation_context = compilation_context }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + std.log.debug("Argument {} - shape: {f}", .{ ctx_.current_argument_id, tensor.shape() }); + _ = ctx_.compilation_context.currentScope().block.addArgument( + tensor.mlirType(ctx_.compilation_context.mlir_ctx), + .unknown(ctx_.compilation_context.mlir_ctx), + ); + ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; + ctx_.current_argument_id += 1; + } + }.cb, &context, &args); + + const input_shapes = try collectShapes(arena.allocator(), &args); + const input_meshes = try collectMesh(arena.allocator(), &args); + const input_attributes = try arena.allocator().alloc(AttributeList, input_shapes.len); + @memset(input_attributes, .{}); + + const func = @field(@TypeOf(args[0]), @tagName(func_name)); + var result = @call(.auto, func, args); + const output_info = try collectOutputInfo(arena.allocator(), &result); + + const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); + @memset(output_attributes, .{}); + const mesh = compilation_context.currentMesh(); + compilation_context.addShardingAttributes(mesh, input_attributes, output_attributes, input_shapes, input_meshes, output_info); + _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ + .name = @tagName(func_name), + .block = compilation_context.currentScope().block, + .location = .unknown(compilation_context.mlir_ctx), + .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), + }).appendTo(compilation_context.module.body()); + _ = mlir_func; // autofix + + const output_types = try arena.allocator().alloc(*const mlir.Type, output_info.values.len); + for (output_info.values, output_types) |value, *t| t.* = value.type_(); + + compilation_context.popBlock(); + + const input_values = try collectValues(arena.allocator(), &args); + const call_op = dialects.func.call(compilation_context.mlir_ctx, @tagName(func_name), input_values, output_types, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + const LocalContext2 = struct { + current_argument_id: usize = 0, + call_op: *const mlir.Operation, + }; + var context2: LocalContext2 = .{ .call_op = call_op }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext2, tensor: *Tensor) void { + tensor._value = ctx_.call_op.operand(ctx_.current_argument_id); + ctx_.current_argument_id += 1; + } + }.cb, &context2, &result); + + return result; +} + +pub fn asyncMain3() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const devices = platform.getDevices(); + + for (devices, 0..) |device, i| { + const desc = device.getDescription(platform.pjrt_api); + log.info(" Device #{d}:", .{i}); + log.info(" - ID: {}", .{desc.getId(platform.pjrt_api)}); + log.info(" - Process Index: {}", .{desc.getProcessIndex(platform.pjrt_api)}); + log.info(" - Kind: {s}", .{desc.getKind(platform.pjrt_api)}); + log.info(" - Debug String: {s}", .{desc.debugString(platform.pjrt_api)}); + log.info(" - To String: {s}", .{desc.toString(platform.pjrt_api)}); + + const attributes = desc.getAttributes(platform.pjrt_api); + if (attributes.len > 0) { + log.info(" - Attributes:", .{}); + for (attributes) |attr| { + log.info(" - {s}:", .{attr.name()}); + switch (attr.kind()) { + .string => log.info(" (string): {s}", .{attr.inner.unnamed_0.string_value[0..attr.inner.value_size]}), + .int64 => { + log.info(" (int64): {}", .{attr.inner.unnamed_0.int64_value}); + if (std.mem.eql(u8, attr.name(), "memory_bandwidth")) { + const bandwidth_gbps: f32 = @as(f32, @floatFromInt(attr.inner.unnamed_0.int64_value)) / (1024.0 * 1024.0 * 1024.0); + log.info(" (Computed Bandwidth: {:.2} GB/s)", .{bandwidth_gbps}); + } + if (std.mem.eql(u8, attr.name(), "core_count")) { + log.info(" (Core Count: {})", .{attr.inner.unnamed_0.int64_value}); + } + }, + .int64list => { + const list_slice = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + log.info(" (int64list): {any}", .{list_slice}); + }, + .float => log.info(" (float): {}", .{attr.inner.unnamed_0.float_value}), + .bool => log.info(" (bool): {any}", .{attr.inner.unnamed_0.bool_value}), + } + } + } else { + log.info(" - No specific attributes.", .{}); + } + } + + const topology: Shape = .init(.{ .x = 2, .y = 2 }, .u8); + + const mesh: Mesh = .init(topology); + + const adder: Adder = .init(allocator, mesh); + + var input = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32).withPartitioning(.{ .m = .x, .n = .y })); + input._mesh = mesh; + const input_shard_shape = zml.Shape.init(.{ .m = 64, .n = 64 }, .f32); + + const exe = try compileModel(allocator, Adder.forward, adder, .{input}, mesh, platform); + defer exe.deinit(); + + var input_buffer: zml.Buffer = .{ ._shape = input.shape(), ._api = platform.pjrt_api, ._shards = .{} }; + var weight_buffer: zml.Buffer = .{ ._shape = adder.weight.shape(), ._api = platform.pjrt_api, ._shards = .{} }; + for (0..mesh.numPartitions()) |partition_index| { + const shard_data = try allocator.alloc(f32, 64 * 64); + defer allocator.free(shard_data); + const value = std.math.pow(f32, 10, @floatFromInt(partition_index)); + @memset(shard_data, value); + + const input_pjrt_buffer = b: { + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = std.mem.sliceAsBytes(shard_data).ptr, + .buffer_type = .f32, + .dims = input_shard_shape.dims(), + .byte_strides = input_shard_shape.computeByteStrides().slice(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .device = platform.getDevices()[partition_index], + }); + + if (event) |ev| ev.deinit(platform.pjrt_api); + break :b pjrt_buffer; + }; + + const weight_pjrt_buffer = b: { + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = std.mem.sliceAsBytes(shard_data).ptr, + .buffer_type = .f32, + .dims = input_shard_shape.dims(), + .byte_strides = input_shard_shape.computeByteStrides().slice(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .device = platform.getDevices()[partition_index], + }); + + if (event) |ev| ev.deinit(platform.pjrt_api); + break :b pjrt_buffer; + }; + + input_buffer._shards.appendAssumeCapacity(input_pjrt_buffer); + weight_buffer._shards.appendAssumeCapacity(weight_pjrt_buffer); + } + defer input_buffer.deinit(); + defer weight_buffer.deinit(); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + args.set(.{ weight_buffer, input_buffer }); + + exe.call(args, &results); + + var output_buffer = results.get(zml.Buffer); + output_buffer._mesh = mesh; + defer output_buffer.deinit(); + + const output_raw_data = try output_buffer.toHost(allocator); + defer allocator.free(output_raw_data); + + log.info("output: {any}", .{std.mem.bytesAsSlice(f32, output_raw_data)}); +} + +pub fn asyncMain4() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const mesh = Mesh.single(); + + const simple: Simple = .init(mesh); + + const input = Tensor.init(zml.Shape.init(.{ .m = 128, .n = 128 }, .f32)).withMesh(mesh); + + const exe = try compileModel(allocator, Simple.forward, simple, .{input}, mesh, platform); + defer exe.deinit(); +} + +pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: llama.LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + if (skip_llama3_encoding) { + // Copy so the ownership is the same in both branches. + return try allocator.dupe(u32, try encoder.encode(prompt)); + } + + const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); + + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline }); + + try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); + + return tokens.toOwnedSlice(allocator); +} + +pub fn asyncMain() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa.deinit(); + //const allocator = std.heap.c_allocator; + const allocator = gpa.allocator(); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const process_args = try std.process.argsAlloc(allocator); + defer std.process.argsFree(allocator, process_args); + + const model_config_path = try std.fs.path.join(allocator, &.{ process_args[1], "config.json" }); + defer allocator.free(model_config_path); + + const model_weights_path = b: { + const simple_path = try std.fs.path.join(allocator, &.{ process_args[1], "model.safetensors" }); + if (asynk.File.access(simple_path, .{})) { + break :b simple_path; + } else |_| { + allocator.free(simple_path); + } + + const sharded_path = try std.fs.path.join(allocator, &.{ process_args[1], "model.safetensors.index.json" }); + break :b sharded_path; + }; + defer allocator.free(model_weights_path); + + const model_tokenizer_path = try std.fs.path.join(allocator, &.{ process_args[1], "tokenizer.json" }); + defer allocator.free(model_tokenizer_path); + + var tokenizer = blk: { + std.log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); + var timer = try stdx.time.Timer.start(); + defer std.log.info("Loaded tokenizer from {s} [{D}]", .{ model_tokenizer_path, timer.read() }); + + break :blk try zml.tokenizer.Tokenizer.fromFile(allocator, model_tokenizer_path); + }; + errdefer tokenizer.deinit(); + + const config = blk: { + var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(&config_json_buffer); + var reader = std.json.Reader.init(allocator, &config_reader.interface); + defer reader.deinit(); + const config_obj = try std.json.parseFromTokenSourceLeaky(llama.LlamaLM.Config, arena.allocator(), &reader, .{ .ignore_unknown_fields = true }); + break :blk config_obj; + }; + + const options: llama.LlamaLM.Options = .{ + .sampling_strategy = null, + .max_seq_len = 256, + .qkv_type = .merged, + }; + + var client: std.http.Client = .{ .allocator = allocator, .write_buffer_size = 8192 }; + try client.initDefaultProxies(arena.allocator()); + defer client.deinit(); + + const buffer_reader = try allocator.alloc(u8, BUF_4_MB); + defer allocator.free(buffer_reader); + + //var resource: Resource = .{ .http = try .init(&client, allocator, std.Uri.parse("https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model.safetensors.index.json") catch unreachable, null) }; + var resource: Resource = .{ .http = try .init(&client, allocator, std.Uri.parse("http://localhost:8000/model.safetensors.index.json") catch unreachable, null) }; + defer resource.deinit(); + + var reader = resource.reader(buffer_reader); + + var resource_index = try parseSafetensorsIndex(arena.allocator(), &resource, reader.interface()); + defer resource_index.deinit(); + + var registry: TensorRegistry = try .initWithMetadata(allocator, resource_index.metadata); + defer registry.deinit(); + + //var registry: TensorRegistry = .init(allocator); + //defer registry.deinit(); + + //var resource: Resource = .{ .http = try .init(&client, allocator, std.Uri.parse("https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/resolve/main/model.safetensors") catch unreachable, null) }; + //defer resource.deinit(); + + //var reader = resource.reader(buffer_reader); + + //try parseSafetensors(®istry, resource.uri(), reader.interface()); + + var seen: std.ArrayHashMapUnmanaged(std.Uri, void, ResourceURIContext, false) = .empty; + + var it = resource_index.map.iterator(); + while (it.next()) |entry| { + if (seen.get(entry.value_ptr.*)) |_| {} else { + try seen.put(arena.allocator(), entry.value_ptr.*, {}); + + std.debug.print("subresource uri: {f}\n", .{entry.value_ptr.fmt(.all)}); + + var subresource: Resource = .{ .http = try .init(&client, allocator, entry.value_ptr.*, null) }; + defer subresource.deinit(); + + std.log.debug("Opened subresource", .{}); + + var reader2 = subresource.reader(buffer_reader); + + try parseSafetensors(®istry, entry.value_ptr.*, reader2.interface()); + } + } + + var buffer_store: BufferStore5 = try .init(allocator, ®istry, &client); + defer buffer_store.deinit(); + + var context = try zml.Context.init(); + defer context.deinit(); + + //zml.tools.tracer.initTracer(); + //defer zml.tools.tracer.deinitTracer(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + var llama_model = try llama.LlamaLM.init(allocator, buffer_store.view(), config, options); + defer llama_model.deinit(allocator); + + const tokens = Tensor.init(zml.Shape.init(.{ .s = 1 }, .u32)); + const token_index = Tensor.init(zml.Shape.init(.{}, .u32)); + const kv_cache = llama.KvCache.init(zml.Shape.init(.{ .layer = config.num_hidden_layers, .k = options.max_seq_len, .h = config.num_key_value_heads, .hd = 128 }, .bf16)); + const rng = Tensor.Rng.init(); + + const exe = try compileModel(allocator, llama.LlamaLM.forward, llama_model, .{ tokens, token_index, kv_cache, rng }, platform); + defer exe.deinit(); + + // Compile executables used when loading buffers + const merge_qkv_exe = try compileMergeQkvExe(allocator, buffer_store.view(), platform); + defer merge_qkv_exe.deinit(); + + const precompute_qkv_exe = try compilePrecomputeQkvExe(allocator, buffer_store.view(), platform); + defer precompute_qkv_exe.deinit(); + + const llama_buffers = try llama.LlamaLM.loadBuffers(allocator, llama_model, buffer_store.view(), platform, .{ .merge_qkv_exe = merge_qkv_exe, .precompute_qkv_exe = precompute_qkv_exe }); + defer llama_buffers.deinit(allocator); + + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + const prompt = "What is the capital of France ?"; + const tokenized_prompt = try tokenizePrompt(allocator, tokenizer, config, prompt, false); + defer allocator.free(tokenized_prompt); + + var rng_buffer = try Tensor.Rng.initBuffer(platform, 0); + defer rng_buffer._state.deinit(); + + var kv_cache_buffer = try llama.KvCache.initBuffer(kv_cache.k.shape(), platform); + defer kv_cache_buffer.layer_index.deinit(); + defer kv_cache_buffer.k.deinit(); + defer kv_cache_buffer.v.deinit(); + + var new_token_id: u32 = undefined; + for (tokenized_prompt[0..], 0..) |token_id, i| { + std.debug.print("Running iteration {} token id: {}\n", .{ i, token_id }); + var token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{@intCast(i)})); + defer token_index_buffer.deinit(); + + var token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{token_id})); + defer token_buffer.deinit(); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + args.set(.{ llama_buffers, token_buffer, token_index_buffer, kv_cache_buffer, rng_buffer }); + std.log.debug("Calling exe", .{}); + exe.call(args, &results); + + results.fill(&.{ &token_buffer, &kv_cache_buffer, &rng_buffer._state }); + + const host = try token_buffer.toHostAlloc(allocator); + defer host.deinit(allocator); + + const predicted_token = host.items(u32)[0]; + const predicted_token_str = try tokenizer_decoder.decode(&.{predicted_token}); + std.debug.print("Predicted: {} {s}\n", .{ predicted_token, predicted_token_str }); + new_token_id = predicted_token; + } + + var token_buffer = try zml.Buffer.fromBytes(platform, tokens.shape(), std.mem.sliceAsBytes(&[1]u32{new_token_id})); + defer token_buffer.deinit(); + + for (0..10) |i| { + std.debug.print("Running iteration {}\n", .{i}); + var token_index_buffer = try zml.Buffer.fromBytes(platform, token_index.shape(), std.mem.sliceAsBytes(&[1]u32{@intCast(i + tokenized_prompt.len)})); + defer token_index_buffer.deinit(); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + args.set(.{ llama_buffers, token_buffer, token_index_buffer, kv_cache_buffer, rng_buffer }); + std.log.debug("Calling exe", .{}); + exe.call(args, &results); + + results.fill(&.{ &token_buffer, &kv_cache_buffer, &rng_buffer._state }); + + const host = try token_buffer.toHostAlloc(allocator); + defer host.deinit(allocator); + + const predicted_token = host.items(u32)[0]; + const predicted_token_str = try tokenizer_decoder.decode(&.{predicted_token}); + std.debug.print("Predicted: {} {s}\n", .{ predicted_token, predicted_token_str }); + } +} + +fn compileMergeQkvExe(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, platform: Platform) !Exe { + const proj: llama.SeparateQkv = .init(buffer_store.withPrefix("model.layers.0.self_attn")); + const exe = try compile(allocator, llama.mergeQkv3, .{proj}, platform); + errdefer exe.deinit(); + + return exe; +} + +fn compilePrecomputeQkvExe(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, platform: Platform) !Exe { + const proj: llama.SeparateQkv = .init(buffer_store.withPrefix("model.layers.0.self_attn")); + const embedding = Tensor.init(buffer_store.getShape("model.embed_tokens.weight").?).withTags(.{ .voc, .d }); + const layer_norm: llama.RmsNorm = .init(buffer_store.withPrefix("model.layers.0.input_layernorm"), 1e-5); + + const exe = try compile(allocator, llama.precomputeQkv0, .{ proj, layer_norm, embedding }, platform); + errdefer exe.deinit(); + + return exe; +} + +pub const CompilationContext = struct { + pub const Scope = struct { + block: *mlir.Block, + id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), + arena: std.heap.ArenaAllocator, + + pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(allocator); + return .{ + .block = block, + .id_to_argument = .empty, + .id_to_donation = .empty, + .arena = arena, + }; + } + + pub fn deinit(self: *Scope) void { + self.arena.deinit(); + } + }; + + allocator: std.mem.Allocator, + + mlir_registry: *mlir.DialectRegistry, + mlir_ctx: *mlir.Context, + mlir_pass_manager: *mlir.PassManager, + //mlir_op_pass_manager: *mlir.OpPassManager, + module: *mlir.Module, + + scopes: stdx.BoundedArray(Scope, 16) = .{}, + _main_mesh: Mesh, + _current_mesh: ?Mesh = null, + + pub fn init(allocator: std.mem.Allocator, mesh: Mesh) CompilationContext { + mlir.registerPasses("Transforms"); + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; + inline for (.{ "func", "stablehlo" }) |d| { + mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); + } + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + //const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); + const module = mlir.Module.init(.unknown(mlir_ctx)); + module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); + + const pass_manager = mlir.PassManager.init(mlir_ctx); + { + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + opm.addPipeline(pass) catch unreachable; + } + } + + return .{ + .allocator = allocator, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = pass_manager, + .module = module, + ._main_mesh = mesh, + }; + } + + pub fn deinit(self: *CompilationContext) void { + self.mlir_pass_manager.deinit(); + self.module.deinit(); + self.mlir_ctx.deinit(); + self.mlir_registry.deinit(); + } + + pub fn currentScope(self: *CompilationContext) *Scope { + return &self.scopes.slice()[self.scopes.len - 1]; + } + + pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { + const scope = Scope.initFromBlock(self.allocator, block); + self.scopes.appendAssumeCapacity(scope); + } + + pub fn popBlock(self: *CompilationContext) void { + var maybe_popped_scope = self.scopes.pop(); + if (maybe_popped_scope) |*popped| { + popped.deinit(); + } + } + + pub fn current() *CompilationContext { + return global_compilation_context.?; + } + + pub fn pushMesh(self: *CompilationContext, mesh: Mesh) void { + if (self._current_mesh) |m| { + if (m.eql(mesh)) { + stdx.debug.panic("Mesh already pushed: {}", .{m}); + } + + stdx.debug.panic("Mesh already exists: {}", .{m}); + } + + self._current_mesh = mesh; + // log.info("Pushed mesh: {}", .{mesh}); + } + + pub fn popMesh(self: *CompilationContext) void { + if (self._current_mesh) |m| { + _ = m; // autofix + self._current_mesh = null; + // log.info("Poped mesh: {}", .{m}); + } else { + stdx.debug.panic("No mesh to pop, main mesh: {}", .{self._main_mesh}); + } + } + + pub fn currentMesh(self: *CompilationContext) Mesh { + return self._current_mesh orelse self._main_mesh; + } + + pub fn addShardingAttributes( + self: *const CompilationContext, + mesh: Mesh, + arg_attrs: []AttributeList, + res_attrs: []AttributeList, + input_shapes: []const Shape, + input_meshes: []const ?Mesh, + output_info: OutputInfo, + ) void { + if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. + + const default_layout = mlir.NamedAttribute.named(self.mlir_ctx, "mhlo.layout_mode", mlir.stringAttribute(self.mlir_ctx, "default")); + for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { + attr.appendAssumeCapacity(default_layout); + if (input_mesh) |m| { + // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); + attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); + } else { + // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); + attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + } + } + + for (res_attrs, output_info.shapes) |*attr, shape| { + attr.appendAssumeCapacity(default_layout); + attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + } + } + + pub fn getShardingAttr(self: *const CompilationContext, sharding: Sharding) *const mlir.Attribute { + var sharding_str: stdx.BoundedArray(u8, 128) = .{}; + sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; + return mlir.stringAttribute(self.mlir_ctx, sharding_str.constSlice()); + } +}; + +fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), mesh: Mesh, platform: Platform) !Exe { + return compile(allocator, func, .{model} ++ args, mesh, platform); +} + +pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx.meta.FnArgs(func), mesh: Mesh, platform: Platform) !Exe { + var compilation_context: CompilationContext = .init(allocator, mesh); + defer compilation_context.deinit(); + + const result = emitMlir(&compilation_context, func, args) catch unreachable; + defer result.output_info.deinit(compilation_context.allocator); + defer compilation_context.allocator.free(result.input_shapes); + defer compilation_context.allocator.free(result.input_meshes); + + _ = result.func.appendTo(compilation_context.module.body()); + compilation_context.module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, mesh.numReplicas())); + compilation_context.module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, mesh.numPartitions())); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), mesh, platform, compilation_context.module, null) catch unreachable; + + log.warn("******** ZML generated MLIR ********", .{}); + log.warn("{f}", .{compilation_context.module.operation()}); + + const exe = try Exe.init(allocator, platform, loaded_executable, result.input_shapes, result.output_info.shapes, mesh.numDevices()); + errdefer exe.deinit(); + + return exe; +} + +fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { + const LocalContext = struct { + list: *std.array_list.Managed(Shape), + }; + var list = std.array_list.Managed(Shape).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.shape()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectMesh(allocator: std.mem.Allocator, v: anytype) ![]?Mesh { + const LocalContext = struct { + list: *std.array_list.Managed(?Mesh), + }; + var list = std.array_list.Managed(?Mesh).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor._mesh); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectValues(allocator: std.mem.Allocator, v: anytype) ![]*const mlir.Value { + const LocalContext = struct { + list: *std.array_list.Managed(*const mlir.Value), + }; + var list = std.array_list.Managed(*const mlir.Value).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.value()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectType(allocator: std.mem.Allocator, v: anytype) ![]*const mlir.Type { + const LocalContext = struct { + list: *std.array_list.Managed(*const mlir.Type), + }; + var list = std.array_list.Managed(*const mlir.Type).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.value().type_()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectTensorDesc(allocator: std.mem.Allocator, buffer_store: BufferStore5.View, v: anytype) ![]TensorDesc { + const LocalContext = struct { + list: *std.array_list.Managed(TensorDesc), + buffer_store: BufferStore5.View, + }; + var list = std.array_list.Managed(TensorDesc).init(allocator); + var context: LocalContext = .{ .list = &list, .buffer_store = buffer_store }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + const tensor_desc = ctx_.buffer_store.store.getPtrFromId(tensor.id).?.*; + ctx_.list.append(tensor_desc) catch unreachable; + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +pub const OutputInfo = struct { + shapes: []Shape, + values: []*const mlir.Value, + donations: []?usize, + + pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + allocator.free(self.values); + allocator.free(self.donations); + } +}; + +fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + value_list: *std.array_list.Managed(*const mlir.Value), + donation_list: *std.array_list.Managed(?usize), + }; + + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); + var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); + errdefer value_list.deinit(); + var donation_list = std.array_list.Managed(?usize).init(allocator); + errdefer donation_list.deinit(); + + var context: LocalContext = .{ .shape_list = &shape_list, .value_list = &value_list, .donation_list = &donation_list }; + + try zml.meta.visit2(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + try ctx_.value_list.append(tensor.value()); + try ctx_.donation_list.append(tensor.donation()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + .values = try value_list.toOwnedSlice(), + .donations = try donation_list.toOwnedSlice(), + }; +} + +const EmitMlirResult = struct { + func: *mlir.Operation, + // TODO: Unify like output_info + input_shapes: []const Shape, + input_meshes: []const ?Mesh, + output_info: OutputInfo, +}; + +fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { + const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); + for (res, attributes) |*r, attr| { + r.* = mlir.dictionaryAttribute(mlir_ctx, attr.constSlice()); + } + return res; +} + +fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { + var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); + defer arena.deinit(); + + const module = mlir.Module.init(.unknown(compilation_context.mlir_ctx)); + errdefer module.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + compilation_context.pushBlock(block); + defer compilation_context.popBlock(); + + const LocalContext = struct { + compilation_context: *CompilationContext, + current_argument_id: usize = 0, + }; + var context: LocalContext = .{ + .compilation_context = compilation_context, + }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + std.log.debug("Argument {} - shape: {f}", .{ ctx_.current_argument_id, tensor.shape() }); + _ = ctx_.compilation_context.currentScope().block.addArgument( + tensor.mlirType(ctx_.compilation_context.mlir_ctx), + .unknown(ctx_.compilation_context.mlir_ctx), + ); + ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; + ctx_.current_argument_id += 1; + } + }.cb, &context, &args); + + const input_shapes = try collectShapes(compilation_context.allocator, &args); + errdefer compilation_context.allocator.free(input_shapes); + const input_meshes = try collectMesh(compilation_context.allocator, &args); + errdefer compilation_context.allocator.free(input_meshes); + + const input_attributes = try arena.allocator().alloc(AttributeList, input_shapes.len); + @memset(input_attributes, .{}); + + const output_info = b: { + global_compilation_context = compilation_context; + defer global_compilation_context = null; + + const result = @call(.auto, func, args); + + const output_info = try collectOutputInfo(compilation_context.allocator, &result); + errdefer output_info.deinit(compilation_context.allocator); + + break :b output_info; + }; + errdefer output_info.deinit(compilation_context.allocator); + + const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); + @memset(output_attributes, .{}); + + const mesh = compilation_context.currentMesh(); + + for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { + input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); + }; + compilation_context.addShardingAttributes(mesh, input_attributes, output_attributes, input_shapes, input_meshes, output_info); + _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + + //self.add + + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ + .name = "main", + .block = compilation_context.currentScope().block, + .location = .unknown(compilation_context.mlir_ctx), + .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), + }); + + compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { + error.MlirUnexpected => { + std.log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_func}); + // user errors should have triggered a panic before we reach this. + @panic("ZML generated invalid mlir. Please open a bug report"); + }, + }; + + return .{ + .func = mlir_func, + .input_shapes = input_shapes, + .input_meshes = input_meshes, + .output_info = output_info, + }; +} + +fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { + const result = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView(flag) }, + .{ .msg_val = blk: { + const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); + switch (@typeInfo(@TypeOf(value))) { + .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), + .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), + .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), + else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), + } + break :blk @ptrCast(field); + } }, + upb_arena, + ); + + if (result == false) { + return std.mem.Allocator.Error.OutOfMemory; + } +} + +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, mesh: Mesh, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { + //const tracer = Tracer.init("ai.zml.compilation"); + //const compile_frame = tracer.frameStart("pjrt compilation"); + //defer tracer.frameEnd(compile_frame, "pjrt compilation"); + + var upb_alloc: upb.Allocator = .init(arena); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const options = blk: { + const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); + c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { + const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); + + c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, mesh.numReplicas()); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, mesh.numPartitions()); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, !mesh.isSinglePartition()); + + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { + const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, mesh.numReplicas()); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, mesh.numPartitions()); + + const ordered_devices = try zml.platform.createDeviceMesh(arena, mesh, platform); + stdx.debug.assert(ordered_devices.len == mesh.numPartitions(), "createDeviceMesh returned {} devices, but mesh requires {}", .{ ordered_devices.len, mesh.numPartitions() }); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, mesh.numPartitions(), upb_arena); + for (ordered_devices, computation_devices[0..mesh.numPartitions()]) |device, *computation_device| { + computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); + const device_id = @as(i64, @intCast(device.getDescription(platform.pjrt_api).getId(platform.pjrt_api))); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(device_id), upb_arena); + } + break :device_assignment_blk device_assignment; + }); + + break :executable_build_options_blk exec_build_options; + }); + + const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); + switch (platform.target) { + .cuda => { + // NVIDIA recommends these settings + // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_libnvptxcompiler", true, upb_arena); + }, + .rocm => { + // Disable Triton GEMM on ROCM. For some reason it's much, much slower when + // enabled on CDNA and it's used on RDNA. Disable it altogether. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + // Use lld from libllvm instead of invoking the ld.lld binary. + // This saves us from having to sandbox it. + try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); + }, + else => {}, + } + + if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { + try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); + if (platform.compilation_options.xla_dump_fusion_visualization) { + try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); + } + if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); + } + } + + break :blk options; + }; + + const loaded_executable = try platform.pjrt_client.compile( + platform.pjrt_api, + arena, + module, + try upb.serialize(options, upb_arena), + ); + errdefer loaded_executable.deinit(); + + return loaded_executable; +} + +pub const Exe = struct { + platform: Platform, + exe: *pjrt.LoadedExecutable, + + context: ?*pjrt.ExecuteContext = null, + + input_shapes: []const Shape, + output_shapes: []const Shape, + + num_devices: u8, + + arena: std.heap.ArenaAllocator, + + pub fn init( + allocator: std.mem.Allocator, + platform: Platform, + exe: *pjrt.LoadedExecutable, + input_shapes: []const Shape, + output_shapes: []const Shape, + num_devices: u8, + ) !Exe { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); + const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); + + return .{ + .platform = platform, + .exe = exe, + .input_shapes = input_shapes_copy, + .output_shapes = output_shapes_copy, + .num_devices = num_devices, + .arena = arena, + }; + } + + pub fn deinit(self: *const Exe) void { + self.arena.deinit(); + } + + pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { + return Arguments.init(allocator, self.input_shapes, self.num_devices); + } + + pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { + return Results.init(allocator, self.output_shapes, self.num_devices, self.platform); + } + + pub const FlatBuffers = struct { + buffers: []const [*]*pjrt.Buffer, + raw_buffers: []const *pjrt.Buffer, + + num_devices: usize, + + pub fn init(allocator: std.mem.Allocator, count: usize, num_devices: usize) !FlatBuffers { + const raw_buffers = try allocator.alloc(*pjrt.Buffer, num_devices * count); + errdefer allocator.free(raw_buffers); + + const buffers = try allocator.alloc([*]*pjrt.Buffer, num_devices); + errdefer allocator.free(buffers); + + for (0..num_devices) |i| { + buffers[i] = raw_buffers[i * count ..].ptr; + } + + return .{ + .buffers = buffers, + .raw_buffers = raw_buffers, + .num_devices = num_devices, + }; + } + + pub fn deinit(self: *const FlatBuffers, allocator: std.mem.Allocator) void { + allocator.free(self.buffers); + allocator.free(self.raw_buffers); + } + }; + + pub const Arguments = struct { + flat_buffers: FlatBuffers, + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Arguments, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn set(self: *Arguments, v: anytype) void { + return self.setPartial(v, 0); + } + + pub fn setPartial(self: *Arguments, v: anytype, offset: usize) void { + const LocalContext = struct { + self: *Arguments, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = offset }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const zml.Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + } + }; + + pub const Results = struct { + platform: Platform, + flat_buffers: FlatBuffers, + + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: Platform) !Results { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .platform = platform, + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Results, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn get(self: *Results, comptime T: type) T { + var result: T = undefined; + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { + var shards: zml.Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = zml.Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, &result); + return result; + } + + pub fn fill(self: *Results, v: anytype) void { + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *zml.Buffer) void { + //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + var shards: zml.Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = zml.Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, v); + } + }; + + pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { + var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; + const sharding = self.platform.sharding(); + + self.exe.execute(self.platform.pjrt_api, .{ + .arguments = arguments.flat_buffers.buffers, + .num_args = arguments.expected_shapes.len, + .results = results_.flat_buffers.buffers, + .events = events[0..sharding.num_partitions], + // this allows to tell a specific buffer shouldn't be donated, + // even if it has been marked as "can be donated" during compilation. + // TODO: expose it ? + .non_donatable_input_indices = &.{}, + .context = self.context, + }) catch |err| { + std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); + }; + + for (events[0..sharding.num_partitions]) |e| { + if (e) |ev| { + ev.await_(self.platform.pjrt_api) catch unreachable; + } + } + } +}; + +pub fn Bufferized(comptime T: type) type { + // TODO: we should strip out the non-buffer fields. + // Currently it's confusing cause the Bufferized struct contains field that are never read. + // Also it will simplify the layout of the Bufferized struct. + // accelerating the calls to execute. + return zml.meta.MapRestrict(Tensor, zml.Buffer).map(T); +} + +pub const BufferStore5 = struct { + registry: *TensorRegistry, + id_map: std.AutoHashMapUnmanaged(usize, *TensorDesc), + allocator: std.mem.Allocator, + // Temporary + http_client: *std.http.Client, + + pub fn init(allocator: std.mem.Allocator, registry: *TensorRegistry, http_client: *std.http.Client) !BufferStore5 { + return .{ + .registry = registry, + .id_map = .empty, + .allocator = allocator, + .http_client = http_client, + }; + } + + pub fn deinit(self: *BufferStore5) void { + self.id_map.deinit(self.allocator); + } + + fn bindIdToKey(self: *BufferStore5, key: []const u8, id: usize) !void { + const tensor_desc_ptr = self.registry.tensors.getPtr(key).?; + + const gop = try self.id_map.getOrPut(self.allocator, id); + if (gop.found_existing) { + stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.value_ptr.* }); + } + errdefer self.id_map.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = tensor_desc_ptr; + } + + fn getPtrFromKey(self: *const BufferStore5, key: []const u8) ?*TensorDesc { + const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; + return tensor_desc_ptr; + } + + fn getPtrFromId(self: *const BufferStore5, id: usize) ?*TensorDesc { + const tensor_desc_ptr = self.id_map.get(id) orelse return null; + return tensor_desc_ptr; + } + + pub fn view(self: *BufferStore5) View { + return .{ .store = self }; + } + + pub const View = struct { + store: *BufferStore5, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + pub fn parent(self: *const View) View { + const slice = self.prefix() orelse unreachable; + const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); + var buffer: [256]u8 = undefined; + @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = index + 1, + }; + } + + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + fn prefix(self: *const View) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + + const tensor = Tensor.init(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensor(self: View, subkey: []const u8) Tensor { + return self.maybeCreateTensor(subkey).?; + } + + pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + ptr.shape = ptr.shape.withTags(tagz).withSharding(.{0}); + + const tensor = Tensor.init(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) Tensor { + return self.maybeCreateTensorWithTags(subkey, tagz).?; + } + + pub fn getShape(self: View, subkey: []const u8) ?Shape { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + + pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { + var buffer: [256]u8 = undefined; + const key = if (opts.no_prefix) + subkey + else b: { + break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + }; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + }; +}; + +const KB = 1024; +const MB = 1024 * KB; +const GB = 1024 * MB; + +const BUF_1_KB = 1 * KB; +const BUF_4_KB = 4 * KB; +const BUF_8_KB = 8 * KB; +const BUF_16_KB = 16 * KB; +const BUF_32_KB = 32 * KB; +const BUF_64_KB = 64 * KB; + +const BUF_1_MB = 1 * MB; +const BUF_4_MB = 4 * MB; +const BUF_8_MB = 8 * MB; +const BUF_16_MB = 16 * MB; +const BUF_32_MB = 32 * MB; +const BUF_64_MB = 64 * MB; +const BUF_128_MB = 128 * MB; +const BUF_256_MB = 256 * MB; + +const BUF_1_GB = 1 * GB; + +pub const MemoryResource = struct { + uri: ResourceURI, + data: []const u8, + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator, data: []const u8) !MemoryResource { + return .{ + .uri = try std.Uri.parse("memory://"), + .data = data, + .allocator = allocator, + }; + } + + pub fn deinit(_: *MemoryResource) void {} +}; + +pub const FileResource = struct { + uri: ResourceURI, + uri_storage: []const u8, + file: std.fs.File, + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator, path: []const u8) !FileResource { + const uri_storage = try std.fmt.allocPrint(allocator, "file://{s}", .{path}); + errdefer allocator.free(uri_storage); + + const uri = std.Uri.parse(uri_storage); + + const file = try std.fs.openFileAbsolute(path, .{ .mode = .read_only }); + errdefer file.close(); + + return .{ + .uri = uri, + .uri_storage = uri_storage, + .file = file, + .allocator = allocator, + }; + } + + // check ownership + pub fn deinit(self: *FileResource) void { + self.allocator.free(self.uri_storage); + self.file.close(); + } +}; + +const Range = struct { + offset: usize, + size: usize, +}; + +pub const HttpResource = struct { + uri: ResourceURI, + + request: std.http.Client.Request, + arena: std.heap.ArenaAllocator, + + pub fn init(client: *std.http.Client, allocator: std.mem.Allocator, uri: std.Uri, maybe_range: ?Range) !HttpResource { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + var request = try client.request(.GET, uri, .{}); + errdefer request.deinit(); + + request.headers.authorization = .{ .override = "Bearer " }; + + const extra_headers = if (maybe_range) |range| b: { + var list: std.ArrayList(std.http.Header) = .empty; + const name = try arena.allocator().dupe(u8, "Range"); + const value = try std.fmt.allocPrint(arena.allocator(), "bytes={}-{}", .{ range.offset, range.offset + range.size - 1 }); + try list.append(arena.allocator(), .{ .name = name, .value = value }); + + break :b try list.toOwnedSlice(arena.allocator()); + } else &.{}; + request.extra_headers = extra_headers; + try request.sendBodiless(); + + const redirect_buffer = try arena.allocator().alloc(u8, 1024 * 1024); + const response = try request.receiveHead(redirect_buffer); + _ = response; + + return .{ + .uri = uri, + .request = request, + .arena = arena, + }; + } + + // check ownership + pub fn deinit(self: *HttpResource) void { + self.request.deinit(); + self.arena.deinit(); + } +}; + +pub const Resource = union(enum) { + memory: MemoryResource, + file: FileResource, + http: HttpResource, + + pub fn deinit(self: *Resource) void { + switch (self.*) { + .memory => |*m| m.deinit(), + .file => |*f| f.deinit(), + .http => |*h| h.deinit(), + } + } + + pub fn reader(self: *Resource, buffer: []u8) IoReader { + return .init(self, buffer); + } + + pub fn uri(self: *Resource) std.Uri { + return switch (self.*) { + inline else => |*r| r.uri, + }; + } +}; + +pub const IoReader = struct { + reader: union(enum) { + memory: std.io.Reader, + file: std.fs.File.Reader, + http: *std.io.Reader, + }, + resource: *Resource, + + pub fn init(resource: *Resource, buffer: []u8) IoReader { + return .{ + .reader = switch (resource.*) { + .memory => |*m| .{ .memory = std.Io.Reader.fixed(m.data) }, + .file => |*f| .{ .file = f.file.reader(buffer) }, + .http => |*h| .{ .http = h.request.reader.bodyReader(buffer, h.request.response_transfer_encoding, h.request.response_content_length) }, + }, + .resource = resource, + }; + } + + pub fn interface(self: *IoReader) *std.io.Reader { + return switch (self.reader) { + .memory => |*m| m, + .file => |*f| &f.interface, + .http => |h| h, + }; + } +}; + +pub const Metadata = union(enum) { + null: void, + int: i64, + float: f64, + bool: bool, + string: []const u8, + + array_bool: []const bool, + array_int: []const i64, + array_float: []const f64, + array_string: []const []const u8, + + pub const ItemType = enum { + int, + float, + bool, + string, + + pub fn toZigType(comptime kind: ItemType) type { + return switch (kind) { + .int => i64, + .float => f64, + .bool => bool, + .string => []const u8, + }; + } + }; + + pub fn wrap(x: anytype) Metadata { + return switch (@TypeOf(x)) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, + inline f16, f32, f64 => .{ .float = @floatCast(x) }, + bool => .{ .bool = x }, + []const u8 => .{ .string = x }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), + }; + } + + pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { + return switch (@TypeOf(any_slice[0])) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => { + const res = try allocator.alloc(i64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @intCast(val); + return .{ .array_int = res }; + }, + inline f16, f32, f64 => { + const res = try allocator.alloc(f64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @floatCast(val); + return .{ .array_float = res }; + }, + bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, + []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), + }; + } + + pub fn format( + self: Metadata, + writer: *std.Io.Writer, + ) !void { + switch (self) { + .null => _ = try writer.write("null"), + .string => |s| try writer.print("{s}", .{s}), + .bool => |b| try writer.print("{}", .{b}), + .int => |i| try writer.print("{d}", .{i}), + .float => |f| try writer.print("{d}", .{f}), + .array_bool => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.write(", "); + try writer.print("{}", .{v}); + } + try writer.writeByte(']'); + }, + .array_int => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.write(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_float => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.write(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_string => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.write(", "); + try writer.print("\"{s}\"", .{v}); + } + try writer.writeByte(']'); + }, + } + } + + pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { + switch (self) { + .string => |s| allocator.free(s), + .array_bool => |s| allocator.free(s), + .array_int => |s| allocator.free(s), + .array_float => |s| allocator.free(s), + .array_string => |s| { + for (s) |str| allocator.free(str); + allocator.free(s); + }, + else => {}, + } + } + + pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { + return switch (self) { + .null => .null, + .int => |v| .{ .int = v }, + .float => |v| .{ .float = v }, + .bool => |v| .{ .bool = v }, + .string => |s| .{ .string = try allocator.dupe(u8, s) }, + .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, + .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, + .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, + .array_string => |s| blk: { + const new_slice = try allocator.alloc([]const u8, s.len); + errdefer allocator.free(new_slice); + for (s, 0..) |str, i| { + new_slice[i] = try allocator.dupe(u8, str); + } + break :blk .{ .array_string = new_slice }; + }, + }; + } +}; + +pub const ResourceURI = std.Uri; +pub const Resources = std.ArrayHashMapUnmanaged(ResourceURI, Resource, ResourceURIContext, false); +pub const Tensors = std.StringArrayHashMapUnmanaged(TensorDesc); +pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); + +const ResourceURIContext = struct { + // todo: implement full uri + pub fn hash(_: ResourceURIContext, uri: ResourceURI) u32 { + return std.array_hash_map.hashString(uri.path.percent_encoded); + } + + pub fn eql(_: ResourceURIContext, a: ResourceURI, b: ResourceURI, _: usize) bool { + return std.mem.eql(u8, a.path.percent_encoded, b.path.percent_encoded); + } +}; + +pub const ResourceType = enum { + index, + safetensors, + unknown, +}; + +const TensorDesc = struct { + resource_uri: ResourceURI, + name: []const u8, + shape: Shape, + offset: u64, + + pub fn byteSize(self: Tensor) u64 { + return self.shape.byteSize(); + } +}; + +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, + + tensors: Tensors, + metadata: Metadatas, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .{}, + }; + } + + pub fn initWithMetadata( + allocator: std.mem.Allocator, + metadata: Metadatas, + ) !TensorRegistry { + var arena = std.heap.ArenaAllocator.init(allocator); + + return .{ + .arena = arena, + .tensors = .{}, + .metadata = blk: { + var arena_allocator = arena.allocator(); + var new_metadata: Metadatas = .{}; + + var it = metadata.iterator(); + while (it.next()) |entry| { + const key = try arena_allocator.dupe(u8, entry.key_ptr.*); + const value = try entry.value_ptr.*.clone(arena_allocator); + try new_metadata.put(arena_allocator, key, value); + } + break :blk new_metadata; + }, + }; + } + + pub fn deinit(self: *TensorRegistry) void { + const allocator = self.arena.allocator(); + self.tensors.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } +}; + +const ResourceIndex = struct { + arena: std.heap.ArenaAllocator, + + map: std.StringArrayHashMapUnmanaged(ResourceURI), + metadata: Metadatas, + + pub fn init(allocator: std.mem.Allocator) ResourceIndex { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .map = .{}, + .metadata = .{}, + }; + } + + pub fn deinit(self: *ResourceIndex) void { + const allocator = self.arena.allocator(); + self.map.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } +}; + +fn parseSafetensorsIndex( + allocator: std.mem.Allocator, + resource: *Resource, + reader: *std.io.Reader, +) !ResourceIndex { + const resource_uri = resource.uri(); + const path = resource_uri.path.percent_encoded; + const basename = path[0 .. std.mem.lastIndexOfScalar(u8, path, '/').? + 1]; + + var resource_index: ResourceIndex = .init(allocator); + errdefer resource_index.deinit(); + + const arena_allocator = resource_index.arena.allocator(); + + var json_reader: std.json.Reader = .init(arena_allocator, reader); + + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + arena_allocator, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + const weight_map = index.object.get("weight_map"); + + if (weight_map) |wm| { + var it = wm.object.iterator(); + + while (it.next()) |entry| { + const weight_name = entry.key_ptr.*; + const filename = entry.value_ptr.string; + + const resource_path = try arena_allocator.alloc(u8, basename.len + filename.len); + + @memcpy(resource_path[0..basename.len], basename); + @memcpy(resource_path[basename.len..], filename); + + const sibling_uri: ResourceURI = .{ + .scheme = resource_uri.scheme, + .user = resource_uri.user, + .password = resource_uri.password, + .host = resource_uri.host, + .port = resource_uri.port, + .path = .{ .percent_encoded = resource_path }, + }; + + _ = try resource_index.map.put(arena_allocator, try arena_allocator.dupe(u8, weight_name), sibling_uri); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + if (index.object.get("__metadata__")) |metadata_val| { + resource_index.metadata = try parseMetadata(arena_allocator, metadata_val); + } + + return resource_index; +} + +fn parseSafetensors( + registry: *TensorRegistry, + resource_uri: ResourceURI, + reader: *std.io.Reader, +) !void { + var arena_allocator = registry.arena.allocator(); + + const json_header_length: usize = @intCast(try reader.takeInt(u64, .little)); + log.debug("json_header_length: {}", .{json_header_length}); + const json_data = try arena_allocator.alloc(u8, json_header_length); + defer arena_allocator.free(json_data); + + try reader.readSliceAll(json_data); + + const data_start_offset = 8 + json_header_length; + const metadata_val = try std.json.parseFromSliceLeaky(std.json.Value, arena_allocator, json_data, .{}); + + var it = metadata_val.object.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + + if (std.mem.eql(u8, key, "__metadata__")) { + registry.metadata = try parseMetadata(arena_allocator, value); + continue; + } + + const shape_field = value.object.get("shape").?.array; + + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); + continue; + } + + const offset_field = value.object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const end: u64 = @intCast(offset_field.array.items[1].integer); + const dtype = try stringToDtype(value.object.get("dtype").?.string); + + var dims: Dims = .{}; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + const shape: Shape = .init(dims.constSlice(), dtype); + const size_in_bytes = end - start; + std.debug.assert(size_in_bytes == shape.byteSize()); + + const tensor_name = try arena_allocator.dupe(u8, key); + const tensor: TensorDesc = .{ + .resource_uri = resource_uri, + .name = tensor_name, + .shape = shape, + .offset = data_start_offset + start, + }; + + try registry.tensors.put(arena_allocator, tensor_name, tensor); + } +} + +fn stringToDtype(safetensor_type: []const u8) !zml.DataType { + const map = std.StaticStringMap(zml.DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, + }); + + return map.get(safetensor_type) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); + return error.UnsupportedDataType; + }; +} + +fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { + const key = prefix.items; + return switch (val) { + .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), + .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), + .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), + .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), + .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), + .array => |v| { + if (v.items.len == 0) return; + if (validSlice(v)) |item_type| { + const data: Metadata = switch (item_type) { + .bool => blk: { + const values = try allocator.alloc(bool, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.bool; + break :blk .{ .array_bool = values }; + }, + .integer => blk: { + const values = try allocator.alloc(i64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.integer; + break :blk .{ .array_int = values }; + }, + .float => blk: { + const values = try allocator.alloc(f64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.float; + break :blk .{ .array_float = values }; + }, + inline .string, .number_string => |tag| blk: { + const values = try allocator.alloc([]const u8, v.items.len); + for (v.items, 0..) |item, i| { + values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); + } + break :blk .{ .array_string = values }; + }, + .null, .array, .object => unreachable, + }; + try metadatas.put(allocator, try allocator.dupe(u8, key), data); + } else { + for (v.items, 0..) |item, i| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + try populateMetadata(allocator, prefix, item, metadatas); + prefix.items.len = old_len; + } + } + }, + .object => |v| { + var obj_iter = v.iterator(); + while (obj_iter.next()) |entry| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.appendSliceAssumeCapacity(entry.key_ptr.*); + try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); + prefix.items.len = old_len; + } + }, + }; +} + +const StringBuilder = std.ArrayListUnmanaged(u8); + +pub fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { + var metadatas: Metadatas = .{}; + var prefix_buf: [BUF_1_KB]u8 = undefined; + var prefix = StringBuilder.initBuffer(&prefix_buf); + + try populateMetadata(allocator, &prefix, val, &metadatas); + + return metadatas; +} + +fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { + if (v.items.len == 0) return null; + + const item_type: std.meta.Tag(std.json.Value) = v.items[0]; + switch (item_type) { + .null, .array, .object => return null, + else => {}, + } + + for (v.items[1..]) |item| { + if (item != item_type) + return null; + } + + return item_type; +} diff --git a/mlir/mlir.h b/mlir/mlir.h index 4ad1523e..633e11f6 100644 --- a/mlir/mlir.h +++ b/mlir/mlir.h @@ -2,8 +2,7 @@ #include #include #include -#include -#include +#include #include #include #include diff --git a/mlir/mlir.zig b/mlir/mlir.zig old mode 100755 new mode 100644 index a222c40a..41b7b5f8 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const c = @import("c"); const stdx = @import("stdx"); @@ -7,9 +6,7 @@ const stdx = @import("stdx"); const log = std.log.scoped(.mlir); test { - std.testing.refAllDecls(@This()); - - _ = try Context.init(); + std.testing.refAllDeclsRecursive(@This()); } const Error = error{ @@ -19,954 +16,1199 @@ const Error = error{ MlirUnexpected, /// A resource/executor was not found. NotFound, - OutOfMemory, -}; +} || std.mem.Allocator.Error; pub inline fn stringRef(str: []const u8) c.MlirStringRef { return .{ .data = str.ptr, .length = str.len }; } -pub inline fn fromStringRef(str: c.MlirStringRef) []const u8 { +pub inline fn string(str: c.MlirStringRef) []const u8 { // Note: mlir.StringRef need not to be null terminated. return str.data[0..str.length]; } +pub const StringCallbackCtx = struct { + writer: *std.Io.Writer, + err: ?std.Io.Writer.Error = null, +}; + +pub fn stringCallback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { + const ctx: *StringCallbackCtx = @ptrCast(@alignCast(ctx_)); + ctx.writer.writeAll(string(str)) catch |err| { + ctx.err = err; + }; +} + +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: StringCallbackCtx = .{ .writer = &writer }; + _ = @call(.auto, streamFn, args ++ .{ stringCallback, &sctx }); + return writer.buffered(); +} + pub fn registerPasses(comptime passes: []const u8) void { @field(c, "mlirRegister" ++ passes ++ "Passes")(); } -pub fn successOr(res: c.MlirLogicalResult, err: anytype) !void { - return if (res.value == 0) err; +fn isPtrConst(comptime T: type) bool { + return @typeInfo(T).pointer.is_const; } -/// Alternative to MlirWrapperType -pub const MlirStrCallback = fn (c.MlirStringRef, ?*anyopaque) callconv(.c) void; - -pub const Registry = struct { - _inner: c.MlirDialectRegistry, - - pub const deinit = helpers.deinit(Registry, c.mlirDialectRegistryDestroy); - - pub fn init() !Registry { - return helpers.init(Registry, c.mlirDialectRegistryCreate(), c.mlirDialectRegistryIsNull) orelse Error.MlirUnexpected; - } -}; - -pub const Context = struct { - _inner: c.MlirContext, - const Self = Context; - pub const deinit = helpers.deinit(Context, c.mlirContextDestroy); - pub const wrapOr = helpers.wrapOr(Context, c.mlirContextIsNull); - - pub fn init() !Self { - return Self.wrapOr(c.mlirContextCreate()) orelse Error.MlirUnexpected; - } - - pub fn initWithRegistry(registry: Registry, threadingEnabled: bool) !Self { - return Self.wrapOr( - c.mlirContextCreateWithRegistry(registry._inner, threadingEnabled), - ) orelse Error.InvalidMlir; - } - - pub fn setMultiThreading(self: *Self, enabled: bool) void { - c.mlirContextEnableMultithreading(self._inner, enabled); - } - - pub fn appendDialectRegistry(self: *Self, registry: Registry) void { - c.mlirContextAppendDialectRegistry(self._inner, registry._inner); - } - - pub fn loadAllAvailableDialects(self: *Self) void { - c.mlirContextLoadAllAvailableDialects(self._inner); - } - - pub fn numRegisteredDialects(self: Self) usize { - return @intCast(c.mlirContextGetNumRegisteredDialects(self._inner)); - } - - pub fn numLoadedDialects(self: Self) usize { - return @intCast(c.mlirContextGetNumLoadedDialects(self._inner)); - } - - pub fn isRegisteredOperation(self: Self, op: [:0]const u8) bool { - return c.mlirContextIsRegisteredOperation(self._inner, stringRef(op)); - } - - pub fn location(self: Self, src: std.builtin.SourceLocation) Location { - return Location.fromSrc(self, src); - } -}; - -pub const Module = struct { - _inner: c.MlirModule, - - pub const deinit = helpers.deinit(Module, c.mlirModuleDestroy); - pub const wrapOr = helpers.wrapOr(Module, c.mlirModuleIsNull); - - const Self = Module; - - pub fn init(loc: Location) Self { - return .{ ._inner = c.mlirModuleCreateEmpty(loc._inner) }; - } - - pub fn parse(ctx: Context, source: [:0]const u8) !Module { - return Module.wrapOr( - c.mlirModuleCreateParse(ctx._inner, stringRef(source)), - ) orelse Error.InvalidMlir; - } - - pub fn fromOperation(operation: Operation) Module { - return .{ ._inner = c.mlirModuleFromOperation(operation._inner) }; - } - - pub fn context(self: Module) Context { - return .{ ._inner = c.mlirModuleGetContext(self._inner) }; - } - - pub fn getBody(self: Module) Block { - return .{ ._inner = c.mlirModuleGetBody(self._inner) }; - } - - pub fn op(self: Module) Operation { - return .{ ._inner = c.mlirModuleGetOperation(self._inner) }; - } - - pub fn hash(self: Module, hasher: *std.hash.XxHash64) void { - return self.op().hash(hasher); - } -}; - -pub const PassManager = struct { - _inner: c.MlirPassManager, - - pub const deinit = helpers.deinit(PassManager, c.mlirPassManagerDestroy); - pub const wrapOr = helpers.wrapOr(PassManager, c.mlirPassManagerIsNull); - - const Self = PassManager; - - pub fn init(ctx: Context) !Self { - return Self.wrapOr( - c.mlirPassManagerCreate(ctx._inner), - ) orelse Error.MlirUnexpected; - } - - pub fn initOnOperation(ctx: Context, op: [:0]const u8) !Self { - return Self.wrapOr( - c.mlirPassManagerCreateOnOperation(ctx._inner, stringRef(op)), - ) orelse Error.MlirUnexpected; - } - - pub fn asOpPassManager(self: Self) OpPassManager { - return .{ ._inner = c.mlirPassManagerGetAsOpPassManager(self._inner) }; - } - - pub fn enableIRPrinting(self: *Self) void { - c.mlirPassManagerEnableIRPrinting(self._inner); - } - - pub fn runOnOp(self: *Self, op: Operation) error{InvalidMlir}!void { - if (c.mlirPassManagerRunOnOp(self._inner, op._inner).value == 0) { - return Error.InvalidMlir; - } - } -}; - -fn _mlir_passpipeline_error(err: c.MlirStringRef, ctx: ?*anyopaque) callconv(.c) void { - _ = ctx; - std.debug.print(">>ERROR: {s}\n", .{err.data}); +fn CastedPtr(comptime T1: type, comptime T2: type) type { + return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; } -pub const OpPassManager = struct { - _inner: c.MlirOpPassManager, - - pub fn addPipeline(self: *OpPassManager, pipeline: [:0]const u8) error{OutOfMemory}!void { - if (c.mlirOpPassManagerAddPipeline( - self._inner, - stringRef(pipeline), - &_mlir_passpipeline_error, - null, - ).value == 0) { - return Error.OutOfMemory; - } - } -}; - -pub const Identifier = struct { - _inner: c.MlirIdentifier, - const Self = Identifier; - - pub fn get(ctx: Context, str_: [:0]const u8) Self { - return .{ ._inner = c.mlirIdentifierGet(ctx._inner, stringRef(str_)) }; - } - - pub fn context(self: Self) Context { - return .{ ._inner = c.mlirIdentifierGetContext(self._inner) }; - } - - pub fn str(self: Self) []const u8 { - return fromStringRef(c.mlirIdentifierStr(self._inner)); - } - - pub fn equals(self: Self, other: Self) bool { - return c.mlirIdentifierEqual(self._inner, other._inner); - } -}; - -pub const AttrTuple = struct { [:0]const u8, Attribute }; - -pub const Attribute = struct { - _inner: c.MlirAttribute, - - pub const dump = helpers.dump(Attribute, c.mlirAttributeDump); - pub const eql = helpers.eql(Attribute, c.mlirAttributeEqual); - pub const format = helpers.format(Attribute, c.mlirAttributePrint); - pub const wrapOr = helpers.wrapOr(Attribute, c.mlirAttributeIsNull); - - pub fn wrap(c_attr: c.MlirAttribute) Attribute { - return .{ ._inner = c_attr }; - } - - pub fn parse(ctx: Context, attr: [:0]const u8) !Attribute { - return Attribute.wrapOr( - c.mlirAttributeParseGet(ctx._inner, stringRef(attr)), - ) orelse Error.InvalidMlir; - } - - pub fn fromAny(SpecificAttr: type) fn (x: SpecificAttr) Attribute { - return struct { - fn cast(x: SpecificAttr) Attribute { - return .{ ._inner = x._inner }; - } - }.cast; - } - - pub fn isA(self: Attribute, SpecificAttr: type) bool { - return SpecificAttr.is_a_fn(self._inner); - } - - // utilities function to built common attributes. - // All attributes are upcasted to the Attribute type, making it easier to chain construct, - // but losing type information. - - pub fn null_() Attribute { - return .wrap(c.mlirAttributeGetNull()); - } - - pub fn string(ctx: Context, str: []const u8) Attribute { - return StringAttribute.init(ctx, str).asAttr(); - } - - pub fn type_(t: Type) Attribute { - return TypeAttribute.init(t).asAttr(); - } - - pub fn unit(ctx: Context) Attribute { - return .wrap(c.mlirUnitAttrGet(ctx._inner)); - } - - pub fn boolean(ctx: Context, value: bool) Attribute { - return BoolAttribute.init(ctx, value).asAttr(); - } - - pub fn i1FromBool(ctx: Context, value: bool) Attribute { - return IntegerAttribute(.i1).init(ctx, @intFromBool(value)).asAttr(); - } - - pub fn int(ctx: Context, comptime int_type: IntegerTypes, value: i64) Attribute { - return IntegerAttribute(int_type).init(ctx, value).asAttr(); - } - - pub fn float(ctx: Context, comptime float_type: FloatTypes, value: f64) Attribute { - return FloatAttribute(float_type).init(ctx, value).asAttr(); - } - - pub fn array(ctx: Context, attrs: []const Attribute) Attribute { - return ArrayAttribute.init(ctx, attrs).asAttr(); - } - - pub fn dense(ctx: Context, comptime dt: DenseArrayTypes, values: []const dt.ZigType()) Attribute { - return DenseArrayAttribute(dt).init(ctx, values).asAttr(); - } - - /// Use a tensor as an attribute. - /// The tensor is specified by dims, dtype and a flat slice of values. - pub fn denseElements(ctx: Context, dims: []const i64, comptime dt: DenseElementsAttributeTypes, values: []const dt.ZigType()) Attribute { - return DenseElementsAttribute(dt).init(.tensor(dims, dt.mlirType(ctx)), values).asAttr(); - } - - pub fn denseElementsFromBytes(ctx: Context, dims: []const i64, dt: DenseElementsAttributeTypes, raw_bytes: []const u8) Attribute { - const shape: Type = .tensor(dims, dt.mlirType(ctx)); - return .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( - shape._inner, - @intCast(raw_bytes.len), - raw_bytes.ptr, - ) }; - } - - pub fn symbol(ctx: Context, flat_name: [:0]const u8) Attribute { - return FlatSymbolRefAttribute.init(ctx, flat_name).asAttr(); - } - - pub fn named(attr: Attribute, ctx: Context, name: [:0]const u8) NamedAttribute { - return NamedAttribute.named(ctx, name, attr); - } - - pub fn dict(ctx: Context, named_attrs: []const AttrTuple) Attribute { - var attr_buf: [32]NamedAttribute = undefined; - stdx.debug.assert(named_attrs.len <= attr_buf.len, ".dict helper only support up to {} attribute, got {}. You will need to call mlir.DictionaryAttribute directly", .{ attr_buf.len, named_attrs.len }); - - const attrs = attr_buf[0..named_attrs.len]; - for (attrs, named_attrs) |*attr, tuple| { - attr.* = .named(ctx, tuple[0], tuple[1]); - } - - return DictionaryAttribute.init(ctx, attrs).asAttr(); - } -}; - -pub const NamedAttribute = extern struct { - _inner: c.MlirNamedAttribute, - - pub fn wrap(c_named_attribute: c.MlirNamedAttribute) NamedAttribute { - return @bitCast(c_named_attribute); - } - - pub fn named(ctx: Context, name: [:0]const u8, attr: Attribute) NamedAttribute { - return .{ ._inner = .{ - .name = c.mlirIdentifierGet(ctx._inner, stringRef(name)), - .attribute = attr._inner, - } }; - } - - pub fn init(name: Identifier, attr: Attribute) NamedAttribute { - return .{ ._inner = .{ - .name = name._inner, - .attribute = attr._inner, - } }; - } -}; - -pub const StringAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAString; - const Self = StringAttribute; - pub const asAttr = Attribute.fromAny(Self); - pub const eql = Attribute.eqlAny(Self); - - pub fn init(ctx: Context, str: []const u8) Self { - return .{ ._inner = c.mlirStringAttrGet(ctx._inner, stringRef(str)) }; - } - - pub fn value(self: Self) []const u8 { - return fromStringRef(c.mlirStringAttrGetValue(self._inner)); - } -}; - -pub const BoolAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsABool; - const Self = BoolAttribute; - pub const asAttr = Attribute.fromAny(Self); - pub const eql = Attribute.eqlAny(Self); - - pub fn init(ctx: Context, value_: bool) Self { - return .{ ._inner = c.mlirBoolAttrGet(ctx._inner, if (value_) 1 else 0) }; - } - - pub fn value(self: Self) bool { - return c.mlirBoolAttrGetValue(self._inner); - } -}; - -pub const TypeAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAType; - pub const eql = Attribute.eqlAny(TypeAttribute); - - pub fn init(type_: Type) TypeAttribute { - return .{ ._inner = c.mlirTypeAttrGet(type_._inner) }; - } - - pub fn typ(self: TypeAttribute) Type { - return .{ ._inner = c.mlirAttributeGetType(self._inner) }; - } - - pub const asAttr = Attribute.fromAny(TypeAttribute); -}; - -pub const ArrayAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAArray; - const Self = ArrayAttribute; - pub const asAttr = Attribute.fromAny(Self); - pub const eql = Attribute.eqlAny(Self); - - pub fn init(ctx: Context, attrs: []const Attribute) Self { - return .{ ._inner = c.mlirArrayAttrGet(ctx._inner, @intCast(attrs.len), @ptrCast(attrs.ptr)) }; - } - - pub fn size(self: Self) usize { - return @intCast(c.mlirArrayAttrGetNumElements(self._inner)); - } - - pub fn get(self: Self, index: usize) Attribute { - return .{ ._inner = c.mlirArrayAttrGetElement(self._inner, @intCast(index)) }; - } -}; - -pub fn IntegerAttribute(comptime it: IntegerTypes) type { - const ZigType, const getter = comptime switch (it) { - .i1, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, - .si4, .si8, .si16, .si32, .si64 => .{ i64, c.mlirIntegerAttrGetValueSInt }, - .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, - .unknown => @compileError("IntegerAttribute(unknown)"), - }; - +pub fn Methods(comptime T: type, comptime NativeT: type) type { return struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAInteger; + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; + pub const Ptr = if (IsPtrConst) *const T else *T; - pub const IntegerTypeType = IntegerType(it); - const IntAttr = @This(); - - pub const asAttr = Attribute.fromAny(IntAttr); - pub const eql = Attribute.eqlAny(IntAttr); - - pub fn init(ctx: Context, value: i64) IntAttr { - return .{ ._inner = c.mlirIntegerAttrGet( - IntegerType(it).init(ctx)._inner, - value, - ) }; + pub fn ptr(self: anytype) NativeT { + return .{ .ptr = @ptrCast(@constCast(self)) }; } - pub fn get(value: IntAttr) ZigType { - return @intCast(getter(value._inner)); + fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return struct { + fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var ctx: StringCallbackCtx = .{ .writer = writer }; + f(self.ptr(), stringCallback, &ctx); + if (ctx.err) |err| { + return err; + } + } + }.format; + } + + pub fn eql(f: anytype) fn (self: *const T, other: *const T) bool { + return struct { + fn eql(self: *const T, other: *const T) bool { + return f(self.ptr(), other.ptr()); + } + }.eql; + } + + pub fn isA(self: *const T, OtherType: type) ?CastedPtr(@TypeOf(self), OtherType) { + return if (OtherType.isAFn(self.ptr())) @ptrCast(self) else null; } }; } -pub fn FloatAttribute(comptime ft: FloatTypes) type { - return struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAFloat; - const FloatAttr = @This(); - pub const asAttr = Attribute.fromAny(FloatAttr); +pub const DialectRegistry = opaque { + const M = Methods(DialectRegistry, c.MlirDialectRegistry); - pub fn init(ctx: Context, value: f64) FloatAttr { - return .{ ._inner = c.mlirFloatAttrDoubleGet( - ctx._inner, - FloatType(ft).init(ctx)._inner, - value, - ) }; - } + pub const ptr = M.ptr; - pub fn get(value: FloatAttr) f64 { - return c.mlirFloatAttrGetValueDouble(value._inner); + pub fn init() std.mem.Allocator.Error!*DialectRegistry { + return @ptrCast(c.mlirDialectRegistryCreate().ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *DialectRegistry) void { + c.mlirDialectRegistryDestroy(self.ptr()); + } + + pub fn registerDialect(self: *DialectRegistry, comptime name: []const u8) void { + DialectHandle.fromString(name).insertDialect(self); + } +}; + +pub const PassManager = opaque { + const M = Methods(PassManager, c.MlirPassManager); + + pub const ptr = M.ptr; + + pub fn init(ctx: *Context) *PassManager { + return @ptrCast(c.mlirPassManagerCreate(ctx.ptr()).ptr); + } + + pub fn deinit(self: *const PassManager) void { + c.mlirPassManagerDestroy(self.ptr()); + } + + pub fn asOpPassManager(self: *PassManager) *OpPassManager { + return @ptrCast(c.mlirPassManagerGetAsOpPassManager(self.ptr()).ptr); + } + + pub fn runOnOp(self: *const PassManager, op: *Operation) !void { + const result = c.mlirPassManagerRunOnOp(self.ptr(), op.ptr()); + if (result.value == 0) { + return error.MlirUnexpected; } + } + + pub const EnableIRPrintingOpts = struct { + printBeforeAll: bool = false, + printAfterAll: bool = false, + printModuleScope: bool = false, + printAfterOnlyOnChange: bool = false, + printAfterOnlyOnFailure: bool = false, + flags: Operation.PrintFlags = .{}, + treePrintingPath: []const u8 = "", }; + + pub fn enableIRPrinting(self: *PassManager, opts: EnableIRPrintingOpts) void { + const flags = opts.flags.createNative(); + defer c.mlirOpPrintingFlagsDestroy(flags); + + c.mlirPassManagerEnableIRPrinting( + self.ptr(), + opts.printBeforeAll, + opts.printAfterAll, + opts.printModuleScope, + opts.printAfterOnlyOnChange, + opts.printAfterOnlyOnFailure, + flags, + stringRef(opts.treePrintingPath), + ); + } + + // pub extern fn mlirPassManagerEnableIRPrinting(passManager: MlirPassManager, printBeforeAll: bool, printAfterAll: bool, printModuleScope: bool, printAfterOnlyOnChange: bool, printAfterOnlyOnFailure: bool, flags: MlirOpPrintingFlags, treePrintingPath: MlirStringRef) void; + // pub extern fn mlirPassManagerEnableVerifier(passManager: MlirPassManager, enable: bool) void; + +}; + +pub const OpPassManager = opaque { + const M = Methods(OpPassManager, c.MlirOpPassManager); + + pub const ptr = M.ptr; + pub const format = M.format(c.mlirPrintPassPipeline); + + pub fn addPass(self: *OpPassManager, pass: *Pass) void { + c.mlirOpPassManagerAddOwnedPass(self.ptr(), pass.ptr()); + } + + pub fn addPasses(self: *OpPassManager, passes: []const *Pass) void { + for (passes) |pass| { + self.addPass(pass); + } + } + + pub fn addPipeline(self: *OpPassManager, pipeline_name: []const u8) !void { + var buffer: [1024]u8 = undefined; + const result = stringFromStream(&buffer, c.mlirOpPassManagerAddPipeline, .{ self.ptr(), stringRef(pipeline_name) }); + if (result.len != 0) { + log.err("mlir error: {s}", .{result}); + return error.MlirUnexpected; + } + } + + // pub fn parsePipeline(self: *OpPassManager, pipeline: []const u8) !void { + // var buffer: [1024]u8 = undefined; + // const result = stringFromStream(&buffer, c.mlirOpPassManagerParsePipeline, .{ self.ptr(), stringRef(pipeline) }); + // if (result.len != 0) { + // log.err("mlir error: {s}", .{result}); + // return error.MlirUnexpected; + // } + // } +}; + +pub const Pass = opaque { + const M = Methods(Pass, c.MlirPass); + + pub const ptr = M.ptr; + + pub fn create(comptime name: @TypeOf(.enum_literal)) *Pass { + return @ptrCast(@field(c, "mlirCreate" ++ @tagName(name))().ptr); + } +}; + +pub const Context = opaque { + const M = Methods(Context, c.MlirContext); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirContextEqual); + + pub const InitOptions = struct { + registry: ?*const DialectRegistry = null, + threading: bool = false, + }; + + pub fn init(opts: InitOptions) std.mem.Allocator.Error!*Context { + const ctx = if (opts.registry) |r| c.mlirContextCreateWithRegistry( + r.ptr(), + opts.threading, + ) else c.mlirContextCreateWithThreading(opts.threading); + return @ptrCast(ctx.ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *Context) void { + c.mlirContextDestroy(self.ptr()); + } + + pub fn setMultiThreading(self: *Context, enabled: bool) void { + c.mlirContextEnableMultithreading(self.ptr(), enabled); + } + + pub fn appendDialectRegistry(self: *Context, registry: *const DialectRegistry) void { + c.mlirContextAppendDialectRegistry(self.ptr(), registry.ptr()); + } + + pub fn loadAllAvailableDialects(self: *Context) void { + c.mlirContextLoadAllAvailableDialects(self.ptr()); + } + + pub fn numRegisteredDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumRegisteredDialects(self.ptr())); + } + + pub fn numLoadedDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumLoadedDialects(self.ptr())); + } + + pub fn numThreads(self: *const Context) usize { + return @intCast(c.mlirContextGetNumThreads(self.ptr())); + } + + pub fn isRegisteredOperation(self: *const Context, op: []const u8) bool { + return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); + } + + pub fn format(self: *const Context, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ + @typeName(Context), + self.numThreads(), + self.numRegisteredDialects(), + self.numLoadedDialects(), + }); + } +}; + +pub const Location = opaque { + const M = Methods(Location, c.MlirLocation); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirLocationEqual); + pub const format = M.format(c.mlirLocationPrint); + + pub fn fromSrc(ctx: *Context, src: std.builtin.SourceLocation) *const Location { + return fileLineCol( + ctx, + src.file, + src.line, + src.column, + ); + } + + pub fn fileLineCol(ctx: *Context, file: []const u8, line: usize, column: usize) *const Location { + return @ptrCast(c.mlirLocationFileLineColGet( + ctx.ptr(), + stringRef(file), + @intCast(line), + @intCast(column), + ).ptr); + } + + pub fn callSite(callee: *const Location, caller: *const Location) *const Location { + return @ptrCast(c.mlirLocationCallSiteGet(callee.ptr(), caller.ptr()).ptr); + } + + pub fn fused(ctx: *Context, locs: []const *const Location, metadata: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFusedGet( + ctx.ptr(), + @intCast(locs.len), + @ptrCast(locs), + metadata.ptr(), + ).ptr); + } + + pub fn named(self: *const Location, ctx: *Context, loc_name: []const u8) *const Location { + return @ptrCast(c.mlirLocationNameGet(ctx.ptr(), stringRef(loc_name), self.ptr()).ptr); + } + + pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { + var buf: [256]u8 = undefined; + return loc.named(loc, ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { + buf[buf.len - 3 ..].* = "...".*; + break :blk &buf; + }); + } + + pub fn fromAttribute(attribute: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFromAttribute(attribute.ptr()).ptr); + } + + pub fn unknown(ctx: *Context) *const Location { + return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); + } +}; + +pub const Type = opaque { + const M = Methods(Type, c.MlirType); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { + return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); + } +}; + +pub const IndexType = opaque { + const M = Methods(IndexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAIndex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context) *const IndexType { + return @ptrCast(c.mlirIndexTypeGet(ctx.ptr()).ptr); + } +}; + +pub fn indexType(ctx: *Context) *const Type { + return @ptrCast(IndexType.get(ctx)); } -pub const DenseArrayTypes = enum { - bool, - i8, - i16, - i32, - i64, - f32, - f64, - - pub fn ZigType(comptime dt: DenseArrayTypes) type { - return switch (dt) { - .bool => i32, - .i8 => i8, - .i16 => i16, - .i32 => i32, - .i64 => i64, - .f32 => f32, - .f64 => f64, - }; - } -}; - -pub fn DenseArrayAttribute(comptime dt: DenseArrayTypes) type { - const _is_a_fn, const get_fn, const get_element_fn = switch (dt) { - .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, - .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, - .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, - .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, - .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, - .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, - .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, - }; - - return struct { - _inner: c.MlirAttribute, - const Attr = @This(); - const ElementType = dt; - const ElementTypeZig = dt.ZigType(); - - pub const asAttr = Attribute.fromAny(Attr); - pub const eql = Attribute.eqlAny(Attr); - pub const is_a_fn = _is_a_fn; - - pub fn init(ctx: Context, values: []const ElementTypeZig) Attr { - return .{ ._inner = get_fn(ctx._inner, @intCast(values.len), @ptrCast(values.ptr)) }; - } - - pub fn get(self: Attr, pos: usize) ElementTypeZig { - return get_element_fn(self._inner, @intCast(pos)); - } - - pub fn len(self: Attr) usize { - return @intCast(c.mlirDenseArrayGetNumElements(self._inner)); - } - }; -} - -pub const DenseElementsAttributeTypes = enum { - bool, +pub const IntegerTypes = enum { + i1, + i4, i8, i16, i32, i64, + i128, + u1, + u4, u8, u16, u32, u64, + u128, + si1, + si4, + si8, + si16, + si32, + si64, + si128, + + pub fn bitwidth(self: IntegerTypes) usize { + return switch (self) { + .i1, .u1, .si1 => 1, + .i4, .u4, .si4 => 4, + .i8, .u8, .si8 => 8, + .i16, .u16, .si16 => 16, + .i32, .u32, .si32 => 32, + .i64, .u64, .si64 => 64, + .i128, .u128, .si128 => 128, + }; + } + + pub fn signedness(self: IntegerTypes) IntegerType.Signedness { + return switch (self) { + .i1, .i4, .i8, .i16, .i32, .i64, .i128 => .signless, + .si1, .si4, .si8, .si16, .si32, .si64, .si128 => .signed, + .u1, .u4, .u8, .u16, .u32, .u64, .u128 => .unsigned, + }; + } +}; + +pub const IntegerType = opaque { + pub const Signedness = enum { + signless, + signed, + unsigned, + }; + + const M = Methods(IntegerType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { + return exact(ctx, it.bitwidth(), it.signedness()); + } + + fn exact(ctx: *Context, bitwidth: usize, sign: Signedness) *const IntegerType { + return @ptrCast(switch (sign) { + .signless => c.mlirIntegerTypeGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .signed => c.mlirIntegerTypeSignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .unsigned => c.mlirIntegerTypeUnsignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + }); + } +}; + +pub fn integerType(ctx: *Context, it: IntegerTypes) *const Type { + return @ptrCast(IntegerType.get(ctx, it)); +} + +pub const FloatTypes = enum { + f4e2m1fn, + f6e2m3fn, + f6e3m2fn, + f8e3m4, + f8e4m3, + f8e4m3b11fnuz, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + f8e8m0fnu, bf16, f16, f32, f64, - index, - - pub fn ZigType(comptime dt: DenseElementsAttributeTypes) type { - return switch (dt) { - .bool => bool, - .i8 => i8, - .i16 => i16, - .i32 => i32, - .i64 => i64, - .u8 => u8, - .u16 => u16, - .u32 => u32, - .u64 => u64, - .bf16 => u16, - .f16 => f16, - .f32 => f32, - .f64 => f64, - .index => usize, - }; - } - - pub fn mlirType(dt: DenseElementsAttributeTypes, ctx: Context) Type { - return switch (dt) { - .bool => .int(ctx, .i1), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .u8 => .int(ctx, .u8), - .u16 => .int(ctx, .u16), - .u32 => .int(ctx, .u32), - .u64 => .int(ctx, .u64), - .bf16 => .float(ctx, .bf16), - .f16 => .float(ctx, .f16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .index => .index(ctx), - }; - } + // f80, + // f128, + tf32, }; -pub fn DenseElementsAttribute(comptime dt: DenseElementsAttributeTypes) type { - return struct { - _inner: c.MlirAttribute, +pub fn FloatType(comptime ft: FloatTypes) type { + const _isAFn, const getter = switch (ft) { + .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, + .f6e2m3fn => .{ c.mlirTypeIsAFloat6E2M3FN, c.mlirFloat6E2M3FNTypeGet }, + .f6e3m2fn => .{ c.mlirTypeIsAFloat6E3M2FN, c.mlirFloat6E3M2FNTypeGet }, + .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, + .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, + .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, + .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, + .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, + .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, + .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, + .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, + .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, + .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, + .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, + .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, + .tf32 => .{ c.mlirTypeIsATF32, c.mlirTF32TypeGet }, + }; - const Attr = @This(); + return opaque { + const Self = @This(); + const M = Methods(Self, c.MlirType); - pub const is_a_fn = c.mlirAttributeIsADenseElements; - pub const asAttr = Attribute.fromAny(Attr); - pub const eql = Attribute.eqlAny(Attr); + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); - pub fn init(shaped_type: Type, slice: []const dt.ZigType()) Attr { - const raw_bytes = std.mem.sliceAsBytes(slice); - const res: Attr = .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( - shaped_type._inner, - @intCast(raw_bytes.len), - @ptrCast(raw_bytes.ptr), - ) }; - std.debug.assert(Attribute.wrapOr(res._inner) != null); - return res; - } - - pub fn len(self: Attr) usize { - return @intCast(c.mlirElementsAttrGetNumElements(self._inner)); - } - - pub fn items(self: Attr) []const dt.ZigType() { - const raw_bytes: [*]const u8 = c.mlirDenseElementsAttrGetRawData(self._inner) orelse unreachable; - const ptr: [*]const dt.ZigType() = @alignCast(@ptrCast(raw_bytes)); - // Note the mlir API returns us the number of elements, not the number of bytes, - // that's why we track the element type at comptime to allow items to work. - return ptr[0..self.len()]; - } - - pub fn bytes(self: Attr) []const u8 { - return std.mem.sliceAsBytes(self.items()); + pub fn get(ctx: *Context) *const Self { + return @ptrCast(getter(ctx.ptr()).ptr); } }; } -pub const FlatSymbolRefAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAFlatSymbolRef; - const Self = FlatSymbolRefAttribute; - pub const eql = Attribute.eqlAny(Self); +pub fn floatType(ctx: *Context, ft: FloatTypes) *const Type { + return switch (ft) { + inline else => |v| @ptrCast(FloatType(v).get(ctx)), + }; +} - pub fn init(ctx: Context, str: [:0]const u8) Self { - return .{ ._inner = c.mlirFlatSymbolRefAttrGet(ctx._inner, stringRef(str)) }; +pub const ComplexTypes = enum { + c64, + c128, +}; + +pub const ComplexType = opaque { + const M = Methods(ComplexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAComplex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context, complex_type: ComplexTypes) *const ComplexType { + return @ptrCast(c.mlirComplexTypeGet(switch (complex_type) { + .c64 => floatType(ctx, .f32).ptr(), + .c128 => floatType(ctx, .f64).ptr(), + }).ptr); + } +}; + +pub fn complexType(ctx: *Context, complex_type: ComplexTypes) *const Type { + return @ptrCast(ComplexType.get(ctx, complex_type)); +} + +pub const Attribute = opaque { + const M = Methods(Attribute, c.MlirAttribute); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, attr: []const u8) Error!*const Attribute { + return @ptrCast(c.mlirAttributeParseGet(ctx.ptr(), stringRef(attr)).ptr orelse return Error.InvalidMlir); } - pub fn value(self: Self) []const u8 { - return fromStringRef(c.mlirFlatSymbolRefAttrGetValue(self._inner)); + pub fn context(self: *const Context) *Context { + return @ptrCast(c.mlirAttributeGetContext(self.ptr()).ptr); } - pub const asAttr = Attribute.fromAny(Self); + pub fn dialect(self: *const Context) *const Dialect { + return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); + } +}; + +pub const StringAttribute = opaque { + const M = Methods(StringAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAString; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, str: []const u8) *const StringAttribute { + return @ptrCast(c.mlirStringAttrGet(ctx.ptr(), stringRef(str)).ptr); + } + + pub fn value(self: *const StringAttribute) []const u8 { + return string(c.mlirStringAttrGetValue(self.ptr())); + } +}; + +pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { + return @ptrCast(StringAttribute.init(ctx, str)); +} + +pub const IntegerAttribute = opaque { + const M = Methods(IntegerAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { + return @ptrCast(c.mlirIntegerAttrGet( + IntegerType.get(ctx, it).ptr(), + @intCast(value_), + ).ptr); + } + + pub fn value(self: *const IntegerAttribute, comptime T: type) T { + const getValue = switch (@typeInfo(T).int.signedness) { + .signed => c.mlirIntegerAttrGetValueInt, + .unsigned => c.mlirIntegerAttrGetValueUInt, + }; + return @intCast(getValue(self.ptr())); + } +}; + +pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, it, value_)); +} + +pub const BoolAttribute = opaque { + const M = Methods(BoolAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsABool; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, value_: bool) *const BoolAttribute { + return @ptrCast(c.mlirBoolAttrGet(ctx.ptr(), @intFromBool(value_)).ptr); + } + + pub fn value(self: *const BoolAttribute) bool { + return c.mlirBoolAttrGetValue(self.ptr()); + } +}; + +pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { + return @ptrCast(BoolAttribute.init(ctx, value)); +} + +pub const TypeAttribute = opaque { + const M = Methods(TypeAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(type_: *const Type) *const TypeAttribute { + return @ptrCast(c.mlirTypeAttrGet(type_.ptr()).ptr); + } + + pub fn value(self: *const TypeAttribute) *const Type { + return @ptrCast(c.mlirTypeAttrGetValue(self.ptr()).ptr); + } +}; + +pub fn typeAttribute(type_: *const Type) *const Attribute { + return @ptrCast(TypeAttribute.init(type_)); +} + +pub const Dialect = struct { + const M = Methods(Dialect, c.MlirDialect); + + pub const ptr = M.ptr; + + pub fn context(self: *const Dialect) *Context { + return @ptrCast(c.mlirDialectGetContext(self.ptr()).ptr); + } + + pub fn namespace(self: *const Dialect) []const u8 { + return string(c.mlirDialectGetNamespace(self.ptr())); + } + + pub fn format(self: *const Dialect, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(Dialect), self.namespace() }); + } +}; + +pub const DialectHandle = opaque { + const M = Methods(DialectHandle, c.MlirDialectHandle); + + pub const ptr = M.ptr; + + pub fn format(self: *const DialectHandle, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(DialectHandle), self.namespace() }); + } + + pub fn namespace(self: *const DialectHandle) []const u8 { + return string(c.mlirDialectHandleGetNamespace(self.ptr())); + } + + pub fn insertDialect(self: *const DialectHandle, registry: *DialectRegistry) void { + c.mlirDialectHandleInsertDialect(self.ptr(), registry.ptr()); + } + + pub fn registerDialect(self: *const DialectHandle, ctx: *Context) void { + c.mlirDialectHandleRegisterDialect(self.ptr(), ctx.ptr()); + } + + pub fn loadDialect(self: *const DialectHandle, ctx: *Context) *Dialect { + return @ptrCast(c.mlirDialectHandleLoadDialect(self.ptr(), ctx.ptr()).ptr); + } + + pub fn fromString(comptime namespace_: []const u8) *const DialectHandle { + return @ptrCast(@field(c, "mlirGetDialectHandle__" ++ namespace_ ++ "__")().ptr); + } +}; + +pub const Identifier = opaque { + const M = Methods(Identifier, c.MlirIdentifier); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirIdentifierEqual); + + pub fn context(self: *const Identifier) *Context { + return @ptrCast(c.mlirIdentifierGetContext(self.ptr()).ptr); + } + + pub fn get(ctx: *Context, str_: []const u8) *const Identifier { + return @ptrCast(c.mlirIdentifierGet(ctx.ptr(), stringRef(str_)).ptr); + } + + pub fn str(self: *const Identifier) []const u8 { + return string(c.mlirIdentifierStr(self.ptr())); + } + + pub fn format(self: *const Identifier, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .str = \"{s}\" }}", .{ @typeName(Identifier), self.str() }); + } +}; + +pub const Module = opaque { + const M = Methods(Module, c.MlirModule); + + pub const ptr = M.ptr; + + pub fn init(loc: *const Location) *Module { + return @ptrCast(@constCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr)); + } + + pub fn deinit(self: *Module) void { + c.mlirModuleDestroy(self.ptr()); + } + + pub fn format(self: *const Module, writer: *std.Io.Writer) !void { + return self.operation().format(writer); + } + + pub fn fromOperation(op: *Operation) *Module { + return @ptrCast(@constCast(c.mlirModuleFromOperation(op.ptr()).ptr)); + } + + pub fn operation(self: *const Module) *Operation { + return @ptrCast(c.mlirModuleGetOperation(self.ptr()).ptr); + } + + pub fn parse(ctx: *Context, source: []const u8) Error!*Module { + return @ptrCast(@constCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir)); + } + + pub fn context(self: *const Module) *Context { + return @ptrCast(c.mlirModuleGetContext(self.ptr()).ptr); + } + + pub fn body(self: *const Module) *Block { + return @ptrCast(c.mlirModuleGetBody(self.ptr()).ptr); + } +}; + +pub const Block = opaque { + const M = Methods(Block, c.MlirBlock); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirBlockEqual); + pub const format = M.format(c.mlirBlockPrint); + + pub fn init(args: []const *const Type, locs: []const *const Location) *Block { + return @ptrCast(c.mlirBlockCreate(@intCast(args.len), @ptrCast(args), @ptrCast(locs)).ptr); + } + + pub fn deinit(self: *Block) void { + std.debug.assert(self.parentOperation() == null); + c.mlirBlockDestroy(self.ptr()); + } + + pub fn argument(self: *const Block, index: usize) *const Value { + return @ptrCast(c.mlirBlockGetArgument(self.ptr(), @intCast(index)).ptr); + } + + pub fn numArguments(self: *const Block) usize { + return @intCast(c.mlirBlockGetNumArguments(self.ptr())); + } + + pub fn addArgument(self: *Block, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockAddArgument(self.ptr(), type_.ptr(), loc.ptr()).ptr); + } + + pub fn insertArgument(self: *Block, index: usize, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockInsertArgument(self.ptr(), @intCast(index), type_.ptr(), loc.ptr()).ptr); + } + + pub fn appendOwnedOperation(self: *Block, op: *Operation) void { + c.mlirBlockAppendOwnedOperation(self.ptr(), op.ptr()); + } + + pub fn appendOwnedOperations(self: *Block, ops: []const *Operation) void { + for (ops) |op| { + self.appendOwnedOperation(op); + } + } + + pub fn parentOperation(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); + } + + pub fn detach(self: *Block) void { + c.mlirBlockDetach(self.ptr()); + } + + pub fn terminator(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetTerminator(self.ptr()).ptr); + } +}; + +pub const NamedAttribute = struct { + inner: c.MlirNamedAttribute, + + pub fn ptr(self: NamedAttribute) c.MlirNamedAttribute { + return self.inner; + } + + pub fn name(self: NamedAttribute) *const Identifier { + return @ptrCast(self.inner.name.ptr); + } + + pub fn attribute(self: NamedAttribute) *const Attribute { + return @ptrCast(self.inner.attribute.ptr); + } + + pub fn named(ctx: *Context, name_: []const u8, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = Identifier.get(ctx, name_).ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn init(name_: *const Identifier, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = name_.ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn format(self: NamedAttribute, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s}{{ .{s} = \"{f}\" }}", .{ + @typeName(NamedAttribute), + self.name().str(), + self.attribute(), + }); + } +}; + +pub const DictionaryAttribute = opaque { + const M = Methods(DictionaryAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADictionary; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attributes: []const NamedAttribute) *const DictionaryAttribute { + return @ptrCast(c.mlirDictionaryAttrGet( + ctx.ptr(), + @intCast(attributes.len), + @ptrCast(attributes), + ).ptr); + } + + pub fn size(self: *const DictionaryAttribute) usize { + return @intCast(c.mlirDictionaryAttrGetNumElements(self.ptr())); + } + + pub fn get(self: *const DictionaryAttribute, pos: usize) NamedAttribute { + return .{ .inner = c.mlirDictionaryAttrGetElement(self.ptr(), @intCast(pos)) }; + } + + pub fn getByName(self: *const DictionaryAttribute, name: []const u8) ?*const Attribute { + return @ptrCast(c.mlirDictionaryAttrGetElementByName(self.ptr(), stringRef(name)).ptr); + } +}; + +pub fn dictionaryAttribute(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { + return @ptrCast(DictionaryAttribute.init(ctx, attributes)); +} + +pub const Value = opaque { + const M = Methods(Value, c.MlirValue); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + pub const isA = M.isA; + + pub fn type_(val: *const Value) *const Type { + return @ptrCast(c.mlirValueGetType(val.ptr()).ptr); + } + + pub fn setType(self: *Value, type__: *const Type) void { + c.mlirValueSetType(self.ptr(), type__.ptr()); + } + + pub fn firstUse(self: *const Value) *const OpOperand { + return @ptrCast(c.mlirValueGetFirstUse(self.ptr()).ptr); + } + + pub fn replaceAllUsesWith(self: *const Value, with: *const Value) void { + c.mlirValueReplaceAllUsesOfWith(self.ptr(), with.ptr()); + } +}; + +pub const OpOperand = opaque { + const M = Methods(OpOperand, c.MlirOpOperand); + + pub const ptr = M.ptr; + + pub fn owner(self: *const OpOperand) *Operation { + return @ptrCast(c.mlirOpOperandGetOwner(self.ptr()).ptr); + } + + pub fn number(self: *const OpOperand) usize { + return @intCast(c.mlirOpOperandGetOperandNumber(self.ptr())); + } + + pub fn nextUse(self: *const OpOperand) ?*const OpOperand { + return @ptrCast(c.mlirOpOperandGetNextUse(self.ptr()).ptr); + } +}; + +pub const OpResult = opaque { + const M = Methods(OpResult, c.MlirValue); + + pub const isAFn = c.mlirValueIsAOpResult; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn owner(self: *const OpResult) *Operation { + return @ptrCast(c.mlirOpResultGetOwner(self.ptr()).ptr); + } + + pub fn resultNumber(self: *const OpResult) usize { + return @intCast(c.mlirOpResultGetResultNumber(self.ptr())); + } +}; + +pub const BlockArgument = opaque { + const M = Methods(BlockArgument, c.MlirValue); + + pub const isAFn = c.mlirValueIsABlockArgument; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn block(arg: *const BlockArgument) *Block { + return @ptrCast(c.mlirBlockArgumentGetOwner(arg.ptr()).ptr); + } + + pub fn number(self: *const BlockArgument) usize { + return @intCast(c.mlirBlockArgumentGetArgNumber(self.ptr())); + } + + pub fn setType(self: *BlockArgument, type_: *const Type) void { + c.mlirBlockArgumentSetType(self.ptr(), type_.ptr()); + } }; pub const OperationState = struct { - _inner: c.MlirOperationState, + inner: c.MlirOperationState, - const Self = OperationState; - - pub fn init(name: [:0]const u8, loc: Location) Self { - return .{ ._inner = c.mlirOperationStateGet(stringRef(name), loc._inner) }; + pub fn init(name: []const u8, location: *const Location) OperationState { + return .{ .inner = c.mlirOperationStateGet(stringRef(name), location.ptr()) }; } - pub fn addResult(self: *Self, type_: Type) void { - c.mlirOperationStateAddResults(&self._inner, 1, &[_]c.MlirType{type_._inner}); + pub fn addResults(self: *OperationState, results: []const *const Type) void { + c.mlirOperationStateAddResults(&self.inner, @intCast(results.len), @ptrCast(results)); } - pub fn addResults(self: *Self, types: []const Type) void { - c.mlirOperationStateAddResults(&self._inner, @intCast(types.len), @ptrCast(types.ptr)); + pub fn addOperands(self: *OperationState, operands: []const *const Value) void { + c.mlirOperationStateAddOperands(&self.inner, @intCast(operands.len), @ptrCast(operands)); } - pub fn addOperand(self: *Self, value: Value) void { - c.mlirOperationStateAddOperands(&self._inner, 1, &[_]c.MlirValue{value._inner}); + pub fn addOwnedRegions(self: *OperationState, regions: []const *Region) void { + c.mlirOperationStateAddOwnedRegions(&self.inner, @intCast(regions.len), @ptrCast(regions)); } - pub fn addOperands(self: *Self, values: []const Value) void { - c.mlirOperationStateAddOperands(&self._inner, @intCast(values.len), @ptrCast(values.ptr)); + pub fn addSuccessors(self: *OperationState, successors: []*const Block) void { + c.mlirOperationStateAddSuccessors(&self.inner, @intCast(successors.len), @ptrCast(successors)); } - pub fn addRegion(self: *Self, region: *Region) void { - c.mlirOperationStateAddOwnedRegions(&self._inner, 1, &[_]c.MlirRegion{region._inner}); + pub fn addAttributes(self: *OperationState, attributes: []const NamedAttribute) void { + c.mlirOperationStateAddAttributes(&self.inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); } - pub fn addRegions(self: *Self, regions: []const Region) void { - c.mlirOperationStateAddOwnedRegions(&self._inner, @intCast(regions.len), @ptrCast(regions.ptr)); - } - - pub fn addAttribute(self: *Self, ctx: Context, name: [:0]const u8, attr: Attribute) void { - c.mlirOperationStateAddAttributes(&self._inner, 1, @ptrCast(&.{ - .{ - .name = Identifier.get(ctx, name)._inner, - .attribute = attr._inner, - }, - })); - } - - pub fn addAttributeRaw(self: *Self, name: Identifier, attr: Attribute) void { - c.mlirOperationStateAddAttributes(&self._inner, 1, @ptrCast(&.{ - .{ - .name = name._inner, - .attribute = attr._inner, - }, - })); - } - - pub fn addAttributes(self: *Self, attributes: []const NamedAttribute) void { - c.mlirOperationStateAddAttributes(&self._inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); - } - - pub fn resultTypeInference(self: *Self, enabled: bool) void { - self._inner.enableResultTypeInference = enabled; + pub fn enableResultTypeInference(self: *OperationState) void { + c.mlirOperationStateEnableResultTypeInference(&self.inner); } }; -pub const DictionaryAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsADictionary; - pub const asAttr = Attribute.fromAny(DictionaryAttribute); - pub const eql = Attribute.eqlAny(DictionaryAttribute); +pub const Operation = opaque { + const M = Methods(Operation, c.MlirOperation); - pub fn init(ctx: Context, attributes: []const NamedAttribute) DictionaryAttribute { - return .{ ._inner = c.mlirDictionaryAttrGet( - ctx._inner, - @intCast(attributes.len), - @ptrCast(attributes.ptr), - ) }; + const MAX_SEGMENTS = 32; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirOperationEqual); + pub const format = M.format(c.mlirOperationPrint); + + pub fn init(state: *OperationState) !*Operation { + defer state.* = undefined; + return @ptrCast(c.mlirOperationCreate(&state.inner).ptr orelse return Error.InvalidMlir); } - pub fn size(self: DictionaryAttribute) usize { - return @intCast(c.mlirDictionaryAttrGetNumElements(self._inner)); + pub fn deinit(self: *Operation) void { + std.debug.assert(self.block() == null); + c.mlirOperationDestroy(self.ptr()); } - pub fn get(self: DictionaryAttribute, pos: usize) NamedAttribute { - return .wrap(c.mlirDictionaryAttrGetElement(self._inner, @bitCast(pos))); + pub fn parse(ctx: *Context, source: []const u8, name_: []const u8) Error!*Operation { + return @ptrCast(c.mlirOperationCreateParse(ctx.ptr(), stringRef(source), stringRef(name_)).ptr orelse return Error.InvalidMlir); } - pub fn getByName(self: DictionaryAttribute, name: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirDictionaryAttrGetElementByName(self._inner, name)); - } -}; - -pub const Operation = struct { - const Self = Operation; - _inner: c.MlirOperation, - - pub const dump = helpers.dump(Operation, c.mlirOperationDestroy); - pub const deinit = helpers.deinit(Operation, c.mlirOperationDestroy); - pub const wrapOr = helpers.wrapOr(Operation, c.mlirOperationIsNull); - - pub const eql = Attribute.eqlAny(Self); - - pub fn init(state: *OperationState) !Self { - return Self.wrapOr(c.mlirOperationCreate(&state._inner)) orelse Error.InvalidMlir; + pub fn name(self: *const Operation) []const u8 { + return string(c.mlirOperationGetName(self.ptr())); } - pub fn make(ctx: Context, op_name: [:0]const u8, args: struct { - operands: ?[]const Value = null, - variadic_operands: ?[]const []const Value = null, - tt_variadic_operands: ?[]const []const Value = null, - results: ?[]const Type = null, - variadic_results: ?[]const []const Type = null, - result_type_inference: ?bool = null, - n_regions: usize = 0, - attributes: ?[]const AttrTuple = null, - blocks: ?[]const Block = null, - verify: bool = true, - location: Location, - }) Self { - var state = OperationState.init(op_name, args.location); - std.debug.assert(!(args.operands != null and args.variadic_operands != null)); - if (args.operands) |operands| { - state.addOperands(operands); - } else if (args.variadic_operands) |operands_segments| { - const MAX_SEGMENTS = 32; - var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + pub fn clone(self: *const Operation) *Operation { + return @ptrCast(c.mlirOperationClone(self.ptr()).ptr); + } - for (operands_segments) |operands| { - state.addOperands(operands); - segments.appendAssumeCapacity(@intCast(operands.len)); - } - state.addAttribute(ctx, "operandSegmentSizes", .denseElements(ctx, &.{@intCast(segments.len)}, .i32, segments.constSlice())); - } else if (args.tt_variadic_operands) |operands_segments| { - // stablehlo and triton seems to disagree on the expected type of operandSegmentSizes, let's fix that. - const MAX_SEGMENTS = 32; - var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + pub fn block(self: *const Operation) ?*Block { + return @ptrCast(c.mlirOperationGetBlock(self.ptr()).ptr); + } - for (operands_segments) |operands| { - state.addOperands(operands); - segments.appendAssumeCapacity(@intCast(operands.len)); - } - state.addAttribute(ctx, "operandSegmentSizes", .dense(ctx, .i32, segments.constSlice())); - } - if (args.result_type_inference) |enable| { - state.resultTypeInference(enable); - } - std.debug.assert(!(args.results != null and args.variadic_results != null)); - if (args.results) |results| { - state.addResults(results); - } else if (args.variadic_results) |result_segments| { - for (result_segments) |results| { - state.addResults(results); - } - } - for (0..args.n_regions) |_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - state.addRegion(®ion_); - } - if (args.attributes) |attrs| { - for (attrs) |attr| { - state.addAttributeRaw( - Identifier.get(ctx, attr[0]), - attr[1], - ); - } - } - if (args.blocks) |blocks_| { - for (blocks_) |block_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - region_.appendBlock(block_); - state.addRegion(®ion_); - } - } + pub fn appendTo(self: *Operation, block_: *Block) *Operation { + block_.appendOwnedOperation(self); + return self; + } - const new_op = Operation.init(&state) catch { - @panic("Failed to create MLIR operation"); + pub fn parent(self: *const Operation) ?*Operation { + return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); + } + + pub fn removeFromParent(self: *Operation) void { + c.mlirOperationRemoveFromParent(self.ptr()); + } + + pub fn numOperands(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumOperands(self.ptr())); + } + + pub fn operand(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetOperand(self.ptr(), @intCast(index)).ptr); + } + + pub fn setOperand(self: *Operation, index: usize, value: *const Value) void { + c.mlirOperationSetOperand(self.ptr(), @intCast(index), value.ptr()); + } + + pub fn numResults(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumResults(self.ptr())); + } + + pub fn result(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetResult(self.ptr(), @intCast(index)).ptr); + } + + pub fn nextInBlock(self: *const Operation) *const Operation { + return @ptrCast(c.mlirOperationGetNextInBlock(self.ptr()).ptr); + } + + pub fn context(self: *const Operation) *Context { + return @ptrCast(c.mlirOperationGetContext(self.ptr()).ptr); + } + + pub fn setAttributeByName(self: *Operation, name_: []const u8, attribute: *const Attribute) void { + c.mlirOperationSetAttributeByName(self.ptr(), stringRef(name_), attribute.ptr()); + } + + pub const PrintFlags = struct { + elide_large_elements_attrs: ?usize = null, + debug_info: bool = false, + debug_info_pretty_form: bool = true, + print_generic_op_form: bool = false, + use_local_scope: bool = false, + assume_verified: bool = false, + + const Ctx = struct { + self: *const Operation, + flags: PrintFlags, }; + + fn createNative(self: PrintFlags) c.MlirOpPrintingFlags { + const flags = c.mlirOpPrintingFlagsCreate(); + if (self.elide_large_elements_attrs) |v| { + c.mlirOpPrintingFlagsElideLargeElementsAttrs(flags, @intCast(v)); + } + c.mlirOpPrintingFlagsEnableDebugInfo( + flags, + self.debug_info, + self.debug_info_pretty_form, + ); + if (self.print_generic_op_form) { + c.mlirOpPrintingFlagsPrintGenericOpForm(flags); + } + if (self.use_local_scope) { + c.mlirOpPrintingFlagsUseLocalScope(flags); + } + if (self.assume_verified) { + c.mlirOpPrintingFlagsAssumeVerified(flags); + } + return flags; + } + }; + + fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + const flags = fctx.flags.createNative(); + defer c.mlirOpPrintingFlagsDestroy(flags); + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationPrintWithFlags(fctx.self.ptr(), flags, stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } + } + + pub fn fmt(self: *const Operation, flags: PrintFlags) std.fmt.Alt(PrintFlags.Ctx, formatWithFlags) { + return .{ .data = .{ + .self = self, + .flags = flags, + } }; + } + + pub const MakeArgs = struct { + pub const AttrTuple = struct { []const u8, *const Attribute }; + + pub const Operands = union(enum) { + flat: []const *const Value, + variadic: []const []const *const Value, + }; + + pub const ResultTypes = union(enum) { + flat: []const *const Type, + variadic: []const []const *const Type, + }; + + operands: ?Operands = null, + results: ?ResultTypes = null, + result_type_inference: bool = false, + attributes: ?[]const NamedAttribute = null, + blocks: ?[]const *Block = null, + verify: bool = true, + location: ?*const Location = null, + }; + + pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { + var state: OperationState = .init(name_, args.location orelse .unknown(ctx)); + if (args.result_type_inference) { + state.enableResultTypeInference(); + } + if (args.operands) |operands| switch (operands) { + .flat => |v| state.addOperands(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_operands| { + state.addOperands(segment_operands); + sizes.appendAssumeCapacity(@intCast(segment_operands.len)); + } + state.addAttributes(&.{ + .named(ctx, "operandSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + }); + }, + }; + if (args.results) |results| switch (results) { + .flat => |v| state.addResults(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_results| { + state.addResults(segment_results); + sizes.appendAssumeCapacity(@intCast(segment_results.len)); + } + state.addAttributes(&.{ + .named(ctx, "resultSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + }); + }, + }; + if (args.attributes) |attrs| { + state.addAttributes(attrs); + } + if (args.blocks) |blocks| { + const region = Region.init(); + for (blocks) |block_| { + region.appendOwnedBlock(block_); + } + state.addOwnedRegions(&.{region}); + } + const new_op = try Operation.init(&state); + errdefer new_op.deinit(); if (args.verify and new_op.verify() == false) { - log.err("Failed to verify MLIR operation:\n{f}", .{new_op.mlirFormatter(.{ .debug_info = true })}); - @panic("Failed to verify MLIR operation"); + log.err("Failed to verify MLIR operation:\n{f}", .{ + new_op.fmt(.{ + .print_generic_op_form = true, + .debug_info = true, + }), + }); + return Error.InvalidMlir; } return new_op; } - pub fn initParse(ctx: Context, str: [:0]const u8) !Self { - return Self.wrapOr( - c.mlirOperationCreateParse(ctx._inner, stringRef(str), stringRef("pouet")), - ) orelse Error.InvalidMlir; - } - - pub fn clone(self: Self) !Self { - return Self.wrapOr( - c.mlirOperationClone(self._inner), - ) orelse Error.InvalidMlir; - } - - pub fn name(self: Self) Identifier { - return .{ ._inner = c.mlirOperationGetName(self._inner) }; - } - - pub fn removeFromParent(self: *Self) void { - c.mlirOperationRemoveFromParent(self._inner); - } - - pub fn numOperands(self: Self) usize { - return @intCast(c.mlirOperationGetNumOperands(self._inner)); - } - - pub fn operand(self: Self, index: usize) Value { - return .{ ._inner = c.mlirOperationGetOperand(self._inner, @intCast(index)) }; - } - - pub fn setOperand(self: *Self, index: usize, value: Value) void { - c.mlirOperationSetOperand(self._inner, @intCast(index), value._inner); - } - - pub fn numResults(self: Self) usize { - return @intCast(c.mlirOperationGetNumResults(self._inner)); - } - - pub fn result(self: Self, index: usize) Value { - return .{ ._inner = c.mlirOperationGetResult(self._inner, @intCast(index)) }; - } - - pub fn nextInBlock(self: Self) Self { - return .{ ._inner = c.mlirOperationGetNextInBlock(self._inner) }; - } - - // pub fn previousInBlock(self: Self) Self { - // return .{ ._inner = c.mlirOperationGetPrevInBlock(self._inner) }; - // } - - pub fn block(self: Self) ?Block { - return Block.wrapOr(c.mlirOperationGetBlock(self._inner)); - } - - pub fn parent(self: Self) ?Self { - return Self.wrapOr(c.mlirOperationGetParentOperation(self._inner)); - } - - pub fn region(self: Self, index: usize) Region { - return .{ ._inner = c.mlirOperationGetRegion(self._inner, @intCast(index)) }; - } - - pub fn context(self: Self) Context { - return .{ ._inner = c.mlirOperationGetContext(self._inner) }; - } - - pub fn writeBytecode(self: Self, writer: anytype) void { - var writer_context = .{ .writer = writer }; - const WriterContext = @TypeOf(writer_context); - - c.mlirOperationWriteBytecode( - self._inner, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch unreachable; - } - }).callback, - &writer_context, - ); - } - - pub fn writeBytecodeWithConfig(self: Self, writer: anytype, config: struct { - desiredEmitedVersion: ?i64 = null, - }) !void { - const cfg = c.mlirBytecodeWriterConfigCreate(); - defer c.mlirBytecodeWriterConfigDestroy(cfg); - if (config.desiredEmitedVersion) |v| { - c.mlirBytecodeWriterConfigDesiredEmitVersion(cfg, v); - } - - const WriterContext = struct { - writer: @TypeOf(writer), - write_error: ?@TypeOf(writer).Error = null, + pub fn make(ctx: *Context, op_name: []const u8, args: MakeArgs) *Operation { + return try_make(ctx, op_name, args) catch { + @panic("Failed to create MLIR operation"); }; - var writer_context: WriterContext = .{ .writer = writer }; - - try successOr(c.mlirOperationWriteBytecodeWithConfig( - self._inner, - cfg, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch |err| { - inner_writer_context.write_error = err; - }; - } - }).callback, - &writer_context, - ), error.InvalidMlirBytecodeVersion); - - if (writer_context.write_error) |err| return err; } - /// Enable a full dump of the IR. - /// Usage `std.log.debug("{}", .{ module.op().mlirFormatter(.{}) }); - pub fn mlirFormatter(self: Operation, flags: OpPrintingFlags) MlirFormatter { - return .{ .op = self, .flags = flags }; + pub fn verify(self: *const Operation) bool { + return c.mlirOperationVerify(self.ptr()); } - const MlirFormatter = struct { - op: Operation, - flags: OpPrintingFlags, - - pub fn format(self: @This(), writer: anytype) !void { - self.op.print(writer, self.flags); - } - }; - - pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) void { - const pflags = flags.create(); - defer c.mlirOpPrintingFlagsDestroy(pflags); - - c.mlirOperationPrintWithFlags( - self._inner, - pflags, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const _writer: *std.Io.Writer = @ptrCast(@alignCast(ctx_)); - _writer.writeAll(str.data[0..str.length]) catch @panic("Mlir print failed"); - } - }).callback, - writer, - ); - } - - pub fn verify(self: Self) bool { - return c.mlirOperationVerify(self._inner); - } - - pub fn getLocation(self: Self) Location { - return .{ ._inner = c.mlirOperationGetLocation(self._inner) }; + pub fn location(self: *const Operation) *const Location { + return @ptrCast(c.mlirOperationGetLocation(self.ptr()).ptr); } pub const WalkOrder = enum(c.MlirWalkOrder) { @@ -980,7 +1222,7 @@ pub const Operation = struct { skip = c.MlirWalkResultSkip, }; - pub fn walk(self: Self, order: WalkOrder, ctx: anytype, walkfn: fn (ctx: anytype, op: Operation) WalkResult) void { + pub fn walk(self: *Operation, order: WalkOrder, ctx: anytype, walkfn: fn (ctx_: @TypeOf(ctx), op: *Operation) WalkResult) void { var inner_ctx = .{ .ctx = ctx }; const ContextType = @TypeOf(inner_ctx); @@ -989,7 +1231,7 @@ pub const Operation = struct { (struct { pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @intFromEnum(walkfn(inner_ctx_.ctx, .{ ._inner = op })); + return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, @@ -997,779 +1239,383 @@ pub const Operation = struct { ); } - pub fn getAttribute(self: Self, pos: usize) NamedAttribute { - return .{ ._inner = c.mlirOperationGetAttribute(self._inner, @intCast(pos)) }; - } + pub const BytecodeWriterConfig = struct { + desired_emit_version: ?usize = null, - pub fn getAttributeByName(self: Self, name_: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirOperationGetAttributeByName(self._inner, stringRef(name_))); - } - - pub fn setAttributeByName(self: Self, name_: [:0]const u8, attr: Attribute) void { - c.mlirOperationSetAttributeByName(self._inner, stringRef(name_), attr._inner); - } - - pub fn removeAttributeByName(self: Self, name_: [:0]const u8) bool { - return c.mlirOperationRemoveAttributeByName(self._inner, stringRef(name_)); - } - - /// Hash the canonicalized IR, without debug information that can change across builds. - pub fn hash(op: Operation, hasher: *std.hash.XxHash64) void { - // Note: before we where using op.writeBytecode(writer), - // but it crashes on some inputs, notably for unused variables. - // So we use the text representation of the mlir. - // See https://github.com/zml/zml/issues/97. - const flags = OpPrintingFlags.create(.{ .debug_info = false }); - defer c.mlirOpPrintingFlagsDestroy(flags); - - c.mlirOperationPrintWithFlags( - op._inner, - flags, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const _hasher: *std.hash.XxHash64 = @ptrCast(@alignCast(ctx_)); - _hasher.update(str.data[0..str.length]); - } - }).callback, - hasher, - ); - } -}; - -pub const OpPrintingFlags = struct { - elide_large_elements_attrs: ?usize = null, - debug_info: bool = false, - debug_info_pretty_form: bool = true, - print_generic_op_form: bool = false, - use_local_scope: bool = false, - assume_verified: bool = false, - - pub fn create(self: OpPrintingFlags) c.MlirOpPrintingFlags { - const pflags = c.mlirOpPrintingFlagsCreate(); - if (self.elide_large_elements_attrs) |v| { - c.mlirOpPrintingFlagsElideLargeElementsAttrs(pflags, @intCast(v)); - } - c.mlirOpPrintingFlagsEnableDebugInfo(pflags, self.debug_info, self.debug_info_pretty_form); - if (self.print_generic_op_form) { - c.mlirOpPrintingFlagsPrintGenericOpForm(pflags); - } - if (self.use_local_scope) { - c.mlirOpPrintingFlagsUseLocalScope(pflags); - } - if (self.assume_verified) { - c.mlirOpPrintingFlagsAssumeVerified(pflags); - } - return pflags; - } -}; - -pub const OpOperand = struct { - _inner: c.MlirOpOperand, - const Self = OpOperand; - - pub fn owner(self: Self) Operation { - return .{ ._inner = c.mlirOpOperandGetOwner(self._inner) }; - } - - pub fn number(self: Self) usize { - return @intCast(c.mlirOpOperandGetOperandNumber(self._inner)); - } - - pub fn nextUse(self: Self) ?Self { - return Self.wrapOr( - c.mlirOpOperandGetNextUse(self._inner), - ); - } -}; - -pub const Region = struct { - _inner: c.MlirRegion, - - pub const eql = helpers.eql(Region, c.mlirBlockEqual); - pub const deinit = helpers.deinit(Region, c.mlirRegionDestroy); - pub const wrapOr = helpers.wrapOr(Region, c.mlirRegionIsNull); - - const Self = Region; - - pub fn init() !Self { - return Self.wrapOr(c.mlirRegionCreate()) orelse Error.InvalidMlir; - } - - pub fn appendBlock(self: *Self, block: Block) void { - c.mlirRegionAppendOwnedBlock(self._inner, block._inner); - } - - pub fn insertBlock(self: *Self, index: isize, block: Block) void { - c.mlirRegionInsertOwnedBlock(self._inner, index, block._inner); - } - - pub fn insertBlockBefore(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockBefore(self._inner, reference._inner, block._inner); - } - - pub fn insertBlockAfter(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockAfter(self._inner, reference._inner, block._inner); - } - - pub fn firstBlock(self: Self) Block { - return .{ ._inner = c.mlirRegionGetFirstBlock(self._inner) }; - } -}; - -pub const Value = struct { - _inner: c.MlirValue, - - pub const dump = helpers.dump(Value, c.mlirValueDump); - pub const eql = helpers.eql(Value, c.mlirValueEqual); - pub const format = helpers.format(Value, c.mlirValuePrint).format; - pub const wrapOr = helpers.wrapOr(Value, c.mlirValueIsNull); - - pub fn getType(val: Value) Type { - return .{ ._inner = c.mlirValueGetType(val._inner) }; - } - - pub fn setType(val: *Value, typ: Type) void { - c.mlirValueSetType(val._inner, typ._inner); - } - - pub fn firstUse(val: Value) OpOperand { - return .{ ._inner = c.mlirValueGetFirstUse(val._inner) }; - } - - pub fn replaceAllUsesWith(val: Value, with: Value) void { - c.mlirValueReplaceAllUsesOfWith(val._inner, with._inner); - } - - pub fn owner(val: Value) Operation { - return .{ ._inner = c.mlirOpResultGetOwner(val._inner) }; - } - - pub fn isABlockArgument(val: Value) bool { - return c.mlirValueIsABlockArgument(val._inner); - } - - pub fn isAOpResult(val: Value) bool { - return c.mlirValueIsAOpResult(val._inner); - } - - pub const Kind = union(enum) { - block_argument: BlockArgument, - op_result: Operation, - null, + const Ctx = struct { + self: *const Operation, + config: ?BytecodeWriterConfig, + }; }; - pub fn kind(val: Value) Kind { - if (val.isAOpResult()) { - return .{ .op_result = val.owner() }; - } - if (val.isABlockArgument()) { - return .{ .block_argument = .{ ._inner = val._inner } }; - } - // From MLIR docs: - // https://mlir.llvm.org/doxygen/classmlir_1_1Value.html#details - // > An SSA value is either a BlockArgument or the result of an operation. - return .null; - } -}; + const WriteBytecodeError = error{InvalidMlirBytecodeVersion} || std.Io.Writer.Error; -pub const BlockArgument = struct { - _inner: c.MlirValue, - - pub fn block(arg: BlockArgument) Block { - return .{ ._inner = c.mlirBlockArgumentGetOwner(arg._inner) }; - } - - pub fn number(arg: BlockArgument) usize { - return @bitCast(c.mlirBlockArgumentGetArgNumber(arg._inner)); - } - - pub fn format(self: BlockArgument, writer: anytype) !void { - const value = Value{ ._inner = self._inner }; - return value.format(writer); - } -}; - -pub const Type = struct { - _inner: c.MlirType, - - pub const dump = helpers.eql(Type, c.mlirTypeDump); - pub const eql = helpers.eql(Type, c.mlirTypeEqual); - pub const format = helpers.format(Type, c.mlirTypePrint); - pub const wrapOr = helpers.wrapOr(Type, c.mlirTypeIsNull); - - pub fn parse(ctx: Context, str: [:0]const u8) !Type { - return Type.wrapOr( - c.mlirTypeParseGet(ctx._inner, stringRef(str)), - ) orelse Error.InvalidMlir; - } - - pub fn as(generic: Type, SpecificType: type) ?SpecificType { - if (@hasDecl(SpecificType, "is_a_fn")) { - return if (SpecificType.is_a_fn(generic._inner)) - .{ ._inner = generic._inner } - else - null; - } - @compileError("Mlir subclass of type need `is_a_fn` attribute: " ++ @typeName(SpecificType)); - } - - pub fn fromAny(SpecificType: type) fn (x: SpecificType) Type { - stdx.debug.assertComptime(@hasDecl(SpecificType, "asType"), "Type.fromAny expects a type subclass, got: {}. Missing `asAttr` declaration.", .{SpecificType}); - return struct { - fn cast(x: SpecificType) Type { - return .{ ._inner = x._inner }; + pub fn writeBytecode(self: *const Operation, config: ?BytecodeWriterConfig, writer: *std.Io.Writer) WriteBytecodeError!void { + if (config) |config_| { + const bc_config = c.mlirBytecodeWriterConfigCreate(); + defer c.mlirBytecodeWriterConfigDestroy(bc_config); + if (config_.desired_emit_version) |version| { + c.mlirBytecodeWriterConfigDesiredEmitVersion(bc_config, @intCast(version)); } - }.cast; - } + var sctx: StringCallbackCtx = .{ .writer = writer }; + const logical_result = c.mlirOperationWriteBytecodeWithConfig(self.ptr(), bc_config, stringCallback, &sctx); - pub fn eqlAny(SpecificType: type) fn (SpecificType, SpecificType) bool { - return struct { - fn eql(a: SpecificType, b: SpecificType) bool { - return a.asType().eql(b.asType()); + if (logical_result.value == 0) { + return WriteBytecodeError.InvalidMlirBytecodeVersion; } - }.eql; - } - pub fn formatAny(SpecificType: type) fn (SpecificType, SpecificType) type { - return struct { - pub fn format(self: SpecificType, writer: anytype) !void { - return try Type.format(self.asType(), writer); + if (sctx.err) |err| { + return err; } - }; + return; + } + + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationWriteBytecode(self.ptr(), stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } } - pub fn index(ctx: Context) Type { - return IndexType.init(ctx).asType(); + fn fmtBytecodeImpl(ctx: BytecodeWriterConfig.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + try ctx.self.writeBytecode(ctx.config, writer); } - pub fn int(ctx: Context, int_type: IntegerTypes) Type { - return switch (int_type) { - .unknown => @panic("Unknown integer type"), - inline else => |t| IntegerType(t).init(ctx).asType(), - }; - } - - pub fn float(ctx: Context, float_type: FloatTypes) Type { - return switch (float_type) { - inline else => |t| FloatType(t).init(ctx).asType(), - }; - } - - pub fn complex(ctx: Context, complex_type: ComplexTypes) Type { - return switch (complex_type) { - .unknown => @panic("Unknown complex type can't be created like this"), // What's the point ? - inline else => |t| ComplexType(t).init(ctx).asType(), - }; - } - - pub fn tuple(ctx: Context, types: []const Type) Type { - return (TupleType.init(ctx, types) catch unreachable).asType(); - } - - pub fn function(ctx: Context, args: []const Type, results: []const Type) Type { - return (FunctionType.init(ctx, args, results) catch unreachable).asType(); - } - - pub fn tensor(dimensions: []const i64, elem_type: Type) Type { - return RankedTensorType.init(dimensions, elem_type).asType(); + pub fn fmtBytecode(self: *const Operation, config: ?BytecodeWriterConfig) std.fmt.Alt(BytecodeWriterConfig.Ctx, fmtBytecodeImpl) { + return .{ .data = .{ + .self = self, + .config = config, + } }; } }; -pub const IndexType = struct { - _inner: c.MlirType, +pub const Region = opaque { + const M = Methods(Region, c.MlirRegion); - pub const asType = Type.fromAny(IndexType); - pub const eql = Type.eqlAny(IndexType); - pub const format = Type.formatAny(IndexType).format; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirRegionEqual); - pub fn init(ctx: Context) IndexType { - return .{ ._inner = c.mlirIndexTypeGet(ctx._inner) }; + pub fn init() *Region { + return @ptrCast(c.mlirRegionCreate().ptr); + } + + pub fn deinit(self: *Region) void { + c.mlirRegionDestroy(self.ptr()); + } + + pub fn appendOwnedBlock(self: *Region, block: *Block) void { + c.mlirRegionAppendOwnedBlock(self.ptr(), block.ptr()); + } + + pub fn insertOwnedBlock(self: *Region, index: usize, block: *Block) void { + c.mlirRegionInsertOwnedBlock(self.ptr(), @intCast(index), block.ptr()); + } + + pub fn insertOwnedBlockBefore(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockBefore(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn insertOwnedBlockAfter(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockAfter(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn firstBlock(self: *const Region) ?*Block { + return @ptrCast(c.mlirRegionGetFirstBlock(self.ptr()).ptr); + } + + pub fn takeBody(self: *Region, from: *Region) void { + c.mlirRegionTakeBody(self.ptr(), from.ptr()); + } + + pub fn nextInOperation(self: *const Region) *Region { + return @ptrCast(c.mlirRegionGetNextInOperation(self.ptr()).ptr); } }; -pub const IntegerTypes = enum { - i1, - i4, +pub const ShapedType = opaque { + const M = Methods(ShapedType, c.MlirType); + + pub const MAX_RANK = 16; + + pub const isAFn = c.mlirTypeIsAShaped; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn elementType(self: *const ShapedType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const ShapedType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const ShapedType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } +}; + +pub const RankedTensorType = opaque { + const M = Methods(RankedTensorType, c.MlirType); + + pub const isAFn = c.mlirTypeIsARankedTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(dimensions: []const i64, elemType: *const Type, encoding: ?*Attribute) *const RankedTensorType { + return @ptrCast(c.mlirRankedTensorTypeGet( + @intCast(dimensions.len), + @ptrCast(dimensions), + elemType.ptr(), + if (encoding) |e| e.ptr() else c.mlirAttributeGetNull(), + ).ptr); + } + + pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return get( + dims.constSlice(), + other.elementType(), + ); + } + + pub fn elementType(self: *const RankedTensorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const RankedTensorType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const RankedTensorType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn shaped(self: *const RankedTensorType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); +} + +pub fn rankedTensorTypeWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); +} + +pub const DenseElementsAttribute = opaque { + const M = Methods(DenseElementsAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADenseElements; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(shaped_type: *const ShapedType, values: anytype) *const DenseElementsAttribute { + const bytes = std.mem.sliceAsBytes(values); + return @ptrCast(c.mlirDenseElementsAttrRawBufferGet( + shaped_type.ptr(), + @intCast(bytes.len), + @ptrCast(bytes), + ).ptr orelse unreachable); + } +}; + +pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: anytype) *const Attribute { + return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); +} + +pub const DenseArrayTypes = enum { + bool, i8, i16, i32, i64, - si4, - si8, - si16, - si32, - si64, - u4, - u8, - u16, - u32, - u64, - - unknown, -}; - -pub fn IntegerType(comptime it: IntegerTypes) type { - const Config = switch (it) { - .i1 => .{ 1, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i4 => .{ 4, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i8 => .{ 8, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i16 => .{ 16, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i32 => .{ 32, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i64 => .{ 64, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .si4 => .{ 4, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si8 => .{ 8, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si16 => .{ 16, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si32 => .{ 32, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si64 => .{ 64, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .u4 => .{ 4, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u8 => .{ 8, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u16 => .{ 16, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u32 => .{ 32, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u64 => .{ 64, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .unknown => .{ 0, null, null }, - }; - - return struct { - _inner: c.MlirType, - - const Int = @This(); - pub const is_a_fn = switch (it) { - .unknown => c.mlirTypeIsAInteger, - else => typeIsAIntegerExact, - }; - - pub const asType = Type.fromAny(Int); - pub const eql = Type.eqlAny(Int); - pub const format = helpers.format(Int, c.mlirTypePrint); - - fn typeIsAIntegerExact(typ: c.MlirType) callconv(.c) bool { - const bit_width = Config[0]; - const is_sign = Config[2]; - return c.mlirTypeIsAInteger(typ) and (c.mlirIntegerTypeGetWidth(typ) == bit_width) and is_sign(typ); - } - - pub const BitWidth = Config[0]; - - pub const init = if (it != .unknown) struct { - pub fn init(ctx: Context) Int { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner, BitWidth) }; - } - }.init else {}; - }; -} - -pub const FloatTypes = enum { - f8e4m3b11fnuz, - f8e4m3fn, - f8e4m3fnuz, - f8e5m2, - f8e5m2fnuz, - bf16, - f16, f32, f64, - pub fn asType(self: FloatTypes, ctx: Context) Type { - return switch (self) { - inline else => |ft| FloatType(ft).init(ctx).asType(), + fn ZigType(comptime ElementType: @This()) type { + return switch (ElementType) { + .bool => i32, + .i8 => i8, + .i16 => i16, + .i32 => i32, + .i64 => i64, + .f32 => f32, + .f64 => f64, }; } }; -pub fn FloatType(comptime ft: FloatTypes) type { - const Config = switch (ft) { - .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, - .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, - .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, - .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, - .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, - .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, - .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, - .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, - .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, +pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { + const ElementTypeZig = ElementType.ZigType(); + + const _isAFn, const getFn, const getElementFn = switch (ElementType) { + .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, + .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, + .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, + .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, + .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, + .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, + .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, }; return struct { - _inner: c.MlirType, - const Self = @This(); - pub const is_a_fn = Config[0]; + const M = Methods(@This(), c.MlirAttribute); - pub const asType = Type.fromAny(Self); - pub const eql = Type.eqlAny(Self); - pub const format = helpers.format(Self, c.mlirTypePrint); + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); - pub fn init(ctx: Context) Self { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner) }; + pub fn init(ctx: *Context, values: []const ElementTypeZig) *const Self { + return @ptrCast(getFn(ctx.ptr(), @intCast(values.len), @ptrCast(values)).ptr); + } + + pub fn element(self: *const Self, pos: usize) ElementTypeZig { + return getElementFn(self.ptr(), @intCast(pos)); + } + + pub fn numElements(self: *const Self) usize { + return @intCast(c.mlirDenseArrayGetNumElements(self.ptr())); } }; } -pub const ComplexTypes = enum { - c64, - c128, - - unknown, -}; - -pub fn ComplexType(comptime ct: ComplexTypes) type { - return struct { - _inner: c.MlirType, - const Complex = @This(); - - fn mlirC64TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF32TypeGet(ctx)); - } - - fn mlirC128TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF64TypeGet(ctx)); - } - - fn mlirTypeIsAC64(typ: c.MlirType) callconv(.c) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF32(element_type); - } - - fn mlirTypeIsAC128(typ: c.MlirType) callconv(.c) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF64(element_type); - } - - const Config = switch (ct) { - .c64 => .{ mlirTypeIsAC64, mlirC64TypeGet }, - .c128 => .{ mlirTypeIsAC128, mlirC128TypeGet }, - .unknown => .{ c.mlirTypeIsAComplex, null }, - }; - - fn typeIsAUnknownComplex(typ: c.MlirType) callconv(.c) bool { - return c.mlirTypeIsAComplex(typ); - } - - pub const is_a_fn = Config[0]; - - pub const asType = Type.fromAny(Complex); - pub const eql = Type.eqlAny(Complex); - pub const format = Type.formatAny(Complex).format; - pub const ComplexTypeType: ComplexTypes = ct; - - pub const init = if (ct != .unknown) struct { - pub fn init(ctx: Context) Complex { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner) }; - } - }.init else {}; - }; +pub fn denseArrayAttribute(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { + return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); } -pub const TupleType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsATuple; +pub const ArrayAttribute = opaque { + const M = Methods(ArrayAttribute, c.MlirAttribute); - const Self = TupleType; + pub const isAFn = c.mlirAttributeIsAArray; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); - pub fn init(ctx: Context, elements: []const Type) !Self { - return Self.wrapOr(c.mlirTupleTypeGet( - ctx._inner, - @intCast(elements.len), - @ptrCast(elements.ptr), - )) orelse Error.InvalidMlir; + pub fn init(ctx: *Context, attrs: []const *const Attribute) *const ArrayAttribute { + return @ptrCast(c.mlirArrayAttrGet(ctx.ptr(), @intCast(attrs.len), @ptrCast(attrs)).ptr); } - pub fn getNumTypes(self: Self) usize { - return @intCast(c.mlirTupleTypeGetNumTypes(self._inner)); + pub fn element(self: *const ArrayAttribute, index: usize) Attribute { + return @ptrCast(c.mlirArrayAttrGetElement(self.ptr(), @intCast(index)).ptr); } - pub fn getElementType(self: Self, index: usize) Type { - return .{ ._inner = c.mlirTupleTypeGetType(self._inner, @intCast(index)) }; + pub fn numElements(self: *const ArrayAttribute) usize { + return @intCast(c.mlirArrayAttrGetNumElements(self.ptr())); } - - pub const asType = Type.fromAny(Self); }; -pub const FunctionType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsAFunction; - const Self = FunctionType; - pub const asType = Type.fromAny(Self); - pub const eql = Type.eqlAny(IndexType); +pub fn arrayAttribute(ctx: *Context, attrs: []const *const Attribute) *const Attribute { + return @ptrCast(ArrayAttribute.init(ctx, attrs)); +} - pub fn init(ctx: Context, args: []const Type, results: []const Type) !Self { - const func = Type.wrapOr(c.mlirFunctionTypeGet( - ctx._inner, +pub const FunctionType = opaque { + const M = Methods(FunctionType, c.MlirType); + pub const isAFn = c.mlirTypeIsAFunction; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(ctx: *Context, args: []const *const Type, results: []const *const Type) *const FunctionType { + return @ptrCast(c.mlirFunctionTypeGet( + ctx.ptr(), @intCast(args.len), @ptrCast(args.ptr), @intCast(results.len), @ptrCast(results.ptr), - )) orelse return Error.InvalidMlir; - return func.as(Self).?; + ).ptr); } }; -pub const RankedTensorType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsARankedTensor; - pub const eql = Type.eqlAny(RankedTensorType); - pub const format = helpers.format(Type, c.mlirTypePrint); +pub fn functionType(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { + return @ptrCast(FunctionType.init(ctx, args, results)); +} - pub fn init(dimensions: []const i64, elemType: Type) RankedTensorType { - return .{ ._inner = c.mlirRankedTensorTypeGet( - @intCast(dimensions.len), - @ptrCast(dimensions.ptr), - elemType._inner, - c.mlirAttributeGetNull(), - ) }; +pub const FlatSymbolRefAttribute = opaque { + const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAFlatSymbolRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, symbol_: []const u8) *const FlatSymbolRefAttribute { + return @ptrCast(c.mlirFlatSymbolRefAttrGet(ctx.ptr(), stringRef(symbol_)).ptr); } - pub fn getElementType(self: RankedTensorType) Type { - return .{ ._inner = c.mlirShapedTypeGetElementType(self._inner) }; - } - - pub fn getRank(self: RankedTensorType) usize { - return @intCast(c.mlirShapedTypeGetRank(self._inner)); - } - - pub fn getDimension(self: RankedTensorType, dim: usize) i64 { - return c.mlirShapedTypeGetDimSize(self._inner, @intCast(dim)); - } - - pub const asType = Type.fromAny(RankedTensorType); -}; - -pub const Dialect = struct { - _inner: c.MlirDialect, - - const Self = Dialect; - - pub fn getContext(self: Self) Context { - return .{ ._inner = c.mlirDialectGetContext(self._inner) }; - } - - pub fn getNamespace(self: Self) []const u8 { - return fromStringRef(c.mlirDialectGetNamespace(self._inner)); + pub fn symbol(self: *const FlatSymbolRefAttribute) []const u8 { + return string(c.mlirFlatSymbolRefAttrGetValue(self.ptr())); } }; -pub const DialectHandle = struct { - _inner: c.MlirDialectHandle, +pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { + return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); +} - pub fn getNamespace(self: DialectHandle) []const u8 { - return fromStringRef(c.mlirDialectHandleGetNamespace(self._inner)); - } +pub const UnitAttribute = opaque { + const M = Methods(UnitAttribute, c.MlirAttribute); - pub fn insertDialect(self: DialectHandle, registry: Registry) void { - c.mlirDialectHandleInsertDialect(self._inner, registry._inner); - } + pub const isAFn = c.mlirAttributeIsAUnit; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); - pub fn registerDialect(self: DialectHandle, ctx: Context) void { - c.mlirDialectHandleRegisterDialect(self._inner, ctx._inner); - } - - pub fn loadDialect(self: DialectHandle, ctx: Context) Dialect { - return .{ ._inner = c.mlirDialectHandleLoadDialect(self._inner, ctx._inner) }; - } - - pub fn fromString(comptime namespace: []const u8) DialectHandle { - return .{ ._inner = @field(c, "mlirGetDialectHandle__" ++ namespace ++ "__")() }; + pub fn get(ctx: *Context) *const UnitAttribute { + return @ptrCast(c.mlirUnitAttrGet(ctx.ptr()).ptr); } }; -pub const Location = struct { - _inner: c.MlirLocation, +pub fn unitAttribute(ctx: *Context) *const Attribute { + return @ptrCast(UnitAttribute.get(ctx)); +} - pub const eql = helpers.eql(Type, c.mlirLocationEqual); - pub const format = helpers.format(Location, c.mlirLocationPrint); +pub const MemRefType = opaque { + const M = Methods(MemRefType, c.MlirType); - pub fn fromSrc(ctx: Context, src: std.builtin.SourceLocation) Location { - return .{ ._inner = c.mlirLocationFileLineColGet( - ctx._inner, - stringRef(src.file), - @intCast(src.line), - @intCast(src.column), - ) }; + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { + return @ptrCast(c.mlirMemRefTypeGet( + element_type.ptr(), + @intCast(shape.len), + @ptrCast(shape), + if (layout) |l| l.ptr() else .{}, + if (memory_space) |m| m.ptr() else .{}, + ).ptr); } - pub fn fileLineCol(ctx: Context, file: []const u8, line: usize, column: usize) Location { - return .{ ._inner = c.mlirLocationFileLineColGet( - ctx._inner, - stringRef(file), - @intCast(line), - @intCast(column), - ) }; - } - - pub fn callSite(callee: Location, caller: Location) Location { - return .{ ._inner = c.mlirLocationCallSiteGet(callee._inner, caller._inner) }; - } - - pub fn fused(ctx: Context, locations: []const Location, metadata: Attribute) Location { - return .{ ._inner = c.mlirLocationFusedGet( - ctx._inner, - @intCast(locations.len), - @ptrCast(locations.ptr), - metadata._inner, - ) }; - } - - pub fn named(loc: Location, ctx: Context, loc_name: [:0]const u8) Location { - return .{ ._inner = c.mlirLocationNameGet(ctx._inner, stringRef(loc_name), loc._inner) }; - } - - pub fn namedFmt(loc: Location, ctx: Context, comptime fmt: [:0]const u8, args: anytype) Location { - var buf: [256]u8 = undefined; - var stream = std.io.fixedBufferStream(&buf); - std.fmt.format(stream.writer(), fmt, args) catch { - buf[256 - 3 ..].* = "...".*; - }; - return loc.named(ctx, @ptrCast(stream.getWritten())); - } - - pub fn unknown(ctx: Context) Location { - return .{ ._inner = c.mlirLocationUnknownGet(ctx._inner) }; - } -}; - -pub const Block = struct { - _inner: c.MlirBlock, - - pub const wrapOr = helpers.wrapOr(Block, c.mlirBlockIsNull); - pub const deinit = helpers.deinit(Block, c.mlirBlockDestroy); - - pub const eql = helpers.eql(Block, c.mlirBlockEqual); - - pub fn init(args: []const Type, locs: []const Location) !Block { - const block = Block.wrapOr( - c.mlirBlockCreate(@intCast(args.len), @ptrCast(args.ptr), @ptrCast(locs.ptr)), + pub fn fromShaped(other: *const ShapedType) *const MemRefType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return init( + other.elementType(), + dims.constSlice(), + null, + null, ); - return block orelse error.InvalidMlir; } - pub fn argument(self: Block, index: usize) Value { - return .{ ._inner = c.mlirBlockGetArgument(self._inner, @intCast(index)) }; - } - - pub fn numArguments(self: Block) usize { - return @intCast(c.mlirBlockGetNumArguments(self._inner)); - } - - pub fn addArgument(self: *Block, typ: Type, loc: Location) Value { - return .{ ._inner = c.mlirBlockAddArgument(self._inner, typ._inner, loc._inner) }; - } - - pub fn insertArgument(self: *Block, index: usize, typ: Type, loc: Location) Value { - return .{ ._inner = c.mlirBlockInsertArgument(self._inner, @intCast(index), typ._inner, loc._inner) }; - } - - pub fn equals(self: Block, other: Block) bool { - return c.mlirBlockEqual(self._inner, other._inner); - } - - pub fn appendOperation(self: Block, op: Operation) void { - c.mlirBlockAppendOwnedOperation(self._inner, op._inner); - } - - pub fn appendOperations(self: *Block, ops: []const Operation) void { - for (ops) |op| { - c.mlirBlockAppendOwnedOperation(self._inner, op._inner); - } - } - - pub const RecursiveOpts = enum { open, hermetic }; - - pub fn appendValueRecursive(self: Block, value: Value, opt: RecursiveOpts) void { - switch (value.kind()) { - .op_result => |parent_op| self.appendOperationRecursive(parent_op, opt), - .block_argument => |arg| { - // Hermetic blocks are not allowed to use arguments from other blocks. - stdx.debug.assert(opt == .open or self.eql(arg.block()), "Can't add {f} from {*} block to {*} block", .{ arg, arg.block()._inner.ptr, self._inner.ptr }); - }, - .null => @panic("InvalidMlir"), - } - } - - pub fn appendOperationRecursive(self: Block, op: Operation, opt: RecursiveOpts) void { - if (op.block()) |prev_block| { - // Hermetic blocks are not allowed to reference values from other blocks. - stdx.debug.assert(opt == .open or self.equals(prev_block), "Can't add {} from {*} block to {*} block", .{ op, prev_block._inner.ptr, self._inner.ptr }); - return; - } - for (0..op.numOperands()) |i| { - self.appendValueRecursive(op.operand(i), opt); - } - self.appendOperation(op); + pub fn shaped(self: *const MemRefType) *const ShapedType { + return @ptrCast(self); } }; -pub const helpers = struct { - pub fn eql(T: type, equal_fn: fn (@FieldType(T, "_inner"), @FieldType(T, "_inner")) callconv(.c) bool) fn (T, T) bool { - return struct { - fn eql(a: T, b: T) bool { - return equal_fn(a._inner, b._inner); - } - }.eql; - } - - pub fn deinit(T: type, deinit_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (*T) void { - return struct { - fn deinit(a: *T) void { - deinit_fn(a._inner); - a.* = undefined; - } - }.deinit; - } - - pub fn dump(T: type, dump_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (T) void { - return struct { - fn dump(a: T) void { - return dump_fn(a._inner); - } - }.dump; - } - - pub fn isNull(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (T) bool { - return struct { - fn isNull(a: T) bool { - return is_null_fn(a._inner); - } - }.isNull; - } - - pub fn format(Any: type, print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void) type { - return struct { - pub fn format(self: Any, writer: *std.Io.Writer) !void { - const WriterWithErr = struct { - writer: *std.Io.Writer, - err: ?std.Io.Writer.Error = null, - fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { - var ctx: *@This() = @alignCast(@ptrCast(opaque_ctx)); - if (ctx.err) |_| return; - _ = ctx.writer.write(mlir_str.data[0..mlir_str.length]) catch |err| { - ctx.err = err; - return; - }; - } - }; - - var context: WriterWithErr = .{ .writer = writer }; - print_fn(self._inner, &WriterWithErr.printCallback, &context); - if (context.err) |err| return err; - } - }; - } - - pub fn wrapOr(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (@FieldType(T, "_inner")) ?T { - return struct { - fn wrapOr(inner: @FieldType(T, "_inner")) ?T { - if (is_null_fn(inner)) return null; - return .{ ._inner = inner }; - } - }.wrapOr; - } - - pub fn init(T: type, inner: @FieldType(T, "_inner"), is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) ?T { - if (is_null_fn(inner)) return null; - return .{ ._inner = inner }; - } -}; +pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { + return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); +} diff --git a/mlir/writer.zig b/mlir/writer.zig new file mode 100644 index 00000000..2f76436a --- /dev/null +++ b/mlir/writer.zig @@ -0,0 +1,194 @@ +const std = @import("std"); +const asynk = @import("async"); + +const stdx = @import("stdx"); +const zml = @import("zml"); +const Shape = zml.Shape; + +pub const ShardWriter = struct { + tensor_shape: Shape, + shard_shape: Shape, + interface: std.Io.Writer, + out: *std.Io.Writer, + + fn init(out: *std.Io.Writer, buffer: []u8, tensor_shape: Shape, shard_shape: Shape) ShardWriter { + return .{ + .tensor_shape = tensor_shape, + .shard_shape = shard_shape, + .interface = .{ + .buffer = buffer, + .vtable = &.{ .drain = drain }, + }, + .out = out, + }; + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *ShardWriter = @alignCast(@fieldParentPtr("interface", w)); + return self.out.writeSplat(data, splat); + } +}; + +fn computeCoordinates(tensor_shape: Shape, shard_shape: Shape, offset: usize) [Shape.MAX_RANK]usize { + var coordinates: [Shape.MAX_RANK]usize = @splat(0); + var offset_in_elem = offset / tensor_shape.dtype().sizeOf(); + const strides = tensor_shape.computeStrides(); + for (0..tensor_shape.rank()) |dim| { + const offset_in_dim = offset_in_elem / @as(usize, @intCast(strides.get(dim))); + const coordinate_in_dim = offset_in_dim / @as(usize, @intCast(shard_shape.dim(dim))); + offset_in_elem -= offset_in_dim * @as(usize, @intCast(strides.get(dim))); + coordinates[dim] = coordinate_in_dim; + } + + return coordinates; +} + +test computeCoordinates { + { + const tensor_shape: Shape = .init(.{ 1024, 1024 }, .u8); + const shard_shape: Shape = .init(.{ 512, 512 }, .u8); + const rank = tensor_shape.rank(); + + try std.testing.expectEqualSlices(usize, &[2]usize{ 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 0)[0..rank]); + try std.testing.expectEqualSlices(usize, &[2]usize{ 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 511)[0..rank]); + try std.testing.expectEqualSlices(usize, &[2]usize{ 0, 1 }, computeCoordinates(tensor_shape, shard_shape, 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[2]usize{ 1, 0 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[2]usize{ 1, 1 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 512 + 512)[0..rank]); + } + + { + const tensor_shape: Shape = .init(.{ 1024, 1024, 1024 }, .u8); + const shard_shape: Shape = .init(.{ 512, 512, 512 }, .u8); + const rank = tensor_shape.rank(); + + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 0)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 511)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 0, 1 }, computeCoordinates(tensor_shape, shard_shape, 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 1, 0 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 0, 1, 1 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 512 + 512)[0..rank]); + try std.testing.expectEqualSlices(usize, &[3]usize{ 1, 0, 0 }, computeCoordinates(tensor_shape, shard_shape, 1024 * 1024 * 512)[0..rank]); + } +} + +pub const TensorWriter = struct { + shard_writers: []ShardWriter, + temp_writers: []std.Io.Writer.Allocating, + interface: std.Io.Writer, + + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator, tensor_shape: Shape, buffer: []u8, platform: zml.Platform) !TensorWriter { + const device_count = platform.getDevices().len; + var dims: stdx.BoundedArray(i64, Shape.MAX_RANK) = try .init(tensor_shape.rank()); + for (0..tensor_shape.rank()) |dim| { + if (tensor_shape._sharding_info[dim]) { + dims.slice()[dim] = @divExact(tensor_shape.dim(dim), @as(i64, @intCast(device_count))); + } else { + dims.slice()[dim] = tensor_shape.dim(dim); + } + } + const shard_shape = Shape.init(dims.slice(), tensor_shape.dtype()); + + const shard_writers = try allocator.alloc(ShardWriter, device_count); + errdefer allocator.free(shard_writers); + + const temp_writers = try allocator.alloc(std.Io.Writer.Allocating, device_count); + errdefer allocator.free(temp_writers); + + for (0..device_count) |i| { + temp_writers[i] = std.Io.Writer.Allocating.init(allocator); + shard_writers[i] = ShardWriter.init(&temp_writers[i].writer, &.{}, tensor_shape, shard_shape); + } + + return .{ + .shard_writers = shard_writers, + .temp_writers = temp_writers, + .interface = .{ + .buffer = buffer, + .vtable = &.{ + .drain = drain, + }, + }, + .allocator = allocator, + }; + } + + pub fn deinit(self: TensorWriter) void { + self.allocator.free(self.shard_writers); + for (self.temp_writers) |*temp_writer| temp_writer.deinit(); + self.allocator.free(self.temp_writers); + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *TensorWriter = @alignCast(@fieldParentPtr("interface", w)); + var maybe_written: ?usize = null; + for (self.shard_writers) |*shard_writer| { + const shard_written = try shard_writer.interface.writeSplat(data, splat); + if (maybe_written) |written| { + std.debug.assert(written == shard_written); + } else { + maybe_written = shard_written; + } + } + + return maybe_written.?; + } +}; + +pub const RandomReader = struct { + random: std.Random, + interface: std.Io.Reader, + + pub fn init(random: std.Random, buffer: []u8) RandomReader { + return .{ + .random = random, + .interface = .{ + .buffer = buffer, + .vtable = &.{ + .stream = stream, + }, + .seek = 0, + .end = 0, + }, + }; + } + + pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + //std.log.info("Calling stream with limit: {any}", .{limit}); + const self: *RandomReader = @alignCast(@fieldParentPtr("interface", r)); + const bytes = w.unusedCapacitySlice(); + const to_write = @min(bytes.len, limit.toInt() orelse bytes.len); + self.random.bytes(bytes[0..to_write]); + //@memset(bytes[0..to_write], 0xca); + + return to_write; + } +}; + +test "random reader" { + try asynk.AsyncThread.main(std.heap.c_allocator, testRandomReader); +} + +fn testRandomReader() !void { + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + + var generator = std.Random.DefaultPrng.init(0); + const random = generator.random(); + + var random_reader: RandomReader = .init(random, &.{}); + _ = &random_reader; // autofix + + const tensor_shape: Shape = .init(.{ 16, 16 }, .u8); + var tensor_writer: TensorWriter = try .init(std.testing.allocator, tensor_shape, &.{}, platform); + defer tensor_writer.deinit(); + + _ = try random_reader.interface.stream(&tensor_writer.interface, .limited(16 * 16)); + try tensor_writer.interface.flush(); + + for (tensor_writer.temp_writers, 0..) |*temp_writer, i| { + std.debug.print("shard: {d} - written: {}\n", .{ i, temp_writer.written().len }); + } +} diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 87f3d174..f4564f22 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -58,7 +58,7 @@ pub const ApiError = error{ fn InnerMixin(comptime innerT: type) type { return struct { fn inner(self: anytype) *innerT { - return @ptrCast(@constCast(@alignCast(self))); + return @ptrCast(@alignCast(@constCast(self))); } }; } @@ -125,10 +125,10 @@ pub const Api = struct { } pub fn lookupExtension(self: *const Api, comptime ExtensionT: type, ext_id: c_int) ?*const ExtensionT { - var cur: [*c]const c.PJRT_Extension_Base = @alignCast(@ptrCast(self.inner.extension_start)); + var cur: [*c]const c.PJRT_Extension_Base = @ptrCast(@alignCast(self.inner.extension_start)); while (cur != null) : (cur = cur.*.next) { if (cur.*.type == ext_id) { - return @alignCast(@ptrCast(cur)); + return @ptrCast(@alignCast(cur)); } } @@ -264,22 +264,22 @@ pub const ShapeSpec = extern struct { inner: c.PJRT_ShapeSpec, - pub fn init(dims_: []const usize, bt: BufferType) ShapeSpec { + pub fn init(dims_: []const i64, bt: BufferType) ShapeSpec { return .{ .inner = pjrtStruct(c.PJRT_ShapeSpec{ .dims = @ptrCast(@constCast(dims_.ptr)), - .num_dims = dims.len, - .buffer_type = @intFromEnum(bt), + .num_dims = dims_.len, + .element_type = @intFromEnum(bt), }), }; } - pub fn dims(self: ShapeSpec) []usize { + pub fn dims(self: ShapeSpec) []const i64 { return self.inner.dims[0..self.inner.num_dims]; } pub fn bufferType(self: ShapeSpec) BufferType { - return @enumFromInt(self.inner.buffer_type); + return @enumFromInt(self.inner.element_type); } }; @@ -432,7 +432,7 @@ pub const Client = opaque { .client = self.inner(), }) catch unreachable; if (ret.addressable_memories) |memories| { - return @constCast(@ptrCast(memories[0..ret.num_addressable_memories])); + return @ptrCast(@constCast(memories[0..ret.num_addressable_memories])); } return &.{}; } @@ -461,7 +461,7 @@ pub const Client = opaque { pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { const ret = try api.call(.PJRT_Client_CreateBuffersForAsyncHostToDevice, .{ .client = self.inner(), - .shape_specs = @ptrCast(args.shape_specs.ptr), + .shape_specs = @ptrCast(@constCast(args.shape_specs.ptr)), .num_shape_specs = args.shape_specs.len, .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts.ptr)) else null, .num_device_layouts = if (args.device_layouts) |layouts| @intCast(layouts.len) else 0, @@ -610,6 +610,18 @@ pub const DeviceDescription = opaque { return @intCast(ret.process_index); } + pub fn getAttributes(self: *const DeviceDescription, api: *const Api) []const NamedValue { + const ret = api.call(.PJRT_DeviceDescription_Attributes, .{ + .device_description = self.inner(), + }) catch unreachable; + + if (ret.attributes) |attrs| { + return @ptrCast(attrs[0..ret.num_attributes]); + } + + return &.{}; + } + pub fn getKind(self: *const DeviceDescription, api: *const Api) []const u8 { const ret = api.call(.PJRT_DeviceDescription_Kind, .{ .device_description = self.inner(), diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 45fddc0b..52c509b6 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -4,6 +4,9 @@ platform( "@platforms//cpu:x86_64", "@platforms//os:linux", ], + flags = [ + "--force_pic=true", + ], visibility = ["//visibility:public"], ) @@ -13,6 +16,9 @@ platform( "@platforms//cpu:aarch64", "@platforms//os:linux", ], + flags = [ + "--force_pic=true", + ], visibility = ["//visibility:public"], ) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 184925a5..7a41737c 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -83,6 +83,7 @@ zig_shared_library( "-fno-stack-check", "-fPIC", ], + features = ["pic"], main = "libneuronxla.zig", shared_lib_name = "libneuronxla.so", visibility = ["@libpjrt_neuron//:__subpackages__"], diff --git a/stdx/io.zig b/stdx/io.zig index 390647b8..e69de29b 100644 --- a/stdx/io.zig +++ b/stdx/io.zig @@ -1,4 +0,0 @@ -const std = @import("std"); - -pub const BufferedAnyWriter = std.io.BufferedWriter(4096, std.io.AnyWriter); -pub const BufferedAnyReader = std.io.BufferedReader(4096, std.io.AnyReader); diff --git a/stdx/meta.zig b/stdx/meta.zig index 0441d689..3dc8db32 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -167,6 +167,10 @@ pub fn FnResult(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } +pub fn FnError(comptime func: anytype) type { + return FnSignature(func, null).ReturnErrorSet.?; +} + pub fn Head(Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { diff --git a/third_party/zls/zls.bzl b/third_party/zls/zls.bzl index 4cc5caa1..9e28da65 100644 --- a/third_party/zls/zls.bzl +++ b/third_party/zls/zls.bzl @@ -1,24 +1,24 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_VERSION = "0.14.0" +_VERSION = "0.15.0" _ARCH = { "x86_64-linux": struct( - sha256 = "661f8d402ba3dc9b04b6e9bc3026495be7b838d2f18d148db2bd98bd699c1360", + sha256 = "508bfe3fd637d2a02f07f3fc7da8900351f407116b03685c5dae26b4f01a30de", exec_compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", ], ), "x86_64-macos": struct( - sha256 = "baee69e4645deeccb42970b4a01f573592209dc1cf72e32893c59ca06af511dc", + sha256 = "46c31838bfef5adcc7fee82428c3ec2b9abbfae38242639afea5f242ee133d93", exec_compatible_with = [ "@platforms//os:macos", "@platforms//cpu:x86_64", ], ), "aarch64-macos": struct( - sha256 = "dfb627e1f9603583678f552d8035a12dce878215c0a507b32d6f1b9d074d6c4d", + sha256 = "76c7a23190f67e67970024065f689c2c49b3c7b0fc16876fb24ef199fb05fc2a", exec_compatible_with = [ "@platforms//os:macos", "@platforms//cpu:aarch64", diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e0722f70..9f61f308 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -24,6 +24,8 @@ zig_library( "aio/json.zig", "aio/safetensors.zig", "aio/tinyllama.zig", + "io.zig", + "io/file.zig", "buffer.zig", "context.zig", "dtype.zig", @@ -37,6 +39,7 @@ zig_library( "nn.zig", "nn/cuda.zig", "ops.zig", + "partitioning.zig", "pjrtx.zig", "platform.zig", "posix.zig", diff --git a/zml/buffer.zig b/zml/buffer.zig index 3553bb94..36d20102 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -8,6 +8,8 @@ const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Sharding = @import("partitioning.zig").Sharding; +const Mesh = @import("partitioning.zig").Mesh; test { std.testing.refAllDecls(@This()); @@ -43,6 +45,7 @@ pub const Buffer = struct { _shape: Shape, _api: *const pjrt.Api, _shards: Shards, + _mesh: ?Mesh = null, // ? todo pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); @@ -314,14 +317,14 @@ pub const Buffer = struct { /// Copies the content of the Buffer back to host, in the given buffer, /// and return a new `HostBuffer` object with the same shape. /// The returned `HostBuffer` doesn't own the memory. - pub fn toHost(self: Buffer, output: []u8) !HostBuffer { - stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); - } + //pub fn toHost(self: Buffer, output: []u8) !HostBuffer { + // stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + // const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); + // if (maybe_event) |event| { + // try event.await_(self._api); + // } + // return HostBuffer.fromBytes(self.shape(), output); + //} /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. @@ -335,6 +338,91 @@ pub const Buffer = struct { return output; } + pub fn toHost(self: Buffer, allocator: std.mem.Allocator) ![]u8 { + const sharding: Sharding = .init(self._mesh.?, self.shape()); + stdx.debug.assert(self._shards.len == sharding.mesh.numPartitions(), "Expected {} PJRT buffers, got {}", .{ sharding.mesh.numPartitions(), self._shards.len }); + + const global_buffer = try allocator.alloc(u8, sharding.global_shape.byteSize()); + errdefer allocator.free(global_buffer); + + if (sharding.getType() == .replicated) { + if (self._shards.len > 0) { + const event = try self._shards.get(0).toHostBuffer(self._api, global_buffer); + if (event) |e| try e.await_(self._api); + } + return global_buffer; + } + + const shard_shape = sharding.shardShape(); + const temp_shard_buffer = try allocator.alloc(u8, shard_shape.byteSize()); + defer allocator.free(temp_shard_buffer); + + const element_size = sharding.global_shape.dtype().sizeOf(); + const global_element_strides = sharding.global_shape.computeElementStrides(); + const rk = sharding.global_shape.rank(); + + var devices: [MAX_NUM_SHARDS]*const pjrt.Device = undefined; + for (self._shards.constSlice(), 0..) |shard, i| { + devices[i] = try shard.getDevice(self._api); + } + + var sharding_iter = sharding.iterator(devices[0..self._shards.len]); + while (sharding_iter.next()) |device_shard| { + const shard_index: usize = @intCast(device_shard.index); + const src_buffer = self._shards.get(shard_index); + + const event = try src_buffer.toHostBuffer(self._api, temp_shard_buffer); + if (event) |e| try e.await_(self._api); + if (rk == 0) { + if (global_buffer.len > 0) { + @memcpy(global_buffer[0..element_size], temp_shard_buffer[0..element_size]); + } + continue; + } + + const last_dim_size = shard_shape.dim(rk - 1); + const contiguous_row_bytes = @as(usize, @intCast(last_dim_size)) * element_size; + const outer_dims_shape = shard_shape.remove(rk - 1); + + var iter = outer_dims_shape.iterator(); + + while (iter.next()) |item| { + const src_flat_index_start_of_row = item.flat_index * @as(usize, @intCast(last_dim_size)); + const src_byte_offset = src_flat_index_start_of_row * element_size; + + var dest_flat_index_start_of_row: i64 = 0; + + for (0..rk) |dim_idx| { + const shard_coord_in_dim = if (dim_idx < rk - 1) item.coords[dim_idx] else 0; + + const global_coord_for_dim = blk: { + const part_spec = sharding.global_shape.partition(dim_idx); + if (part_spec != .axis) break :blk shard_coord_in_dim; // Replicated axis + const mesh_axis_tag = part_spec.toTag(); + if (device_shard.indices.hasTag(mesh_axis_tag) != null) { + const coord = device_shard.indices.dim(mesh_axis_tag); + break :blk (coord * shard_shape.dim(dim_idx)) + shard_coord_in_dim; + } else { + break :blk shard_coord_in_dim; + } + }; + + dest_flat_index_start_of_row += global_coord_for_dim * global_element_strides.get(dim_idx); + } + + const dest_byte_offset = @as(usize, @intCast(dest_flat_index_start_of_row)) * element_size; + + if (contiguous_row_bytes > 0) { + @memcpy( + global_buffer[dest_byte_offset..][0..contiguous_row_bytes], + temp_shard_buffer[src_byte_offset..][0..contiguous_row_bytes], + ); + } + } + } + + return global_buffer; + } /// Frees the accelerator memory. /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. @@ -387,7 +475,7 @@ pub const Buffer = struct { fn hasShardedAxis(self: Buffer) bool { if (self._shards.len == 1) return false; - return @reduce(.Or, self._shape._sharding_info); + return self._shape.hasAtLeastOnePartitionedAxis(); } pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { diff --git a/zml/dtype.zig b/zml/dtype.zig index b0deb8f4..033c6032 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -1,9 +1,15 @@ const std = @import("std"); const floats = @import("floats.zig"); +pub const Float8E4M3B11FNUZ = floats.Float8E4M3B11FNUZ; +pub const Float8E4M3FN = floats.Float8E4M3FN; +pub const Float8E4M3FNUZ = floats.Float8E4M3FNUZ; +pub const Float8E5M2 = floats.Float8E5M2; +pub const Float8E5M2FNUZ = floats.Float8E5M2FNUZ; +pub const BFloat16 = floats.BFloat16; -const C64 = std.math.Complex(f32); -const C128 = std.math.Complex(f64); +pub const C64 = std.math.Complex(f32); +pub const C128 = std.math.Complex(f64); test { std.testing.refAllDecls(@This()); @@ -70,12 +76,12 @@ pub const DataType = enum(u8) { pub fn fromZigType(comptime T: type) DataType { return switch (T) { - floats.Float8E4M3B11FNUZ => .f8e4m3b11fnuz, - floats.Float8E4M3FN => .f8e4m3fn, - floats.Float8E4M3FNUZ => .f8e4m3fnuz, - floats.Float8E5M2 => .f8e5m2, - floats.Float8E5M2FNUZ => .f8e5m2fnuz, - floats.BFloat16 => .bf16, + Float8E4M3B11FNUZ => .f8e4m3b11fnuz, + Float8E4M3FN => .f8e4m3fn, + Float8E4M3FNUZ => .f8e4m3fnuz, + Float8E5M2 => .f8e5m2, + Float8E5M2FNUZ => .f8e5m2fnuz, + BFloat16 => .bf16, f16 => .f16, f32 => .f32, f64 => .f64, @@ -112,7 +118,7 @@ pub const DataType = enum(u8) { } pub fn toZigType(comptime dtype: DataType) type { - return @FieldType(Data, @tagName(dtype)); + return @FieldType(Value, @tagName(dtype)); } pub fn isSignedInt(dtype: DataType) bool { @@ -170,165 +176,165 @@ pub const DataType = enum(u8) { try std.testing.expectEqual(DataType.c64.resolvePeerType(.f32), null); } - pub fn zero(dtype: DataType) Data { - return Data.init(dtype, 0); + pub fn zero(dtype: DataType) Value { + return .init(dtype, 0); } - pub fn one(dtype: DataType) Data { - return Data.init(dtype, 1); + pub fn one(dtype: DataType) Value { + return .init(dtype, 1); } - pub fn minValue(dtype: DataType) Data { + pub fn minValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = false }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).zero), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).minus_inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), -std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.minInt(@FieldType(Data, @tagName(tag)))), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).zero), + inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).minus_inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), -std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.minInt(@FieldType(Value, @tagName(tag)))), inline else => |tag| @panic("Unsupported type: " ++ @tagName(tag)), }; } - pub fn maxValue(dtype: DataType) Data { + pub fn maxValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = true }, inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.maxInt(@FieldType(Value, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), }; } - pub fn constant(dtype: DataType, value: anytype) Data { - return Data.init(dtype, value); + pub fn constant(dtype: DataType, value: anytype) Value { + return .init(dtype, value); } -}; -pub const Data = union(DataType) { - bool: bool, - f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, - f8e4m3fn: floats.Float8E4M3FN, - f8e4m3fnuz: floats.Float8E4M3FNUZ, - f8e5m2: floats.Float8E5M2, - f8e5m2fnuz: floats.Float8E5M2FNUZ, - bf16: floats.BFloat16, - f16: f16, - f32: f32, - f64: f64, - i4: i4, - i8: i8, - i16: i16, - i32: i32, - i64: i64, - u4: u4, - u8: u8, - u16: u16, - u32: u32, - u64: u64, - c64: C64, - c128: C128, + pub const Value = union(DataType) { + bool: bool, + f8e4m3b11fnuz: Float8E4M3B11FNUZ, + f8e4m3fn: Float8E4M3FN, + f8e4m3fnuz: Float8E4M3FNUZ, + f8e5m2: Float8E5M2, + f8e5m2fnuz: Float8E5M2FNUZ, + bf16: BFloat16, + f16: f16, + f32: f32, + f64: f64, + i4: i4, + i8: i8, + i16: i16, + i32: i32, + i64: i64, + u4: u4, + u8: u8, + u16: u16, + u32: u32, + u64: u64, + c64: C64, + c128: C128, - /// Creates `Data` from a `value`. - /// - /// If the `dtype` and `@TypeOf(value)` are incompatible - /// or a cast from `value` to `FieldType(dtype)` would - /// be lossy, a panic occurs. - pub fn init(dtype_: DataType, value: anytype) Data { - const T = @TypeOf(value); - const Ti = @typeInfo(T); + /// Creates `Data` from a `value`. + /// + /// If the `dtype` and `@TypeOf(value)` are incompatible + /// or a cast from `value` to `FieldType(dtype)` would + /// be lossy, a panic occurs. + pub fn init(dtype_: DataType, value: anytype) Value { + const T = @TypeOf(value); + const Ti = @typeInfo(T); - return switch (dtype_) { - .bool => switch (Ti) { - .bool => .{ .bool = value }, - .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, - else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), - }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), - else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), - }, - inline .f16, .f32, .f64 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @floatFromInt(value)), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { - .comptime_int => blk: { - const OutT = @FieldType(Data, @tagName(tag)); - if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { - break :blk @unionInit(Data, @tagName(tag), @intCast(value)); - } else { - @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); - } + return switch (dtype_) { + .bool => switch (Ti) { + .bool => .{ .bool = value }, + .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, + else => @panic("Could not create Value of type bool from value of type " ++ @typeName(T)), }, - .int => @unionInit(Data, @tagName(tag), @intCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - .c64 => switch (T) { - C64 => .{ .c64 = value }, - C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - else => @panic("Could not create Data of type c64 from value of type " ++ @typeName(T)), - }, - .c128 => switch (T) { - C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - C128 => .{ .c128 = value }, - else => @panic("Could not create Data of type c128 from value of type " ++ @typeName(T)), - }, - }; - } - - test init { - try std.testing.expectEqual(20.0, Data.init(.f16, 20).f16); - try std.testing.expectEqual(20.5, Data.init(.f16, 20.5).f16); - try std.testing.expectEqual(20, Data.init(.f16, @as(u8, 20)).f16); - try std.testing.expectEqual(-20, Data.init(.f16, @as(i8, -20)).f16); - try std.testing.expectEqual(2000.5, Data.init(.f16, @as(f32, 2000.5)).f16); - - try std.testing.expectEqual(true, Data.init(.bool, true).bool); - - try std.testing.expectEqual(10, Data.init(.u8, 10).u8); - try std.testing.expectEqual(10, Data.init(.u8, @as(u16, 10)).u8); - - try std.testing.expectEqual(10, Data.init(.i8, 10).i8); - try std.testing.expectEqual(10, Data.init(.i8, @as(u16, 10)).i8); - try std.testing.expectEqual(-10, Data.init(.i8, -10).i8); - try std.testing.expectEqual(-10, Data.init(.i8, @as(i16, -10)).i8); - - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C64.init(1, 2)).c64); - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C128.init(1, 2)).c64); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C128.init(1, 2)).c128); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C64.init(1, 2)).c128); - } - - pub fn dtype(self: Data) DataType { - return std.meta.activeTag(self); - } - - pub fn constSlice(data: *const Data) []const u8 { - return switch (data.*) { - inline else => |*value| std.mem.asBytes(value), - }; - } - - pub fn as(self: Data, comptime T: type) T { - // TODO allow more lossless conversions - switch (@typeInfo(T)) { - .bool => return self.bool, - .float => switch (self) { - inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), - else => {}, - }, - .int => switch (self) { - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), - else => {}, - }, - else => {}, + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatFromInt(value))), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatCast(value))), + else => @panic("Could not create Value of type bf16 from value of type " ++ @typeName(T)), + }, + inline .f16, .f32, .f64 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @floatFromInt(value)), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @floatCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + .comptime_int => blk: { + const OutT = @FieldType(Value, @tagName(tag)); + if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { + break :blk @unionInit(Value, @tagName(tag), @intCast(value)); + } else { + @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + } + }, + .int => @unionInit(Value, @tagName(tag), @intCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + .c64 => switch (T) { + C64 => .{ .c64 = value }, + C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + else => @panic("Could not create Value of type c64 from value of type " ++ @typeName(T)), + }, + .c128 => switch (T) { + C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + C128 => .{ .c128 = value }, + else => @panic("Could not create Value of type c128 from value of type " ++ @typeName(T)), + }, + }; } - std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); - } + + test init { + try std.testing.expectEqual(20.0, Value.init(.f16, 20).f16); + try std.testing.expectEqual(20.5, Value.init(.f16, 20.5).f16); + try std.testing.expectEqual(20, Value.init(.f16, @as(u8, 20)).f16); + try std.testing.expectEqual(-20, Value.init(.f16, @as(i8, -20)).f16); + try std.testing.expectEqual(2000.5, Value.init(.f16, @as(f32, 2000.5)).f16); + + try std.testing.expectEqual(true, Value.init(.bool, true).bool); + + try std.testing.expectEqual(10, Value.init(.u8, 10).u8); + try std.testing.expectEqual(10, Value.init(.u8, @as(u16, 10)).u8); + + try std.testing.expectEqual(10, Value.init(.i8, 10).i8); + try std.testing.expectEqual(10, Value.init(.i8, @as(u16, 10)).i8); + try std.testing.expectEqual(-10, Value.init(.i8, -10).i8); + try std.testing.expectEqual(-10, Value.init(.i8, @as(i16, -10)).i8); + + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C64.init(1, 2)).c64); + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C128.init(1, 2)).c64); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C128.init(1, 2)).c128); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C64.init(1, 2)).c128); + } + + pub fn dtype(self: Value) DataType { + return std.meta.activeTag(self); + } + + pub fn constSlice(data: *const Value) []const u8 { + return switch (data.*) { + inline else => |*value| std.mem.asBytes(value), + }; + } + + pub fn as(self: Value, comptime T: type) T { + // TODO allow more lossless conversions + switch (@typeInfo(T)) { + .bool => return self.bool, + .float => switch (self) { + inline .f16, .f32, .f64 => |v| return @floatCast(v), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), + else => {}, + }, + .int => switch (self) { + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), + else => {}, + }, + else => {}, + } + std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); + } + }; }; pub fn mantissaSize(dtype: DataType) usize { @@ -342,3 +348,5 @@ pub fn mantissaSize(dtype: DataType) usize { else => @panic("Can't get mantissa size for a non-float dtype"), }; } + +pub const Data = DataType.Value; diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 117cb1e4..cce98b22 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -30,7 +30,7 @@ pub const HostBuffer = struct { pub fn empty(allocator: std.mem.Allocator, sh: Shape) error{OutOfMemory}!HostBuffer { return .{ ._shape = sh, - ._strides = sh.computeStrides().buffer, + ._strides = sh.computeByteStrides().buffer, ._data = (try allocator.alignedAlloc(u8, .@"64", sh.byteSize())).ptr, ._memory = .{ .managed = .@"64" }, }; @@ -43,7 +43,7 @@ pub const HostBuffer = struct { stdx.debug.assert(shape_.byteSize() == data_.len, "shape {} and data {} don't match", .{ shape_.byteSize(), data_.len }); return .{ ._shape = shape_, - ._strides = shape_.computeStrides().buffer, + ._strides = shape_.computeByteStrides().buffer, ._data = data_.ptr, ._memory = .unmanaged, }; @@ -172,7 +172,7 @@ pub const HostBuffer = struct { // TODO we should allow interpreting the output as @Vector(8, f32) when the tensor is f32. stdx.debug.assert(DataType.fromZigType(T) == self.dtype(), "Can't reinterpret {f} as {s}", .{ self, @typeName(T) }); stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); - const ptr: [*]const T = @alignCast(@ptrCast(self._data)); + const ptr: [*]const T = @ptrCast(@alignCast(self._data)); return ptr[0..self._shape.count()]; } @@ -225,7 +225,7 @@ pub const HostBuffer = struct { pub fn isContiguous(self: HostBuffer) bool { const _strides = self._strides; - const cont_strides = self._shape.computeStrides(); + const cont_strides = self._shape.computeByteStrides(); for (self._shape.dims(), _strides[0..self.rank()], cont_strides.constSlice()) |d, stride, cont_stride| { if (d != 1 and stride != cont_stride) return false; } diff --git a/zml/io.zig b/zml/io.zig new file mode 100644 index 00000000..26227210 --- /dev/null +++ b/zml/io.zig @@ -0,0 +1,59 @@ +const std = @import("std"); +pub const file = @import("io/file.zig"); + +pub const ResourceKind = enum { file }; + +pub const Loader = struct { + file: ?*file.Loader = null, + + pub fn init(allocator: std.mem.Allocator) Loader { + _ = allocator; // autofix + return .{}; + } + + pub fn deinit(self: Loader) void { + _ = self; // autofix + } + + pub fn open(self: Loader, allocator: std.mem.Allocator, uri: std.Uri) !Resource { + inline for (std.meta.tags(ResourceKind)) |tag| { + if (std.mem.eql(u8, uri.scheme, @tagName(tag))) { + const maybe_loader = @field(self, @tagName(tag)); + if (maybe_loader) |loader| { + return @unionInit(Resource, @tagName(tag), try loader.open(allocator, uri)); + } + } + } + return error.NoLoaderFound; + } + + pub fn register(self: *Loader, kind: ResourceKind, loader: anytype) void { + @field(self, @tagName(kind)) = loader; + } +}; + +pub const Resource = union(ResourceKind) { + file: file.Resource, + + pub fn deinit(self: Resource) void { + switch (self) { + inline else => |v| v.deinit(), + } + } + + pub fn reader(self: Resource, buffer: []u8) Reader { + return switch (self) { + .file => |v| .{ .file = v.reader(buffer) }, + }; + } +}; + +pub const Reader = union(ResourceKind) { + file: file.Reader, + + pub fn interface(self: *Reader) *std.io.Reader { + return switch (self.*) { + .file => |*v| &v.interface, + }; + } +}; diff --git a/zml/io/file.zig b/zml/io/file.zig new file mode 100644 index 00000000..8c215cbe --- /dev/null +++ b/zml/io/file.zig @@ -0,0 +1,107 @@ +const std = @import("std"); + +pub const Reader = struct { + reader: std.fs.File.Reader, + remaining: std.Io.Limit, + interface: std.Io.Reader, + + pub fn init(reader: std.fs.File.Reader, limit: std.Io.Limit) Reader { + return .{ + .reader = reader, + .remaining = limit, + .interface = .{ + .vtable = &.{ + .stream = stream, + .discard = discard, + }, + .buffer = &.{}, + .seek = 0, + .end = 0, + }, + }; + } + + fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *Reader = @fieldParentPtr("interface", r); + const combined_limit = limit.min(self.remaining); + const n = try self.reader.interface.stream(w, combined_limit); + self.remaining = self.remaining.subtract(n).?; + return n; + } + + fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { + const self: *Reader = @fieldParentPtr("interface", r); + const combined_limit = limit.min(self.remaining); + const n = try self.reader.interface.discard(combined_limit); + self.remaining = self.remaining.subtract(n).?; + return n; + } +}; + +pub const Loader = struct { + pub fn init() Loader { + return .{}; + } + + pub fn deinit(self: Loader) void { + _ = self; // autofix + } + + pub fn open(self: Loader, allocator: std.mem.Allocator, uri: std.Uri) !Resource { + _ = self; // autofix + const resource: Resource = try .init(allocator, uri); + + return resource; + } +}; + +pub const Resource = struct { + file: std.fs.File, + size: usize, + start: usize, + + pub fn init(allocator: std.mem.Allocator, uri: std.Uri) !Resource { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const raw_query = if (uri.query) |query| try query.toRawMaybeAlloc(arena.allocator()) else ""; + var it = std.mem.tokenizeScalar(u8, raw_query, '&'); + var start: ?usize = null; + var end: ?usize = null; + while (it.next()) |component| { + var it2 = std.mem.tokenizeScalar(u8, component, '='); + const key = it2.next().?; + const value = it2.next() orelse ""; + if (std.mem.eql(u8, key, "start")) { + start = try std.fmt.parseInt(usize, value, 10); + } else if (std.mem.eql(u8, key, "end")) { + end = try std.fmt.parseInt(usize, value, 10); + } + } + + std.debug.print("start: {any} - end: {any}\n", .{ start, end }); + + const path = try uri.path.toRawMaybeAlloc(arena.allocator()); + + const file = try std.fs.openFileAbsolute(path, .{}); + + const size = end.? - start.?; + + return .{ + .file = file, + .size = size, + .start = start.?, + }; + } + + pub fn deinit(self: Resource) void { + self.file.close(); + } + + pub fn reader(self: Resource, buffer: []u8) Reader { + var r = std.fs.File.Reader.init(self.file, buffer); + r.seekTo(self.start) catch unreachable; + const r2 = Reader.init(r, .limited(self.size)); + return r2; + } +}; diff --git a/zml/meta.zig b/zml/meta.zig index ee2fbc68..88516cef 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -3,6 +3,7 @@ const testing = std.testing; const stdx = @import("stdx"); const FnParam = stdx.meta.FnParam; +const FnError = stdx.meta.FnError; const asSlice = stdx.meta.asSlice; test { @@ -588,6 +589,92 @@ test visit { } } +pub fn VisitReturn(comptime cb: anytype) type { + return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| + @Type(std.builtin.Type{ .error_union = .{ .error_set = error_set, .payload = void } }) + else + void; +} + +pub fn visit2(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn(cb) { + const Callback = @TypeOf(cb); + const Ptr = @TypeOf(v); + const type_info_v = @typeInfo(Ptr); + if (type_info_v != .pointer) { + stdx.debug.compileError("zml.meta.visit2({}) is expecting a pointer/slice input, but received: {}", .{ Callback, Ptr }); + } + const ptr_info = type_info_v.pointer; + const Child = ptr_info.child; + + const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { + .pointer => |info| .{ info.child, !info.is_const }, + else => stdx.debug.compileError("zml.meta.visit2 is expecting a callback with a pointer as second argument but found {}", .{FnParam(cb, 1)}), + }; + // Abort if v doesnt' contain any K. + if (comptime !Contains(Ptr, K)) return; + + // Handle simple cases. + switch (Ptr) { + *const K, *K => return try cb(ctx, v), + *const ?K, *?K => return if (v.*) |*val| try cb(ctx, val) else {}, + []const K, []K => { + for (v) |*v_elem| try cb(ctx, v_elem); + return; + }, + else => {}, + } + + // Handle stdx.BoundedArray that contains uninitalized data. + if (@typeInfo(Child) == .@"struct" and @hasDecl(Child, "constSlice") and @hasDecl(Child, "slice")) { + return visit2(cb, ctx, if (mutating_cb) v.slice() else v.constSlice()); + } + + // Recursively visit2 fields of v. + switch (ptr_info.size) { + .one => switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + switch (field_type_info) { + .pointer => try visit2(cb, ctx, @field(v, field.name)), + .array, .optional, .@"union", .@"struct" => try visit2(cb, ctx, &@field(v, field.name)), + else => {}, + } + }, + .array => |_| for (v) |*elem| try visit2(cb, ctx, elem), + .optional => if (v.* != null) try visit2(cb, ctx, &v.*.?), + .@"union" => switch (v.*) { + inline else => |*v_field| try visit2(cb, ctx, v_field), + }, + else => stdx.debug.compileError("zml.meta.visit2({}) doesn't support fields of type: {}", .{ Callback, Child }), + }, + .slice => { + for (v) |*v_elem| { + switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + if (field_type_info == .pointer) { + try visit2(cb, ctx, @field(v_elem, field.name)); + } else { + try visit2(cb, ctx, &@field(v_elem, field.name)); + } + }, + .array => |_| for (v) |*elem| try visit2(cb, ctx, elem), + .optional => if (v.* != null) try visit2(cb, ctx, &v.*.?), + .@"union" => switch (v_elem.*) { + inline else => |*v_field| try visit2(cb, ctx, v_field), + }, + else => stdx.debug.compileError("zml.meta.visit2({}) doesn't support fields of type: {}", .{ Callback, Child }), + } + } + }, + .many, .c => stdx.debug.compileError("zml.meta.visit2({}) doesn't support [*] style pointers, got: {}", .{ Callback, Ptr }), + } +} + pub fn count(T: type, value: anytype) u32 { var counter: u32 = 0; visit(struct { diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 28f40383..e18eb04e 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -7,11 +7,6 @@ const Shape = @import("shape.zig").Shape; const mlirx = @This(); -/// Returns the mlir.Type corresponding to a given zml.Shape. -pub fn tensorType(ctx: mlir.Context, sh: Shape) mlir.Type { - return .tensor(sh.dims(), mlirx.Type.fromDType(ctx, sh.dtype())); -} - pub fn denseElementAttrType(dt: dtype.DataType) ?mlir.DenseElementsAttributeTypes { return switch (dt) { .bool => .bool, @@ -32,34 +27,36 @@ pub fn denseElementAttrType(dt: dtype.DataType) ?mlir.DenseElementsAttributeType } pub const Type = struct { - pub fn fromDType(ctx: mlir.Context, dt: dtype.DataType) mlir.Type { + pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { return switch (dt) { - .bool => .int(ctx, .i1), - .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), - .f8e4m3fn => .float(ctx, .f8e4m3fn), - .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), - .f8e5m2 => .float(ctx, .f8e5m2), - .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), - .bf16 => .float(ctx, .bf16), - .f16 => .float(ctx, .f16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .i4 => .int(ctx, .i4), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .u4 => .int(ctx, .u4), - .u8 => .int(ctx, .u8), - .u16 => .int(ctx, .u16), - .u32 => .int(ctx, .u32), - .u64 => .int(ctx, .u64), - .c64 => .complex(ctx, .c64), - .c128 => .complex(ctx, .c128), + .bool => mlir.integerType(ctx, .i1), + .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), + .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), + .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), + .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), + .bf16 => mlir.floatType(ctx, .bf16), + .f16 => mlir.floatType(ctx, .f16), + .f32 => mlir.floatType(ctx, .f32), + .f64 => mlir.floatType(ctx, .f64), + .i4 => mlir.integerType(ctx, .i4), + .i8 => mlir.integerType(ctx, .i8), + .i16 => mlir.integerType(ctx, .i16), + .i32 => mlir.integerType(ctx, .i32), + .i64 => mlir.integerType(ctx, .i64), + .u4 => mlir.integerType(ctx, .u4), + .u8 => mlir.integerType(ctx, .u8), + .u16 => mlir.integerType(ctx, .u16), + .u32 => mlir.integerType(ctx, .u32), + .u64 => mlir.integerType(ctx, .u64), + .c64 => mlir.complexType(ctx, .c64), + .c128 => mlir.complexType(ctx, .c128), }; } - pub fn toDType(mlir_type: mlir.Type) dtype.DataType { + // TODO(Corentin): Maybe remove that as it now requires a mlir context. + // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. + pub fn toDType(mlir_type: *const mlir.Type) dtype.DataType { const mapping = .{ .{ .bool, mlir.IntegerType(.i1) }, @@ -91,7 +88,7 @@ pub const Type = struct { inline for (mapping) |entry| { const dt, const mlirT = entry; - if (mlirT.is_a_fn(mlir_type._inner)) { + if (mlirT.isAFn(mlir_type._inner)) { return dt; } } diff --git a/zml/module.zig b/zml/module.zig index 4e40882e..9f3858d8 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -29,10 +29,10 @@ pub const MlirFn = struct { name: []const u8, args_shapes: []Shape, res_tensors: *const anyopaque, - res_types: []mlir.Type, + res_types: []*const mlir.Type, res_shapes: []Shape, res_donations: []Tensor._Donation, - mlir_fn: mlir.Operation, + mlir_fn: *mlir.Operation, pub const Kind = enum { main, @@ -40,16 +40,18 @@ pub const MlirFn = struct { }; }; +const RecursiveOpts = enum { open, hermetic }; + pub const CompilationContext = struct { _platform: Platform, _name: []const u8, _arena: std.heap.ArenaAllocator, - _mlir_ctx: mlir.Context, - _mlir_registry: mlir.Registry, - _mlir_canonicalizer: mlir.PassManager, + _mlir_ctx: *mlir.Context, + _mlir_registry: *mlir.DialectRegistry, + _mlir_canonicalizer: *mlir.PassManager, - _module: mlir.Module, + _module: *mlir.Module, _blocks: stdx.BoundedArray(TaggedBlock, 64) = .{}, _fn_cache: FnCache = .{}, @@ -61,16 +63,16 @@ pub const CompilationContext = struct { _previous: ?*CompilationContext = null, threadlocal var _current: ?*CompilationContext = null; - const TaggedBlock = struct { mlir.Block, mlir.Block.RecursiveOpts }; - const TensorToBlockArg = std.AutoHashMapUnmanaged(Tensor._Id, struct { mlir.Value, Tensor._Donation }); + const TaggedBlock = struct { *mlir.Block, RecursiveOpts }; + const TensorToBlockArg = std.AutoHashMapUnmanaged(Tensor._Id, struct { *const mlir.Value, Tensor._Donation }); const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); pub fn init(allocator_: std.mem.Allocator, full_name: []const u8, platform: Platform) !CompilationContext { - const mlir_registry = mlir.Registry.init() catch unreachable; + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); } - var mlir_ctx = mlir.Context.initWithRegistry(mlir_registry, false) catch unreachable; + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; mlir_ctx.loadAllAvailableDialects(); // Too long module names create too long file paths and files failed to create. @@ -79,11 +81,11 @@ pub const CompilationContext = struct { const max_name_len = @divFloor(std.fs.max_path_bytes, 2) - 17; const name = full_name[0..@min(max_name_len, full_name.len)]; - const loc = mlir_ctx.location(@src()).named(mlir_ctx, "main"); + const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); const module = mlir.Module.init(loc); - module.op().setAttributeByName("sym_name", .string(mlir_ctx, "zml")); + module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); - var canonicalizer = try mlir.PassManager.init(mlir_ctx); + var canonicalizer = mlir.PassManager.init(mlir_ctx); { var opm = canonicalizer.asOpPassManager(); try opm.addPipeline("canonicalize"); @@ -138,7 +140,7 @@ pub const CompilationContext = struct { return self._platform.target; } - pub fn mlirCtx(self: *const CompilationContext) mlir.Context { + pub fn mlirCtx(self: *const CompilationContext) *mlir.Context { return self._mlir_ctx; } @@ -164,12 +166,12 @@ pub const CompilationContext = struct { // Run in a dedicated thread because compilation relies on `threadlocal`. const f = try asynk.callBlocking(CompilationContext.emitMlir, .{ self, func, &tensor_args, CompilationContext.EmitMlirOpts{ .name = "main", .kind = .main } }); const module = self._module; - module.getBody().appendOperation(f.mlir_fn); + _ = f.mlir_fn.appendTo(module.body()); const sharding = self._platform.sharding(); const mlir_ctx = self._mlir_ctx; - module.op().setAttributeByName("mhlo.num_replicas", .int(mlir_ctx, .i32, sharding.num_replicas)); - module.op().setAttributeByName("mhlo.num_partitions", .int(mlir_ctx, .i32, sharding.num_partitions)); + module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(mlir_ctx, .i32, sharding.num_replicas)); + module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(mlir_ctx, .i32, sharding.num_partitions)); const module_hash = computeModuleHash(self._platform, module); var module_dir: ?[]const u8 = null; @@ -244,8 +246,8 @@ pub const CompilationContext = struct { return if (self._blocks.len > 0) self._blocks.get(self._blocks.len - 1) else null; } - pub fn openBlock(self: *CompilationContext, kind: mlir.Block.RecursiveOpts, args: []const mlir.Type, locs: []const mlir.Location) !TaggedBlock { - const block: TaggedBlock = .{ try mlir.Block.init(args, locs), kind }; + pub fn openBlock(self: *CompilationContext, kind: RecursiveOpts, args: []const *const mlir.Type, locs: []const *const mlir.Location) !TaggedBlock { + const block: TaggedBlock = .{ mlir.Block.init(args, locs), kind }; self.pushBlock(block); return block; } @@ -336,17 +338,19 @@ pub const CompilationContext = struct { const tensor_count = meta.count(Tensor, args); const mlir_ctx = self.mlirCtx(); - const loc = mlir_ctx.location(@src()); + const loc: *const mlir.Location = .fromSrc(mlir_ctx, @src()); - const locations = try arena.alloc(mlir.Location, tensor_count); - @memset(locations, mlir.Location.unknown(mlir_ctx)); + const locations = try arena.alloc(*const mlir.Location, tensor_count); + @memset(locations, .unknown(mlir_ctx)); var input_shapes: std.array_list.Managed(Shape) = try .initCapacity(res_allocator, tensor_count); meta.collect(Tensor.shape, {}, &input_shapes, args) catch unreachable; stdx.debug.internalAssert(input_shapes.items.len == tensor_count, "args have changed ?", .{}); - const input_types = try arena.alloc(mlir.Type, tensor_count); - for (input_types, input_shapes.items) |*t, sh| t.* = mlirx.tensorType(mlir_ctx, sh); + const input_types = try arena.alloc(*const mlir.Type, tensor_count); + for (input_types, input_shapes.items) |*t, sh| { + t.* = mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlir_ctx, sh.dtype())); + } const og_block_args = self._block_args; defer { @@ -364,7 +368,7 @@ pub const CompilationContext = struct { // Those are returned to caller so we don't put them in the arena, but in the module allocator. const fn_res = try res_allocator.create(ReturnT); - const fn_res_types = try res_allocator.alloc(mlir.Type, out_tensor_count); + const fn_res_types = try res_allocator.alloc(*const mlir.Type, out_tensor_count); const fn_res_shapes = try res_allocator.alloc(Shape, out_tensor_count); const fn_res_donations = try res_allocator.alloc(Tensor._Donation, out_tensor_count); const fn_res_output_memory_kind = try res_allocator.alloc(Buffer.Memory, out_tensor_count); @@ -382,10 +386,10 @@ pub const CompilationContext = struct { break :forward @call(.auto, func, args.*); }; - var fn_res_values: [out_tensor_count]mlir.Value = undefined; + var fn_res_values: [out_tensor_count]*const mlir.Value = undefined; self.extractValuesAndTypes(fn_res, &fn_res_values, fn_res_types, fn_res_shapes, fn_res_donations, fn_res_output_memory_kind); - const fn_ret = dialect.func.return_(mlir_ctx, &fn_res_values, loc); + const fn_ret = dialect.func.returns(mlir_ctx, &fn_res_values, loc); fn_body[0].appendOperationRecursive(fn_ret, fn_body[1]); } @@ -690,10 +694,10 @@ pub const CompilationContext = struct { /// This is done so that we have a mapping between the arguments of the kernel associated with a module and the actual Tensors /// stored in the Module. /// Caller need to allocate required memory in self._block_args. - pub fn mapBlockArguments(self: *CompilationContext, v: anytype, block: mlir.Block, start: usize) usize { + pub fn mapBlockArguments(self: *CompilationContext, v: anytype, block: *mlir.Block, start: usize) usize { const LocalContext = struct { index: usize, - block: mlir.Block, + block: *mlir.Block, self: *CompilationContext, }; var context = LocalContext{ .self = self, .block = block, .index = start }; @@ -736,8 +740,8 @@ pub const CompilationContext = struct { pub fn extractValuesAndTypes( self: *const CompilationContext, v: anytype, - values: []mlir.Value, - types: []mlir.Type, + values: []*const mlir.Value, + types: []*const mlir.Type, shapes: []Shape, donations: []Tensor._Donation, output_memory_kind: []Buffer.Memory, @@ -746,8 +750,8 @@ pub const CompilationContext = struct { const LocalContext = struct { self: *const CompilationContext, index: usize = 0, - values: []mlir.Value, - types: []mlir.Type, + values: []*const mlir.Value, + types: []*const mlir.Type, shapes: []Shape, donations: []Tensor._Donation, output_memory_kind: []Buffer.Memory, diff --git a/zml/partitioning.zig b/zml/partitioning.zig new file mode 100644 index 00000000..cb2e8027 --- /dev/null +++ b/zml/partitioning.zig @@ -0,0 +1,906 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const stdx = @import("stdx"); + +const pjrtx = @import("pjrtx.zig"); +//const slice = @import("slice.zig"); +//const Shaped = slice.Shaped; +const Buffer = @import("buffer.zig").Buffer; +const Context = @import("context.zig").Context; +const DataType = @import("dtype.zig").DataType; +const Device = @import("pjrtx.zig").Device; +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; + +const log = std.log.scoped(.@"zml/partitioning"); + +test { + std.testing.refAllDecls(Mesh); + std.testing.refAllDecls(DeviceShard); + std.testing.refAllDecls(Sharding); +} + +pub const MaxMeshAxes: u8 = 3; +pub const MaxMeshSize: u8 = MaxMeshAxes * Shape.MAX_RANK; + +/// Iterates through all possible multi-dimensional indices of a topology. +/// The iteration order is row-major. +pub const TopologyIndicesIterator = struct { + index: usize = 0, + topology: Shape, + indices: Shape, + + pub fn next(self: *TopologyIndicesIterator) ?Shape { + if (self.index >= self.topology.count()) return null; + + const current_indices = self.indices; + + // Prepare for the next iteration + self.index += 1; + + // Use a standard row-major increment logic + if (self.index < self.topology.count()) { + const rank = self.topology.rank(); + var i: usize = rank; + + while (i > 0) { + i -= 1; + const dim: u4 = @intCast(i); + + const current_value = self.indices.dim(dim); + if (current_value < self.topology.dim(dim) - 1) { + self.indices = self.indices.setDim(dim, current_value + 1); + return current_indices; + } else { + self.indices = self.indices.setDim(dim, 0); + } + } + } + + return current_indices; + } +}; + +pub const Mesh = struct { + topology: Shape, + + pub fn init(topology: anytype) Mesh { + const self: Mesh = .{ + .topology = .init(topology, .u8), + }; + + if (self.rank() == 0) { + stdx.debug.panic("Mesh must have at least one tagged axis defined, got: {any}", .{topology}); + } + + if (!self.topology.isFullyTagged()) { + stdx.debug.panic("Mesh must be fully tagged, got: {any}", .{topology}); + } + + if (self.rank() > MaxMeshAxes) { + stdx.debug.panic("Mesh rank ({}) exceeds maximum allowed axes ({})", .{ self.rank(), MaxMeshAxes }); + } + + return self; + } + + pub fn single() Mesh { + const topology: Shape = .init(.{ .x = 1 }, .u8); + return .init(topology); + } + + pub fn auto(platform: Platform) Mesh { + const num_devices = platform.getDevices().len; + + if (num_devices == 0) { + stdx.debug.panic("No devices available in the platform: {}", .{platform.target}); + } + + if (num_devices > MaxMeshSize) { + stdx.debug.panic("Too many devices ({}) for a mesh, max is {}", .{ num_devices, MaxMeshSize }); + } + + const topology: Shape = .init(.{ .x = num_devices }, .u8); + + return .init(topology); + } + + pub fn reshape(self: Mesh, new_shape: anytype) Mesh { + const new_topology = self.topology.reshape(new_shape); + return .init(new_topology); + } + + pub fn flatten(self: Mesh, new_axis_name: anytype) Mesh { + const total_devices = self.numDevices(); + + var new_topology = Shape.init(.{}, .u8); + new_topology = new_topology.appendDim(total_devices, @tagName(new_axis_name)); + return Mesh.init(new_topology); + } + + test flatten { + { + // Flatten a 2D mesh + const mesh = Mesh.init(.{ .x = 2, .y = 4 }); + const flattened = mesh.flatten(.all); + const expected = Mesh.init(.{ .all = 8 }); + + try std.testing.expect(flattened.eql(expected)); + try std.testing.expectEqual(1, flattened.rank()); + try std.testing.expectEqual(8, flattened.numDevices()); + } + + { + // Flatten a 3D mesh + const mesh = Mesh.init(.{ .data = 2, .model = 3, .pipeline = 4 }); + const flattened = mesh.flatten(.devices); + const expected = Mesh.init(.{ .devices = 24 }); + try std.testing.expect(flattened.eql(expected)); + try std.testing.expectEqual(1, flattened.rank()); + try std.testing.expectEqual(24, flattened.numDevices()); + } + { + // Flattening a 1D mesh should just rename the axis + const mesh = Mesh.init(.{ .x = 16 }); + const flattened = mesh.flatten(.y); + const expected = Mesh.init(.{ .y = 16 }); + try std.testing.expect(flattened.eql(expected)); + try std.testing.expectEqual(1, expected.rank()); + } + } + + pub fn rank(self: Mesh) i64 { + return @intCast(self.topology.rank()); + } + + pub fn is1D(self: Mesh) bool { + return self.rank() == 1; + } + + pub fn is2D(self: Mesh) bool { + return self.rank() == 2; + } + + pub fn is3D(self: Mesh) bool { + return self.rank() == 3; + } + + pub fn axis(self: Mesh, ax: anytype) i64 { + return self.topology.dim(ax); + } + + pub fn hasManyPartitions(self: Mesh) bool { + return self.numPartitions() > 1; + } + + pub fn isSinglePartition(self: Mesh) bool { + return self.numPartitions() == 1; + } + + pub fn iterator(self: Mesh) TopologyIndicesIterator { + var indices = self.topology; + + for (0..indices.rank()) |dim| { + indices = indices.setDim(dim, 0); + } + + return .{ + .topology = self.topology, + .indices = indices, + }; + } + + pub fn numPartitions(self: Mesh) u8 { + return @intCast(self.topology.count()); + } + + pub fn numReplicas(_: Mesh) u8 { + return 1; + } + + pub fn numDevices(self: Mesh) u8 { + return self.numPartitions() * self.numReplicas(); + } + + pub fn eql(self: Mesh, other: Mesh) bool { + return self.topology.eql(other.topology); + } + + pub fn format( + self: Mesh, + writer: anytype, + ) !void { + try writer.print("Mesh(topology={f} rank={} numDevices={})", .{ self.topology, self.rank(), self.numDevices() }); + } +}; + +test "Mesh / 1D mesh with 1 partition" { + const mesh: Mesh = .init(.{ .x = 1 }); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / 1D mesh with 8 partitions" { + const mesh: Mesh = .init(.{ .x = 8 }); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 8); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(mesh.hasManyPartitions()); + try std.testing.expect(!mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 8); + try std.testing.expect(mesh.numDevices() == 8); +} + +test "Mesh / 2D mesh with 1 partition" { + const mesh: Mesh = .init(.{ .x = 1, .y = 1 }); + try std.testing.expect(mesh.rank() == 2); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.axis(.y) == 1); + try std.testing.expect(!mesh.is1D()); + try std.testing.expect(mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / 2D mesh with 6 partitions" { + const mesh: Mesh = .init(.{ .x = 2, .y = 3 }); + try std.testing.expect(mesh.rank() == 2); + try std.testing.expect(mesh.axis(.x) == 2); + try std.testing.expect(mesh.axis(.y) == 3); + try std.testing.expect(!mesh.is1D()); + try std.testing.expect(mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(mesh.hasManyPartitions()); + try std.testing.expect(!mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 6); + try std.testing.expect(mesh.numDevices() == 6); +} + +test "Mesh / 3D mesh with 1 partition" { + const mesh: Mesh = .init(.{ .x = 1, .y = 1, .z = 1 }); + try std.testing.expect(mesh.rank() == 3); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.axis(.y) == 1); + try std.testing.expect(mesh.axis(.z) == 1); + try std.testing.expect(!mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / 3D mesh with 64 partitions" { + const mesh_3d: Mesh = .init(.{ .x = 4, .y = 4, .z = 4 }); + try std.testing.expect(mesh_3d.rank() == 3); + try std.testing.expect(mesh_3d.axis(.x) == 4); + try std.testing.expect(mesh_3d.axis(.y) == 4); + try std.testing.expect(mesh_3d.axis(.z) == 4); + try std.testing.expect(!mesh_3d.is1D()); + try std.testing.expect(!mesh_3d.is2D()); + try std.testing.expect(mesh_3d.is3D()); + try std.testing.expect(mesh_3d.hasManyPartitions()); + try std.testing.expect(!mesh_3d.isSinglePartition()); + try std.testing.expect(mesh_3d.numPartitions() == 64); + try std.testing.expect(mesh_3d.numDevices() == 64); +} + +test "Mesh / single" { + const mesh: Mesh = .single(); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 1); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(!mesh.hasManyPartitions()); + try std.testing.expect(mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 1); + try std.testing.expect(mesh.numDevices() == 1); +} + +test "Mesh / auto with 4 devices" { + const platform = env(.{ .cpu = .{ .cpu_device_count = 4 } }); + const mesh: Mesh = .auto(platform); + try std.testing.expect(mesh.rank() == 1); + try std.testing.expect(mesh.axis(.x) == 4); + try std.testing.expect(mesh.is1D()); + try std.testing.expect(!mesh.is2D()); + try std.testing.expect(!mesh.is3D()); + try std.testing.expect(mesh.hasManyPartitions()); + try std.testing.expect(!mesh.isSinglePartition()); + try std.testing.expect(mesh.numPartitions() == 4); + try std.testing.expect(mesh.numDevices() == 4); +} + +test "Mesh / 1D mesh iterator" { + const mesh = Mesh.init(.{ .x = 4 }); + var iter = mesh.iterator(); + var count: u8 = 0; + + while (iter.next()) |indices| : (count += 1) { + const expected_shape = Shape.init(.{ .x = count }, .u8); + try std.testing.expect(indices.eqlWithTags(expected_shape)); + } + + try std.testing.expectEqual(4, count); + try std.testing.expect(iter.next() == null); +} + +test "Mesh / 2D mesh iterator" { + const mesh = Mesh.init(.{ .x = 2, .y = 3 }); + var iter = mesh.iterator(); + var count: u8 = 0; + const expected_indices = [_]Shape{ + .init(.{ .x = 0, .y = 0 }, .u8), + .init(.{ .x = 0, .y = 1 }, .u8), + .init(.{ .x = 0, .y = 2 }, .u8), + .init(.{ .x = 1, .y = 0 }, .u8), + .init(.{ .x = 1, .y = 1 }, .u8), + .init(.{ .x = 1, .y = 2 }, .u8), + }; + + while (iter.next()) |indices| : (count += 1) { + try std.testing.expect(indices.eqlWithTags(expected_indices[count])); + } + try std.testing.expectEqual(6, count); + try std.testing.expect(iter.next() == null); +} + +test "Mesh / 3D mesh iterator" { + const mesh = Mesh.init(.{ .x = 2, .y = 2, .z = 2 }); + var iter = mesh.iterator(); + var count: u8 = 0; + const expected_indices = [_]Shape{ + .init(.{ .x = 0, .y = 0, .z = 0 }, .u8), + .init(.{ .x = 0, .y = 0, .z = 1 }, .u8), + .init(.{ .x = 0, .y = 1, .z = 0 }, .u8), + .init(.{ .x = 0, .y = 1, .z = 1 }, .u8), + .init(.{ .x = 1, .y = 0, .z = 0 }, .u8), + .init(.{ .x = 1, .y = 0, .z = 1 }, .u8), + .init(.{ .x = 1, .y = 1, .z = 0 }, .u8), + .init(.{ .x = 1, .y = 1, .z = 1 }, .u8), + }; + + while (iter.next()) |indices| : (count += 1) { + try std.testing.expect(indices.eqlWithTags(expected_indices[count])); + } + + try std.testing.expectEqual(8, count); + try std.testing.expect(iter.next() == null); +} + +pub const DeviceShard = struct { + index: u64, + topology: Shape, + global_shape: Shape, + indices: Shape, + shard_shape: Shape, + device: *const Device, + + pub const SliceSpec = struct { + start_offset: usize, + byte_strides: [Shape.MAX_RANK]i64, + dims: [Shape.MAX_RANK]i64, + num_dims: u4, + }; + + pub fn specs(self: DeviceShard) SliceSpec { + const rank = self.global_shape.rank(); + + const host_byte_strides_ba = self.global_shape.computeByteStrides(); + + var shard_start_offset_bytes: i64 = 0; + for (0..rank) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + + // Check if the device's indices actually have this axis. + // If not, this dimension is effectively replicated for this device. + if (self.indices.hasTag(mesh_axis_tag) != null) { + const device_coord_on_mesh_axis = self.indices.dim(mesh_axis_tag); + const shard_dim_size = self.shard_shape.dim(i); + const start_element_in_dim = device_coord_on_mesh_axis * shard_dim_size; + shard_start_offset_bytes += start_element_in_dim * host_byte_strides_ba.get(i); + } + } + } + + var transfer_dims_buffer: [Shape.MAX_RANK]i64 = undefined; + for (0..rank) |i| { + transfer_dims_buffer[i] = self.shard_shape.dim(i); + } + + return .{ + .start_offset = @intCast(shard_start_offset_bytes), + .byte_strides = host_byte_strides_ba.buffer, + .dims = transfer_dims_buffer, + .num_dims = rank, + }; + } + + pub fn format( + self: DeviceShard, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + const slice_specs = self.specs(); + try writer.print("DeviceShard(index={d}/{d} topology={} indices={} shard_shape={} ({d}B) global_shape={} ({d}B) start_offset={d} byte_strides={any} device={})", .{ + self.index + 1, + self.topology.count(), + self.topology, + self.indices, + self.shard_shape, + self.shard_shape.byteSize(), + self.global_shape, + self.global_shape.byteSize(), + slice_specs.start_offset, + slice_specs.byte_strides[0..slice_specs.num_dims], + self.device, + }); + } +}; + +pub const DeviceShardIterator = struct { + index: usize = 0, + indices_iterator: TopologyIndicesIterator, + sharding: Sharding, + devices: []const *const pjrtx.Device, + + pub fn next(self: *DeviceShardIterator) ?DeviceShard { + const num_partitions = self.sharding.mesh.numPartitions(); // todo proxy numPartitions + + if (self.index >= num_partitions) return null; + + defer self.index += 1; + + return .{ + .index = self.index, + .topology = self.sharding.mesh.topology, + .global_shape = self.sharding.global_shape, + .indices = self.indices_iterator.next().?, + .shard_shape = self.sharding.shardShape(), + .device = self.devices[self.index], + }; + } +}; + +pub const Sharding = struct { + mesh: Mesh, + global_shape: Shape, + + pub const Type = enum { + replicated, + maximal, + manual, + }; + + pub fn init(mesh: Mesh, shape: Shape) Sharding { + // Validate that the shape is divisible by the mesh where partitioned. + for (0..shape.rank()) |dim_idx| { + const part_spec = shape.partition(dim_idx); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + + if (mesh.topology.hasTag(mesh_axis_tag) == null) { + // This partitioning is invalid for this mesh, so we skip the divisibility check. + // The dimension will effectively be replicated. + log.debug("Partitioning axis '{s}' not found in mesh {f}. Treating dimension {s} as replicated.", .{ + std.mem.span(mesh_axis_tag), + mesh, + shape.debugTag(dim_idx), + }); + continue; + } + + const global_dim = shape.dim(dim_idx); + const mesh_dim = mesh.topology.dim(mesh_axis_tag); + if (@rem(global_dim, mesh_dim) != 0) { + stdx.debug.panic( + "Tensor dimension {s} (size={d}) is not divisible by mesh axis {s} (size={d})", + .{ shape.debugTag(dim_idx), global_dim, std.mem.span(mesh_axis_tag), mesh_dim }, + ); + } + } + } + + return .{ + .mesh = mesh, + .global_shape = shape, + }; + } + + test "init / non existent mesh axis" { + const mesh: Mesh = .init(.{ .x = 4 }); + const shape: Shape = .init(.{ .m = 8, .n = 12 }, .i32); + const sharding = Sharding.init(mesh, shape.withPartitioning(.{ .m = .x, .n = .y })); + const shard_shape = sharding.shardShape(); + try std.testing.expectEqual(2, shard_shape.dim(.m)); + try std.testing.expectEqual(12, shard_shape.dim(.n)); + } + + pub fn getType(self: Sharding) Type { + var has_any_partitioned_axis = false; + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + has_any_partitioned_axis = true; + break; + } + } + } + + if (!has_any_partitioned_axis) { + return .replicated; + } + + var is_fully_partitioned = true; + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + switch (part_spec) { + .replicated, .open, .unknown => { + is_fully_partitioned = false; + break; + }, + .axis => |tag| { + if (self.mesh.topology.hasTag(tag) == null) { + is_fully_partitioned = false; + break; + } + }, + } + } + + if (is_fully_partitioned) { + return .maximal; + } + + return .manual; + } + + pub fn shardShape(self: Sharding) Shape { + var shard_: Shape = .init(.{}, self.global_shape.dtype()); + const rank = self.global_shape.rank(); + + for (0..rank) |dim| { + const part_spec = self.global_shape.partition(dim); + const global_dim_size = self.global_shape.dim(dim); + const tag = self.global_shape.tag(dim); + + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + + // If the specified mesh axis exists, partition. Otherwise, replicate. + if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + const mesh_dim = self.mesh.topology.dim(mesh_axis_tag); + const shard_dim_size = if (mesh_dim > 1) @divExact(global_dim_size, mesh_dim) else global_dim_size; + shard_ = shard_.appendDim(shard_dim_size, tag); + } else { + // Mesh axis does not exist, so this dimension is replicated. + shard_ = shard_.appendDim(global_dim_size, tag); + } + } else { + // For replicated, open, or unknown, the shard dimension is the full global dimension. + shard_ = shard_.appendDim(global_dim_size, tag); + } + } + + return shard_; + } + + pub fn iterator(self: Sharding, devices: []const *const pjrtx.Device) DeviceShardIterator { + const indices_iterator = self.mesh.iterator(); + + return .{ .indices_iterator = indices_iterator, .sharding = self, .devices = devices }; + } + + pub fn getShardingAttr(self: Sharding) std.BoundedArray(u8, 128) { + var sharding_str: std.BoundedArray(u8, 128) = .{}; + self.writeShardingRepresentation(sharding_str.writer()) catch unreachable; + return sharding_str; + } + + pub fn writeShardingRepresentation(self: Sharding, writer: anytype) !void { + if (self.getType() == .replicated) { + try writer.writeAll("{replicated}"); + return; + } + + var product_of_tensor_tiles: i64 = 1; + var has_partitioned_axis = false; + + // Calculate the product of explicitly defined tile dimensions + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + if (part_spec == .axis) { + const mesh_axis_tag = part_spec.toTag(); + if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + product_of_tensor_tiles *= self.mesh.topology.dim(mesh_axis_tag); + has_partitioned_axis = true; + } + } + } + + if (!has_partitioned_axis) { + try writer.writeAll("{replicated}"); + return; + } + + const is_fully_sharded_on_mesh = (product_of_tensor_tiles == self.mesh.numPartitions()); + + try writer.writeAll("{devices=["); + // The tile array must have the same rank as the tensor, for both partial and full sharding. + for (0..self.global_shape.rank()) |i| { + const part_spec = self.global_shape.partition(i); + var tile_dim: i64 = 1; + if (part_spec == .axis) { + if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + tile_dim = self.mesh.topology.dim(part_spec.toTag()); + } + } + try writer.print("{d}", .{tile_dim}); + if (i < self.global_shape.rank() - 1) { + try writer.writeByte(','); + } + } + + try writer.writeAll("]<=["); + try writer.print("{}", .{self.mesh.numDevices()}); + //const mesh_dims = self.mesh.topology.dims(); + //for (mesh_dims, 0..) |dim, i| { + // try writer.print("{d}", .{dim}); + // if (i < mesh_dims.len - 1) { + // try writer.writeByte(','); + // } + //} + + if (is_fully_sharded_on_mesh) { + try writer.writeAll("]}"); + } else { + try writer.writeAll("] last_tile_dim_replicate}"); + } + } + + // pub fn writeShardingRepresentation(self: Sharding, writer: anytype) !void { + // if (self.getType() == .replicated) { + // try writer.writeAll("{replicated}"); + // return; + // } + + // var product_of_tensor_tiles: i64 = 1; + // var num_partitioned_dims: u32 = 0; + + // // Calculate the product of explicitly defined tile dimensions + // for (0..self.global_shape.rank()) |i| { + // const part_spec = self.global_shape.partition(i); + // if (part_spec == .axis) { + // const mesh_axis_tag = part_spec.toTag(); + // if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + // product_of_tensor_tiles *= self.mesh.topology.dim(mesh_axis_tag); + // num_partitioned_dims += 1; + // } + // } + // } + + // const is_fully_sharded_on_mesh = (product_of_tensor_tiles == self.mesh.numPartitions()); + + // if (is_fully_sharded_on_mesh) { + // // This is the easy case. The tensor is partitioned across all mesh devices. + // // The tile array must have the same rank as the tensor. + // try writer.writeAll("{devices=["); + // for (0..self.global_shape.rank()) |i| { + // const part_spec = self.global_shape.partition(i); + // var tile_dim: i64 = 1; + // if (part_spec == .axis) { + // if (self.mesh.topology.hasTag(part_spec.toTag()) != null) { + // tile_dim = self.mesh.topology.dim(part_spec.toTag()); + // } + // } + // try writer.print("{d}", .{tile_dim}); + // if (i < self.global_shape.rank() - 1) { + // try writer.writeByte(','); + // } + // } + // try writer.print("]<=[{d}]}}", .{self.mesh.numPartitions()}); + // } else { + // // This is partial sharding. This is more complex. + // // The `devices` array lists the tiling only for the *partitioned* dimensions. + // // The remaining devices are handled by `last_tile_dim_replicate`. + // try writer.writeAll("{devices=["); + // var first = true; + // for (0..self.global_shape.rank()) |i| { + // const part_spec = self.global_shape.partition(i); + // if (part_spec == .axis) { + // const mesh_axis_tag = part_spec.toTag(); + // if (self.mesh.topology.hasTag(mesh_axis_tag) != null) { + // if (!first) try writer.writeByte(','); + // try writer.print("{d}", .{self.mesh.topology.dim(mesh_axis_tag)}); + // first = false; + // } + // } + // } + // try writer.print("]<=[{d}] last_tile_dim_replicate}}", .{self.mesh.numPartitions()}); + // } + // } + + test "Sharding MLIR representation" { + // Case 1: Fully Replicated + const mesh_rep = Mesh.init(.{ .x = 4 }); + const shape_rep = Shape.init(.{ .m = 8, .k = 8 }, .i32); + const sharding_rep = Sharding.init(mesh_rep, shape_rep); + var attr_rep_ba = sharding_rep.getShardingAttr(); + try std.testing.expectEqualStrings("{replicated}", attr_rep_ba.constSlice()); + + // Case 2: Partially partitioned + const mesh_part = Mesh.init(.{ .x = 2, .y = 4 }); + const shape_part = Shape.init(.{ .m = 8, .k = 12 }, .i32).withPartitioning(.{ .m = .x }); + const sharding_part = Sharding.init(mesh_part, shape_part); + var attr_part_ba = sharding_part.getShardingAttr(); + // dim 'm' is partitioned on 'x' (size 2), dim 'k' is replicated (size 1) + try std.testing.expectEqualStrings("{devices=[2,1]<=[8]}", attr_part_ba.constSlice()); + + // Case 3: Fully partitioned + const mesh_full = Mesh.init(.{ .x = 2, .y = 4 }); + const shape_full = Shape.init(.{ .m = 8, .k = 12 }, .i32).withPartitioning(.{ .m = .x, .k = .y }); + const sharding_full = Sharding.init(mesh_full, shape_full); + var attr_full_ba = sharding_full.getShardingAttr(); + // dim 'm' partitioned on 'x' (size 2), dim 'k' partitioned on 'y' (size 4) + try std.testing.expectEqualStrings("{devices=[2,4]<=[8]}", attr_full_ba.constSlice()); + + // Case 4: More complex 3D case with a replicated dimension in the middle + const mesh_3d = Mesh.init(.{ .x = 2, .y = 2, .z = 3 }); + const shape_3d = Shape.init(.{ .b = 4, .h = 8, .w = 6 }, .i32).withPartitioning(.{ .b = .x, .w = .z }); + const sharding_3d = Sharding.init(mesh_3d, shape_3d); + var attr_3d_ba = sharding_3d.getShardingAttr(); + // b -> x (size 2), h -> replicated (size 1), w -> z (size 3) + try std.testing.expectEqualStrings("{devices=[2,1,3]<=[12]}", attr_3d_ba.constSlice()); + } + + pub fn format( + self: Sharding, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + try writer.print("Sharding(global_shape={} mesh={})", .{ self.global_shape, self.mesh }); + } +}; + +//fn testShardingCase(allocator: std.mem.Allocator, mesh: Mesh, shape: Shape, partition_spec: anytype, verbose: bool) !void { +// _platform = null; // Reset the global platform +// const platform = env(.{ .cpu = .{ .cpu_device_count = @intCast(mesh.numDevices()) } }); +// +// const shape_partitioned = shape.withPartitioning(partition_spec); +// const sharding = Sharding.init(mesh, shape_partitioned); +// std.debug.print("\n--- Testing Case (verbose={any}) --- {}\n", .{ verbose, sharding }); +// +// const original_data = try slice.arange(allocator, shape_partitioned, .{}); +// defer allocator.free(original_data); +// if (verbose) { +// //std.debug.print("Original full slice of data: {any}\n\n", .{Shaped(i32, shape_partitioned, original_data)}); +// } +// +// // Scatter +// var pjrt_buffers = std.ArrayList(*pjrtx.Buffer).init(allocator); +// defer pjrt_buffers.deinit(); +// +// var sharding_iter = sharding.iterator(platform.getDevices()); +// while (sharding_iter.next()) |device_shard| { +// if (verbose) std.debug.print("{}\n", .{device_shard}); +// const specs = device_shard.specs(); +// const args = pjrtx.Client.BufferFromHostBufferArgs{ +// .data = original_data[specs.start_offset..].ptr, +// .buffer_type = bufferTypeFromDtype(device_shard.shard_shape.dtype()), +// .dims = specs.dims[0..specs.num_dims], +// .byte_strides = specs.byte_strides[0..specs.num_dims], +// .host_buffer_semantics = .ImmutableUntilTransferCompletes, +// .device = device_shard.device, +// }; +// const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); +// if (event) |e| try e.await_(platform.pjrt_api); +// try pjrt_buffers.append(pjrt_buffer); +// +// // Optional: verify the transferred data for debugging +// if (verbose) { +// const shard_shape = device_shard.shard_shape; +// const fetched_data = try allocator.alloc(u8, shard_shape.byteSize()); +// defer allocator.free(fetched_data); +// const fetch_event = try pjrt_buffer.toHostBuffer(platform.pjrt_api, fetched_data, .{}); +// if (fetch_event) |e| try e.await_(platform.pjrt_api); +// //std.debug.print("Fetched shard data: {any}\n\n", .{Shaped(i32, shard_shape, fetched_data)}); +// } +// } +// +// // Gather +// if (verbose) std.debug.print("--- Reassembling from shards ---\n", .{}); +// const sharded_buffer = Buffer.fromPjrtBuffers(platform, sharding, pjrt_buffers.items); +// defer sharded_buffer.deinit(); // This will deinit all the underlying PJRT buffers. +// +// const reassembled_data = try sharded_buffer.toHost(allocator); +// defer allocator.free(reassembled_data); +// +// //if (verbose) std.debug.print("Reassembled full slice of data: {any}\n\n", .{Shaped(i32, sharding.global_shape, reassembled_data)}); +// +// // Verify +// try std.testing.expectEqualSlices(u8, original_data, reassembled_data); +// std.debug.print("✅ Verification successful for: {}\n\n", .{sharding}); +//} + +//test "Sharding All Cases" { +// const allocator = std.testing.allocator; +// const verbose = true; +// +// // Fully replicated +// try testShardingCase(allocator, .init(.{ .x = 4 }), .init(.{ .m = 8, .k = 8 }, .i32), .{}, verbose); +// // 1D sharding on a 1D mesh +// try testShardingCase(allocator, .init(.{ .x = 8 }), .init(.{ .m = 16, .k = 16 }, .i32), .{ .k = .x }, verbose); +// // 2D sharding on a 2D mesh +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 4 }), .init(.{ .m = 8, .k = 12 }, .i32), .{ .m = .x, .k = .y }, verbose); +// // 3D tensor on a 2D mesh, partitioned on non-contiguous dimensions +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .b = 4, .h = 8, .w = 6 }, .i32), .{ .b = .x, .w = .y }, verbose); +// // 3D sharding on a 3D mesh +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 4, .z = 3 }), .init(.{ .m = 4, .k = 8, .j = 6 }, .i32), .{ .m = .x, .k = .y, .j = .z }, verbose); +// // 2D tensor, 2D mesh, but only partitioned on one axis +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 3 }), .init(.{ .m = 10, .k = 6 }, .i32), .{ .k = .x }, verbose); +// // Replicated on a 2D mesh +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .m = 4, .k = 4 }, .i32), .{}, verbose); +// // 3D tensor, 2D mesh, one replicated dimension in the middle +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .batch = 4, .seq_len = 10, .features = 8 }, .i32), .{ .batch = .x, .features = .y }, verbose); +// // 2D mesh where one dimension is 1 +// try testShardingCase(allocator, .init(.{ .x = 1, .y = 4 }), .init(.{ .m = 8, .k = 8 }, .i32), .{ .m = .x, .k = .y }, verbose); +// // Partially partitioned, non-divisible on replicated axis +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 3 }), .init(.{ .m = 8, .k = 10 }, .i32), .{ .m = .x }, verbose); +// // Single device mesh +// try testShardingCase(allocator, .init(.{ .x = 1 }), .init(.{ .m = 8, .k = 10 }, .i32), .{ .m = .x }, verbose); +// // NEW: 3D tensor, 2D mesh, partitioned on first two axes. Good test for new reassemble logic. +// try testShardingCase(allocator, .init(.{ .x = 2, .y = 2 }), .init(.{ .b = 4, .h = 8, .w = 6 }, .i32), .{ .b = .x, .h = .y }, verbose); +//} + +// todo: temp +pub fn bufferTypeFromDtype(dt: DataType) pjrtx.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrtx.BufferType, @tagName(tag)), + }; +} + +pub fn dtypeFromBufferType(pjrt_type: pjrtx.BufferType) DataType { + return switch (pjrt_type) { + .invalid => @panic("Found an invalid pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; +} + +// todo: temp (zig deps and compilation story...) +var _platform: ?Platform = null; + +pub fn env(opts: Platform.CreateOptions) Platform { + if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); + if (_platform == null) { + var ctx = Context.init() catch unreachable; + _platform = ctx.autoPlatform(opts).withCompilationOptions(.{ + .xla_dump_to = "/tmp/zml/tests-partitioning/", + .sharding_enabled = true, + }); + } + + return _platform.?; +} diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 84343a63..0b3cda57 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -23,6 +23,7 @@ pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; pub const ExecuteError = ApiError; pub const Memory = pjrt.Memory; pub const Stream = pjrt.Stream; +pub const ShapeSpec = pjrt.ShapeSpec; const log = std.log.scoped(.zml); @@ -74,12 +75,16 @@ pub const Client = opaque { return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); } - fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { var bytecode: std.array_list.Managed(u8) = .init(allocator); defer bytecode.deinit(); - module.op().writeBytecodeWithConfig(bytecode.writer(), .{ .desiredEmitedVersion = 1 }) catch |err| { + var bytecode_adapter = bytecode.writer().adaptToNewApi(&.{}); + module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode_adapter.new_interface) catch |err| { log.err("failed to write module bytecode: {}", .{err}); - return err; + return switch (err) { + error.InvalidMlirBytecodeVersion => error.InvalidMlirBytecodeVersion, + error.WriteFailed => error.OutOfMemory, + }; }; var serialized_buffer: std.array_list.Managed(u8) = .init(allocator); @@ -87,14 +92,18 @@ pub const Client = opaque { const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { - break :blk dialects.stablehlo.stablehloGetSmallerVersion(requested_version, dialects.stablehlo.getCurrentVersion()); + break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); } - break :blk dialects.stablehlo.getMinimumVersion(); + break :blk dialects.stablehlo.minimumVersion(); }; - dialects.stablehlo.serializePortableArtifact(bytecode.items, stablehlo_version, serialized_buffer.writer()) catch |err| { + var serialized_adapter = serialized_buffer.writer().adaptToNewApi(&.{}); + dialects.stablehlo.serializePortableArtifact(bytecode.items, stablehlo_version, &serialized_adapter.new_interface) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); - return err; + return switch (err) { + error.InvalidMlirBytecodeVersion => error.InvalidMlirBytecodeVersion, + error.WriteFailed => error.OutOfMemory, + }; }; return @ptrCast(try self.inner().compile(api, .{ @@ -104,7 +113,7 @@ pub const Client = opaque { })); } - pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { return try asynk.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); } @@ -126,6 +135,11 @@ pub const Client = opaque { pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); } + + pub const CreateBuffersForAsyncHostToDeviceArgs = pjrt.Client.CreateBuffersForAsyncHostToDeviceArgs; + pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { + return @ptrCast(try self.inner().createBuffersForAsyncHostToDevice(api, args)); + } }; pub const Buffer = opaque { @@ -298,11 +312,11 @@ pub const AsyncHostToDeviceTransferManager = opaque { return @ptrCast(self.inner().device(api)); } - pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) usize { + pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!usize { return self.inner().bufferCount(api); } - pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) usize { + pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!usize { return self.inner().bufferSize(api, buffer_index); } diff --git a/zml/platform.zig b/zml/platform.zig index 0bd8a2a5..b59d49a5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,6 +3,8 @@ const std = @import("std"); const runtimes = @import("runtimes"); pub const Target = runtimes.Platform; const stdx = @import("stdx"); +const Mesh = @import("partitioning.zig").Mesh; +const Shape = @import("shape.zig").Shape; const pjrt = @import("pjrtx.zig"); @@ -10,6 +12,305 @@ const log = std.log.scoped(.zml); pub const available_targets = std.enums.values(Target); +/// Represents the physical arrangement of hardware devices. +/// It contains the devices ordered according to their physical topology and the shape of that topology. +pub const PhysicalMesh = struct { + allocator: std.mem.Allocator, + devices: []const *const pjrt.Device, + shape: Shape, + + pub fn deinit(self: *PhysicalMesh) void { + self.allocator.free(self.devices); + self.* = undefined; + } +}; + +// Optimal orderings for TPU collectives, from JAX. +const tpu_v5_lite_2x2_ring_order = [_]u8{ 0, 1, 3, 2 }; +const tpu_v5_lite_2x2x2_iota_order = [_]u8{ 0, 4, 2, 6, 1, 5, 3, 7 }; +// Same ring order for v5e tray, v6e tray, and v7x 2x4 tray +const tpu_tray_ring_order_8 = [_]u8{ 0, 1, 2, 3, 7, 6, 5, 4 }; + +/// Sorts devices primarily by their coordinates (lexicographically, reversed) and then by core_on_chip. +/// This matches JAX's z,y,x,core sorting for creating a physical mesh. +/// This function is allocation-free. +fn sortDevicesForPhysicalMesh(context: *const pjrt.Api, lhs_dev: *const pjrt.Device, rhs_dev: *const pjrt.Device) bool { + const api: *const pjrt.Api = context; + + const lhs_desc = lhs_dev.getDescription(api); + const rhs_desc = rhs_dev.getDescription(api); + + var lhs_coords: ?[]const i64 = null; + var lhs_core: i64 = 0; + for (lhs_desc.getAttributes(api)) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + lhs_coords = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + lhs_core = attr.inner.unnamed_0.int64_value; + } + } + + var rhs_coords: ?[]const i64 = null; + var rhs_core: i64 = 0; + for (rhs_desc.getAttributes(api)) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + rhs_coords = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + rhs_core = attr.inner.unnamed_0.int64_value; + } + } + + // TPUs are expected to have coordinates. + // If not, we can't sort them topologically, so we treat them as equal and let the original order prevail. + if (lhs_coords == null or rhs_coords == null) { + return false; + } + + const lhs_coords_slice = lhs_coords.?; + const rhs_coords_slice = rhs_coords.?; + + stdx.debug.assert(lhs_coords_slice.len == rhs_coords_slice.len, "Devices have different coordinate ranks", .{}); + var i = lhs_coords_slice.len; + while (i > 0) { + i -= 1; + if (lhs_coords_slice[i] != rhs_coords_slice[i]) { + return lhs_coords_slice[i] < rhs_coords_slice[i]; + } + } + + return lhs_core < rhs_core; +} + +/// Sorts devices topologically for consistent ordering across platforms. +fn getTopologicallySortedDevices( + allocator: std.mem.Allocator, + devices: []const *const pjrt.Device, + api: *const pjrt.Api, +) ![]*const pjrt.Device { + const sorted_devices = try allocator.dupe(*const pjrt.Device, devices); + std.sort.pdq(*const pjrt.Device, sorted_devices, api, sortDevicesForPhysicalMesh); + return sorted_devices; +} + +/// Creates a device mesh that arranges devices for optimal collective performance, +/// similar to `jax.experimental.mesh_utils.create_device_mesh`. +pub fn createDeviceMesh( + allocator: std.mem.Allocator, + logical_mesh: Mesh, + platform: Platform, +) ![]const *const pjrt.Device { + const api = platform.pjrt_api; + const devices = platform.getDevices(); + + const num_logical_devices: u64 = logical_mesh.topology.count(); + stdx.debug.assert(num_logical_devices <= devices.len, "Number of devices {} must be >= the product of logical_mesh_shape {f}", .{ devices.len, logical_mesh.topology }); + const active_devices = devices[0..@intCast(num_logical_devices)]; + + if (active_devices.len == 0) return &.{}; + + // Always sort devices topologically first to ensure a consistent base order + // for mapping logical devices to physical devices across all platforms. + const sequential_devices = try getTopologicallySortedDevices(allocator, active_devices, api); + + // For TPUs, we might apply an additional reordering for performance. + // For other platforms, we will use the topologically sorted list directly. + switch (platform.target) { + .tpu => { + // Keep the TPU-specific reordering logic, but operate on the sorted list. + const first_device_kind = active_devices[0].getDescription(api).getKind(api); + + if (std.mem.eql(u8, first_device_kind, "TPU v5 lite") or std.mem.eql(u8, first_device_kind, "TPU v6 lite")) { + if (try createTpuV5LiteMesh(allocator, logical_mesh, sequential_devices, api)) |ordered_devices| { + allocator.free(sequential_devices); // Free the intermediate sorted list + return ordered_devices; + } + } else if (std.mem.eql(u8, first_device_kind, "TPU7x")) { + if (try createTpu7xMesh(allocator, logical_mesh, sequential_devices, api)) |ordered_devices| { + allocator.free(sequential_devices); // Free the intermediate sorted list + return ordered_devices; + } + } + + // Fallback for other TPU kinds (e.g., v2, v3, v4, v5p). + log.warn("Using generic TPU physical mesh creation for {s}. For optimal performance, a specific handler may be needed.", .{first_device_kind}); + var physical_mesh = try getPhysicalTpuMesh(allocator, sequential_devices, api); + defer physical_mesh.deinit(); + return try createDeviceMeshForNdTorus(allocator, physical_mesh, logical_mesh); + }, + .cuda, .rocm, .cpu, .neuron => { + // For other platforms, the topologically sorted list is the final list. + return sequential_devices; + }, + } +} + +/// Creates a device mesh for TPU v5 lite and v6 lite, applying optimal orderings. +fn createTpuV5LiteMesh( + allocator: std.mem.Allocator, + logical_mesh: Mesh, + devices: []const *const pjrt.Device, + api: *const pjrt.Api, +) !?[]const *const pjrt.Device { + const sequential_devices = try allocator.dupe(*const pjrt.Device, devices); + defer allocator.free(sequential_devices); + std.sort.pdq(*const pjrt.Device, sequential_devices, api, sortDevicesForPhysicalMesh); + + const reordered_devices = try allocator.alloc(*const pjrt.Device, devices.len); + + const order: ?[]const u8 = switch (devices.len) { + 4 => &tpu_v5_lite_2x2_ring_order, + 8 => &tpu_v5_lite_2x2x2_iota_order, // JAX uses iota order for v5e 2x2x2 cubes + else => null, + }; + + if (order) |o| { + for (o, 0..) |old_idx, new_idx| { + reordered_devices[new_idx] = sequential_devices[old_idx]; + } + // Reshaping is conceptual; the flat ordered array is what matters for XLA's device_assignment. + _ = logical_mesh; + return reordered_devices; + } + + allocator.free(reordered_devices); + return null; +} + +/// Creates a device mesh for TPU v7x. +fn createTpu7xMesh( + allocator: std.mem.Allocator, + logical_mesh: Mesh, + devices: []const *const pjrt.Device, + api: *const pjrt.Api, +) !?[]const *const pjrt.Device { + const sequential_devices = try allocator.dupe(*const pjrt.Device, devices); + defer allocator.free(sequential_devices); + std.sort.pdq(*const pjrt.Device, sequential_devices, api, sortDevicesForPhysicalMesh); + + if (devices.len == 8) { + const reordered_devices = try allocator.alloc(*const pjrt.Device, devices.len); + for (tpu_tray_ring_order_8, 0..) |old_idx, new_idx| { + reordered_devices[new_idx] = sequential_devices[old_idx]; + } + _ = logical_mesh; + return reordered_devices; + } + + return null; +} + +/// Constructs a PhysicalMesh representing the N-D torus of a TPU slice. +fn getPhysicalTpuMesh(allocator: std.mem.Allocator, devices: []const *const pjrt.Device, api: *const pjrt.Api) !PhysicalMesh { + stdx.debug.assert(devices.len > 0, "Cannot create physical mesh from zero devices", .{}); + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + _ = arena; // autofix + + var min_coords: [3]i64 = .{ std.math.maxInt(i64), std.math.maxInt(i64), std.math.maxInt(i64) }; + var max_coords: [3]i64 = .{ std.math.minInt(i64), std.math.minInt(i64), std.math.minInt(i64) }; + var min_core: i64 = std.math.maxInt(i64); + var max_core: i64 = std.math.minInt(i64); + + for (devices) |device| { + const desc = device.getDescription(api); + const attrs = desc.getAttributes(api); + for (attrs) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + const coords = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + stdx.debug.assert(coords.len == 3, "Expected 3D coordinates for TPU device, got {any}", .{coords}); + for (0..3) |i| { + min_coords[i] = @min(min_coords[i], coords[i]); + max_coords[i] = @max(max_coords[i], coords[i]); + } + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + min_core = @min(min_core, attr.inner.unnamed_0.int64_value); + max_core = @max(max_core, attr.inner.unnamed_0.int64_value); + } + } + } + + const dims: [3]u64 = .{ + @intCast(max_coords[0] - min_coords[0] + 1), + @intCast(max_coords[1] - min_coords[1] + 1), + @intCast(max_coords[2] - min_coords[2] + 1), + }; + const cores_per_chip: u64 = @intCast(max_core - min_core + 1); + stdx.debug.assert(devices.len == dims[0] * dims[1] * dims[2] * cores_per_chip, "Device list does not form a contiguous cuboid.", .{}); + + var physical_shape: Shape = undefined; + var physical_dims: Shape.DimsArray = .{}; + if (cores_per_chip > 1) { + try physical_dims.appendSlice(&.{ @intCast(dims[0]), @intCast(dims[1]), @intCast(cores_per_chip) }); + physical_shape = Shape.init(physical_dims.constSlice(), .u8).withTags(.{ .x, .y, .c }); + } else { + try physical_dims.appendSlice(&.{ @intCast(dims[0]), @intCast(dims[1]), @intCast(dims[2]) }); + physical_shape = Shape.init(physical_dims.constSlice(), .u8).withTags(.{ .x, .y, .z }); + } + + const ordered_devices = try allocator.alloc(*const pjrt.Device, devices.len); + // @memset(ordered_devices, null); + + for (devices) |device| { + const desc = device.getDescription(api); + const attrs = desc.getAttributes(api); + var coords: [3]i64 = undefined; + var core_on_chip: i64 = 0; + for (attrs) |attr| { + if (std.mem.eql(u8, attr.name(), "coords")) { + const coords_slice = attr.inner.unnamed_0.int64_array_value[0..attr.inner.value_size]; + @memcpy(coords[0..coords_slice.len], coords_slice); + } else if (std.mem.eql(u8, attr.name(), "core_on_chip")) { + core_on_chip = attr.inner.unnamed_0.int64_value; + } + } + const x = @as(usize, @intCast(coords[0] - min_coords[0])); + const y = @as(usize, @intCast(coords[1] - min_coords[1])); + const z = @as(usize, @intCast(coords[2] - min_coords[2])); + const c = @as(usize, @intCast(core_on_chip - min_core)); + + const index = if (cores_per_chip > 1) (x * @as(usize, @intCast(dims[1])) * @as(usize, @intCast(cores_per_chip))) + (y * @as(usize, @intCast(cores_per_chip))) + c else (x * @as(usize, @intCast(dims[1])) * @as(usize, @intCast(dims[2]))) + (y * @as(usize, @intCast(dims[2]))) + z; + stdx.debug.assert(index < ordered_devices.len, "Index out of bounds", .{}); + // stdx.debug.assert(ordered_devices[index] == null, "Duplicate device position in physical mesh", .{}); + ordered_devices[index] = device; + } + + return PhysicalMesh{ + .allocator = allocator, + .devices = ordered_devices, + .shape = physical_shape, + }; +} + +/// Maps a logical mesh onto a physical N-D torus mesh. +fn createDeviceMeshForNdTorus( + allocator: std.mem.Allocator, + physical_mesh: PhysicalMesh, + logical_mesh: Mesh, +) ![]const *const pjrt.Device { + // This is a complex combinatorial problem. For now, we implement a simplified greedy approach + // that assumes logical axes map to single physical axes. A full implementation would explore + // combinations and splitting physical axes. + // TODO: Implement the full JAX axis assignment algorithm. + log.warn("Using simplified logical to physical mesh mapping. Performance may not be optimal.", .{}); + + var physical_axes = physical_mesh.shape._dims; + var logical_axes = logical_mesh.topology._dims; + std.sort.pdq(i64, physical_axes.slice(), {}, std.sort.desc(i64)); + std.sort.pdq(i64, logical_axes.slice(), {}, std.sort.desc(i64)); + + if (!std.mem.eql(i64, physical_axes.constSlice(), logical_axes.constSlice())) { + log.err("Physical mesh shape {any} cannot be mapped to logical mesh shape {any} with the current simplified algorithm.", .{ + physical_mesh.shape, + logical_mesh.topology, + }); + return error.MeshMappingNotSupported; + } + + // Since shapes are identical after sorting, a simple copy is sufficient for this simplified version. + return allocator.dupe(*const pjrt.Device, physical_mesh.devices); +} + pub const CompilationOptions = struct { xla_dump_to: ?[]const u8 = null, xla_dump_fusion_visualization: bool = false, @@ -113,7 +414,7 @@ const _CreateOptions = struct { /// "Best-Fit with Coalescing" algorithm bfc: Options, /// use cudaMallocAsync - @"async": Options, + async: Options, /// use raw cuMalloc platform, @@ -129,7 +430,7 @@ const _CreateOptions = struct { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); }, - .bfc, .@"async" => |opt| { + .bfc, .async => |opt| { values.appendAssumeCapacity(pjrt.NamedValue.from("allocator", self.allocator)); values.appendAssumeCapacity(pjrt.NamedValue.from("preallocate", opt.preallocate)); if (opt.memory_fraction > 0) { diff --git a/zml/shape.zig b/zml/shape.zig index bd5f2a22..b2d61975 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -11,28 +11,159 @@ const EnumLiteral = @TypeOf(.enum_literal); const log = std.log.scoped(.shape); test { - std.testing.refAllDecls(Shape); + testing.refAllDecls(Shape); } /// Represent the shape of a tensor. pub const Shape = struct { pub const MAX_RANK: u8 = 8; + pub const PartitionSpec = union(enum) { + axis: Tag, + open, + replicated, + unknown, + + pub fn init(comptime value: anytype) PartitionSpec { + const T = @TypeOf(value); + + if (T == PartitionSpec) { + return value; + } + + if (comptime T == EnumLiteral) { + const name = @tagName(value); + inline for (std.meta.fields(PartitionSpec)) |field| { + if (field.type == void and std.mem.eql(u8, field.name, name)) { + return @field(PartitionSpec, field.name); + } + } + } + + if (comptime isTagConvertible(T)) { + const tag_ = Shape.toTag(value); + + return .{ .axis = tag_ }; + } + + stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); + } + + test "PartitionSpec.init" { + const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; + try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); + + const spec_axis_open: PartitionSpec = .open; + try testing.expect(spec_axis_open.eql(.init(.open))); + } + + pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { + if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { + return false; + } + + switch (self) { + .axis => |t1| { + const t2 = other.axis; + return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); + }, + else => return true, + } + } + + test "PartitionSpec.eql" { + const spec_axis: PartitionSpec = .init(.a); + const spec_axis_b: PartitionSpec = .init(.b); + const spec_open: PartitionSpec = .open; + const spec_replicated: PartitionSpec = .replicated; + const spec_unknown: PartitionSpec = .unknown; + + try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); + try testing.expect(!spec_axis.eql(spec_axis_b)); + try testing.expect(!spec_axis.eql(.open)); + try testing.expect(!spec_axis.eql(.replicated)); + try testing.expect(!spec_axis.eql(.unknown)); + + try testing.expect(spec_open.eql(.open)); + try testing.expect(!spec_open.eql(spec_axis)); + try testing.expect(!spec_open.eql(.replicated)); + try testing.expect(!spec_open.eql(.unknown)); + + try testing.expect(spec_replicated.eql(.replicated)); + try testing.expect(!spec_replicated.eql(spec_axis)); + try testing.expect(!spec_replicated.eql(.open)); + try testing.expect(!spec_replicated.eql(.unknown)); + + try testing.expect(spec_unknown.eql(.unknown)); + try testing.expect(!spec_unknown.eql(spec_axis)); + try testing.expect(!spec_unknown.eql(.open)); + try testing.expect(!spec_unknown.eql(.replicated)); + } + + pub fn isClosed(self: PartitionSpec) bool { + return switch (self) { + .axis => true, + .open => false, + .replicated => true, + .unknown => false, + }; + } + + test isClosed { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.isClosed()); + + const spec_open: PartitionSpec = .open; + try testing.expect(!spec_open.isClosed()); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.isClosed()); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(!spec_unknown.isClosed()); + } + + pub fn toTag(self: PartitionSpec) Tag { + return switch (self) { + .axis => |t| return t, + .open => TagOpen, + .replicated => TagReplicated, + .unknown => TagUnknown, + }; + } + + test "PartitionSpec.toTag" { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); + + const spec_open: PartitionSpec = .open; + try testing.expect(spec_open.toTag() == TagOpen); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.toTag() == TagReplicated); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(spec_unknown.toTag() == TagUnknown); + } + }; + pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; const TagLast = "last".ptr; + const TagReplicated = "replicated".ptr; + const TagOpen = "open".ptr; pub const DimsArray = stdx.BoundedArray(i64, MAX_RANK); pub const TagsArray = stdx.BoundedArray(Tag, MAX_RANK); pub const AxesArray = stdx.BoundedArray(u3, MAX_RANK); - pub const ShardingInfo = @Vector(MAX_RANK, bool); + pub const PartitionArray = stdx.BoundedArray(PartitionSpec, MAX_RANK); const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** MAX_RANK }; _dtype: DataType, _dims: DimsArray = .{}, _tags: TagsArray = UnknownTags, - _sharding_info: ShardingInfo = @splat(false), + _partitioning: PartitionArray = .{}, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -129,6 +260,8 @@ pub const Shape = struct { pub fn init(dimz: anytype, dt: DataType) Shape { var res: Shape = .{ ._dtype = dt }; res._dims, res._tags = parseDimensions(dimz); + res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); + return res; } @@ -145,6 +278,7 @@ pub const Shape = struct { }; res._tags.append(TagUnknown) catch unreachable; } + res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -153,17 +287,17 @@ pub const Shape = struct { } pub fn rank(self: Shape) u4 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return @intCast(self._dims.len); } pub fn dim(self: Shape, ax: anytype) i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.get(self.axis(ax)); } pub fn dims(self: *const Shape) []const i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.constSlice(); } @@ -190,12 +324,12 @@ pub const Shape = struct { }; } - inline fn ensureDimsAndTagsAreSync(self: Shape) void { - stdx.debug.assert(self._dims.len == self._tags.len, "Tags and dims have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + inline fn ensureAttributesAreSync(self: Shape) void { + stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); } pub fn tag(self: Shape, ax: anytype) Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.get(self.axis(ax)); } @@ -215,7 +349,7 @@ pub const Shape = struct { } pub fn tags(self: *const Shape) []const Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.constSlice(); } @@ -255,7 +389,7 @@ pub const Shape = struct { } pub fn axis(self: Shape, axis_: anytype) u3 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axis_); if (comptime stdx.meta.isInteger(T)) { @@ -270,7 +404,7 @@ pub const Shape = struct { } pub fn axes(self: Shape, axes_: anytype) AxesArray { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axes_); @@ -300,7 +434,7 @@ pub const Shape = struct { fn axisFromInt(self: Shape, a: isize) u3 { const rk: i8 = self.rank(); if (a < -rk or a > rk) { - stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); + stdx.debug.panic("Tensor {f} doesn't have dimension: {}", .{ self, a }); } return if (a < 0) @intCast(a + rk) @@ -400,13 +534,16 @@ pub const Shape = struct { if (need_comma) try writer.writeByte(','); const t = self.tag(i); if (t != TagUnknown) { - try writer.print("{s}={d}", .{ t, d }); + try writer.print("{s}={}", .{ t, d }); } else { - try writer.print("{d}", .{d}); + try writer.print("{}", .{d}); } - if (self._sharding_info[i]) { - try writer.writeByte('!'); + + const part = self._partitioning.get(i); + if (part.toTag() != TagUnknown) { + try writer.print("/{s}", .{part.toTag()}); } + need_comma = true; } if (need_comma) try writer.writeByte(','); @@ -439,13 +576,14 @@ pub const Shape = struct { pub fn reshape(self: Shape, new_shape_: anytype) Shape { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); + new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); new_shape.inferMissingAxis(self.count()); - stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); + stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {d} to {d}", .{ self.dims(), new_shape.dims() }); return new_shape; } fn inferMissingAxis(self: *Shape, n_: usize) void { - stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); + stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {}", .{self.*}); const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; // We can't use `self.count()` yet cause we have negative dims. @@ -513,6 +651,11 @@ pub const Shape = struct { var res = self; res._dims.insertSlice(ax, dims_.constSlice()) catch unreachable; res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + for (0..dims_.len) |_| { + res._partitioning.insert(ax, .unknown) catch unreachable; + } + return res; } @@ -523,7 +666,7 @@ pub const Shape = struct { } pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { - stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); + stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {}, it's already at max rank.", .{self}); const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) self.rank() @@ -533,6 +676,8 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; + res._partitioning.insert(ax, .unknown) catch unreachable; + return res; } @@ -541,6 +686,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); + res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); return res; } @@ -562,6 +708,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); + res._partitioning.appendAssumeCapacity(.unknown); return res; } @@ -570,14 +717,15 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); + _ = res._partitioning.orderedRemove(a); return res; } pub const drop = remove; test remove { - try std.testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); - try std.testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); } pub fn removeMany(self: Shape, axes_: anytype) Shape { @@ -595,20 +743,22 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; + sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; + sh._partitioning.len = rk - to_remove.len; return sh; } test removeMany { - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{12}, Shape.init(.{ 10, 11, 12 }, .f32).removeMany(.{ 0, 1 }).dims(), ); - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{ 10, 11 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).removeMany(.{ -1, -2 }).dims(), @@ -619,10 +769,16 @@ pub const Shape = struct { std.debug.assert(self.rank() == permutations.len); const permutations_ = self.axes(permutations); var res = self; + const original_dims = self._dims; + const original_tags = self._tags; + const original_parts = self._partitioning; + for (permutations_.constSlice(), 0..) |permutation, i| { - res._dims.set(i, self.dim(permutation)); - res._tags.set(i, self.tag(permutation)); + res._dims.set(i, original_dims.get(permutation)); + res._tags.set(i, original_tags.get(permutation)); + res._partitioning.set(i, original_parts.get(permutation)); } + return res; } @@ -638,6 +794,10 @@ pub const Shape = struct { Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32).transpose(.{ 0, 2, 1, 3 }), ), ); + + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); } /// Tag each ax of this shape with tags from a tuple. @@ -659,7 +819,7 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, tagz }); + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {any}", .{ self, tagz }); inline for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -698,7 +858,7 @@ pub const Shape = struct { var res = self; if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {any}", .{ self, tagz }); + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {}, got {any}", .{ self, tagz }); for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -706,7 +866,7 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {}", .{ self, tagz }); + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {}, got {}", .{ self, tagz }); inline for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -746,16 +906,233 @@ pub const Shape = struct { return res; } - pub fn withSharding(self: Shape, axes_: anytype) Shape { + pub fn withPartitioning(self: Shape, specs: anytype) Shape { + const T = @TypeOf(specs); + + var res = self.withDefaultPartitioning(); // todo add test for this new change + + if (stdx.meta.isStruct(T)) { + inline for (std.meta.fields(T)) |field| { + const partition_axis = @field(specs, field.name); + const axis_ = res.axisFromTagMaybe(toTag(field)); + + if (axis_) |ax| { + res._partitioning.set(ax, PartitionSpec.init(partition_axis)); + } else { + stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); + } + } + } else { + stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + } + + // Check that no mesh axis is used to partition multiple tensor dimensions. + var used_mesh_axes: stdx.BoundedArray(Tag, MAX_RANK) = .{ .len = 0 }; + + for (res._partitioning.constSlice()) |spec| { + const axis_tag = spec.toTag(); + + if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { + stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); + } + + switch (spec) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), + } + } + + return res; + } + + test withPartitioning { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); + } + + pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { + const T = @TypeOf(mapping); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); + var res = self; - // Reset sharding. - res._sharding_info = @splat(false); - for (self.axes(axes_).constSlice()) |ax| { - res._sharding_info[ax] = true; + for (res._partitioning.slice()) |*spec| { + switch (spec.*) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => |*old_axis_tag| { + inline for (std.meta.fields(T)) |field| { + const logical_axis_in_map = toTag(field); + + if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { + const physical_axis_from_map = @field(mapping, field.name); + spec.* = PartitionSpec.init(physical_axis_from_map); + break; + } + } + }, + } } return res; } + test mapPartitioningAxes { + const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ + .batch = .data, + .seq = .replicated, + .features = .model, + }); + + const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); + + try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); + try testing.expect(physical_shape.partition(.features).eql(.init(.y))); + try testing.expect(physical_shape.partition(.seq).eql(.replicated)); + + const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); + try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); + try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); + } + + pub fn withDefaultPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withDefaultPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + } + + pub fn withReplicatedPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withReplicatedPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withReplicatedPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); + } + + pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec == .axis) return true; + } + return false; + } + + test hasAtLeastOnePartitionedAxis { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + } + + pub fn isFullyPartitioned(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + switch (spec) { + .replicated, .open, .unknown => return false, + .axis => {}, + } + } + return true; + } + + test isFullyPartitioned { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); + try testing.expect(shape.isFullyPartitioned()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyPartitioned()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.isFullyPartitioned()); + } + + pub fn isFullyReplicated(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec != .replicated and spec != .unknown) { + return false; + } + } + return true; + } + + test isFullyReplicated { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyReplicated()); + } + + pub fn partition(self: Shape, ax: anytype) PartitionSpec { + return self._partitioning.get(self.axis(ax)); + } + + test partition { + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); + try testing.expectEqual(.open, shape.partition(.b)); + try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); + } + + pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { + for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { + if (dim_part.eql(part)) return true; + } + return false; + } + + test containsPartitionSpec { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.open)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(shape.containsPartitionSpec(.unknown)); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.open)); + } + /// Renames some of the tags in this shape. /// Shape.init(.{ .a = 10, .b = 20 }).rename(.{ .b = .batch }); // .{ .a = 10, .batch = 20 }; pub fn rename(self: Shape, renames: anytype) Shape { @@ -764,7 +1141,7 @@ pub const Shape = struct { var res = self; inline for (std.meta.fields(T)) |field| { const new_field = @field(renames, field.name); - stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); + stdx.debug.assert(self.hasTag(new_field) == null, "{}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); res._tags.set(self.axis(field), toTag(new_field)); } return res; @@ -784,7 +1161,103 @@ pub const Shape = struct { } } - pub fn computeStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + /// Computes the strides of the shape in units of elements. + /// For a shape {d0, d1, d2}, the strides would be {d1*d2, d2, 1}. + pub fn computeElementStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + return self.computeStrides(1); + } + + test computeElementStrides { + { + const s = Shape.init(.{ 4, 1024 }, .bf16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 2, 3, 4 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12, 4, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .i16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 6, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{10}, .u8); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{1}, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1, 5 }, .i32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 5, 5, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ .n = 8, .c = 3, .h = 224, .w = 224 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 3 * 224 * 224, 224 * 224, 224, 1 }, strides.constSlice()); + } + } + + /// Computes the strides of the shape in units of bytes. + /// For a shape {d0, d1, d2} of type T, the strides would be + /// {(d1*d2)*sizeof(T), d2*sizeof(T), 1*sizeof(T)}. + pub fn computeByteStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + return self.computeStrides(self.dtype().sizeOf()); + } + + test computeByteStrides { + { + const s = Shape.init(.{ 2, 3, 4 }, .f16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12 * 2, 4 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1024 }, .bf16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .c64); // 8 bytes (f32 + f32) + const strides = s.computeByteStrides(); + try std.testing.expectEqual(@sizeOf(std.math.Complex(f32)), 8); + try std.testing.expectEqualSlices(i64, &.{ 6 * 8, 1 * 8 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 128, 512 }, .i8); // 1 byte + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 512, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 1, 4096 }, .u32); // 4 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 4096 * 4, 1 * 4 }, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); // 8 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + } + + fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, MAX_RANK) { const rk = self.rank(); var strides: stdx.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; if (rk == 0) return strides; @@ -795,7 +1268,7 @@ pub const Shape = struct { // and the element size in bytes. var d: V = @bitCast(self._dims.buffer); d = @select(i64, rank_mask, d, @as(V, @splat(1))); - d = std.simd.shiftElementsLeft(d, 1, self.dtype().sizeOf()); + d = std.simd.shiftElementsLeft(d, 1, element_size); d = std.simd.prefixScan(.Mul, -1, d); strides.buffer = @bitCast(d); @@ -854,6 +1327,12 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; + _ = new_shape._partitioning.orderedRemove(ax); + + for (0..dims_.len) |_| { + new_shape._partitioning.insert(ax, .unknown) catch unreachable; + } + new_shape.inferMissingAxis(self.count()); return new_shape; } @@ -898,22 +1377,43 @@ pub const Shape = struct { /// Merge the given axes into one axis. /// eg: `Shape.init(.{.a1 = 5, .a2 = 2, .b = 3}).merge(.{ .a = .{ .a1, .a2 }); -> .{ .a = 10, .b = 3 }` pub fn mergeAxis(self: Shape, axis_: anytype, axes_: anytype) Shape { - const axes__ = self.axes(axes_); - - const first_axis = axes__.get(0); - const last_axis = axes__.get(axes__.len - 1); + const axes_to_merge = self.axes(axes_); + stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); var new_dim: i64 = 1; - for (axes__.constSlice(), first_axis..) |ax, counter| { - new_dim *= self.dim(ax); - stdx.debug.assert(ax == counter, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); + // Merging partitioned axes is complex. If ANY axis being merged is partitioned, + // the resulting sharding is ambiguous. A propagation pass must resolve this. + // Marking it 'open' is the correct signal. If all are replicated/unknown, the + // result is also replicated/unknown. + // rule: .replicated wins over .unknown. .open wins over everything. + var resulting_partition_spec: PartitionSpec = .unknown; + + for (0..axes_to_merge.len) |i| { + const current_axis = axes_to_merge.get(i); + new_dim *= self.dim(current_axis); + + if (i > 0) { + stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {} along non-contiguous axes {any}", .{ self, axes_ }); + } + + switch (self.partition(current_axis)) { + .axis, .open => resulting_partition_spec = .open, + .replicated => if (resulting_partition_spec != .open) { + resulting_partition_spec = .replicated; + }, + .unknown => {}, + } } + const first_axis_to_merge = axes_to_merge.get(0); + const num_axes_to_merge = axes_to_merge.len; + var new_shape = self; - new_shape._dims.set(first_axis, new_dim); - new_shape._dims.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.dims()[last_axis + 1 ..]) catch unreachable; - new_shape._tags.set(first_axis, if (comptime isTagConvertible(@TypeOf(axis_))) toTag(axis_) else TagUnknown); - new_shape._tags.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.tags()[last_axis + 1 ..]) catch unreachable; + + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); + new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); + new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); + return new_shape; } @@ -945,6 +1445,28 @@ pub const Shape = struct { ); } + test "mergeAxis partitioning" { + // Merging a partitioned axis with a replicated one -> open + const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); + const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged1.partition(.ab).eql(.open)); + try testing.expect(merged1.dim(.ab) == 10); + try testing.expect(merged1.rank() == 2); + + // Merging two replicated axes -> replicated + const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); + const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged2.partition(.ab).eql(.replicated)); + try testing.expect(merged2.partition(.c).eql(.init(.y))); + + // Merging two unknown axes -> unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown + const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged3.partition(.ab).eql(.unknown)); + } + pub fn mergeAxes(self: Shape, axes_: anytype) Shape { const T = @TypeOf(axes_); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); @@ -990,10 +1512,10 @@ pub const Shape = struct { return res; } - pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, MAX_RANK), TagsArray } { + pub fn parseStruct(T: type, v: anytype) struct { std.BoundedArray(T, MAX_RANK), TagsArray } { const V = @TypeOf(v); - var vals_: stdx.BoundedArray(T, MAX_RANK) = .{}; + var vals_: std.BoundedArray(T, MAX_RANK) = .{}; var tags_: TagsArray = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { @@ -1028,10 +1550,10 @@ pub const Shape = struct { } /// Parses a struct literal into a list of options for each axes. - pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, MAX_RANK) { + pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) std.BoundedArray(T, MAX_RANK) { const V = @TypeOf(options); - var res: stdx.BoundedArray(T, MAX_RANK) = .{}; + var res: std.BoundedArray(T, MAX_RANK) = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); for (options) |d| { @@ -1063,17 +1585,17 @@ pub const Shape = struct { test "comptimeShape" { comptime { const s = Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } // comptime only the shape { const s = comptime Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } } @@ -1083,7 +1605,7 @@ pub const Shape = struct { for (0..other.rank()) |ax| { if (other.tag(ax) != Shape.TagUnknown) { if (self.hasTag(other.tag(ax))) |batching_ax| { - stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({f}, {f})", .{ self, other }); + stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({}, {})", .{ self, other }); batching_axes += 1; } } @@ -1092,4 +1614,135 @@ pub const Shape = struct { } return res_shape; } + + pub fn iterator(self: Shape) MultiDimIterator { + return .{ + .shape = self, + .current_coords = .{0} ** Shape.MAX_RANK, + .flat_index = 0, + }; + } }; + +/// Iterates over all coordinates of a shape, yielding the multi-dimensional +/// coordinates and the corresponding flat index for each element. +pub const MultiDimIterator = struct { + shape: Shape, + current_coords: [Shape.MAX_RANK]i64, + flat_index: usize, + + pub const Item = struct { + coords: [Shape.MAX_RANK]i64, + rank: u4, + flat_index: usize, + }; + + pub fn next(self: *MultiDimIterator) ?Item { + if (self.flat_index >= self.shape.count()) { + return null; + } + + const rank = self.shape.rank(); + const item = Item{ + .coords = self.current_coords, + .rank = rank, + .flat_index = self.flat_index, + }; + + self.flat_index += 1; + + var i: usize = rank; + while (i > 0) { + i -= 1; + self.current_coords[i] += 1; + if (self.current_coords[i] < self.shape.dim(i)) { + return item; + } + self.current_coords[i] = 0; + } + + return item; + } +}; + +test "MultiDimIterator / Scalar" { + const shape = Shape.scalar(.f32); + var iter = shape.iterator(); + const item = iter.next().?; + + try std.testing.expectEqual(0, item.rank); + try std.testing.expectEqual(0, item.flat_index); + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / Empty" { + const shape = Shape.init(.{ .x = 5, .y = 0, .z = 2 }, .f32); + var iter = shape.iterator(); + + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / 1D shape" { + const shape = Shape.init(.{5}, .f32); + var iter = shape.iterator(); + var count: i64 = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.coords[0]); + } + try std.testing.expectEqual(5, count); +} + +test "MultiDimIterator / 2D shape" { + const shape = Shape.init(.{ .rows = 2, .cols = 3 }, .f32); + var iter = shape.iterator(); + var items = std.ArrayList(MultiDimIterator.Item).init(std.testing.allocator); + defer items.deinit(); + + while (iter.next()) |item| { + try items.append(item); + } + + try std.testing.expectEqual(6, items.items.len); + try std.testing.expectEqualSlices(i64, &.{ 0, 0 }, items.items[0].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 1 }, items.items[1].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 2 }, items.items[2].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 0 }, items.items[3].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 1 }, items.items[4].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 2 }, items.items[5].coords[0..2]); + try std.testing.expectEqual(5, items.items[5].flat_index); +} + +test "MultiDimIterator / 3D shape" { + const shape = Shape.init(.{ .d = 2, .h = 2, .w = 2 }, .f32); + var iter = shape.iterator(); + const expected_coords = [_][3]i64{ + .{ 0, 0, 0 }, + .{ 0, 0, 1 }, + .{ 0, 1, 0 }, + .{ 0, 1, 1 }, + .{ 1, 0, 0 }, + .{ 1, 0, 1 }, + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + }; + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.flat_index); + try std.testing.expectEqualSlices(i64, &expected_coords[count], item.coords[0..3]); + } + + try std.testing.expectEqual(8, count); +} + +test "MultiDimIterator / Shape with dimension of size 1" { + const shape = Shape.init(.{ .a = 2, .b = 1, .c = 3 }, .f32); + var iter = shape.iterator(); + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(0, item.coords[1]); + } + try std.testing.expectEqual(6, count); +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 26f670d7..2dbfe9a2 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -42,7 +42,7 @@ pub const Tensor = struct { _output_memory_kind: Memory = .device, pub const _Donation = union(enum) { no_buffer, input_buffer, arg: u16 }; - pub const _Id = union(enum) { mlir: mlir.Value, buffer_id: u64, arg_id: u64 }; + pub const _Id = union(enum) { mlir: *const mlir.Value, buffer_id: u64, arg_id: u64 }; pub const MAX_RANK = Shape.MAX_RANK; /// Returns the current compilation context. diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 06bcd6fa..70d787ff 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -2,11 +2,32 @@ const builtin = @import("builtin"); const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { - .macos => MacOsTracer, + .macos => FakeTracer, .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, else => FakeTracer, }; +var _tracer: Tracer = undefined; +var _tracer_initialized: bool = false; + +pub fn tracer() *Tracer { + if (!_tracer_initialized) { + unreachable; + } + return &_tracer; +} + +pub fn initTracer() void { + if (_tracer_initialized) return; + _tracer = Tracer.init("tracer"); + _tracer_initialized = true; +} + +pub fn deinitTracer() void { + if (!_tracer_initialized) return; + _tracer.deinit(); +} + const CudaTracer = struct { // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so diff --git a/zml/zml.zig b/zml/zml.zig index b5cd8039..db6da357 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -42,9 +42,14 @@ pub const Target = @import("platform.zig").Target; pub const Tensor = @import("tensor.zig").Tensor; pub const testing = @import("testing.zig"); pub const torch = @import("torch.zig"); +pub const upb = @import("upb"); +pub const c = @import("c"); +pub const partitioning = @import("partitioning.zig"); +pub const io = @import("io.zig"); // pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { + pub const tracer = @import("tools/tracer.zig"); pub const Tracer = @import("tools/tracer.zig").Tracer; };