From 428836eb69e65b16cf02644b72a1ffc7aec815fa Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 11 Sep 2025 10:56:54 +0200 Subject: [PATCH 01/21] remove last `writer: anytype` --- async/coro.zig | 26 ++--- mlir/dialects/stablehlo/stablehlo.zig | 6 +- mlir/mlir.zig | 109 +++++++++++--------- pjrt/ffi.zig | 12 +-- pjrt/pjrt.zig | 6 +- zml/aio.zig | 9 +- zml/aio/torch/py.zig | 15 +-- zml/buffer.zig | 5 +- zml/exe.zig | 6 +- zml/hostbuffer.zig | 5 +- zml/mlirx.zig | 2 +- zml/module.zig | 42 ++++---- zml/nn.zig | 1 + zml/pjrtx.zig | 15 +-- zml/shape.zig | 12 +-- zml/tensor.zig | 16 +-- zml/tokenizer/hftokenizers/hftokenizers.zig | 3 +- 17 files changed, 130 insertions(+), 160 deletions(-) diff --git a/async/coro.zig b/async/coro.zig index a4d0345e..67042967 100644 --- a/async/coro.zig +++ b/async/coro.zig @@ -159,26 +159,21 @@ const Coro = struct { fn runcoro(from: *base.Coro, this: *base.Coro) callconv(.c) noreturn { const from_coro: *Coro = @fieldParentPtr("impl", from); const this_coro: *Coro = @fieldParentPtr("impl", this); - log(.debug, "coro start {any}", .{this_coro.id}); + log(.debug, "coro start {f}", .{this_coro.*}); @call(.auto, this_coro.func, .{}); this_coro.status = .Done; thread_state.switchOut(from_coro); // Never returns - stdx.debug.panic("Cannot resume an already completed coroutine {any}", .{this_coro.id}); + stdx.debug.panic("Cannot resume an already completed coroutine {f}", .{this_coro.*}); } pub fn getStorage(self: Coro, comptime T: type) *T { return @ptrCast(@alignCast(self.storage)); } - pub fn format(self: Coro, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { - _ = fmt; - _ = options; - try writer.print("Coro{{.id = {any}, .status = {s}}}", .{ - self.id, - @tagName(self.status), - }); + pub fn format(self: Coro, writer: *std.Io.Writer) !void { + try writer.print("Coro{{.id = {any}, .status = {t}}}", .{ self.id, self.status }); } }; @@ -292,7 +287,7 @@ const ThreadState = struct { /// Called from resume fn switchIn(self: *ThreadState, target: Frame) void { - log(.debug, "coro resume {any} from {any}", .{ target.id, self.current().id }); + log(.debug, "coro resume {f} from {f}", .{ target.id, self.current().id }); // Switch to target, setting this coro as the resumer. self.switchTo(target, true); @@ -307,7 +302,7 @@ const ThreadState = struct { /// Called from suspend fn switchOut(self: *ThreadState, target: Frame) void { - log(.debug, "coro suspend {any} to {any}", .{ self.current().id, target.id }); + log(.debug, "coro suspend {f} to {f}", .{ self.current().id, target.id }); self.switchTo(target, false); } @@ -384,13 +379,8 @@ const CoroId = struct { self.invocation += 1; } - pub fn format(self: @This(), comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { - _ = fmt; - _ = options; - try writer.print("CoroId{{.cid={d}, .i={d}}}", .{ - self.id.coro, - self.invocation, - }); + pub fn format(self: @This(), writer: *std.Io.Writer) !void { + try writer.print("CoroId{{.cid={d}, .i={d}}}", .{ self.id.coro, self.invocation }); } }; }; diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 0c60fd01..39a9fb16 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -750,13 +750,13 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa if (@intFromEnum(opts.api_version) < @intFromEnum(CustomCallOpts.ApiVersion.typed_ffi)) { stdx.debug.assert( backend_config.isA(mlir.StringAttribute), - "API version < 4 requires a string as backend_config, got {}", + "API version < 4 requires a string as backend_config, got {f}", .{backend_config}, ); } else { stdx.debug.assert( backend_config.isA(mlir.DictionaryAttribute), - "API version >= 4 requires a dictionary as backend_config, got {}", + "API version >= 4 requires a dictionary as backend_config, got {f}", .{backend_config}, ); } @@ -1353,7 +1353,7 @@ pub fn getMinimumVersion() []const u8 { return state.str; } -pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: anytype) !void { +pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: *std.Io.Writer) !void { var context = .{ .writer = writer }; const WriterContext = @TypeOf(context); diff --git a/mlir/mlir.zig b/mlir/mlir.zig index a6721914..310eb299 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -881,7 +881,7 @@ pub const Operation = struct { return .{ ._inner = c.mlirOperationGetContext(self._inner) }; } - pub fn writeBytecode(self: Self, writer: anytype) void { + pub fn writeBytecode(self: Self, writer: *std.Io.Writer) void { var writer_context = .{ .writer = writer }; const WriterContext = @TypeOf(writer_context); @@ -897,7 +897,7 @@ pub const Operation = struct { ); } - pub fn writeBytecodeWithConfig(self: Self, writer: anytype, config: struct { + pub fn writeBytecodeWithConfig(self: Self, writer: *std.Io.Writer, config: struct { desiredEmitedVersion: ?i64 = null, }) !void { const cfg = c.mlirBytecodeWriterConfigCreate(); @@ -906,27 +906,14 @@ pub const Operation = struct { c.mlirBytecodeWriterConfigDesiredEmitVersion(cfg, v); } - const WriterContext = struct { - writer: @TypeOf(writer), - write_error: ?@TypeOf(writer).Error = null, - }; - var writer_context: WriterContext = .{ .writer = writer }; - + var writer_with_err: WriterWithErr = .{ .writer = writer }; try successOr(c.mlirOperationWriteBytecodeWithConfig( self._inner, cfg, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch |err| { - inner_writer_context.write_error = err; - }; - } - }).callback, - &writer_context, + &WriterWithErr.printCallback, + &writer_with_err, ), error.InvalidMlirBytecodeVersion); - - if (writer_context.write_error) |err| return err; + return writer_with_err.check(); } /// Enable a full dump of the IR. @@ -939,26 +926,33 @@ pub const Operation = struct { op: Operation, flags: OpPrintingFlags, - pub fn format(self: @This(), writer: anytype) !void { - self.op.print(writer, self.flags); + pub fn format(self: @This(), writer: *std.Io.Writer) !void { + try self.op.print(writer, self.flags); } }; - pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) void { + pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) !void { const pflags = flags.create(); defer c.mlirOpPrintingFlagsDestroy(pflags); + var writer_err: WriterWithErr = .{ .writer = writer }; + c.mlirOperationPrintWithFlags( self._inner, pflags, (struct { pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const _writer: *std.Io.Writer = @ptrCast(@alignCast(ctx_)); - _writer.writeAll(str.data[0..str.length]) catch @panic("Mlir print failed"); + const _writer_err: *WriterWithErr = @ptrCast(@alignCast(ctx_)); + _writer_err.writer.writeAll(str.data[0..str.length]) catch |err| { + _writer_err.err = err; + }; } }).callback, - writer, + &writer_err, ); + + if (writer_err.err) |err| return err; + try writer_err.writer.flush(); } pub fn verify(self: Self) bool { @@ -1121,7 +1115,7 @@ pub const Value = struct { pub const dump = helpers.dump(Value, c.mlirValueDump); pub const eql = helpers.eql(Value, c.mlirValueEqual); - pub const format = helpers.format(Value, c.mlirValuePrint).format; + pub const format = helpers.format(Value, c.mlirValuePrint); pub const wrapOr = helpers.wrapOr(Value, c.mlirValueIsNull); pub fn getType(val: Value) Type { @@ -1183,7 +1177,7 @@ pub const BlockArgument = struct { return @bitCast(c.mlirBlockArgumentGetArgNumber(arg._inner)); } - pub fn format(self: BlockArgument, writer: anytype) !void { + pub fn format(self: BlockArgument, writer: *std.Io.Writer) !void { const value = Value{ ._inner = self._inner }; return value.format(writer); } @@ -1232,7 +1226,7 @@ pub const Type = struct { pub fn formatAny(SpecificType: type) fn (SpecificType, SpecificType) type { return struct { - pub fn format(self: SpecificType, writer: anytype) !void { + pub fn format(self: SpecificType, writer: *std.Io.Writer) !void { return try Type.format(self.asType(), writer); } }; @@ -1280,7 +1274,7 @@ pub const IndexType = struct { pub const asType = Type.fromAny(IndexType); pub const eql = Type.eqlAny(IndexType); - pub const format = Type.formatAny(IndexType).format; + pub const format = Type.formatAny(IndexType); pub fn init(ctx: Context) IndexType { return .{ ._inner = c.mlirIndexTypeGet(ctx._inner) }; @@ -1452,7 +1446,7 @@ pub fn ComplexType(comptime ct: ComplexTypes) type { pub const asType = Type.fromAny(Complex); pub const eql = Type.eqlAny(Complex); - pub const format = Type.formatAny(Complex).format; + pub const format = Type.formatAny(Complex); pub const ComplexTypeType: ComplexTypes = ct; pub const init = if (ct != .unknown) struct { @@ -1736,27 +1730,15 @@ pub const helpers = struct { }.isNull; } - pub fn format(Any: type, print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void) type { + pub fn format( + Any: type, + print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void, + ) fn (Any, *std.Io.Writer) std.Io.Writer.Error!void { return struct { - pub fn format(self: Any, writer: *std.Io.Writer) !void { - const WriterWithErr = struct { - writer: *std.Io.Writer, - err: ?std.Io.Writer.Error = null, - fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { - var ctx: *@This() = @ptrCast(@alignCast(opaque_ctx)); - if (ctx.err) |_| return; - _ = ctx.writer.write(mlir_str.data[0..mlir_str.length]) catch |err| { - ctx.err = err; - return; - }; - } - }; - - var context: WriterWithErr = .{ .writer = writer }; - print_fn(self._inner, &WriterWithErr.printCallback, &context); - if (context.err) |err| return err; + pub fn format(self: Any, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try callPrintFn(Any, self, print_fn, writer); } - }; + }.format; } pub fn wrapOr(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (@FieldType(T, "_inner")) ?T { @@ -1773,3 +1755,32 @@ pub const helpers = struct { return .{ ._inner = inner }; } }; + +pub fn callPrintFn( + T: type, + value: T, + print_fn: fn (@FieldType(T, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void, + writer: *std.Io.Writer, +) std.Io.Writer.Error!void { + var writer_with_err: WriterWithErr = .{ .writer = writer }; + print_fn(value._inner, &WriterWithErr.printCallback, &writer_with_err); + return writer_with_err.check(); +} + +const WriterWithErr = struct { + writer: *std.Io.Writer, + err: ?std.Io.Writer.Error = null, + + fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { + var ctx: *WriterWithErr = @ptrCast(@alignCast(opaque_ctx)); + if (ctx.err) |_| return; + _ = ctx.writer.write(mlir_str.data[0..mlir_str.length]) catch |err| { + ctx.err = err; + return; + }; + } + + fn check(self: WriterWithErr) std.Io.Writer.Error!void { + if (self.err) |err| return err; + } +}; diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 2ca639a7..6a49294e 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -265,16 +265,8 @@ pub const Buffer = extern struct { return self._dims[0..self.rank]; } - pub fn format( - buffer: Buffer, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; - - try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data) }); + pub fn format(buffer: Buffer, writer: *std.Io.Writer) !void { + try writer.print("FfiBuffer({any}, .{t})@0x{x}", .{ buffer.dims(), buffer.dtype, @intFromPtr(buffer.data) }); } }; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 08ccf217..fccf31f1 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1259,12 +1259,8 @@ pub const NamedValue = extern struct { pub fn format( self: NamedValue, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, + writer: *std.Io.Writer, ) !void { - _ = fmt; - _ = options; try writer.print("{s}{{ .name = {s},", .{ @typeName(NamedValue), self.inner.name[0..self.inner.name_size] }); const u = self.inner.unnamed_0; switch (self.kind()) { diff --git a/zml/aio.zig b/zml/aio.zig index a778a627..a705e498 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -315,14 +315,7 @@ pub const Metadata = union(enum) { }; } - pub fn format( - self: Metadata, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; + pub fn format(self: Metadata, writer: *std.Io.Writer) !void { switch (self) { .null => _ = try writer.write("null"), inline .bool, .array_bool => |b| try writer.print("{any}", .{b}), diff --git a/zml/aio/torch/py.zig b/zml/aio/torch/py.zig index c523f280..539c28a0 100644 --- a/zml/aio/torch/py.zig +++ b/zml/aio/torch/py.zig @@ -1,9 +1,10 @@ const std = @import("std"); const math = std.math; -const log = std.log.scoped(.@"zml/aio"); const pickle = @import("pickle.zig"); +const log = std.log.scoped(.@"zml/aio"); + /// Correspond to a function/constructor call pub const Object = struct { member: Any, @@ -206,12 +207,16 @@ pub const Any = union(Kind) { self.* = undefined; } - inline fn writeIndents(indents: usize, writer: anytype) !void { + inline fn writeIndents(indents: usize, writer: *std.Io.Writer) !void { try writer.writeBytesNTimes(" ", indents); // resolve tab = 2 spaces // try writer.writeByteNTimes('\t'); } - fn internalFormat(value: Any, indents: usize, writer: anytype) !void { + pub fn format(self: Any, writer: *std.Io.Writer) !void { + return internalFormat(self, 0, writer); + } + + fn internalFormat(value: Any, indents: usize, writer: *std.Io.Writer) !void { try writeIndents(indents, writer); try writer.writeAll(".{\n"); try writeIndents(indents + 1, writer); @@ -303,10 +308,6 @@ pub const Any = union(Kind) { try writer.writeByte('}'); } - pub fn format(self: Any, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { - return internalFormat(self, 0, writer); - } - pub fn clone(self: Any, allocator: std.mem.Allocator) !Any { return switch (self) { inline .raw, .raw_num => |v, tag| @unionInit(Any, @tagName(tag), try v.clone(allocator)), diff --git a/zml/buffer.zig b/zml/buffer.zig index f502d703..b356b9b7 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -384,10 +384,7 @@ pub const Buffer = struct { return res; } - pub fn format( - self: Buffer, - writer: anytype, - ) !void { + pub fn format(self: Buffer, writer: *std.Io.Writer) !void { try writer.print("Buffer({f})", .{self._shape}); } diff --git a/zml/exe.zig b/zml/exe.zig index dffc16c2..f3df0c2e 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -95,6 +95,8 @@ pub fn compileFn( ) !FnExe(func) { var pretty_name = try prettyFnName(func, allocator); defer pretty_name.deinit(allocator); + log.info("Compiling {s} with {f}", .{ pretty_name.items, stdx.fmt.any(args) }); + var context = try CompilationContext.init(allocator, pretty_name.items, platform); defer context.deinit(); @@ -306,7 +308,7 @@ pub const BaseExe = struct { } } - pub fn serialize(self: BaseExe, writer: anytype) !void { + pub fn serialize(self: BaseExe, writer: *std.Io.Writer) !void { var executable = try self.exe.getExecutable(self.platform.pjrt_api); var serialize_result = try executable.serialize(self.platform.pjrt_api); defer serialize_result.deinit(); @@ -377,7 +379,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { try self.inner.bind(T, value); } - pub fn serialize(self: Self, writer: anytype) !void { + pub fn serialize(self: Self, writer: *std.Io.Writer) !void { return try self.inner.serialize(writer); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 848fee86..88419b7d 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -321,10 +321,7 @@ pub const HostBuffer = struct { }; } - pub fn format( - self: HostBuffer, - writer: anytype, - ) !void { + pub fn format(self: HostBuffer, writer: *std.Io.Writer) !void { try writer.print("HostBuffer(.{f})", .{self._shape}); } diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 28f40383..f50a2efa 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -96,6 +96,6 @@ pub const Type = struct { } } - std.debug.panic("Could not convert mlir.Type to DataType: {}", .{mlir_type}); + std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); } }; diff --git a/zml/module.zig b/zml/module.zig index 1b63ac7f..eae9b361 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -187,7 +187,7 @@ pub const CompilationContext = struct { if (cache_dir.createFile(mlir_name, .{ .truncate = true })) |file| { var write_buf: [4096]u8 = undefined; var writer = file.writer(&write_buf); - module.op().print(&writer.interface, .{ .debug_info = true, .debug_info_pretty_form = false }); + try module.op().print(&writer.interface, .{ .debug_info = true, .debug_info_pretty_form = false }); log.info("Wrote MLIR to {s}/{s}", .{ module_dir.?, mlir_name }); } else |_| { log.warn("Failed to open {s}", .{mlir_name}); @@ -506,9 +506,9 @@ pub const CompilationContext = struct { var tensor_args = .{ model, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1234 } }, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1235 } } }; const f = try comp.emitMlir(Local._fwd, &tensor_args, .{ .name = "test.emitMlir.Local.forward", .kind = .main }); - var mlir_bytecode = std.array_list.Managed(u8).init(std.testing.allocator); - defer mlir_bytecode.deinit(); - try mlir_bytecode.writer().print("{f}", .{f.mlir_fn.mlirFormatter(.{})}); + var mlir_code: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer mlir_code.deinit(); + try f.mlir_fn.print(&mlir_code.writer, .{}); // Check that the `x` input argument gives its buffer to the result tensor. // `%arg0` is the bias of the model, `%arg1` is `x`, `%arg2` is `y`. @@ -518,8 +518,8 @@ pub const CompilationContext = struct { var buf = template.*; for (0..2) |i| { const alias_attr = std.fmt.bufPrint(&buf, template, .{i}) catch unreachable; - std.testing.expect(std.mem.indexOf(u8, mlir_bytecode.items, alias_attr) != null) catch |err| { - log.warn("Didn't produced the expected IR:\n{s}", .{mlir_bytecode.items}); + std.testing.expect(std.mem.indexOf(u8, mlir_code.written(), alias_attr) != null) catch |err| { + log.warn("Didn't produced the expected IR:\n{s}", .{mlir_code.written()}); return err; }; } @@ -547,12 +547,14 @@ pub const CompilationContext = struct { pub fn getShardingAttr(self: CompilationContext, shape: Shape) mlir.Attribute { const ctx = self.mlirCtx(); const num_partitions = self.numPartitions(); - var sharding_str: stdx.BoundedArray(u8, 128) = .{}; - writeShardingRepresentation(shape, num_partitions, sharding_str.writer()) catch unreachable; - return mlir.Attribute.string(ctx, sharding_str.constSlice()); + // This is big enough, see test below for examples values + var sharding_str_buf: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&sharding_str_buf); + writeShardingRepresentation(shape, num_partitions, &writer) catch unreachable; + return mlir.Attribute.string(ctx, writer.buffered()); } - fn writeShardingRepresentation(shape: Shape, num_partitions: u8, writer: anytype) @TypeOf(writer).Error!void { + fn writeShardingRepresentation(shape: Shape, num_partitions: u8, writer: *std.Io.Writer) std.Io.Writer.Error!void { const n_sharded: u8 = @popCount(@as(u8, @bitCast(shape._sharding_info))); if (n_sharded == 0 or num_partitions == 1) { try writer.writeAll("{replicated}"); @@ -567,26 +569,26 @@ pub const CompilationContext = struct { } test writeShardingRepresentation { - var rule: [64]u8 = undefined; + var attr_buf: [64]u8 = undefined; const x = Shape.init(.{ 16, 8 }, .f32); // By default tensors are replicated. { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x, 4, fbs.writer()); - try std.testing.expectEqualStrings("{replicated}", fbs.getWritten()); + var writer: std.Io.Writer = .fixed(&attr_buf); + try writeShardingRepresentation(x, 4, &writer); + try std.testing.expectEqualStrings("{replicated}", writer.buffered()); } // Shard along first axis. { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x.withSharding(.{0}), 4, fbs.writer()); - try std.testing.expectEqualStrings("{devices=[4,1]<=[4]}", fbs.getWritten()); + var writer: std.Io.Writer = .fixed(&attr_buf); + try writeShardingRepresentation(x.withSharding(.{0}), 4, &writer); + try std.testing.expectEqualStrings("{devices=[4,1]<=[4]}", writer.buffered()); } // Also shard along second axis. { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x.withSharding(.{ 0, 1 }), 2, fbs.writer()); - try std.testing.expectEqualStrings("{devices=[2,2]<=[2]}", fbs.getWritten()); + var writer: std.Io.Writer = .fixed(&attr_buf); + try writeShardingRepresentation(x.withSharding(.{ 0, 1 }), 2, &writer); + try std.testing.expectEqualStrings("{devices=[2,2]<=[2]}", writer.buffered()); } } diff --git a/zml/nn.zig b/zml/nn.zig index 631bce16..33354e53 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -22,6 +22,7 @@ test { pub const Linear = struct { weight: Tensor, + // weight_scale_inv: ?Tensor = null, bias: ?Tensor = null, pub fn forward(self: Linear, x: Tensor) Tensor { diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 8658aebe..8f63b9bf 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -75,14 +75,17 @@ pub const Client = opaque { } fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { - var bytecode: std.array_list.Managed(u8) = .init(allocator); + var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer bytecode.deinit(); - module.op().writeBytecodeWithConfig(bytecode.writer(), .{ .desiredEmitedVersion = 1 }) catch |err| { + module.op().writeBytecodeWithConfig(&bytecode.writer, .{ .desiredEmitedVersion = 1 }) catch |err| { log.err("failed to write module bytecode: {}", .{err}); - return err; + return switch (err) { + std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + else => |e| e, + }; }; - var serialized_buffer: std.array_list.Managed(u8) = .init(allocator); + var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer serialized_buffer.deinit(); const stablehlo_version = blk: { @@ -92,13 +95,13 @@ pub const Client = opaque { break :blk dialects.stablehlo.getMinimumVersion(); }; - dialects.stablehlo.serializePortableArtifact(bytecode.items, stablehlo_version, serialized_buffer.writer()) catch |err| { + dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); return err; }; return @ptrCast(try self.inner().compile(api, .{ - .bytecode = serialized_buffer.items, + .bytecode = serialized_buffer.written(), .bytecode_format = .mlir, .compile_options_pb = compile_options_pb, })); diff --git a/zml/shape.zig b/zml/shape.zig index bd5f2a22..a9daffa7 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -386,14 +386,8 @@ pub const Shape = struct { /// Format the shape. /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" - pub fn format( - self: Shape, - writer: anytype, - ) !void { - // TODO: impl alternative format - // const bare_fmt = fmt.len == 1 and fmt[0] == '_'; - const bare_fmt = true; - _ = try writer.write(if (bare_fmt) "{" else "Shape({"); + pub fn format(self: Shape, writer: *std.Io.Writer) !void { + _ = try writer.writeByte('{'); var need_comma = false; for (self.dims(), 0..) |d, i| { @@ -411,7 +405,7 @@ pub const Shape = struct { } if (need_comma) try writer.writeByte(','); _ = try writer.write(@tagName(self.dtype())); - _ = try writer.write(if (bare_fmt) "}" else "})"); + _ = try writer.writeByte('}'); } /// Broadcasts a Tensor to the given shape, extending dimensions if needed. diff --git a/zml/tensor.zig b/zml/tensor.zig index 9d55ebf3..69a20ffb 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -52,10 +52,7 @@ pub const Tensor = struct { return CompilationContext.current(); } - pub fn format( - self: Tensor, - writer: anytype, - ) !void { + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { // TODO(0.15.0) handle format // const bare_fmt = fmt.len == 1 and fmt[0] == '_'; const bare_fmt = false; @@ -1521,14 +1518,7 @@ pub const Tensor = struct { const to_the_end = std.math.maxInt(i64); - pub fn format( - self: Slice, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; + pub fn format(self: Slice, writer: *std.Io.Writer) !void { if (self.singleton) { try writer.print("[{}]", .{self.start}); } else if (self.end == to_the_end and self.step == 1) { @@ -2582,7 +2572,7 @@ pub const Tensor = struct { /// that requires host<->device synchronization. /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { - scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); + scoped_log.debug("scatterSlices({f}, {any}, {f})", .{ self, indices, updates }); const UpdateType = @TypeOf(ScatterOpts.increment); diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig index 7b36dbe8..01b9c997 100644 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ b/zml/tokenizer/hftokenizers/hftokenizers.zig @@ -110,6 +110,7 @@ pub const HFTokenizer = opaque { } pub fn tokenToId(self: *HFTokenizer, token: []const u8) ?u32 { - return c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); + const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); + return if (id == std.math.maxInt(u32)) null else id; } }; From 5c038a4167f81cbcfd352ca9c497750790a87de1 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 11 Sep 2025 11:30:44 +0200 Subject: [PATCH 02/21] remove managed array list --- examples/modernbert/main.zig | 17 +++++++-------- runtimes/neuron/libneuronxla.zig | 4 ++-- zml/exe.zig | 2 +- zml/meta.zig | 5 +++-- zml/module.zig | 11 +++++----- zml/ops.zig | 35 +++++++++++++------------------ zml/tokenizer/homemade.zig | 36 ++++++++++++++++---------------- 7 files changed, 52 insertions(+), 58 deletions(-) diff --git a/examples/modernbert/main.zig b/examples/modernbert/main.zig index 95d5aff1..7eab9c5a 100644 --- a/examples/modernbert/main.zig +++ b/examples/modernbert/main.zig @@ -216,27 +216,26 @@ pub fn unmask( } pub fn tokenize(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - var tokens = std.array_list.Managed(u32).init(allocator); var encoder = try tokenizer.encoder(); defer encoder.deinit(); const bos = tokenizer.tokenToId("[CLS]") orelse return error.NoSuchToken; const eos = tokenizer.tokenToId("[SEP]") orelse return error.NoSuchToken; - try tokens.append(bos); - try tokens.appendSlice(try encoder.encode(prompt)); - try tokens.append(eos); + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.append(allocator, bos); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.append(allocator, eos); - return tokens.toOwnedSlice(); + return tokens.toOwnedSlice(allocator); } fn findMaskPositions(allocator: std.mem.Allocator, tokens: []const u32, mask_token: u32) ![]usize { - var mask_positions = std.array_list.Managed(usize).init(allocator); - defer mask_positions.deinit(); + var mask_positions: std.ArrayList(usize) = .empty; for (tokens, 0..) |token, i| { if (token == mask_token) { - try mask_positions.append(i); + try mask_positions.append(allocator, i); } } @@ -247,7 +246,7 @@ fn findMaskPositions(allocator: std.mem.Allocator, tokens: []const u32, mask_tok if (mask_positions.items.len > 1) log.warn("Currently only supporting one [MASK] per input", .{}); - return mask_positions.toOwnedSlice(); + return mask_positions.toOwnedSlice(allocator); } fn prepareTensorInputs( diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig index 0010d28e..59d8ae6b 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/runtimes/neuron/libneuronxla.zig @@ -117,8 +117,8 @@ fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff }; { - var operand_ids: std.ArrayListUnmanaged(i64) = .initBuffer(c.xla_HloInstructionProto_resize_operand_ids(fused_root, parameters_len + 1, upb_arena)[0 .. parameters_len + 1]); - var new_instructions: std.ArrayListUnmanaged(*const c.xla_HloInstructionProto) = .initBuffer(@ptrCast(c.xla_HloComputationProto_resize_instructions(entry, parameters_len + 1, upb_arena)[0 .. parameters_len + 1])); + var operand_ids: std.ArrayList(i64) = .initBuffer(c.xla_HloInstructionProto_resize_operand_ids(fused_root, parameters_len + 1, upb_arena)[0 .. parameters_len + 1]); + var new_instructions: std.ArrayList(*const c.xla_HloInstructionProto) = .initBuffer(@ptrCast(c.xla_HloComputationProto_resize_instructions(entry, parameters_len + 1, upb_arena)[0 .. parameters_len + 1])); for (entry_instructions) |instruction| { if (std.mem.eql(u8, upb.slice(c.xla_HloInstructionProto_opcode(instruction)) orelse continue, "parameter")) { const id = c.xla_HloInstructionProto_id(instruction); diff --git a/zml/exe.zig b/zml/exe.zig index f3df0c2e..849d54ff 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -439,7 +439,7 @@ fn fillBuffers(v: anytype, shapes: []const Shape, buffers: []const [*]*pjrt.Buff fn prettyFnName( comptime func: anytype, allocator: std.mem.Allocator, -) !std.ArrayListUnmanaged(u8) { +) !std.ArrayList(u8) { const full_noisy_name = @typeName(@TypeOf(func)); const og_len = full_noisy_name.len; const buffer = try allocator.alloc(u8, og_len); diff --git a/zml/meta.zig b/zml/meta.zig index 855318a5..499c466b 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -679,7 +679,7 @@ test zip { /// Given a func(X) -> Y or a func(Ctx, X) -> Y, /// finds all X in the given object, and write the result of func(X) into an arraylist. -pub fn collect(func: anytype, func_ctx: _CollectCtx(func), out: *std.array_list.Managed(stdx.meta.FnSignature(func, null).ReturnT), obj: anytype) error{OutOfMemory}!void { +pub fn collectAlloc(func: anytype, func_ctx: _CollectCtx(func), out: *std.array_list.Managed(stdx.meta.FnSignature(func, null).ReturnT), obj: anytype) error{OutOfMemory}!void { stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { func_ctx: _CollectCtx(func), @@ -700,7 +700,8 @@ pub fn collect(func: anytype, func_ctx: _CollectCtx(func), out: *std.array_list. } /// Given a func(X) -> Y or a func(Ctx, X) -> Y, -/// finds all X in the given object, and write the result of func(X) into an arraylist. +/// finds all X in the given object, and write the result of func(X) into a given slice. +/// Asserts that the number of X found is equal to the slice len. pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnResult(func)) void { stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { diff --git a/zml/module.zig b/zml/module.zig index eae9b361..1c825159 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -341,12 +341,11 @@ pub const CompilationContext = struct { const locations = try arena.alloc(mlir.Location, tensor_count); @memset(locations, mlir.Location.unknown(mlir_ctx)); - var input_shapes: std.array_list.Managed(Shape) = try .initCapacity(res_allocator, tensor_count); - meta.collect(Tensor.shape, {}, &input_shapes, args) catch unreachable; - stdx.debug.internalAssert(input_shapes.items.len == tensor_count, "args have changed ?", .{}); + const input_shapes = try res_allocator.alloc(Shape, tensor_count); + meta.collectBuf(Tensor.shape, {}, args, input_shapes); const input_types = try arena.alloc(mlir.Type, tensor_count); - for (input_types, input_shapes.items) |*t, sh| t.* = mlirx.tensorType(mlir_ctx, sh); + for (input_types, input_shapes) |*t, sh| t.* = mlirx.tensorType(mlir_ctx, sh); const og_block_args = self._block_args; defer { @@ -399,7 +398,7 @@ pub const CompilationContext = struct { self.addDonationsAttributes(arg_attrs, fn_res_donations); self.addOutputMemoryKindAttributes(res_attrs, fn_res_output_memory_kind); if (self._platform.sharding().num_partitions > 1) { - self.addShardingAttributes(arg_attrs, res_attrs, input_shapes.items, fn_res_shapes); + self.addShardingAttributes(arg_attrs, res_attrs, input_shapes, fn_res_shapes); } } @@ -427,7 +426,7 @@ pub const CompilationContext = struct { return .{ .mlir_fn = mlir_fn, .name = opts.name, - .args_shapes = input_shapes.items, + .args_shapes = input_shapes, .res_tensors = fn_res, .res_types = fn_res_types, .res_shapes = fn_res_shapes, diff --git a/zml/ops.zig b/zml/ops.zig index ee125086..6077575f 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -448,18 +448,11 @@ pub fn if_( var false_shapes = std.array_list.Managed(Shape).init(ctx.allocator()); defer false_shapes.deinit(); - var failed_to_collect = false; - meta.collect(Tensor.shape, {}, &true_shapes, &true_branch_res) catch { - failed_to_collect = true; - }; - meta.collect(Tensor.shape, {}, &false_shapes, &false_branch_res) catch { - failed_to_collect = true; - }; - if (!failed_to_collect) { - stdx.debug.assert(true_shapes.items.len == false_shapes.items.len, "zml.ops.if_ expects the true and false branch to produce the same number of tensors. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); - for (true_shapes.items, false_shapes.items) |true_shape, false_shape| { - stdx.debug.assert(true_shape.eqlWithTags(false_shape), "zml.ops.if_ expects the true and false branch to produce tensors of the same shape. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); - } + meta.collectAlloc(Tensor.shape, {}, &true_shapes, &true_branch_res) catch @panic("OOM"); + meta.collectAlloc(Tensor.shape, {}, &false_shapes, &false_branch_res) catch @panic("OOM"); + stdx.debug.assert(true_shapes.items.len == false_shapes.items.len, "zml.ops.if_ expects the true and false branch to produce the same number of tensors. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); + for (true_shapes.items, false_shapes.items) |true_shape, false_shape| { + stdx.debug.assert(true_shape.eqlWithTags(false_shape), "zml.ops.if_ expects the true and false branch to produce tensors of the same shape. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); } const scalar_pred = if (pred.rank() == 0) pred else pred.flattenAll().squeeze(0); @@ -958,18 +951,20 @@ pub fn scatter( const UpdateS = BlockSign(update_fn); const update_block, _ = ctx.makeBlock(.hermetic, UpdateS, update_fn, blkctx, .{ _scalar, _scalar }); - var input_values = std.array_list.Managed(mlir.Value).initCapacity(ctx.allocator(), n_inputs) catch @panic("OOM"); - defer input_values.deinit(); - meta.collect(CompilationContext.getValue, ctx, &input_values, &inputs) catch unreachable; - var updates_values = std.array_list.Managed(mlir.Value).initCapacity(ctx.allocator(), n_updates) catch @panic("OOM"); - defer updates_values.deinit(); - meta.collect(CompilationContext.getValue, ctx, &updates_values, &updates) catch unreachable; + const arena = ctx.allocator(); + const input_values = arena.alloc(mlir.Value, n_inputs) catch @panic("OOM"); + defer arena.free(input_values); + meta.collectBuf(CompilationContext.getValue, ctx, &inputs, input_values); + + const updates_values = arena.alloc(mlir.Value, n_updates) catch @panic("OOM"); + defer arena.free(updates_values); + meta.collectBuf(CompilationContext.getValue, ctx, &updates, updates_values); const op = dialect.stablehlo.scatter( mlir_ctx, - input_values.items, + input_values, &.{indices.value()}, - updates_values.items, + updates_values, update_block, .{ .update_window_dims = _collectAxes(AxisKind, config.up_kind, .update_window).constSlice(), diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig index 9b5898d8..91eb1df9 100644 --- a/zml/tokenizer/homemade.zig +++ b/zml/tokenizer/homemade.zig @@ -189,10 +189,9 @@ pub const Tokenizer = struct { // Step by step visualization of the progress. if (options.debug) { var _debug_buf: [256]u8 = undefined; - var _debug_alloc = std.heap.FixedBufferAllocator.init(&_debug_buf); - var debug_progress = std.array_list.Managed(u8).init(_debug_alloc.allocator()); - self.decodeWithOpts(&debug_progress, tok_buff[0..num_tokens], .{ .sep = "|" }) catch {}; - log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.items }); + var debug_progress: std.Io.Writer = .fixed(&_debug_buf); + self.decodeWithOpts(tok_buff[0..num_tokens], &debug_progress, .{ .sep = "|" }) catch {}; + log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.buffered() }); } var best_score: f32 = -1e10; var best_token: u32 = 0; @@ -311,22 +310,19 @@ pub const Tokenizer = struct { /// Converts the given slice of tokens back into bytes. /// Note that if the tokenizer allows sub-unicode bytes, it's possible /// the output is not valid utf8. - pub fn decode(self: *const Tokenizer, allocator: std.mem.Allocator, input: []const u32) error{OutOfMemory}![]u8 { - var output = std.array_list.Managed(u8).init(allocator); - errdefer output.deinit(); - - try self.decodeWithOpts(&output, input, .{}); - return output.toOwnedSlice(); + pub fn decode(self: *const Tokenizer, input: []const u32, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try self.decodeWithOpts(input, writer, .{}); } pub fn decodeWithOpts( self: *const Tokenizer, - output: *std.array_list.Managed(u8), input: []const u32, + output: *std.Io.Writer, opts: struct { sep: []const u8 = "" }, - ) error{OutOfMemory}!void { + ) std.Io.Writer.Error!void { const escaped = if (self.normalizer) |n| n.escapedSpace() else null; // Flag used to indicate if the first dummy whitespace has been consumed. + var first_token: bool = true; for (input) |id| { // Retrieve the slice corresponding to the id. var piece = self.lookupPiece(id); @@ -337,12 +333,13 @@ pub const Tokenizer = struct { while (std.mem.startsWith(u8, piece, escspc)) { piece = piece[escspc.len..]; // don't output a space at beginning of text. - if (output.items.len > 0) try output.append(' '); + if (!first_token) try output.writeByte(' '); } } - try output.appendSlice(piece); - if (opts.sep.len > 0) try output.appendSlice(opts.sep); + try output.writeAll(piece); + first_token = false; + try output.writeAll(opts.sep); } } @@ -472,10 +469,13 @@ pub const Decoder = struct { } pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { + // TODO: revisite tokenizer api to use std.Io.Writer. self.reset(); - const res = try self.inner.decode(self.arena.allocator(), ids); - self.current_string = res; - return res; + // Reuse the same warmup than in init, to maximize contiguous writes. + var arena_writer = std.Io.Writer.Allocating.initCapacity(self.arena.allocator(), 4096) catch unreachable; + try self.inner.decode(ids, &arena_writer.writer); + self.current_string = arena_writer.written(); + return self.current_string.?; } pub fn string(self: *const Decoder) []const u8 { From a82e618b029cb1a1226639a3a1adcf1fdf386aca Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 11 Sep 2025 11:45:31 +0200 Subject: [PATCH 03/21] remove managed arraylist from meta collectAlloc --- stdx/fmt.zig | 4 ++++ zml/meta.zig | 27 ++++++++++++++++++--------- zml/ops.zig | 23 ++++++++++++----------- zml/tensor.zig | 2 +- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 03bf7a35..4e38488a 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -9,6 +9,10 @@ fn FmtSlice(T: type) type { return struct { slice: []const T, + pub fn format(f: @This(), writer: *std.io.Writer) std.io.Writer.Error!void { + return try formatSliceAny(f.slice, .{}, writer); + } + pub fn formatNumber(f: @This(), writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { return switch (@typeInfo(T)) { .comptime_float, .float => try formatFloatSlice(f.slice, n, writer), diff --git a/zml/meta.zig b/zml/meta.zig index 499c466b..891a27ec 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -679,24 +679,33 @@ test zip { /// Given a func(X) -> Y or a func(Ctx, X) -> Y, /// finds all X in the given object, and write the result of func(X) into an arraylist. -pub fn collectAlloc(func: anytype, func_ctx: _CollectCtx(func), out: *std.array_list.Managed(stdx.meta.FnSignature(func, null).ReturnT), obj: anytype) error{OutOfMemory}!void { +pub fn collectAlloc( + func: anytype, + func_ctx: _CollectCtx(func), + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]stdx.meta.FnSignature(func, null).ReturnT { stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); - const LocalContext = struct { + + const CollectAllocCtx = struct { func_ctx: _CollectCtx(func), - out: *std.array_list.Managed(stdx.meta.FnSignature(func, null).ReturnT), + allocator: std.mem.Allocator, + out: std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT) = .empty, oom: bool = false, - }; - var context = LocalContext{ .func_ctx = func_ctx, .out = out }; - visit((struct { - fn cb(ctx: *LocalContext, val: *const _CollectArg(func)) void { + + fn cb(ctx: *@This(), val: *const _CollectArg(func)) void { if (ctx.oom) return; const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); - ctx.out.append(res) catch { + ctx.out.append(ctx.allocator, res) catch { ctx.oom = true; }; } - }).cb, &context, obj); + }; + var context = CollectAllocCtx{ .func_ctx = func_ctx, .allocator = allocator }; + visit(CollectAllocCtx.cb, &context, obj); if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); } /// Given a func(X) -> Y or a func(Ctx, X) -> Y, diff --git a/zml/ops.zig b/zml/ops.zig index 6077575f..edbcd4a2 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -443,22 +443,23 @@ pub fn if_( const true_branch_block, const true_branch_res = ctx.makeBlock(.open, TrueBlockSignature, &true_branch_fn, blkctx, {}); const false_branch_block, const false_branch_res = ctx.makeBlock(.open, TrueBlockSignature, &false_branch_fn, blkctx, {}); - var true_shapes = std.array_list.Managed(Shape).init(ctx.allocator()); - defer true_shapes.deinit(); - var false_shapes = std.array_list.Managed(Shape).init(ctx.allocator()); - defer false_shapes.deinit(); + check: { + const arena = ctx.allocator(); + const true_shapes = meta.collectAlloc(Tensor.shape, {}, arena, &true_branch_res) catch break :check; + defer arena.free(true_shapes); - meta.collectAlloc(Tensor.shape, {}, &true_shapes, &true_branch_res) catch @panic("OOM"); - meta.collectAlloc(Tensor.shape, {}, &false_shapes, &false_branch_res) catch @panic("OOM"); - stdx.debug.assert(true_shapes.items.len == false_shapes.items.len, "zml.ops.if_ expects the true and false branch to produce the same number of tensors. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); - for (true_shapes.items, false_shapes.items) |true_shape, false_shape| { - stdx.debug.assert(true_shape.eqlWithTags(false_shape), "zml.ops.if_ expects the true and false branch to produce tensors of the same shape. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); + const false_shapes = meta.collectAlloc(Tensor.shape, {}, arena, &false_branch_res) catch break :check; + defer arena.free(false_shapes); + + stdx.debug.assert(true_shapes.len == false_shapes.len, "zml.ops.if_ expects the true and false branch to produce the same number of tensors. Got: \n - true branch: {f}\n -false branch: {f}", .{ stdx.fmt.slice(true_shapes), stdx.fmt.slice(false_shapes) }); + for (true_shapes, false_shapes) |true_shape, false_shape| { + stdx.debug.assert(true_shape.eqlWithTags(false_shape), "zml.ops.if_ expects the true and false branch to produce tensors of the same shape. Got: \n - true branch: {f}\n -false branch: {f}", .{ stdx.fmt.slice(true_shapes), stdx.fmt.slice(false_shapes) }); + } } - const scalar_pred = if (pred.rank() == 0) pred else pred.flattenAll().squeeze(0); const loc = ctx.mlirCtx().location(@src()); const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.if", .{ - .operands = &.{scalar_pred.value()}, + .operands = &.{pred.asScalar().value()}, .result_type_inference = true, .blocks = &.{ true_branch_block, false_branch_block }, // We can't verify right away, cause the weights captured by the if haven't been added yet. diff --git a/zml/tensor.zig b/zml/tensor.zig index 69a20ffb..3027f6a6 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2033,7 +2033,7 @@ pub const Tensor = struct { /// Converts the given 1 element Tensor into a 0-rank Tensor. pub fn asScalar(self: Tensor) Tensor { stdx.debug.assert(self.count() == 1, "Tensor.asScalar expects an input with exactly 1-element got {f}", .{self}); - return self.reshape(.{}); + return if (self.rank() == 0) self else self.reshape(.{}); } pub const Pad = struct { From 3a6eba2ac10fef97fb2fdc525f4a0b4dd0147472 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 11 Sep 2025 12:01:33 +0200 Subject: [PATCH 04/21] flush file --- mlir/mlir.zig | 23 ++++------------------- zml/module.zig | 1 + 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 310eb299..3602d2ee 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -926,33 +926,18 @@ pub const Operation = struct { op: Operation, flags: OpPrintingFlags, - pub fn format(self: @This(), writer: *std.Io.Writer) !void { + pub fn format(self: MlirFormatter, writer: *std.Io.Writer) !void { try self.op.print(writer, self.flags); } }; - pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) !void { + pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) std.Io.Writer.Error!void { const pflags = flags.create(); defer c.mlirOpPrintingFlagsDestroy(pflags); var writer_err: WriterWithErr = .{ .writer = writer }; - - c.mlirOperationPrintWithFlags( - self._inner, - pflags, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const _writer_err: *WriterWithErr = @ptrCast(@alignCast(ctx_)); - _writer_err.writer.writeAll(str.data[0..str.length]) catch |err| { - _writer_err.err = err; - }; - } - }).callback, - &writer_err, - ); - - if (writer_err.err) |err| return err; - try writer_err.writer.flush(); + c.mlirOperationPrintWithFlags(self._inner, pflags, WriterWithErr.printCallback, &writer_err); + return writer_err.check(); } pub fn verify(self: Self) bool { diff --git a/zml/module.zig b/zml/module.zig index 1c825159..38038758 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -188,6 +188,7 @@ pub const CompilationContext = struct { var write_buf: [4096]u8 = undefined; var writer = file.writer(&write_buf); try module.op().print(&writer.interface, .{ .debug_info = true, .debug_info_pretty_form = false }); + try writer.interface.flush(); log.info("Wrote MLIR to {s}/{s}", .{ module_dir.?, mlir_name }); } else |_| { log.warn("Failed to open {s}", .{mlir_name}); From 8c89794c56b2db4e28ab0bac4a96b8e846d6e9ae Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 9 Sep 2025 11:09:06 +0200 Subject: [PATCH 05/21] drop input_layernorm --- examples/llama/llama.zig | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 5d0e87d5..9dfc9c38 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -65,9 +65,6 @@ pub const LlamaLM = struct { self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); - var input_layernorm = try zml.aio.populateModelWithPrefix(RmsNorm, allocator, store, prefix.concat("input_layernorm")); - input_layernorm.eps = config.rms_norm_eps; - var post_attention_layernorm = try zml.aio.populateModelWithPrefix(RmsNorm, allocator, store, prefix.concat("post_attention_layernorm")); post_attention_layernorm.eps = config.rms_norm_eps; @@ -78,9 +75,9 @@ pub const LlamaLM = struct { layer.* = .{ .self_attn = self_attn, - .input_layernorm = input_layernorm, - .post_attention_layernorm = post_attention_layernorm, .mlp = mlp, + .post_attention_layernorm = post_attention_layernorm, + .rms_norm_eps = config.rms_norm_eps, }; } @@ -200,11 +197,12 @@ pub const Llama = struct { }; pub const TransformerLayer = struct { - input_layernorm: RmsNorm, self_attn: SelfAttn, post_attention_layernorm: RmsNorm, mlp: Mlp, + rms_norm_eps: f32, + pub fn forward( self: TransformerLayer, x0: Tensor, @@ -215,7 +213,7 @@ pub const TransformerLayer = struct { //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); - const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); + const x0_normalized = zml.nn.rmsNorm(x0, .d, self.rms_norm_eps); const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); const x1 = x0.add(delta0); From d319ad2578492e7585d5f743e9555aeb30fd6d9a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 9 Sep 2025 15:00:49 +0200 Subject: [PATCH 06/21] merge layer norms --- examples/llama/llama.zig | 19 ++++++++++++++-- examples/llama/main.zig | 47 ++++++++++++++++++++++++++++++++++++++++ zml/aio.zig | 5 +++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 9dfc9c38..0d258303 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -218,7 +218,7 @@ pub const TransformerLayer = struct { const x1 = x0.add(delta0); // Fully Connected - const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); + const x1_normalized = zml.nn.rmsNorm(x1, .d, self.rms_norm_eps); const x2 = zml.call(self.mlp, .forward, .{x1_normalized}).add(x1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; @@ -237,7 +237,7 @@ const RmsNorm = struct { } }; -const Mlp = struct { +pub const Mlp = struct { up_proj: zml.nn.Linear, // (dim -> hidden_dim) gate_proj: zml.nn.Linear, // (dim -> hidden_dim) down_proj: zml.nn.Linear, // (hidden_dim -> dim) @@ -248,6 +248,13 @@ const Mlp = struct { output = output.silu().mul(proj); return zml.call(self.down_proj, .forward, .{output}); } + + pub fn mergePostAttentionLayerNorm(self: Mlp, weight: Tensor) [2]Tensor { + return .{ + self.up_proj.weight.mul(weight.broadcast(self.up_proj.weight.shape(), &.{1})).reuseBuffer(self.up_proj.weight), + self.gate_proj.weight.mul(weight.broadcast(self.gate_proj.weight.shape(), &.{1})).reuseBuffer(self.gate_proj.weight), + }; + } }; pub const SelfAttn = struct { @@ -313,6 +320,14 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; } + + pub fn mergeInputLayerNorm(self: SelfAttn, weight: Tensor) [3]Tensor { + return .{ + self.q_proj.weight.mul(weight.broadcast(self.q_proj.weight.shape(), &.{1})).reuseBuffer(self.q_proj.weight), + self.k_proj.weight.mul(weight.broadcast(self.k_proj.weight.shape(), &.{1})).reuseBuffer(self.k_proj.weight), + self.v_proj.weight.mul(weight.broadcast(self.v_proj.weight.shape(), &.{1})).reuseBuffer(self.v_proj.weight), + }; + } }; pub const KvCache = struct { diff --git a/examples/llama/main.zig b/examples/llama/main.zig index c1c8d805..8936605c 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -298,12 +298,59 @@ pub fn asyncMain() !void { platform, }); + var fut_self_attn_merge_mod = try asynk.asyncc(zml.compileModel, .{ + allocator, llama.SelfAttn.mergeInputLayerNorm, llama_tensors.model.layers[0].self_attn, + .{ + store.get("model.layers.0.input_layernorm.weight").?.shape(), + }, + platform, + }); + + var fut_mlp_merge_mod = try asynk.asyncc(zml.compileModel, .{ + allocator, llama.Mlp.mergePostAttentionLayerNorm, llama_tensors.model.layers[0].mlp, + .{ + store.get("model.layers.0.post_attention_layernorm.weight").?.shape(), + }, + platform, + }); + // While we are still compiling load the weights to the device. log.info("\tLoading Llama weights from {s}...", .{model_weights_path}); var llama_buffers = try store.loadModelById(llama.LlamaLM, compiler_arena.allocator(), llama_tensors, platform); defer zml.aio.unloadBuffers(&llama_buffers); log.info("✅\tLoaded weights in {D}", .{start.read()}); + { + var self_attn_merge_mod = try fut_self_attn_merge_mod.awaitt(); + defer self_attn_merge_mod.deinit(); + + var norm_name_buf: [1024]u8 = undefined; + for (0.., llama_buffers.model.layers) |i, *layer| { + const norm_name = try std.fmt.bufPrint(&norm_name_buf, "model.layers.{d}.input_layernorm.weight", .{i}); + const norm = try store.loadBufferByName(norm_name, platform); + defer norm.deinit(); + const wq, const wk, const wv = self_attn_merge_mod.call(.{ layer.self_attn, norm }); + layer.self_attn.q_proj.weight = wq; + layer.self_attn.k_proj.weight = wk; + layer.self_attn.v_proj.weight = wv; + } + } + + { + var mlp_merge_mod = try fut_mlp_merge_mod.awaitt(); + defer mlp_merge_mod.deinit(); + + var norm_name_buf: [1024]u8 = undefined; + for (0.., llama_buffers.model.layers) |i, *layer| { + const norm_name = try std.fmt.bufPrint(&norm_name_buf, "model.layers.{d}.post_attention_layernorm.weight", .{i}); + const norm = try store.loadBufferByName(norm_name, platform); + defer norm.deinit(); + const wu, const wg = mlp_merge_mod.call(.{ layer.mlp, norm }); + layer.mlp.up_proj.weight = wu; + layer.mlp.gate_proj.weight = wg; + } + } + var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_buffers); defer llama_module_prefill.deinit(); var llama_module = (try fut_mod.awaitt()).prepare(llama_buffers); diff --git a/zml/aio.zig b/zml/aio.zig index a705e498..70ba2b7d 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -110,6 +110,11 @@ pub const BufferStore = struct { return self.buffers.get(key); } + pub fn loadBufferByName(self: BufferStore, key: []const u8, platform: zml.Platform) !zml.Buffer { + const host_buffer: zml.HostBuffer = self.buffers.get(key) orelse return error.NotFound; + return try host_buffer.toDevice(platform); + } + pub fn loadBufferById(self: BufferStore, x: zml.Tensor, platform: zml.Platform) !zml.Buffer { var host_buffer: zml.HostBuffer = switch (x._id) { .buffer_id => |id| hb: { From 74311dc9588fe734beab2c69f599deed3789f5f8 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 9 Sep 2025 15:01:48 +0200 Subject: [PATCH 07/21] remove post_attention_layernorm --- examples/llama/llama.zig | 5 ----- 1 file changed, 5 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 0d258303..ade7a916 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -65,9 +65,6 @@ pub const LlamaLM = struct { self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); - var post_attention_layernorm = try zml.aio.populateModelWithPrefix(RmsNorm, allocator, store, prefix.concat("post_attention_layernorm")); - post_attention_layernorm.eps = config.rms_norm_eps; - var mlp = try zml.aio.populateModelWithPrefix(Mlp, allocator, store, prefix.concat("mlp")); mlp.up_proj.weight = mlp.up_proj.weight.withSharding(.{0}); mlp.gate_proj.weight = mlp.gate_proj.weight.withSharding(.{0}); @@ -76,7 +73,6 @@ pub const LlamaLM = struct { layer.* = .{ .self_attn = self_attn, .mlp = mlp, - .post_attention_layernorm = post_attention_layernorm, .rms_norm_eps = config.rms_norm_eps, }; } @@ -198,7 +194,6 @@ pub const Llama = struct { pub const TransformerLayer = struct { self_attn: SelfAttn, - post_attention_layernorm: RmsNorm, mlp: Mlp, rms_norm_eps: f32, From 4e4da5faaf15d0ed9c3d9141aa3b8e0028576572 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 9 Sep 2025 15:02:33 +0200 Subject: [PATCH 08/21] expose has_side_effect --- zml/callback.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zml/callback.zig b/zml/callback.zig index 4d6ab90e..8cd3e7df 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -99,7 +99,7 @@ pub fn call( .api_version = .typed_ffi, .backend_config = .dict(mlir_ctx, &.{}), .additional_attributes = &.{.{ "mhlo.frontend_attributes", .dict(mlir_ctx, &.{}) }}, - .has_side_effect = true, + .has_side_effect = Callback.callback_config.has_side_effect, .output_operand_aliases = Callback.callback_config.output_operand_aliases, }, output_types, @@ -123,6 +123,7 @@ pub const Config = struct { // TODO: document precisely what `command_buffer_compatible` is doing and its limitations. traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, // TODO: handle sharded inputs + has_side_effect: bool = true, }; /// Compile-time check that a callback has all informations we require. @@ -282,6 +283,7 @@ pub const Print = struct { .copy_inputs_to_host_pinned = true, // Print is fairly predictable and can be captured in an execution graph. .traits = .{ .command_buffer_compatible = false }, + .has_side_effect = false, }; platform: Platform, From c6cf125a3c535ec26d1ae6036b70db35862589bc Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 11 Sep 2025 10:29:25 +0200 Subject: [PATCH 09/21] support qwen chat template --- examples/llama/main.zig | 50 ++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 8936605c..34800617 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -42,22 +42,46 @@ pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok return try allocator.dupe(u32, try encoder.encode(prompt)); } - const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); + if (tokenizer.tokenToId("<|start_header_id|>") != null and tokenizer.tokenToId("<|end_header_id|>") != null) { + // Llama chat template + const start_header = tokenizer.tokenToId("<|start_header_id|>").?; + const end_header = tokenizer.tokenToId("<|end_header_id|>").?; + const user = tokenizer.tokenToId("user").?; + const assistant = tokenizer.tokenToId("assistant").?; + const eot = tokenizer.tokenToId("<|eot_id|>").?; + const newline = tokenizer.tokenToId("\n").?; - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline }); + try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); - try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline }); - return tokens.toOwnedSlice(allocator); + try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); + + return tokens.toOwnedSlice(allocator); + } + + if (tokenizer.tokenToId("<|im_start|>") != null and tokenizer.tokenToId("<|im_end|>") != null) { + // Qwen3 chat template + const start_message = tokenizer.tokenToId("<|im_start|>").?; + const end_message = tokenizer.tokenToId("<|im_end|>").?; + const user = tokenizer.tokenToId("user").?; + const assistant = tokenizer.tokenToId("assistant").?; + const newline = tokenizer.tokenToId("\n").?; + + try tokens.appendSlice(allocator, &.{ start_message, user, newline }); + + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ end_message, newline }); + + try tokens.appendSlice(allocator, &.{ start_message, assistant, newline }); + + return tokens.toOwnedSlice(allocator); + } + + log.err("Tokenizer not recognized", .{}); + return error.NotSupported; } pub fn generateText( @@ -264,7 +288,7 @@ pub fn asyncMain() !void { const gen_tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); const token_idx_shape = zml.Shape.init(.{}, .u32); - const dtype = llama_tensors.model.embed_tokens.weight.dtype(); + const dtype = llama_tensors.model.layers[0].self_attn.q_proj.weight.dtype(); const kv_shape = zml.Shape.init(.{ .layer = llama_tensors.model.layers.len, .k = seq_len, From a067e3efae8d6f217d534ac2a30065ce71e33ea1 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 11 Sep 2025 10:30:41 +0200 Subject: [PATCH 10/21] handle llama 3.1 having no head dim --- examples/llama/llama.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index ade7a916..140784f2 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -18,7 +18,7 @@ pub const LlamaLM = struct { int: u32, ints: []u32, }), - head_dim: ?u32, + head_dim: ?u32 = null, hidden_size: u32, num_hidden_layers: u32, num_attention_heads: u32, From b3fd69caf3a5afb8d874e7d12ac0c25c211b12fe Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 12 Sep 2025 17:38:27 +0200 Subject: [PATCH 11/21] fixup! support qwen chat template --- examples/llama/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 34800617..997d219d 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -68,7 +68,7 @@ pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok const end_message = tokenizer.tokenToId("<|im_end|>").?; const user = tokenizer.tokenToId("user").?; const assistant = tokenizer.tokenToId("assistant").?; - const newline = tokenizer.tokenToId("\n").?; + const newline = tokenizer.tokenToId("\\n").?; try tokens.appendSlice(allocator, &.{ start_message, user, newline }); From b4e71bfb54e9dbbb6b8eef5c30222bc30e95c536 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 12 Sep 2025 17:38:43 +0200 Subject: [PATCH 12/21] support float parsing --- stdx/flags.zig | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/stdx/flags.zig b/stdx/flags.zig index 467cea69..667814f2 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -41,8 +41,9 @@ //! caller to manage the lifetime. The caller should be skipping program name. const std = @import("std"); -const builtin = @import("builtin"); const assert = std.debug.assert; +const builtin = @import("builtin"); + const debug = @import("debug.zig"); /// Format and print an error message to stderr, then exit with an exit code of 1. @@ -285,7 +286,7 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { fn assert_valid_value_type(comptime T: type) void { comptime { - if (T == []const u8 or T == [:0]const u8 or T == ByteSize or @typeInfo(T) == .int) return; + if (T == []const u8 or T == [:0]const u8 or T == ByteSize or @typeInfo(T) == .int or @typeInfo(T) == .float) return; if (@typeInfo(T) == .@"enum") { const info = @typeInfo(T).@"enum"; @@ -347,6 +348,7 @@ fn parse_value(comptime T: type, flag: []const u8, value: [:0]const u8) T { if (V == []const u8 or V == [:0]const u8) return value; if (V == ByteSize) return parse_value_size(flag, value); if (@typeInfo(V) == .int) return parse_value_int(V, flag, value); + if (@typeInfo(V) == .float) return parse_value_float(V, flag, value); if (@typeInfo(V) == .@"enum") return parse_value_enum(V, flag, value); comptime unreachable; } @@ -515,6 +517,20 @@ fn parse_value_int(comptime T: type, flag: []const u8, value: [:0]const u8) T { }; } +/// Parse string value into a float, providing a nice error message for the user. +fn parse_value_float(comptime T: type, flag: []const u8, value: [:0]const u8) T { + assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); + + return std.fmt.parseFloat(T, value) catch |err| { + switch (err) { + error.InvalidCharacter => fatal( + "{s}: expected a decimal value, but found '{s}' (invalid character)", + .{ flag, value }, + ), + } + }; +} + fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); comptime assert(@typeInfo(E).@"enum".is_exhaustive); From 79cd9e5faaf743553c295bd8edb6ef26dad32470 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 12 Sep 2025 17:53:46 +0200 Subject: [PATCH 13/21] Tensor.bytes --- zml/tensor.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zml/tensor.zig b/zml/tensor.zig index 3027f6a6..ce1cf412 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -311,6 +311,11 @@ pub const Tensor = struct { return _result(res_shape, op.result(0)); } + /// Returns the given tensor as one contiguous buffer of bytes. + pub fn bytes(self: Tensor) Tensor { + return self.bitCast(.u8).flattenAll().withTags(.{.bytes}); + } + /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. pub fn countLeadingZeros(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); From a129439a440e7614dc631a41f0cbbd4c28e2a3a1 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 12 Sep 2025 17:57:35 +0200 Subject: [PATCH 14/21] wip: add target tag to buffer struct # Conflicts: # zml/buffer.zig # zml/hostbuffer.zig --- docs/howtos/add_weights.md | 2 +- zml/callback.zig | 6 +++--- zml/exe.zig | 4 ++++ zml/hostbuffer.zig | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/howtos/add_weights.md b/docs/howtos/add_weights.md index 1c151c44..2caf2fc9 100644 --- a/docs/howtos/add_weights.md +++ b/docs/howtos/add_weights.md @@ -44,7 +44,7 @@ Then, in your `BUILD.bazel`, you can refer to the files you defined above, in the following way: ```python -zig_cc_binary( +zig_binary( name = "mnist", args = [ "$(location @mnist//:mnist.pt)", diff --git a/zml/callback.zig b/zml/callback.zig index 8cd3e7df..76e0a8cd 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -191,12 +191,12 @@ fn CallbackImpl(comptime Callback: type, call_frame: *pjrt.ffi.CallFrame) ?*pjrt else .asViewOfDeviceBuffer(platform, shape, null, ffi_buffer.data); if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { - log.debug("Copying argument {d} {f} {*} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer() }); + log.debug("Copying argument {d} {f} {x} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.devicePtr() }); zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch |err| { - log.err("Failed to copy input buffer {d} {f} {*} to host_pinned: {}", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), err }); + log.err("Failed to copy input buffer {d} {f} {x} to host_pinned: {}", .{ i, zml_buffer, zml_buffer.devicePtr(), err }); return .create(call_frame.api, .resource_exhausted, "host pinned OOM"); }; - log.debug("--> {f} {*} ({})", .{ zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), @as(*const f32, @ptrCast(@alignCast(zml_buffer.opaqueDeviceMemoryDataPointer()))).* }); + log.debug("--> {f} {x}", .{ zml_buffer, zml_buffer.devicePtr() }); } callback_args[i] = zml_buffer; } diff --git a/zml/exe.zig b/zml/exe.zig index 849d54ff..71db84d7 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -395,6 +395,10 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { self.inner._unsafeAssignResults(Bufferized(ReturnT), &result); return result; } + + pub fn clone(self: Self, allocator: std.mem.Allocator) !Self { + return .{ .inner = try self.inner.clone(allocator) }; + } }; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 88419b7d..3185348c 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -322,7 +322,7 @@ pub const HostBuffer = struct { } pub fn format(self: HostBuffer, writer: *std.Io.Writer) !void { - try writer.print("HostBuffer(.{f})", .{self._shape}); + try writer.print("HostBuffer(.{f})@{x}", .{ self._shape, @intFromPtr(self._data) }); } pub fn formatNumber(self: HostBuffer, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { From b9001f7a982d69b66e351aaf9af19ec1172e2fb0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 15 Sep 2025 14:35:19 +0200 Subject: [PATCH 15/21] refactor memory handling --- pjrt/pjrt.zig | 31 ++++++--- zml/buffer.zig | 164 +++++++++++++++++++++-------------------------- zml/platform.zig | 38 +++++++++++ zml/tensor.zig | 7 +- 4 files changed, 135 insertions(+), 105 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index fccf31f1..e0477731 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -430,7 +430,7 @@ pub const Client = opaque { pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { const ret = api.call(.PJRT_Client_AddressableMemories, .{ .client = self.inner(), - }) catch unreachable; + }) catch return &.{}; if (ret.addressable_memories) |memories| { return @ptrCast(@constCast(memories[0..ret.num_addressable_memories])); } @@ -556,10 +556,11 @@ pub const Device = opaque { return @intCast(ret.local_hardware_id); } - pub fn addressableMemories(self: *const Device, api: *const Api) ApiError![]const *Memory { - const ret = try api.call(.PJRT_Device_AddressableMemories, .{ - .device = self.inner(), - }); + pub fn addressableMemories(self: *const Device, api: *const Api) []const *Memory { + const ret = api.call( + .PJRT_Device_AddressableMemories, + .{ .device = self.inner() }, + ) catch return &.{}; return @ptrCast(ret.memories[0..ret.num_memories]); } @@ -1048,8 +1049,16 @@ pub const Event = opaque { pub const Memory = opaque { pub const Kind = enum { device, - pinned_host, - unpinned_host, + host_pinned, + host_unpinned, + + pub fn pjrtName(k: Kind) []const u8 { + return switch (k) { + .device => "device", + .host_pinned => "pinned_host", + .host_unpinned => "unpinned_host", + }; + } }; const inner = InnerMixin(c.PJRT_Memory).inner; @@ -1061,8 +1070,12 @@ pub const Memory = opaque { pub fn kind(self: *const Memory, api: *const Api) Kind { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable; - return std.meta.stringToEnum(Kind, kind_[0..ret.kind_size]) orelse unreachable; + return switch (ret.kind_size) { + "device".len => .device, + "pinned_host".len => .host_pinned, + "unpinned_host".len => .host_unpinned, + else => @panic("Memory kind not supported"), + }; } pub fn kindId(self: *const Memory, api: *const Api) u32 { diff --git a/zml/buffer.zig b/zml/buffer.zig index b356b9b7..8f7b4eee 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -8,6 +8,7 @@ const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Target = @import("platform.zig").Target; test { std.testing.refAllDecls(@This()); @@ -22,40 +23,22 @@ const log = std.log.scoped(.zml); /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { - pub const Memory = enum { - host, - host_pinned, - device, - - pub fn toPjrtMemory(self: Memory) pjrt.Memory.Kind { - return switch (self) { - .host => .unpinned_host, - .host_pinned => .pinned_host, - .device => .device, - }; - } - - pub fn pjrtName(self: Memory) []const u8 { - return @tagName(self.toPjrtMemory()); - } - }; - _shape: Shape, _api: *const pjrt.Api, + _target: Target, _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); - pub const FromOptions = struct { - wait: bool = true, - memory: ?Memory = null, - }; + pub const Memory = pjrt.Memory.Kind; + pub const FromOptions = struct { wait: bool = true, memory: Memory = .device }; /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer, opts: FromOptions) !Buffer { var res: Buffer = .{ ._api = platform.pjrt_api, + ._target = platform.target, ._shape = host_buffer.shape(), ._shards = .{}, }; @@ -89,27 +72,15 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (platform.target == .cpu or opts.memory == null) { + if (opts.memory == .device) { args.device = devices[i]; } else { - const memory = opts.memory.?; - const device_memories = try devices[i].addressableMemories(platform.pjrt_api); - // TODO measure the cost of this and consider caching on Zig side inside the platform. - const selected_memory = for (device_memories) |m| { - const kind = m.kind(platform.pjrt_api); - if (kind == memory.toPjrtMemory()) break m; - } else { - log.warn("Platform {s} doesn't have memory {s}", .{ @tagName(platform.target), @tagName(memory) }); - return error.NotFound; - }; - args.memory = selected_memory; + args.memory = platform.memoryForDevice(opts.memory, devices[i]); } const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); - if (event) |ev| { - ev.deinit(platform.pjrt_api); - } + if (event) |ev| ev.deinit(platform.pjrt_api); res._shards.appendAssumeCapacity(pjrt_buffer); } @@ -139,6 +110,7 @@ pub const Buffer = struct { shards.appendSliceAssumeCapacity(pjrt_buffers); return .{ ._api = platform.pjrt_api, + ._target = platform.target, ._shape = shape_, ._shards = shards, }; @@ -185,9 +157,10 @@ pub const Buffer = struct { } pub fn asHostBuffer(self: Buffer) HostBuffer { - // TODO: skip this check on cpu - // const memory = self.getMemory().kind(self._api); - // stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); + if (self._target != .cpu) { + const memory = self.getMemory().kind(self._api); + stdx.debug.assert((memory == .host_pinned) or (memory == .host_unpinned), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); + } const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } @@ -200,7 +173,7 @@ pub const Buffer = struct { } /// Creates a Buffer with a single element repeated manytime. - pub fn constant(platform: Platform, shape_: Shape, val: anytype) !Buffer { + pub fn constant(platform: Platform, shape_: Shape, val: anytype, opts: FromOptions) !Buffer { var start = try std.time.Timer.start(); defer { const duration_ms = stdx.math.divFloat(f32, start.read(), std.time.ns_per_ms); @@ -245,14 +218,15 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, ._data = &bytes }; - return try from(platform, host_buffer, .{ .wait = true }); + std.debug.assert(opts.wait == true); + return try from(platform, host_buffer, opts); } test constant { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x = try constant(platform, Shape.init(.{ 4, 3, 2 }, .u16), 42); + const x = try constant(platform, Shape.init(.{ 4, 3, 2 }, .u16), 42, .{ .wait = true }); const y = try x.getValue([4 * 3 * 2]u16); try std.testing.expectEqual([_]u16{42} ** (4 * 3 * 2), y); } @@ -271,14 +245,6 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const pjrt.Stream, device_data: *anyopaque) Buffer { - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var res: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - res[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk res; - }; - const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, .element_type = bufferTypeFromDtype(shape_.dtype()), @@ -287,7 +253,7 @@ pub const Buffer = struct { .device = platform.getDevices()[0], .layout = .{ .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], + .minor_to_major = minorToMajor(shape_.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, @@ -299,15 +265,16 @@ pub const Buffer = struct { shards.appendAssumeCapacity(pjrt_buffer); return .{ ._api = platform.pjrt_api, + ._target = platform.target, ._shape = shape_, ._shards = shards, }; } - pub fn opaqueDeviceMemoryDataPointer(self: Buffer) [*]u8 { + pub fn devicePtr(self: Buffer) u64 { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); const opaque_ptr: *anyopaque = self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; - return @ptrCast(opaque_ptr); + return @intFromPtr(opaque_ptr); } /// Fetches the content of the given buffer into a stack variable of the given type. @@ -385,7 +352,7 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try writer.print("Buffer({f})", .{self._shape}); + try writer.print("Buffer({f})@{x}", .{ self._shape, self.devicePtr() }); } pub fn getMemory(self: Buffer) *const pjrt.Memory { @@ -402,10 +369,10 @@ pub const Buffer = struct { wait: bool = true, }; - pub fn copyToMemory(self: Buffer, platform: Platform, memory: Memory, opts: CopyToMemoryOpts) !Buffer { - const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory.toPjrtMemory()); + pub fn copyToMemory(self: Buffer, platform: Platform, memory: pjrt.Memory.Kind, opts: CopyToMemoryOpts) !Buffer { + const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory); if (pjrt_memory == null) { - stdx.debug.panic("Memory destination `{s}` for {f}", .{ memory.pjrtName(), self }); + stdx.debug.panic("Memory destination `{t}` for {f}", .{ memory, self }); } var new_shards: Buffer.Shards = .{}; @@ -423,35 +390,26 @@ pub const Buffer = struct { } } - return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; + return Buffer{ ._api = self._api, ._target = platform.target, ._shape = self._shape, ._shards = new_shards }; } - pub const UnitializedOptions = struct { - memory: ?pjrt.Memory.Kind = null, - }; + pub const UnitializedOptions = struct { memory: Memory = .device }; pub fn uninitialized(platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { var res: Buffer = .{ ._api = platform.pjrt_api, ._shape = shape_, ._shards = .{}, + ._target = platform.target, }; errdefer for (res._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); }; - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var minor_to_major: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - minor_to_major[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk minor_to_major; - }; - - // TODO: support more advanced sharding specs stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); const n_partitions = platform.sharding().num_partitions; + const shard_shape = if (sharding_ax) |ax| s: { // This kind of sharding error should be detected earlier on. stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); @@ -459,37 +417,47 @@ pub const Buffer = struct { break :s shard_shape; } else shape_; - const buffer_type = bufferTypeFromDtype(shape_.dtype()); + var args = pjrt.Client.CreateUninitializedBufferArgs{ + .dims = shard_shape.dims(), + .element_type = bufferTypeFromDtype(shape_.dtype()), + .layout = .{ + .tiled = .{ + .minor_to_major = minorToMajor(shape_.rank()), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + }; + const devices = platform.getDevices(); for (0..n_partitions) |i| { - var args = pjrt.Client.CreateUninitializedBufferArgs{ - .dims = shard_shape.dims(), - .element_type = buffer_type, - .layout = .{ - .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - }; - if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); - const memory = for (memories) |m| { - const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; - } else return error.NotFound; - args.memory = memory; - } else { + if (opts.memory == .device) { args.device = devices[i]; + } else { + args.memory = platform.memoryForDevice(opts.memory, devices[i]); } - const pjrt_buffer = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); + const pjrt_buffer = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); res._shards.appendAssumeCapacity(pjrt_buffer); } return res; } + + test uninitialized { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const host_visible_memories: []const Memory = &.{ .host_pinned, .host_unpinned }; + for (host_visible_memories) |memory| { + const x = try uninitialized(platform, .init(.{6}, .u8), .{ .memory = memory }); + const x_ptr: [*]u8 = @ptrFromInt(x.devicePtr()); + @memcpy(x_ptr, &[_]u8{ 104, 101, 108, 108, 111, 33 }); + + const y = try x.getValue([6]u8); + try std.testing.expectEqualStrings("hello!", &y); + } + } }; pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { @@ -517,3 +485,15 @@ test bufferTypeFromDtype { try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); } } + +const _MINOR_TO_MAJOR = blk: { + var ret: [Shape.MAX_RANK]i64 = undefined; + for (0..Shape.MAX_RANK) |i| { + ret[i] = @intCast(Shape.MAX_RANK - i - 1); + } + break :blk ret; +}; + +fn minorToMajor(rank: u8) []const i64 { + return _MINOR_TO_MAJOR[_MINOR_TO_MAJOR.len - rank ..]; +} diff --git a/zml/platform.zig b/zml/platform.zig index 6e13d7f0..2c0f34f9 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -10,6 +10,10 @@ const log = std.log.scoped(.zml); pub const available_targets = std.enums.values(Target); +test { + std.testing.refAllDecls(@This()); +} + pub const CompilationOptions = struct { xla_dump_to: ?[]const u8 = null, xla_dump_fusion_visualization: bool = false, @@ -79,6 +83,40 @@ pub const Platform = struct { pub fn deinit(self: *Platform) void { self.pjrt_client.deinit(self.pjrt_api); } + + pub fn memoryForDevice(platform: Platform, memory: pjrt.Memory.Kind, device: *const pjrt.Device) *const pjrt.Memory { + const memory_target: pjrt.Memory.Kind = switch (memory) { + .host_unpinned => switch (platform.target) { + // Cuda doesn't have host_unpinned. + .cuda => .host_pinned, + else => .host_unpinned, + }, + inline else => |t| t, + }; + // TODO measure the cost of this and consider caching. + const device_memories = device.addressableMemories(platform.pjrt_api); + // for (device_memories) |m| { + // log.warn("Platform {t}, device {*} -> memory {t}", .{platform.target, device, m.kind(platform.pjrt_api)}); + // } + for (device_memories) |m| { + if (memory_target == m.kind(platform.pjrt_api)) { + return m; + } + } + log.err("Platform {t} doesn't have memory {t}", .{ platform.target, memory }); + @panic("Memory kind not found"); + } + + test memoryForDevice { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const memory_fields = @typeInfo(pjrt.Memory.Kind).@"enum".fields; + inline for (memory_fields) |field| { + for (platform.getDevices()) |dev| { + _ = platform.memoryForDevice(@field(pjrt.Memory.Kind, field.name), dev); + } + } + } }; const _CreateOptions = struct { diff --git a/zml/tensor.zig b/zml/tensor.zig index ce1cf412..4020ddb4 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -202,10 +202,8 @@ pub const Tensor = struct { const mlir_ctx = ctx.mlirCtx(); if (ctx.target() == .cpu) return self; - const memory_kind = @tagName(kind.toPjrtMemory()); - const frontend_attributes = mlir.Attribute.dict(mlir_ctx, &.{ - .{ "_xla_buffer_placement", .string(mlir_ctx, memory_kind) }, + .{ "_xla_buffer_placement", .string(mlir_ctx, kind.pjrtName()) }, }); const op = dialect.stablehlo.custom_call(mlir_ctx, &.{self.value()}, .{ @@ -2688,7 +2686,7 @@ pub const Tensor = struct { } { // Test with actual values and batching along axis .a - const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0); + const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0, .{}); defer operand.deinit(); const start_indices = (try zml.Buffer.fromArray( platform, @@ -2709,6 +2707,7 @@ pub const Tensor = struct { platform, Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16), 1, + .{}, ); defer values.deinit(); From 90ea3b407d06cc4b548002c2ac6cb6c79f79e65f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 15 Sep 2025 14:38:24 +0200 Subject: [PATCH 16/21] remove uninitialized --- zml/buffer.zig | 49 +++---------------------------------------------- 1 file changed, 3 insertions(+), 46 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 8f7b4eee..99fa7b6c 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -396,52 +396,9 @@ pub const Buffer = struct { pub const UnitializedOptions = struct { memory: Memory = .device }; pub fn uninitialized(platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { - var res: Buffer = .{ - ._api = platform.pjrt_api, - ._shape = shape_, - ._shards = .{}, - ._target = platform.target, - }; - errdefer for (res._shards.slice()) |shard| { - shard.deinit(platform.pjrt_api); - }; - - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); - const n_partitions = platform.sharding().num_partitions; - - const shard_shape = if (sharding_ax) |ax| s: { - // This kind of sharding error should be detected earlier on. - stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); - const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); - break :s shard_shape; - } else shape_; - - var args = pjrt.Client.CreateUninitializedBufferArgs{ - .dims = shard_shape.dims(), - .element_type = bufferTypeFromDtype(shape_.dtype()), - .layout = .{ - .tiled = .{ - .minor_to_major = minorToMajor(shape_.rank()), - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - }; - - const devices = platform.getDevices(); - for (0..n_partitions) |i| { - if (opts.memory == .device) { - args.device = devices[i]; - } else { - args.memory = platform.memoryForDevice(opts.memory, devices[i]); - } - - const pjrt_buffer = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); - res._shards.appendAssumeCapacity(pjrt_buffer); - } - - return res; + // XLA uninitialized is broken see https://github.com/openxla/xla/pull/31292 + // TODO: use uninitialized when it works again. + return try constant(platform, shape_, 0, .{ .wait = true, .memory = opts.memory }); } test uninitialized { From a471ef3887eb8759b903153bff838c3d3bc852b7 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 15 Sep 2025 14:51:26 +0200 Subject: [PATCH 17/21] fixup! wip: add target tag to buffer struct --- examples/llama/main.zig | 2 +- zml/aio.zig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 997d219d..9d7d7ec6 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -117,7 +117,7 @@ pub fn generateText( var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); defer prefill_tokens.deinit(); - var prefill_token_pos = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0); + var prefill_token_pos = try zml.Buffer.scalar(platform, 0, .u32); defer prefill_token_pos.deinit(); const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); diff --git a/zml/aio.zig b/zml/aio.zig index 70ba2b7d..06545694 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -708,7 +708,7 @@ pub fn loadModelBuffersWithPrefix( var res: zml.Bufferized(Model) = undefined; try zml.meta.mapAlloc(struct { pub fn initBuffer(_: void, tensor: zml.Tensor) zml.Buffer { - return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined }; + return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined, ._target = undefined }; } }.initBuffer, allocator, {}, model, &res); From f78e2c15ca8f7589aa5018b0cce87edf2c4b43b0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 15 Sep 2025 16:18:51 +0200 Subject: [PATCH 18/21] tokenization fixup --- examples/llama/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 9d7d7ec6..e58b4cb6 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -50,7 +50,7 @@ pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok const user = tokenizer.tokenToId("user").?; const assistant = tokenizer.tokenToId("assistant").?; const eot = tokenizer.tokenToId("<|eot_id|>").?; - const newline = tokenizer.tokenToId("\n").?; + const newline = (try encoder.encode("\n"))[0]; try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); @@ -68,7 +68,7 @@ pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok const end_message = tokenizer.tokenToId("<|im_end|>").?; const user = tokenizer.tokenToId("user").?; const assistant = tokenizer.tokenToId("assistant").?; - const newline = tokenizer.tokenToId("\\n").?; + const newline = (try encoder.encode("\n"))[0]; try tokens.appendSlice(allocator, &.{ start_message, user, newline }); From 0b741213c1579ddd80c5f108da7d51b299ee48d0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 15 Sep 2025 16:18:59 +0200 Subject: [PATCH 19/21] add tokenizer test --- zml/tokenizer/BUILD.bazel | 13 ++++++++++++- zml/tokenizer/homemade.zig | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 9b2f9c71..b7af9e29 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") zig_library( name = "tokenizer", @@ -25,3 +25,14 @@ zig_binary( "//stdx", ], ) + +zig_test( + name = "test", + main = "homemade.zig", + visibility = ["//visibility:public"], + deps = [ + "//async", + "//ffi:zig", + "//stdx", + ], +) diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig index 91eb1df9..26db53b6 100644 --- a/zml/tokenizer/homemade.zig +++ b/zml/tokenizer/homemade.zig @@ -1039,6 +1039,7 @@ test Gpt2TextDecoder { try testing.expectEqualStrings(" UINavigationController", try decoder.decode(&out, "ĠUINavigationController")); // Russian is wild try testing.expectEqualStrings(" работ", try decoder.decode(&out, "ĠÑĢабоÑĤ")); + try testing.expectEqualStrings("\n", try decoder.decode(&out, "Ċ")); } /// Open a json file in HF format and load the vocab from it. From 2b54dcccc2e6e4d766cc2355438a5615176564b4 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 16 Sep 2025 11:13:03 +0200 Subject: [PATCH 20/21] fix Platform.CreateOptions --- pjrt/pjrt.zig | 1 - zml/aio.zig | 5 +++++ zml/buffer.zig | 1 + zml/exe.zig | 1 + zml/module.zig | 6 ++++++ zml/platform.zig | 14 +++++++++----- 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index e0477731..4247a799 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -729,7 +729,6 @@ pub const LoadedExecutable = opaque { _ = api.call(.PJRT_LoadedExecutable_Destroy, .{ .executable = self.inner(), }) catch {}; - self.* = undefined; } pub fn delete(self: *LoadedExecutable, api: *const Api) void { diff --git a/zml/aio.zig b/zml/aio.zig index 06545694..29a19945 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -121,12 +121,17 @@ pub const BufferStore = struct { if (id < self._unique_id or self._unique_id + _store_id_range <= id) { @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`, using the same store object."); } + if (platform.target != .cpu) mem_debug: { + const stats = platform.getDevices()[0].memoryStats(platform.pjrt_api) catch break :mem_debug; + log.debug("Loading {s} -> {f} {d:>10} bytes ({d:>10} allocated / {d:>10} reserved)", .{ self.buffers.keys()[id - self._unique_id], x._shape, x.shape().byteSize(), stats.bytes_in_use, stats.bytes_reserved }); + } break :hb self.buffers.values()[id - self._unique_id]; }, else => @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`"), }; // Use the sharding information stored in the tensor. + std.debug.assert(host_buffer.shape().eql(x.shape())); host_buffer._shape = x.shape(); return try host_buffer.toDevice(platform); } diff --git a/zml/buffer.zig b/zml/buffer.zig index 99fa7b6c..817b636f 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -317,6 +317,7 @@ pub const Buffer = struct { /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *const Buffer) void { + // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); for (self._shards.constSlice()) |buffer| { buffer.deinit(self._api); } diff --git a/zml/exe.zig b/zml/exe.zig index 71db84d7..214e245c 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -235,6 +235,7 @@ pub const BaseExe = struct { if (self.execute_context) |ctx| { ctx.deinit(self.platform.pjrt_api); } + self.exe.deinit(self.platform.pjrt_api); self._arena.deinit(); } diff --git a/zml/module.zig b/zml/module.zig index 38038758..04cf7bb3 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -221,6 +221,12 @@ pub const CompilationContext = struct { break :blk loaded_executable; }; + { + const exe = try loaded_executable.getExecutable(self._platform.pjrt_api); + const stats = try exe.getCompiledMemoryStats(self._platform.pjrt_api); + log.debug("Compiled {s}: {any}", .{ self._name, stats }); + } + log.debug("******** ZML generated MLIR ********", .{}); log.debug("{f}", .{module.op().mlirFormatter(.{})}); diff --git a/zml/platform.zig b/zml/platform.zig index 2c0f34f9..88beed98 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -165,17 +165,21 @@ const _CreateOptions = struct { fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { - values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); + values.appendAssumeCapacity(.fromString("allocator", "platform")); }, .bfc, .async => |opt| { - values.appendAssumeCapacity(pjrt.NamedValue.from("allocator", self.allocator)); - values.appendAssumeCapacity(pjrt.NamedValue.from("preallocate", opt.preallocate)); + values.appendAssumeCapacity(.fromString("allocator", switch (self.allocator) { + .bfc => "bfc", + .async => "cuda_async", + .platform => unreachable, + })); + values.appendAssumeCapacity(.from("preallocate", opt.preallocate)); if (opt.memory_fraction > 0) { - values.appendAssumeCapacity(pjrt.NamedValue.from("memory_fraction", opt.memory_fraction)); + values.appendAssumeCapacity(.from("memory_fraction", opt.memory_fraction)); } if (opt.collective_memory_size_mb > 0) { const collective = @as(i64, opt.collective_memory_size_mb) * 1024 * 1024; - values.appendAssumeCapacity(pjrt.NamedValue.from("collective_memory_size", collective)); + values.appendAssumeCapacity(.from("collective_memory_size", collective)); } }, } From 42f76c1391b5766f4612259edf56ad9b00b7095c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 16 Sep 2025 11:37:48 +0200 Subject: [PATCH 21/21] add device_memory_size --- zml/module.zig | 4 +++- zml/platform.zig | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/zml/module.zig b/zml/module.zig index 04cf7bb3..0a8b5f0e 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -887,6 +887,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + if (platform.compilation_options.device_memory_size > 0) { + c.xla_ExecutableBuildOptionsProto_set_device_memory_size(exec_build_options, @intCast(platform.compilation_options.device_memory_size)); + } c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); @@ -901,7 +904,6 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m } break :device_assignment_blk device_assignment; }); - break :executable_build_options_blk exec_build_options; }); diff --git a/zml/platform.zig b/zml/platform.zig index 88beed98..4f39d00a 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -20,6 +20,7 @@ pub const CompilationOptions = struct { xla_dump_hlo_pass_re: ?[]const u8 = null, sharding_enabled: bool = false, sharding_axes: stdx.BoundedArray([*:0]const u8, 8) = .{}, + device_memory_size: u64 = 0, }; pub const Platform = struct {