From ffbb293331fe6dada30eac8eda7f9fba32da445d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 27 May 2025 17:20:58 +0200 Subject: [PATCH 01/67] zml: custom call v2 --- examples/custom_call/BUILD | 11 ++ examples/custom_call/main.zig | 154 +++++++++++++++++++++++ pjrt/ffi.zig | 2 +- pjrt/pjrt.zig | 8 +- zml/BUILD.bazel | 1 + zml/buffer.zig | 46 +++++-- zml/context.zig | 69 +---------- zml/custom_call.zig | 224 ++++++++++++++++++++++++++++++++++ zml/exe.zig | 38 ++++++ zml/hostbuffer.zig | 10 ++ zml/module.zig | 8 +- zml/ops.zig | 27 ---- zml/pjrtx.zig | 7 ++ zml/tensor.zig | 19 +-- zml/zml.zig | 1 + 15 files changed, 494 insertions(+), 131 deletions(-) create mode 100644 examples/custom_call/BUILD create mode 100644 examples/custom_call/main.zig create mode 100644 zml/custom_call.zig diff --git a/examples/custom_call/BUILD b/examples/custom_call/BUILD new file mode 100644 index 00000000..a2aa32fb --- /dev/null +++ b/examples/custom_call/BUILD @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "custom_call", + copts = ["-freference-trace=30"], + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig new file mode 100644 index 00000000..e5a17443 --- /dev/null +++ b/examples/custom_call/main.zig @@ -0,0 +1,154 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +const cuda = zml.context.cuda; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +fn add_op_host_func(data: *anyopaque) callconv(.c) void { + const ctx = @as(*AddOp, @alignCast(@ptrCast(data))); + + const result = std.mem.bytesAsValue(f32, ctx.result.asHostBuffer().mutBytes()); + result.* = ctx.a.asHostBuffer().item(f32, 0) + ctx.b.asHostBuffer().item(f32, 0); +} + +pub const AddOp = struct { + pub var type_id: i64 = undefined; + const Self = @This(); + + allocator: std.mem.Allocator, + platform: zml.Platform, + + a: zml.Buffer, + b: zml.Buffer, + result: zml.Buffer, + + results: [1]zml.Buffer = undefined, + stream: *anyopaque = undefined, + + pub fn init( + allocator: std.mem.Allocator, + platform: zml.Platform, + io_shape: zml.Shape, + ) !AddOp { + const host_buffer = try zml.HostBuffer.empty(allocator, io_shape); + defer host_buffer.deinit(allocator); + + const a = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + const b = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + const result = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + + return .{ + .a = a, + .b = b, + .result = result, + .platform = platform, + .allocator = allocator, + }; + } + + pub fn deinit(self: *Self) void { + self.a.deinit(); + self.b.deinit(); + self.result.deinit(); + } + + pub fn call(self: *Self, a: zml.Buffer, b: zml.Buffer) !void { + if (self.platform.target == .cuda) { + cuda.memcpyToHostAsync(self.a.asHostBuffer().mutBytes(), a.opaqueDeviceMemoryDataPointer(), self.stream); + cuda.memcpyToHostAsync(self.b.asHostBuffer().mutBytes(), b.opaqueDeviceMemoryDataPointer(), self.stream); + + _ = cuda.cuLaunchHostFunc(self.stream, @ptrCast(&add_op_host_func), @ptrCast(self)); + + cuda.memcpyToDeviceAsync(self.results[0].opaqueDeviceMemoryDataPointer(), self.result.asHostBuffer().bytes(), self.stream); + return; + } + + if (self.platform.target == .cpu) { + const result = std.mem.bytesAsValue(f32, self.results[0].asHostBuffer().mutBytes()); + result.* = a.asHostBuffer().item(f32, 0) + b.asHostBuffer().item(f32, 0); + return; + } + + return error.UnsupportedTarget; + } +}; + +const Layer = struct { + pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + const result = zml.custom_call(AddOp, .{ a.print(), b.print() }, &[_]zml.Shape{a.shape()}, .{}); + return result[0]; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const shape = zml.Shape.init(.{}, .f32); + + const buffers: zml.aio.BufferStore.Buffers = .{}; + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); + + const compiled = try compilation.awaitt(); + + var executable = compiled.prepare(model_weights); + defer executable.deinit(); + + executable = try executable.withExecutionContext(); + + try platform.registerFFIType(AddOp); + var add_op: AddOp = try .init(allocator, platform, shape); + defer add_op.deinit(); + try executable.bind(AddOp, &add_op); + + var input_a = [1]f32{1.0}; + var input_buffer_a = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_a), .{}); + defer input_buffer_a.deinit(); + + var input_b = [1]f32{1.0}; + var input_buffer_b = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_b), .{}); + defer input_buffer_b.deinit(); + + var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + defer result.deinit(); + + const cpu_result = try result.toHostAlloc(arena); + _ = cpu_result; // autofix + + // log.warn( + // "\nThe result of {} + {} = {}\n", + // .{ &input_a, &input_b, cpu_result.items(f32) }, + // ); +} diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index d2517a04..b3ce2a74 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -169,7 +169,7 @@ pub const ExecutionContext = opaque { const type_id: TypeId = .{ .type_id = T.type_id }; var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ .ctx = @constCast(self.inner()), - .type_id = @constCast(&type_id.toCStruct()), + .type_id = @constCast(&type_id), }); const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 6e358562..aead3bd3 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1293,7 +1293,7 @@ pub const FFI = extern struct { } }; - pub const RegisterFfiOptions = struct { + pub const RegisterOptions = struct { traits: RegisterHandlerTraits = @enumFromInt(0), }; @@ -1305,7 +1305,7 @@ pub const FFI = extern struct { target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, - options: RegisterFfiOptions, + options: RegisterOptions, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .target_name = target_name.ptr, @@ -1357,7 +1357,3 @@ pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { command_buffer_compatible = c.PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE, _, }; - -pub const CustomCallRegistry = extern struct { - inner: *const c.PJRT_FFI_Register_Handler, -}; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index aaa44f04..88876478 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -31,6 +31,7 @@ zig_library( "aio/torch/py.zig", "buffer.zig", "context.zig", + "custom_call.zig", "dtype.zig", "exe.zig", "floats.zig", diff --git a/zml/buffer.zig b/zml/buffer.zig index 3553bb94..259cb262 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -49,7 +49,7 @@ pub const Buffer = struct { pub const FromOptions = struct { wait: bool = true, - memory: ?pjrt.Memory.Kind = null, + memory: ?Memory = null, }; /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -89,13 +89,13 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); - const memory = for (memories) |m| { + if (opts.memory) |memory| { + const device_memories = try devices[i].addressableMemories(platform.pjrt_api); + const selected_memory = for (device_memories) |m| { const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; + if (kind == memory.toPjrtMemory()) break m; } else return error.NotFound; - args.memory = memory; + args.memory = selected_memory; } else { args.device = devices[i]; } @@ -179,10 +179,9 @@ pub const Buffer = struct { return try from(platform, host_buffer, opts); } - pub fn asPinnedHostBuffer(self: Buffer) HostBuffer { - // TODO restore assert - // const memory = self.getMemory().kind(self._api); - // stdx.debug.assert(memory == .pinned_host, "asPinnedHostBuffer({}) expects a buffer allocated on host memory, got {}. see `toMemory`", .{ self, memory }); + pub fn asHostBuffer(self: Buffer) HostBuffer { + const memory = self.getMemory().kind(self._api); + stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({any}) expects a buffer allocated on host memory, got {any}. see `copyToMemory`", .{ self, memory }); const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } @@ -299,6 +298,11 @@ pub const Buffer = struct { }; } + pub fn opaqueDeviceMemoryDataPointer(self: Buffer) *anyopaque { + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); + return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + } + /// Fetches the content of the given buffer into a stack variable of the given type. pub fn getValue(self: Buffer, T: type) !T { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); @@ -390,13 +394,31 @@ pub const Buffer = struct { return @reduce(.Or, self._shape._sharding_info); } - pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { + pub const CopyToMemoryOpts = struct { + wait: bool = true, + }; + + pub fn copyToMemory(self: Buffer, platform: Platform, memory: Memory, opts: CopyToMemoryOpts) !Buffer { + const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory.toPjrtMemory()); + if (pjrt_memory == null) { + stdx.debug.panic("Memory destination `{s}` for {any}", .{ memory.pjrtName(), self }); + } + var new_shards: Buffer.Shards = .{}; for (self._shards.slice()) |shard| { - const new_shard = try shard.copyToMemory(self._api, memory); + const new_shard = try shard.copyToMemory(self._api, pjrt_memory.?); new_shards.appendAssumeCapacity(new_shard); } + if (opts.wait) { + for (new_shards.constSlice()) |shard| { + const event = shard.getReadyEvent(self._api); + if (event) |e| { + try e.awaitBlocking(self._api); + } + } + } + return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; } diff --git a/zml/context.zig b/zml/context.zig index 15acb93b..f2e01f40 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -13,6 +13,7 @@ const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Target = @import("platform.zig").Target; +const custom_call = @import("custom_call.zig"); const zml_platform = @import("platform.zig"); const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); @@ -36,6 +37,7 @@ pub const Context = struct { inline for (comptime std.enums.values(runtimes.Platform)) |t| { if (runtimes.load(t)) |api| { Context.apis.set(t, api); + if (t == .cuda) cuda.init(); } else |_| {} } } @@ -176,9 +178,9 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); + try custom_call.registerInternalCustomCalls(p); + return p; } @@ -211,71 +213,8 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; }; -const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { - const ffi = platform.pjrt_api.ffi() orelse { - log.warn("Registering custom calls failed: No FFI Extension found in {s} PJRT Plugin.", .{@tagName(platform.target)}); - return; - }; - try ffi.register(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback, .{}); - } - - fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; - std.debug.assert(callback_attr.dtype == .u64); - const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); - - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; - std.debug.assert(user_ctx_ptr.dtype == .u64); - const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.args.buffers()[i]); - } - - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.results.buffers()[i]); - } - - callback(user_ctx, input_buffers, output_buffers); - return null; - } -}; - -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { - // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { - .invalid => @panic("invalid ffi"), - .pred => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), - }; - return Shape.init(buffer_desc.dims(), dt); -} - -/// Create a HostBuffer from a ffi description of a buffer. -/// Normally the ffi describe device buffer but we assume they are located in pinned memory, -/// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { - const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( - buffer_shape, - buffer_desc.data[0..buffer_shape.byteSize()], - ); -} - pub const cuda = struct { pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); pub var cuLaunchHostFunc: CuLaunchHostFunc = @ptrFromInt(0xdeadc00da00); diff --git a/zml/custom_call.zig b/zml/custom_call.zig new file mode 100644 index 00000000..b6cc8f0e --- /dev/null +++ b/zml/custom_call.zig @@ -0,0 +1,224 @@ +const std = @import("std"); + +const asynk = @import("async"); +const stdx = @import("stdx"); + +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; +const mlir = @import("mlir"); +const mlirx = @import("mlirx.zig"); +const pjrt = @import("pjrt"); +const pjrtx = @import("pjrtx.zig"); +const ffi = pjrtx.ffi; +const cuda = @import("context.zig").cuda; +const Buffer = @import("buffer.zig").Buffer; +const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const CompilationContext = @import("module.zig").CompilationContext; +const DataType = @import("dtype.zig").DataType; +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/custom_call"); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn CustomCallInputsType(custom_op: type) type { + const ArgsT = stdx.meta.FnArgs(custom_op.call); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + + const args = @typeInfo(ArgsT).@"struct".fields; + + for (args[1..args.len]) |field| { + if (field.type != Buffer) { + @compileError("Expected all arguments in custom call `call` to be of type zml.Buffer"); + } + } + + return [args.len - 1]Tensor; +} + +pub const CustomCallOptions = struct { + output_operand_aliases: []const i64 = &.{}, + copy_inputs_to_host_pinned: bool = false, + register_ffi_options: pjrt.FFI.RegisterOptions = .{ + .traits = @enumFromInt(0), + }, +}; + +pub fn custom_call( + comptime custom_op: type, + inputs: CustomCallInputsType(custom_op), + res_shapes_: []const Shape, + comptime opts: CustomCallOptions, +) []Tensor { + stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); + const op_name = @typeName(custom_op); + + const ctx = CompilationContext.current(); + const allocator = ctx.allocator(); + const mlir_ctx = ctx.mlirCtx(); + const platform = ctx._platform; + const pjrt_api = platform.pjrt_api; + const ffi_ = pjrt_api.ffi(); + + if (ffi_ == null) { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); + } + + const ffi_func = proxy(custom_op, opts); + const target_name = "callback_" ++ op_name; + + ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.register_ffi_options) catch unreachable; + log.info("{s} / Registered custom call with target name \"{s}\" with proxy func {*}", .{ op_name, target_name, &ffi_func }); + + const custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; + for (inputs, 0..) |t, i| { + custom_call_inputs[i] = t.value(); + } + + const res_types = allocator.alloc(mlir.Type, 8) catch unreachable; + for (res_shapes_, 0..) |sh, i| { + res_types[i] = mlirx.tensorType(mlir_ctx, sh); + } + + const frontend_attributes: mlir.Attribute = .dict(mlir_ctx, &.{}); + + const op = dialect.stablehlo.custom_call( + mlir_ctx, + custom_call_inputs[0..inputs.len], + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, + .has_side_effect = true, + .output_operand_aliases = opts.output_operand_aliases, + }, + res_types[0..res_shapes_.len], + mlir_ctx.location(@src()), + ); + + var custom_call_outputs = allocator.alloc(Tensor, 8) catch unreachable; + + for (custom_call_outputs[0..res_shapes_.len], res_shapes_, 0..) |*t, sh, i| { + t.* = Tensor._result(sh, op.result(i)); + } + + return custom_call_outputs[0..res_shapes_.len]; +} + +fn proxy(comptime custom_op: type, opts: CustomCallOptions) ffi.Handler { + return struct { + const Self = @This(); + + pub fn proxy(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { + if (call_frame.registeringHook()) return null; + const execution_context = call_frame.ctx; + const user_ctx: *custom_op = ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api) catch unreachable; + const platform: Platform = user_ctx.platform; + + if (@hasField(custom_op, "stream") and platform.target != .cpu) { + const stream = call_frame.api.stream(execution_context); + user_ctx.stream = stream; + } else { + log.info("No stream field provided in container {s} for custom call", .{@typeName(custom_op)}); + } + + var callback_args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; + callback_args[0] = user_ctx; + + inline for (1..callback_args.len) |i| { + const ffi_buffer = call_frame.args.buffers()[i - 1]; + const ffi_buffer_shape = getShape(ffi_buffer); + var zml_buffer: Buffer = undefined; + + if (platform.target == .cpu) { + zml_buffer = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + zml_buffer = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch unreachable; + } + callback_args[i] = zml_buffer; + } + + for (0..call_frame.results.len) |i| { + const ffi_buffer = call_frame.results.buffers()[i]; + const ffi_buffer_shape = getShape(ffi_buffer); + + if (platform.target == .cpu) { + user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + } + + @call(.auto, custom_op.call, callback_args) catch |err| { + stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(custom_op), err }); + }; + + return null; + } + }.proxy; +} + +pub fn getShape(ffi_buffer: *const ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .pred => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} + +/// Internal custom calls. +/// These are not meant to be used by users, but rather by the library itself. +pub const custom_call_internal_types = [_]type{ + Print, +}; + +pub fn registerInternalCustomCalls( + platform: Platform, +) !void { + inline for (custom_call_internal_types) |custom_call_type| { + try platform.registerFFIType(custom_call_type); + log.info("Registered internal custom call {s} with type_id {any}", .{ @typeName(custom_call_type), custom_call_type.type_id }); + } +} + +pub const Print = struct { + pub var type_id: i64 = undefined; + const Self = @This(); + + allocator: std.mem.Allocator, + platform: Platform, + + results: [1]Buffer = undefined, + + pub fn init( + allocator: std.mem.Allocator, + platform: Platform, + ) !Print { + return .{ + .allocator = allocator, + .platform = platform, + }; + } + + pub fn call(_: *Self, input: Buffer) !void { + std.log.defaultLog(.info, .zml, "Device buffer: {any}: {any}", .{ input.shape(), input.asHostBuffer().pretty() }); + } +}; diff --git a/zml/exe.zig b/zml/exe.zig index 625fecf9..7a303fa3 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -8,6 +8,7 @@ const Bufferized = @import("tensor.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); +const custom_call = @import("custom_call.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; @@ -285,6 +286,21 @@ pub const BaseExe = struct { stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); } + pub fn bind(self: BaseExe, comptime T: type, value: *T) !void { + stdx.debug.assert(self.context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = self.platform.pjrt_api; + + if (pjrt_api.ffi()) |ffi| { + const type_id = T.type_id; + const user_data: *anyopaque = @ptrCast(@constCast(value)); + + try ffi.addUserData(pjrt_api, self.context.?, .{ .type_id = type_id, .user_data = user_data }); + log.info("Bound {s}@{any} with type id {any} on {any}", .{ @typeName(T), user_data, T.type_id, self.context.? }); + } else { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(self.platform.target)}); + } + } + pub fn serialize(self: BaseExe, writer: anytype) !void { var executable = try self.exe.getExecutable(self.platform.pjrt_api); var serialize_result = try executable.serialize(self.platform.pjrt_api); @@ -348,6 +364,28 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + pub fn withExecutionContext(self: Self) !Self { + stdx.debug.assert(self.inner.context == null, "Exe already has an execution context", .{}); + var new: Self = .{ .inner = self.inner }; + var arena = &new.inner._arena; + var allocator = arena.allocator(); + + const pjrt_execute_context = try new.inner.platform.pjrt_api.createExecuteContext(); + new.inner.context = pjrt_execute_context; + inline for (custom_call.custom_call_internal_types) |custom_call_internal_type| { + const value_ptr = try allocator.create(custom_call_internal_type); + value_ptr.* = try .init(allocator, new.platform()); + try new.inner.bind(custom_call_internal_type, value_ptr); + } + log.info("Created context execution {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); + + return new; + } + + pub fn bind(self: Self, comptime T: type, value: *T) !void { + try self.inner.bind(T, value); + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index c2099b0d..ee598880 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -176,10 +176,20 @@ pub const HostBuffer = struct { return ptr[0..self._shape.count()]; } + pub fn item(self: HostBuffer, comptime T: type, index: usize) T { + stdx.debug.assert(index < self._shape.count(), "index {any} out of bounds for shape {any}", .{ index, self._shape }); + return self.items(T)[index]; + } + pub fn mutItems(self: HostBuffer, comptime T: type) []T { return @constCast(self.items(T)); } + pub fn mutitem(self: HostBuffer, comptime T: type, index: usize) T { + stdx.debug.assert(index < self._shape.count(), "index {any} out of bounds for shape {any}", .{ index, self._shape }); + return self.mutItems(T)[index]; + } + pub fn bytes(self: HostBuffer) []const u8 { stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); return self._data[0..self._shape.byteSize()]; diff --git a/zml/module.zig b/zml/module.zig index 33d22f78..6263d3fd 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -1178,9 +1178,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1196,7 +1196,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/ops.zig b/zml/ops.zig index c26eac3c..ee125086 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -764,33 +764,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - return customCall( - "zmlHostBufferCallback", - inputs, - output_shapes, - .{ - .callback = @intFromPtr(callback), - .user_context = @intFromPtr(blkctx), - }, - .{ - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = opts.output_operand_aliases, - }, - ); -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 84343a63..9239bc3c 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -207,6 +207,13 @@ pub const Event = opaque { return self.inner().getEventError(api); } + pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { + if (self.isReady(api)) { + return; + } + try self.inner().await_(api); + } + pub fn await_(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); diff --git a/zml/tensor.zig b/zml/tensor.zig index 26f670d7..04874214 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -13,6 +13,8 @@ const Memory = @import("buffer.zig").Buffer.Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); +const custom_call_ = @import("custom_call.zig"); +const custom_call = custom_call_.custom_call; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -3824,22 +3826,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - // TODO: find a way of doing print that doesn't involve a H2D copy. - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.log.defaultLog(.info, .zml, "Device buffer: {f}: {f}", .{ host_buffer.shape(), host_buffer.pretty() }); - // This is true because of the operand aliases. - // Since the result is already pointing to the input we don't need to modify the buffer. - std.debug.assert(host_buffer._data == outputs[0]._data); + return custom_call(custom_call_.Print, .{input}, &[_]Shape{input.shape()}, .{ .output_operand_aliases = &.{0}, .copy_inputs_to_host_pinned = true })[0]; } }; diff --git a/zml/zml.zig b/zml/zml.zig index b5cd8039..05602e9a 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -17,6 +17,7 @@ pub const Context = @import("context.zig").Context; pub const Data = @import("dtype.zig").Data; pub const DataType = @import("dtype.zig").DataType; pub const exe = @import("exe.zig"); +pub const custom_call = @import("custom_call.zig").custom_call; pub const compile = exe.compile; pub const compileWithPrefix = exe.compileWithPrefix; pub const compileFn = exe.compileFn; From 3e49bfee282d6e912e159af936f6155cecb17256 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 11 Jun 2025 15:47:50 +0000 Subject: [PATCH 02/67] async exe --- zml/context.zig | 2 -- zml/custom_call.zig | 20 ++++++++++---------- zml/exe.zig | 12 ++++++------ zml/pjrtx.zig | 12 ++++++++++-- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/zml/context.zig b/zml/context.zig index f2e01f40..47c86708 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -7,8 +7,6 @@ const runfiles = @import("runfiles"); const runtimes = @import("runtimes"); const stdx = @import("stdx"); -const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; diff --git a/zml/custom_call.zig b/zml/custom_call.zig index b6cc8f0e..849a854e 100644 --- a/zml/custom_call.zig +++ b/zml/custom_call.zig @@ -1,25 +1,25 @@ const std = @import("std"); const asynk = @import("async"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); const stdx = @import("stdx"); -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; -}; -const mlir = @import("mlir"); +const Buffer = @import("buffer.zig").Buffer; +const CompilationContext = @import("module.zig").CompilationContext; +const cuda = @import("context.zig").cuda; +const DataType = @import("dtype.zig").DataType; +const HostBuffer = @import("hostbuffer.zig").HostBuffer; const mlirx = @import("mlirx.zig"); -const pjrt = @import("pjrt"); const pjrtx = @import("pjrtx.zig"); const ffi = pjrtx.ffi; -const cuda = @import("context.zig").cuda; -const Buffer = @import("buffer.zig").Buffer; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; -const CompilationContext = @import("module.zig").CompilationContext; -const DataType = @import("dtype.zig").DataType; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; const log = std.log.scoped(.@"zml/custom_call"); pub const std_options: std.Options = .{ diff --git a/zml/exe.zig b/zml/exe.zig index 7a303fa3..37fff489 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -250,11 +250,11 @@ pub const BaseExe = struct { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn _unsafeAssignResults(self: BaseExe, T: type, result: *T) void { @@ -330,7 +330,7 @@ pub const BaseExe = struct { pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ - .n_in = self.input_buffer_count, + .input_shapes = self.input_shapes, .result_shapes = self.result_shapes, .n_devices = self.num_devices, }); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 9239bc3c..b8ef9b3f 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -271,14 +271,22 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, .context = args.context, - } }); + }); + // try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + // .num_args = args.num_args, + // .arguments = @ptrCast(args.arguments), + // .results = @ptrCast(args.results), + // .events = @ptrCast(args.events), + // .non_donatable_input_indices = args.non_donatable_input_indices, + // .context = args.context, + // } }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 5c2dc6c37797ce86655c8eab1e05fb9f4fc52e29 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 15 Jul 2025 09:58:40 +0200 Subject: [PATCH 03/67] handle memories on CPU --- examples/custom_call/main.zig | 2 +- zml/buffer.zig | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig index e5a17443..a58a4b11 100644 --- a/examples/custom_call/main.zig +++ b/examples/custom_call/main.zig @@ -12,7 +12,7 @@ pub const std_options: std.Options = .{ const log = std.log.scoped(.@"examples/custom_call"); fn add_op_host_func(data: *anyopaque) callconv(.c) void { - const ctx = @as(*AddOp, @alignCast(@ptrCast(data))); + const ctx = @as(*AddOp, @ptrCast(@alignCast(data))); const result = std.mem.bytesAsValue(f32, ctx.result.asHostBuffer().mutBytes()); result.* = ctx.a.asHostBuffer().item(f32, 0) + ctx.b.asHostBuffer().item(f32, 0); diff --git a/zml/buffer.zig b/zml/buffer.zig index 259cb262..1da9a094 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -89,15 +89,20 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (opts.memory) |memory| { + if (platform.target == .cpu or opts.memory == null) { + args.device = devices[i]; + } else { + const memory = opts.memory.?; const device_memories = try devices[i].addressableMemories(platform.pjrt_api); + // TODO measure the cost of this and consider caching on Zig side inside the platform. const selected_memory = for (device_memories) |m| { const kind = m.kind(platform.pjrt_api); if (kind == memory.toPjrtMemory()) break m; - } else return error.NotFound; + } else { + log.warn("Platform {s} doesn't have memory {s}", .{ @tagName(platform.target), @tagName(memory) }); + return error.NotFound; + }; args.memory = selected_memory; - } else { - args.device = devices[i]; } const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); From 4cf8e55612c9fd95ccdc691e863ad44f298652da Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Jul 2025 12:20:30 +0200 Subject: [PATCH 04/67] refactor imports --- zml/context.zig | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/zml/context.zig b/zml/context.zig index 47c86708..832d7fe0 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -8,14 +8,18 @@ const runtimes = @import("runtimes"); const stdx = @import("stdx"); const pjrt = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Target = @import("platform.zig").Target; -const custom_call = @import("custom_call.zig"); -const zml_platform = @import("platform.zig"); -const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); -const PlatformsMap = std.EnumArray(Target, ?Platform); +const zml = struct { + pub const Platform = @import("platform.zig").Platform; + pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; + pub const Shape = @import("shape.zig").Shape; + pub const Target = @import("platform.zig").Target; + pub const custom_call = @import("custom_call.zig"); + pub const platform = @import("platform.zig"); +}; + +const PjrtApiMap = std.EnumArray(zml.Target, ?*const pjrt.Api); +const PlatformsMap = std.EnumArray(zml.Target, ?zml.Platform); const log = std.log.scoped(.@"zml/context"); test { @@ -94,7 +98,7 @@ pub const Context = struct { return .{ .platforms = PlatformsMap.initFill(null) }; } - fn platformToLibrary(comptime target: Target) []const u8 { + fn platformToLibrary(comptime target: zml.Target) []const u8 { const ext = switch (builtin.os.tag) { .windows => ".dll", .macos, .ios, .watchos => ".dylib", @@ -105,7 +109,7 @@ pub const Context = struct { }; } - pub fn pjrtApi(target: Target) *const pjrt.Api { + pub fn pjrtApi(target: zml.Target) *const pjrt.Api { return Context.apis.get(target).?; } @@ -119,12 +123,12 @@ pub const Context = struct { self.* = undefined; } - const prefered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; + const prefered_targets = [_]zml.Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; /// Automatically selects the best Platform loaded in the current Context. /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn autoPlatform(self: *Context, opts: Platform.CreateOptions) Platform { + pub fn autoPlatform(self: *Context, opts: zml.Platform.CreateOptions) zml.Platform { stdx.debug.assert(prefered_targets.len == apis.values.len, "New target need to be inserted inside `zml.Context.preferred_targets`", .{}); return self.platformByPreferences(opts, &prefered_targets); @@ -133,7 +137,7 @@ pub const Context = struct { /// Given a list of preferred targets to select the best Platform /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn platformByPreferences(self: *Context, opts: Platform.CreateOptions, prefered: []const Target) Platform { + pub fn platformByPreferences(self: *Context, opts: zml.Platform.CreateOptions, prefered: []const zml.Target) zml.Platform { // Try prefered targets. for (prefered) |target| { if (apis.get(target) == null) continue; @@ -150,7 +154,7 @@ pub const Context = struct { // CPU should only be use as fallback. if (target == .cpu) continue; if (entry.value.* == null) continue; - if (std.mem.indexOfScalar(Target, prefered, target) != null) continue; + if (std.mem.indexOfScalar(zml.Target, prefered, target) != null) continue; return self.platform(target, opts) catch |err| { log.err("Failed to load platform .{s}: {}", .{ @tagName(target), err }); continue; @@ -164,7 +168,7 @@ pub const Context = struct { }; } - pub fn platform(self: *Context, target: Target, opts: Platform.CreateOptions) !Platform { + pub fn platform(self: *Context, target: zml.Target, opts: zml.Platform.CreateOptions) !Platform { if (self.platforms.get(target)) |p| { return p; } @@ -182,7 +186,7 @@ pub const Context = struct { return p; } - pub fn printAvailablePlatforms(self: Context, selected: Platform) void { + pub fn printAvailablePlatforms(self: Context, selected: zml.Platform) void { // List available targets log.info("Available Platforms:", .{}); const selected_prefix = "✅"; @@ -190,7 +194,7 @@ pub const Context = struct { const selected_postfix = "(AUTO-SELECTED)"; const not_selected_postfix = ""; - for (zml_platform.available_targets) |target| { + for (zml.platform.available_targets) |target| { log.info(" {s} {s} {s}", .{ if (target == selected.target) selected_prefix else not_selected_prefix, @tagName(target), From b7cd493e569b89af6bedfa4fc3d5085d9b3d64ee Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Aug 2025 16:07:03 +0200 Subject: [PATCH 05/67] update to 0.15.0 --- examples/custom_call/{BUILD => BUILD.bazel} | 0 pjrt/ffi.zig | 91 +++++++------- pjrt/pjrt.zig | 2 +- stdx/fmt.zig | 94 ++------------- zml/buffer.zig | 4 +- zml/context.zig | 6 +- zml/custom_call.zig | 124 ++++++++++---------- zml/exe.zig | 51 ++++---- zml/hostbuffer.zig | 25 +--- zml/testing.zig | 2 +- 10 files changed, 154 insertions(+), 245 deletions(-) rename examples/custom_call/{BUILD => BUILD.bazel} (100%) diff --git a/examples/custom_call/BUILD b/examples/custom_call/BUILD.bazel similarity index 100% rename from examples/custom_call/BUILD rename to examples/custom_call/BUILD.bazel diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index b3ce2a74..1ff2daf2 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -2,12 +2,17 @@ const std = @import("std"); const c = @import("c"); +const TypeId = c.XLA_FFI_TypeId; const stdx = @import("stdx"); -const pjrtStruct = @import("pjrt.zig").pjrtStruct; +const pjrt = @import("pjrt.zig"); +const Stream = @import("pjrt.zig").Stream; const log = std.log.scoped(.pjrt); +/// The signature of a generic custom call. +pub const Handler = fn (*CallFrame) callconv(.c) ?*Error; + pub const ApiVersion = extern struct { pub const major = c.XLA_FFI_API_MAJOR; pub const minor = c.XLA_FFI_API_MINOR; @@ -91,8 +96,8 @@ fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; - pub fn stream(self: *const Api, context: *const ExecutionContext) *Stream { - var ret = pjrtStruct(c.XLA_FFI_Stream_Get_Args{ + pub fn stream(self: *const Api, context: *const ExecutionContext) *pjrt.Stream { + var ret = pjrt.pjrtStruct(c.XLA_FFI_Stream_Get_Args{ .ctx = @constCast(context.inner()), }); const result = self.inner().XLA_FFI_Stream_Get.?(&ret); @@ -108,7 +113,7 @@ pub const Api = opaque { } pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) ApiError!*anyopaque { - var ret = pjrtStruct(c.XLA_FFI_DeviceMemory_Allocate_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Allocate_Args{ .ctx = @constCast(context.inner()), .size = size, .alignment = alignment, @@ -128,7 +133,7 @@ pub const Api = opaque { } pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_DeviceMemory_Free_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Free_Args{ .ctx = @constCast(context.inner()), .size = size, .data = data, @@ -163,33 +168,33 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - pub fn Context(comptime T: type) type { - return struct { - pub fn get(self: *const ExecutionContext, api: *const Api) ApiError!*T { - const type_id: TypeId = .{ .type_id = T.type_id }; - var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = @constCast(self.inner()), - .type_id = @constCast(&type_id), - }); - const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); - - if (result) |ffi_error| { - const err = Error.fromInner(ffi_error); - defer err.destroy(api); - log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); - - // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - return error.Unknown; - } - - if (ret.data == null) return error.NotFound; - return @ptrCast(@alignCast(ret.data.?)); - } + pub fn getContext(self: *const ExecutionContext, T: type, api: *const Api) ApiError!*T { + const type_id: TypeId = .{ .type_id = T.type_id }; + log.warn("[ExecutionContext.get] {*} api: {}", .{ self, api.inner().* }); + var ret: c.XLA_FFI_ExecutionContext_Get_Args = .{ + .struct_size = pjrt.pjrtStructSize(c.XLA_FFI_ExecutionContext_Get_Args), + .extension_start = api.inner().extension_start, + .ctx = @ptrCast(@constCast(self)), + .type_id = @constCast(&type_id), + .data = undefined, // set by XLA_FFI_ExecutionContext_Get. }; + const maybe_err = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + + if (maybe_err) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + if (ret.data == null) return error.NotFound; + return @ptrCast(@alignCast(ret.data.?)); } pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { - var ret = pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ .ctx = @constCast(self.inner()), }); const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(&ret); @@ -207,7 +212,7 @@ pub const ExecutionContext = opaque { } pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ .ctx = @constCast(self.inner()), .task = @ptrCast(@alignCast(task)), .data = @ptrCast(@alignCast(data)), @@ -225,22 +230,10 @@ pub const ExecutionContext = opaque { return error.Unknown; } } - - fn getTypeId(type_name: []const u8) TypeId { - const id: i64 = @bitCast(std.hash.Fnv1a_64.hash(type_name)); - - return .{ - .type_id = id, - }; - } }; -const TypeId = c.XLA_FFI_TypeId; - const Task = fn (*anyopaque) void; -const Stream = @import("pjrt.zig").Stream; - const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -399,6 +392,8 @@ pub const Attrs = extern struct { } }; +/// All informations needed by the user callback, +/// including the list of input/ouput buffers to work on. pub const CallFrame = extern struct { struct_size: usize, extension_start: ?*ExtensionBase, @@ -424,8 +419,6 @@ pub const CallFrame = extern struct { } }; -pub const Handler = fn (*CallFrame) callconv(.c) ?*Error; - pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { cancelled = c.XLA_FFI_Error_Code_CANCELLED, unknown = c.XLA_FFI_Error_Code_UNKNOWN, @@ -471,7 +464,7 @@ pub const Error = opaque { pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { - var ret = pjrtStruct(c.XLA_FFI_Error_Create_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_Create_Args{ .message = message.ptr, .errc = @intFromEnum(error_code), }); @@ -479,12 +472,12 @@ pub const Error = opaque { } pub fn destroy(err: *Error, api: *const Api) void { - var ret = pjrtStruct(c.XLA_FFI_Error_Destroy_Args{ .@"error" = err.inner() }); + var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_Destroy_Args{ .@"error" = err.inner() }); api.inner().XLA_FFI_Error_Destroy.?(&ret); } pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { - var ret = pjrtStruct(c.XLA_FFI_Error_GetMessage_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_GetMessage_Args{ .@"error" = err.inner(), }); api.inner().XLA_FFI_Error_GetMessage.?(&ret); @@ -497,7 +490,7 @@ pub const Future = opaque { pub const fromInner = TransmuteMixin(Future, c.XLA_FFI_Future).from; pub fn create(api: *const Api) ApiError!*Future { - var ret = pjrtStruct(c.XLA_FFI_Future_Create_Args{}); + var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_Create_Args{}); const result = api.inner().XLA_FFI_Future_Create.?(&ret); if (result) |ffi_error| { @@ -513,7 +506,7 @@ pub const Future = opaque { } pub fn setAvailable(self: *Future, api: *const Api) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_Future_SetAvailable_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetAvailable_Args{ .future = self.inner(), }); @@ -530,7 +523,7 @@ pub const Future = opaque { } pub fn setError(self: *Future, api: *const Api, err: *Error) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_Future_SetError_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetError_Args{ .future = self.inner(), .@"error" = err.inner(), }); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index aead3bd3..4c6d9251 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -20,7 +20,7 @@ test { // as the way PJRT does it is not very robust. // // 1. https://github.com/openxla/xla/issues/10032 -fn pjrtStructSize(comptime T: type) usize { +pub fn pjrtStructSize(comptime T: type) usize { // unsafe on purpose, we want this to fail if that ever changes const typedef_name = comptime blk: { const needle = ".struct_"; diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 259b3cd8..b3ec64fe 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -1,69 +1,6 @@ const std = @import("std"); -pub const Fmt = union(enum) { - int: IntFmt, - float: FloatFmt, - generic: void, - - pub fn parse(T: type, comptime fmt_: []const u8) Fmt { - return switch (@typeInfo(T)) { - .float, .comptime_float => .{ .float = FloatFmt.parseComptime(fmt_) }, - .int, .comptime_int => .{ .int = IntFmt.parseComptime(fmt_) }, - else => .{ .generic = {} }, - }; - } -}; - -pub const FullFormatOptions = struct { - fmt: Fmt, - options: std.fmt.FormatOptions, -}; - -pub const IntFmt = struct { - base: u8, - case: std.fmt.Case = .lower, - - pub fn parseComptime(comptime fmt_: []const u8) IntFmt { - return parse(fmt_) catch @panic("invalid fmt for int: " ++ fmt_); - } - - pub fn parse(fmt_: []const u8) error{InvalidArgument}!IntFmt { - return if (fmt_.len == 0 or std.mem.eql(u8, fmt_, "d")) - .{ .base = 10, .case = .lower } - else if (std.mem.eql(u8, fmt_, "x")) - .{ .base = 16, .case = .lower } - else if (std.mem.eql(u8, fmt_, "X")) - .{ .base = 16, .case = .upper } - else if (std.mem.eql(u8, fmt_, "o")) - .{ .base = 8, .case = .upper } - else - // TODO: unicode/ascii - error.InvalidArgument; - } -}; - -pub const FloatFmt = enum(u8) { - scientific = @intFromEnum(std.fmt.Number.Mode.scientific), - decimal = @intFromEnum(std.fmt.Number.Mode.decimal), - hex, - - pub fn parseComptime(comptime fmt_: []const u8) FloatFmt { - return parse(fmt_) catch @panic("invalid fmt for float: " ++ fmt_); - } - - pub fn parse(fmt_: []const u8) error{InvalidArgument}!FloatFmt { - return if (fmt_.len == 0 or std.mem.eql(u8, fmt_, "e")) - .scientific - else if (std.mem.eql(u8, fmt_, "d")) - .decimal - else if (std.mem.eql(u8, fmt_, "x")) - .hex - else - error.InvalidArgument; - } -}; - -pub fn formatValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { +pub fn formatValue(value: anytype, full: std.fmt.Number, writer: anytype) !void { return switch (@typeInfo(@TypeOf(value))) { .comptime_float, .float => try formatFloatValue(value, full, writer), .comptime_int, .int => try formatIntValue(value, full, writer), @@ -71,40 +8,35 @@ pub fn formatValue(value: anytype, full: FullFormatOptions, writer: anytype) !vo }; } -pub fn formatFloatValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { +pub fn formatFloatValue(value: anytype, full: std.fmt.Number, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), .float => value, else => @compileError("formatFloatValue expects a float, got: " ++ @typeName(@TypeOf(value))), }; - try switch (full.fmt.float) { - .scientific => writer.printFloat(x, .{ .mode = .scientific, .precision = full.options.precision }), - .decimal => writer.printFloat(x, .{ .mode = .decimal, .precision = full.options.precision }), - .hex => writer.printFloatHexOptions(x, .{ .mode = .hex }), - }; + return writer.printFloat(x, full); } -pub fn formatIntValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { +pub fn formatIntValue(value: anytype, full: std.fmt.Number, writer: *std.Io.Writer) !void { switch (@typeInfo(@TypeOf(value))) { .int => {}, else => @compileError("formatIntValue expects an int, got: " ++ @typeName(@TypeOf(value))), } - return writer.printInt(value, full.fmt.int.base, full.fmt.int.case, full.options); + return writer.printInt(value, full.mode.base().?, full.case, .{ .alignment = full.alignment, .fill = full.fill }); } -pub fn formatAnyValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { +pub fn formatAnyValue(value: anytype, full: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; const s = std.fmt.bufPrint(&buf, "{any}", .{value}) catch blk: { buf[45..].* = "...".*; break :blk buf[0..]; }; - return try writer.alignBufferOptions(s, full.options); + return try writer.alignBufferOptions(s, .{ .alignment = full.alignment, .fill = full.fill }); } -pub fn formatSliceCustom(fmt_func: anytype, values: anytype, full: FullFormatOptions, writer: anytype) !void { - +pub fn formatSliceCustom(fmt_func: anytype, values: anytype, full: std.fmt.Number, writer: anytype) !void { // Write first rows - const num_cols: usize = full.options.width orelse 12; + const num_cols: usize = full.width orelse 12; const n: usize = values.len; _ = try writer.write("{"); if (n <= num_cols) { @@ -128,18 +60,18 @@ pub fn formatSliceCustom(fmt_func: anytype, values: anytype, full: FullFormatOpt _ = try writer.write("}"); } -pub fn formatAny(values: anytype, full: FullFormatOptions, writer: anytype) !void { +pub fn formatAny(values: anytype, full: std.fmt.Number, writer: anytype) !void { return try formatSliceCustom(formatAnyValue, values, full, writer); } -pub fn formatFloatSlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { +pub fn formatFloatSlice(values: anytype, full: std.fmt.Number, writer: anytype) !void { return try formatSliceCustom(formatFloatValue, values, full, writer); } -pub fn formatIntSlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { +pub fn formatIntSlice(values: anytype, full: std.fmt.Number, writer: anytype) !void { return try formatSliceCustom(formatIntValue, values, full, writer); } -pub fn formatAnySlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { +pub fn formatAnySlice(values: anytype, full: std.fmt.Number, writer: anytype) !void { return try formatSliceCustom(formatAnyValue, values, full, writer); } diff --git a/zml/buffer.zig b/zml/buffer.zig index 1da9a094..3f1ed616 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -186,7 +186,7 @@ pub const Buffer = struct { pub fn asHostBuffer(self: Buffer) HostBuffer { const memory = self.getMemory().kind(self._api); - stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({any}) expects a buffer allocated on host memory, got {any}. see `copyToMemory`", .{ self, memory }); + stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } @@ -406,7 +406,7 @@ pub const Buffer = struct { pub fn copyToMemory(self: Buffer, platform: Platform, memory: Memory, opts: CopyToMemoryOpts) !Buffer { const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory.toPjrtMemory()); if (pjrt_memory == null) { - stdx.debug.panic("Memory destination `{s}` for {any}", .{ memory.pjrtName(), self }); + stdx.debug.panic("Memory destination `{s}` for {f}", .{ memory.pjrtName(), self }); } var new_shards: Buffer.Shards = .{}; diff --git a/zml/context.zig b/zml/context.zig index 832d7fe0..6a284cfb 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -168,20 +168,20 @@ pub const Context = struct { }; } - pub fn platform(self: *Context, target: zml.Target, opts: zml.Platform.CreateOptions) !Platform { + pub fn platform(self: *Context, target: zml.Target, opts: zml.Platform.CreateOptions) !zml.Platform { if (self.platforms.get(target)) |p| { return p; } const api = Context.apis.get(target); if (api == null) return error.PlatformNotCompiled; - const p = try Platform.init(target, api.?, opts); + const p = try zml.Platform.init(target, api.?, opts); if (p.getDevices().len == 0) { log.err("No device found for platform {} !", .{target}); return error.NoDevicesFound; } self.platforms.set(target, p); - try custom_call.registerInternalCustomCalls(p); + try zml.custom_call.registerInternalCustomCalls(p); return p; } diff --git a/zml/custom_call.zig b/zml/custom_call.zig index 849a854e..7390db2a 100644 --- a/zml/custom_call.zig +++ b/zml/custom_call.zig @@ -72,7 +72,7 @@ pub fn custom_call( stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); } - const ffi_func = proxy(custom_op, opts); + const ffi_func = customCallProxy(custom_op, opts); const target_name = "callback_" ++ op_name; ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.register_ffi_options) catch unreachable; @@ -114,60 +114,65 @@ pub fn custom_call( return custom_call_outputs[0..res_shapes_.len]; } -fn proxy(comptime custom_op: type, opts: CustomCallOptions) ffi.Handler { +fn customCallProxy(CustomOp: type, comptime opts: CustomCallOptions) ffi.Handler { return struct { - const Self = @This(); - - pub fn proxy(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { - if (call_frame.registeringHook()) return null; - const execution_context = call_frame.ctx; - const user_ctx: *custom_op = ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api) catch unreachable; - const platform: Platform = user_ctx.platform; - - if (@hasField(custom_op, "stream") and platform.target != .cpu) { - const stream = call_frame.api.stream(execution_context); - user_ctx.stream = stream; - } else { - log.info("No stream field provided in container {s} for custom call", .{@typeName(custom_op)}); - } - - var callback_args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; - callback_args[0] = user_ctx; - - inline for (1..callback_args.len) |i| { - const ffi_buffer = call_frame.args.buffers()[i - 1]; - const ffi_buffer_shape = getShape(ffi_buffer); - var zml_buffer: Buffer = undefined; - - if (platform.target == .cpu) { - zml_buffer = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); - } else { - zml_buffer = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); - } - if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { - zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch unreachable; - } - callback_args[i] = zml_buffer; - } - - for (0..call_frame.results.len) |i| { - const ffi_buffer = call_frame.results.buffers()[i]; - const ffi_buffer_shape = getShape(ffi_buffer); - - if (platform.target == .cpu) { - user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); - } else { - user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); - } - } - - @call(.auto, custom_op.call, callback_args) catch |err| { - stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(custom_op), err }); - }; - - return null; + pub fn cb(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { + return customCallImpl(CustomOp, opts, call_frame); } - }.proxy; + }.cb; +} + +fn customCallImpl(comptime CustomOp: type, opts: CustomCallOptions, call_frame: *ffi.CallFrame) ?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const execution_context = call_frame.ctx; + log.info("Custom call {s} called ! {*}", .{ @typeName(CustomOp), call_frame }); + const user_ctx: *CustomOp = execution_context.getContext(CustomOp, call_frame.api) catch { + return .create(call_frame.api, .internal, "failed to fetch user context for custom_call" ++ @typeName(CustomOp)); + }; + const platform: Platform = user_ctx.platform; + + if (@hasField(CustomOp, "stream") and platform.target != .cpu) { + const stream = call_frame.api.stream(execution_context); + user_ctx.stream = stream; + } + + var callback_args: std.meta.ArgsTuple(@TypeOf(CustomOp.call)) = undefined; + callback_args[0] = user_ctx; + + inline for (1..callback_args.len) |i| { + const ffi_buffer = call_frame.args.buffers()[i - 1]; + const ffi_buffer_shape = getShape(ffi_buffer); + var zml_buffer: Buffer = undefined; + + // TODO reconder using PJRT apis at all from pjrt callbacks. + if (platform.target == .cpu) { + zml_buffer = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + zml_buffer = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch unreachable; + } + callback_args[i] = zml_buffer; + } + + for (0..call_frame.results.len) |i| { + const ffi_buffer = call_frame.results.buffers()[i]; + const ffi_buffer_shape = getShape(ffi_buffer); + + if (platform.target == .cpu) { + user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + } + + @call(.auto, CustomOp.call, callback_args) catch |err| { + stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(CustomOp), err }); + }; + + return null; } pub fn getShape(ffi_buffer: *const ffi.Buffer) Shape { @@ -201,24 +206,21 @@ pub fn registerInternalCustomCalls( pub const Print = struct { pub var type_id: i64 = undefined; - const Self = @This(); allocator: std.mem.Allocator, platform: Platform, + stream: *pjrt.Stream = undefined, results: [1]Buffer = undefined, - pub fn init( - allocator: std.mem.Allocator, - platform: Platform, - ) !Print { + pub fn init(platform: Platform) !Print { return .{ - .allocator = allocator, + .allocator = std.heap.smp_allocator, .platform = platform, }; } - pub fn call(_: *Self, input: Buffer) !void { - std.log.defaultLog(.info, .zml, "Device buffer: {any}: {any}", .{ input.shape(), input.asHostBuffer().pretty() }); + pub fn call(_: *Print, input: Buffer) !void { + std.log.defaultLog(.info, .zml, "Device buffer: {f}: {d}", .{ input, input.asHostBuffer() }); } }; diff --git a/zml/exe.zig b/zml/exe.zig index 37fff489..c9b5d3b6 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -6,9 +6,9 @@ const aio = @import("aio.zig"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("tensor.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; +const custom_call = @import("custom_call.zig"); const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); -const custom_call = @import("custom_call.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; @@ -155,7 +155,7 @@ pub const BaseExe = struct { exe: *pjrt.LoadedExecutable, /// The execution context for this executable. - context: ?*pjrt.ExecuteContext = null, + execute_context: ?*pjrt.ExecuteContext, /// Pre-allocated slice of buffers to use as inputs when the module is called. input_per_device: []const [*]*pjrt.Buffer, @@ -206,9 +206,26 @@ pub const BaseExe = struct { const all_shapes = try allocator.alloc(Shape, n_in + n_out); @memcpy(all_shapes[0..n_in], args.input_shapes); @memcpy(all_shapes[n_in..], args.result_shapes); + + var execute_context: ?*pjrt.ExecuteContext = null; + if (platform.pjrt_api.ffi()) |ffi| { + log.info("Created context execution {*} for {*}", .{ execute_context, exe }); + execute_context = try platform.pjrt_api.createExecuteContext(); + + // Add extra data needed by the ZML provided custom calls. + // Atm we don't have a mechanism to detect if the user need this or not. + inline for (custom_call.custom_call_internal_types) |T| { + const value_ptr = try allocator.create(T); + value_ptr.* = try .init(platform); + try ffi.addUserData(platform.pjrt_api, execute_context.?, .{ .type_id = T.type_id, .user_data = @ptrCast(@constCast(value_ptr)) }); + log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), @intFromPtr(value_ptr), T.type_id, execute_context }); + } + } + return .{ .platform = platform, .exe = exe, + .execute_context = execute_context, .ready_buffer_count = 0, .input_buffer_count = @intCast(n_in), .num_devices = args.n_devices, @@ -221,7 +238,7 @@ pub const BaseExe = struct { } pub fn deinit(self: BaseExe) void { - if (self.context) |ctx| { + if (self.execute_context) |ctx| { ctx.deinit(self.platform.pjrt_api); } self._arena.deinit(); @@ -245,7 +262,7 @@ pub const BaseExe = struct { // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, - .context = self.context, + .context = self.execute_context, }) catch |err| { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; @@ -287,15 +304,15 @@ pub const BaseExe = struct { } pub fn bind(self: BaseExe, comptime T: type, value: *T) !void { - stdx.debug.assert(self.context != null, "Exe doesn't have an execution context", .{}); + stdx.debug.assert(self.execute_context != null, "Exe doesn't have an execution context", .{}); const pjrt_api = self.platform.pjrt_api; if (pjrt_api.ffi()) |ffi| { const type_id = T.type_id; const user_data: *anyopaque = @ptrCast(@constCast(value)); - try ffi.addUserData(pjrt_api, self.context.?, .{ .type_id = type_id, .user_data = user_data }); - log.info("Bound {s}@{any} with type id {any} on {any}", .{ @typeName(T), user_data, T.type_id, self.context.? }); + try ffi.addUserData(pjrt_api, self.execute_context.?, .{ .type_id = type_id, .user_data = user_data }); + log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), user_data, T.type_id, self.execute_context.? }); } else { stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(self.platform.target)}); } @@ -334,7 +351,7 @@ pub const BaseExe = struct { .result_shapes = self.result_shapes, .n_devices = self.num_devices, }); - exe.context = self.context; + exe.execute_context = self.execute_context; return exe; } }; @@ -364,24 +381,6 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } - pub fn withExecutionContext(self: Self) !Self { - stdx.debug.assert(self.inner.context == null, "Exe already has an execution context", .{}); - var new: Self = .{ .inner = self.inner }; - var arena = &new.inner._arena; - var allocator = arena.allocator(); - - const pjrt_execute_context = try new.inner.platform.pjrt_api.createExecuteContext(); - new.inner.context = pjrt_execute_context; - inline for (custom_call.custom_call_internal_types) |custom_call_internal_type| { - const value_ptr = try allocator.create(custom_call_internal_type); - value_ptr.* = try .init(allocator, new.platform()); - try new.inner.bind(custom_call_internal_type, value_ptr); - } - log.info("Created context execution {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); - - return new; - } - pub fn bind(self: Self, comptime T: type, value: *T) !void { try self.inner.bind(T, value); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index ee598880..350efbf6 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -340,32 +340,15 @@ pub const HostBuffer = struct { try writer.print("HostBuffer(.{f})", .{self._shape}); } - /// Formatter for a HostBuffer that also print the values not just the shape. - /// Usage: `std.log.info("my buffer: {}", .{buffer.pretty()});` - pub fn pretty(self: HostBuffer) PrettyPrinter { - return .{ .x = self }; + pub fn formatNumber(self: HostBuffer, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + return self.prettyPrintIndented(writer, 4, 0, n); } - pub const PrettyPrinter = struct { - x: HostBuffer, - - // TODO(0.15.0) revisit pretty printer - pub fn format(self: PrettyPrinter, writer: anytype) !void { - const fmt_: stdx.fmt.Fmt = switch (self.x.dtype().class()) { - .integer => .parse(i32, "d"), - .float => .parse(f32, "d"), - else => .parse(void, ""), - }; - const options: std.fmt.FormatOptions = .{}; - try prettyPrint(self.x, writer, .{ .fmt = fmt_, .options = options }); - } - }; - - pub fn prettyPrint(self: HostBuffer, writer: *std.Io.Writer, options: stdx.fmt.FullFormatOptions) !void { + pub fn prettyPrint(self: HostBuffer, writer: *std.Io.Writer, options: std.fmt.Number) !void { return self.prettyPrintIndented(writer, 4, 0, options); } - fn prettyPrintIndented(self: HostBuffer, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: stdx.fmt.FullFormatOptions) !void { + fn prettyPrintIndented(self: HostBuffer, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: std.fmt.Number) !void { if (self.rank() == 0) { // Special case input tensor is a scalar return switch (self.dtype()) { diff --git a/zml/testing.zig b/zml/testing.zig index b236400b..246f1131 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -51,7 +51,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { if (should_free_left) left.deinit(allocator); if (should_free_right) right.deinit(allocator); } - errdefer log.err("\n--> Left: {f}\n--> Right: {f}", .{ left.pretty(), right.pretty() }); + errdefer log.err("\n--> Left: {f}\n--> Right: {f}", .{ left, right }); if (!std.mem.eql(i64, left.shape().dims(), right.shape().dims())) { log.err("left.shape() {f} != right.shape() {f}", .{ left.shape(), right.shape() }); From 9b96da659bdf17a768a2f6d588abbc99f436972b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Aug 2025 17:02:37 +0200 Subject: [PATCH 06/67] avoid ZML specific stuff in pjrt bindings --- pjrt/ffi.zig | 52 +++++++++++++++------------------------------ pjrt/pjrt.zig | 5 ++--- zml/custom_call.zig | 52 +++++++++++++++++++++++---------------------- zml/exe.zig | 19 ++++++++--------- zml/platform.zig | 12 +++++------ 5 files changed, 61 insertions(+), 79 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 1ff2daf2..c145887b 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -2,7 +2,10 @@ const std = @import("std"); const c = @import("c"); -const TypeId = c.XLA_FFI_TypeId; +pub const TypeId = c.XLA_FFI_TypeId; +comptime { + if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); +} const stdx = @import("stdx"); const pjrt = @import("pjrt.zig"); @@ -54,25 +57,6 @@ pub const MetadataExtension = extern struct { metadata: ?*Metadata, }; -pub const ApiError = error{ - Cancelled, - Unknown, - InvalidArgument, - DeadlineExceeded, - NotFound, - AlreadyExists, - PermissionDenied, - ResourceExhausted, - FailedPrecondition, - Aborted, - OutOfRange, - Unimplemented, - Internal, - Unavailable, - DataLoss, - Unauthenticated, -}; - fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { return struct { pub fn to(self: anytype) switch (@TypeOf(self)) { @@ -112,7 +96,7 @@ pub const Api = opaque { return @ptrCast(ret.stream.?); } - pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) ApiError!*anyopaque { + pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) pjrt.ApiError!*anyopaque { var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Allocate_Args{ .ctx = @constCast(context.inner()), .size = size, @@ -132,7 +116,7 @@ pub const Api = opaque { return ret.data.?; } - pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) ApiError!void { + pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) pjrt.ApiError!void { var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Free_Args{ .ctx = @constCast(context.inner()), .size = size, @@ -168,9 +152,7 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - pub fn getContext(self: *const ExecutionContext, T: type, api: *const Api) ApiError!*T { - const type_id: TypeId = .{ .type_id = T.type_id }; - log.warn("[ExecutionContext.get] {*} api: {}", .{ self, api.inner().* }); + pub fn getContext(self: *const ExecutionContext, type_id: TypeId, api: *const Api) pjrt.ApiError!*anyopaque { var ret: c.XLA_FFI_ExecutionContext_Get_Args = .{ .struct_size = pjrt.pjrtStructSize(c.XLA_FFI_ExecutionContext_Get_Args), .extension_start = api.inner().extension_start, @@ -190,10 +172,10 @@ pub const ExecutionContext = opaque { } if (ret.data == null) return error.NotFound; - return @ptrCast(@alignCast(ret.data.?)); + return ret.data.?; } - pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { + pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) pjrt.ApiError!i32 { var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ .ctx = @constCast(self.inner()), }); @@ -211,7 +193,9 @@ pub const ExecutionContext = opaque { return ret.device_ordinal; } - pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { + const Task = fn (*anyopaque) void; + + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) pjrt.ApiError!void { var ret = pjrt.pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ .ctx = @constCast(self.inner()), .task = @ptrCast(@alignCast(task)), @@ -232,9 +216,7 @@ pub const ExecutionContext = opaque { } }; -const Task = fn (*anyopaque) void; - -const ByteSpan = extern struct { +pub const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -437,7 +419,7 @@ pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { data_loss = c.XLA_FFI_Error_Code_DATA_LOSS, unauthenticated = c.XLA_FFI_Error_Code_UNAUTHENTICATED, - pub fn toApiError(code: ErrorCode) ApiError { + pub fn toApiError(code: ErrorCode) pjrt.ApiError { return switch (code) { .cancelled => error.Cancelled, .unknown => error.Unknown, @@ -489,7 +471,7 @@ pub const Future = opaque { pub const inner = TransmuteMixin(Future, c.XLA_FFI_Future).to; pub const fromInner = TransmuteMixin(Future, c.XLA_FFI_Future).from; - pub fn create(api: *const Api) ApiError!*Future { + pub fn create(api: *const Api) pjrt.ApiError!*Future { var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_Create_Args{}); const result = api.inner().XLA_FFI_Future_Create.?(&ret); @@ -505,7 +487,7 @@ pub const Future = opaque { return fromInner(ret.future.?); } - pub fn setAvailable(self: *Future, api: *const Api) ApiError!void { + pub fn setAvailable(self: *Future, api: *const Api) pjrt.ApiError!void { var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetAvailable_Args{ .future = self.inner(), }); @@ -522,7 +504,7 @@ pub const Future = opaque { } } - pub fn setError(self: *Future, api: *const Api, err: *Error) ApiError!void { + pub fn setError(self: *Future, api: *const Api, err: *Error) pjrt.ApiError!void { var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetError_Args{ .future = self.inner(), .@"error" = err.inner(), diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 4c6d9251..d8c29166 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1323,8 +1323,7 @@ pub const FFI = extern struct { } } - pub fn registerTypeId(self: *const FFI, api: *const Api, T: type) ApiError!void { - const type_name = @typeName(T); + pub fn registerTypeId(self: *const FFI, api: *const Api, type_name: []const u8) ApiError!ffi.TypeId { var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ .type_name = type_name.ptr, .type_name_size = type_name.len, @@ -1336,7 +1335,7 @@ pub const FFI = extern struct { return pjrt_error.getCode(api).toApiError(); } - T.type_id = ret.type_id; + return .{ .type_id = ret.type_id }; } pub fn addUserData(self: *const FFI, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { diff --git a/zml/custom_call.zig b/zml/custom_call.zig index 7390db2a..0c28a059 100644 --- a/zml/custom_call.zig +++ b/zml/custom_call.zig @@ -27,8 +27,8 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; -pub fn CustomCallInputsType(custom_op: type) type { - const ArgsT = stdx.meta.FnArgs(custom_op.call); +pub fn CustomCallInputsType(CustomOp: type) type { + const ArgsT = stdx.meta.FnArgs(CustomOp.call); if (@typeInfo(ArgsT) != .@"struct") { @compileError("Expected struct type"); } @@ -52,14 +52,20 @@ pub const CustomCallOptions = struct { }, }; +/// Insert a user-defined callback into the computation graph. +/// The callback is defined with a struct. +/// +/// ```zig +/// const MyCallback pub fn custom_call( - comptime custom_op: type, - inputs: CustomCallInputsType(custom_op), + comptime CustomOp: type, + inputs: CustomCallInputsType(CustomOp), res_shapes_: []const Shape, comptime opts: CustomCallOptions, ) []Tensor { - stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); - const op_name = @typeName(custom_op); + const op_name = @typeName(CustomOp); + stdx.debug.assertComptime(@hasDecl(CustomOp, "call"), "{s} must have a call method", .{op_name}); + stdx.debug.assertComptime(@hasDecl(CustomOp, "type_id") and @TypeOf(CustomOp.type_id) == pjrt.ffi.TypeId, "{s} must have a field `pub var type_id: pjrt.ffi.TypeId`", .{op_name}); const ctx = CompilationContext.current(); const allocator = ctx.allocator(); @@ -72,11 +78,11 @@ pub fn custom_call( stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); } - const ffi_func = customCallProxy(custom_op, opts); - const target_name = "callback_" ++ op_name; + const ffi_func = customCallProxy(CustomOp, opts); + const target_name = "zml$" ++ op_name; ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.register_ffi_options) catch unreachable; - log.info("{s} / Registered custom call with target name \"{s}\" with proxy func {*}", .{ op_name, target_name, &ffi_func }); + log.info("Registered custom call {s} with target name \"{s}\"", .{ op_name, target_name }); const custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; for (inputs, 0..) |t, i| { @@ -126,12 +132,14 @@ fn customCallImpl(comptime CustomOp: type, opts: CustomCallOptions, call_frame: if (call_frame.registeringHook()) return null; const execution_context = call_frame.ctx; - log.info("Custom call {s} called ! {*}", .{ @typeName(CustomOp), call_frame }); - const user_ctx: *CustomOp = execution_context.getContext(CustomOp, call_frame.api) catch { + // log.info("Custom call {s} called !", .{ @typeName(CustomOp) }); + const user_ctx_opaque = execution_context.getContext(CustomOp.type_id, call_frame.api) catch { return .create(call_frame.api, .internal, "failed to fetch user context for custom_call" ++ @typeName(CustomOp)); }; + const user_ctx: *CustomOp = @ptrCast(@alignCast(user_ctx_opaque)); const platform: Platform = user_ctx.platform; + // Undocumented hook to get a cuda stream in the callback. if (@hasField(CustomOp, "stream") and platform.target != .cpu) { const stream = call_frame.api.stream(execution_context); user_ctx.stream = stream; @@ -145,7 +153,7 @@ fn customCallImpl(comptime CustomOp: type, opts: CustomCallOptions, call_frame: const ffi_buffer_shape = getShape(ffi_buffer); var zml_buffer: Buffer = undefined; - // TODO reconder using PJRT apis at all from pjrt callbacks. + // TODO reconsider using PJRT apis at all from pjrt callbacks. if (platform.target == .cpu) { zml_buffer = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); } else { @@ -195,29 +203,23 @@ pub const custom_call_internal_types = [_]type{ Print, }; -pub fn registerInternalCustomCalls( - platform: Platform, -) !void { +pub fn registerInternalCustomCalls(platform: Platform) !void { inline for (custom_call_internal_types) |custom_call_type| { - try platform.registerFFIType(custom_call_type); - log.info("Registered internal custom call {s} with type_id {any}", .{ @typeName(custom_call_type), custom_call_type.type_id }); + try platform.registerCustomCall(custom_call_type); + log.info("Registered internal custom call {s} with type_id {d}", .{ @typeName(custom_call_type), custom_call_type.type_id.type_id }); } } +/// The print callback pub const Print = struct { - pub var type_id: i64 = undefined; + // a unique type_id will be set by the PJRT plugin during registration. + pub var type_id: pjrt.ffi.TypeId = undefined; - allocator: std.mem.Allocator, platform: Platform, - stream: *pjrt.Stream = undefined, - results: [1]Buffer = undefined, pub fn init(platform: Platform) !Print { - return .{ - .allocator = std.heap.smp_allocator, - .platform = platform, - }; + return .{ .platform = platform }; } pub fn call(_: *Print, input: Buffer) !void { diff --git a/zml/exe.zig b/zml/exe.zig index c9b5d3b6..265aab7e 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -217,8 +217,8 @@ pub const BaseExe = struct { inline for (custom_call.custom_call_internal_types) |T| { const value_ptr = try allocator.create(T); value_ptr.* = try .init(platform); - try ffi.addUserData(platform.pjrt_api, execute_context.?, .{ .type_id = T.type_id, .user_data = @ptrCast(@constCast(value_ptr)) }); - log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), @intFromPtr(value_ptr), T.type_id, execute_context }); + try ffi.addUserData(platform.pjrt_api, execute_context.?, .{ .type_id = T.type_id.type_id, .user_data = @ptrCast(@constCast(value_ptr)) }); + log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), @intFromPtr(value_ptr), T.type_id.type_id, execute_context }); } } @@ -303,18 +303,17 @@ pub const BaseExe = struct { stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); } - pub fn bind(self: BaseExe, comptime T: type, value: *T) !void { - stdx.debug.assert(self.execute_context != null, "Exe doesn't have an execution context", .{}); - const pjrt_api = self.platform.pjrt_api; + pub fn bind(exe: BaseExe, comptime CustomOp: type, op: *CustomOp) !void { + stdx.debug.assert(exe.execute_context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = exe.platform.pjrt_api; if (pjrt_api.ffi()) |ffi| { - const type_id = T.type_id; - const user_data: *anyopaque = @ptrCast(@constCast(value)); + const type_id: i64 = CustomOp.type_id.type_id; - try ffi.addUserData(pjrt_api, self.execute_context.?, .{ .type_id = type_id, .user_data = user_data }); - log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), user_data, T.type_id, self.execute_context.? }); + try ffi.addUserData(pjrt_api, exe.execute_context.?, .{ .type_id = type_id, .user_data = op }); + log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(CustomOp), @intFromPtr(op), type_id, exe.execute_context.? }); } else { - stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(self.platform.target)}); + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(exe.platform.target)}); } } diff --git a/zml/platform.zig b/zml/platform.zig index 838cac48..9b67f2db 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -71,14 +71,14 @@ pub const Platform = struct { return res; } - pub fn registerFFIType(self: Platform, comptime T: type) !void { + pub fn registerCustomCall(self: Platform, comptime CustomOp: type) pjrt.ApiError!void { if (self.pjrt_api.ffi()) |ffi| { - if (!@hasDecl(T, "type_id")) { - stdx.debug.panic("registerFFIType requires type {s} to have a `type_id` i64 field ", .{@typeName(T)}); - } - try ffi.registerTypeId(self.pjrt_api, T); + stdx.debug.assertComptime(@hasDecl(CustomOp, "call"), "{} must have a call method", .{CustomOp}); + stdx.debug.assertComptime(@hasDecl(CustomOp, "type_id") and @TypeOf(CustomOp.type_id) == pjrt.ffi.TypeId, "{} must have a field `pub var type_id: pjrt.ffi.TypeId`", .{CustomOp}); + + CustomOp.type_id = try ffi.registerTypeId(self.pjrt_api, @typeName(CustomOp)); } else { - stdx.debug.panic("registerFFIType is not available for target {s}", .{@tagName(self.target)}); + return error.Unavailable; } } From 523b3cf913125d8dda54f1ca5ce27d68d211002f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 28 Aug 2025 17:03:00 +0200 Subject: [PATCH 07/67] remove dead code --- zml/hostbuffer.zig | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 350efbf6..2fc72142 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -177,7 +177,7 @@ pub const HostBuffer = struct { } pub fn item(self: HostBuffer, comptime T: type, index: usize) T { - stdx.debug.assert(index < self._shape.count(), "index {any} out of bounds for shape {any}", .{ index, self._shape }); + stdx.debug.assert(index < self._shape.count(), "index {} out of bounds for shape {f}", .{ index, self._shape }); return self.items(T)[index]; } @@ -185,11 +185,6 @@ pub const HostBuffer = struct { return @constCast(self.items(T)); } - pub fn mutitem(self: HostBuffer, comptime T: type, index: usize) T { - stdx.debug.assert(index < self._shape.count(), "index {any} out of bounds for shape {any}", .{ index, self._shape }); - return self.mutItems(T)[index]; - } - pub fn bytes(self: HostBuffer) []const u8 { stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); return self._data[0..self._shape.byteSize()]; From a6392412617a7b3805e00988bfab7705245e4736 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 3 Sep 2025 21:44:51 +0200 Subject: [PATCH 08/67] HandlerTraits fixup! fix mlir mixed precisions # Conflicts: # pjrt/ffi.zig # pjrt/pjrt.zig # Conflicts: # zml/context.zig --- pjrt/ffi.zig | 15 ++++++++++----- pjrt/pjrt.zig | 23 +++++++---------------- zml/custom_call.zig | 11 ++--------- zml/tensor.zig | 15 ++++++++++++--- 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index c145887b..f80b11f5 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -3,9 +3,6 @@ const std = @import("std"); const c = @import("c"); pub const TypeId = c.XLA_FFI_TypeId; -comptime { - if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); -} const stdx = @import("stdx"); const pjrt = @import("pjrt.zig"); @@ -13,6 +10,10 @@ const Stream = @import("pjrt.zig").Stream; const log = std.log.scoped(.pjrt); +comptime { + if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); +} + /// The signature of a generic custom call. pub const Handler = fn (*CallFrame) callconv(.c) ?*Error; @@ -43,7 +44,7 @@ pub const HandlerTraits = packed struct(u32) { /// that can be captured and then replayed. command_buffer_compatible: u1, - __unassigned__: u31, + __unassigned__: u31 = 0, }; pub const Metadata = extern struct { @@ -227,7 +228,7 @@ pub const ByteSpan = extern struct { pub const DataType = enum(c.XLA_FFI_DataType) { invalid = c.XLA_FFI_DataType_INVALID, - pred = c.XLA_FFI_DataType_PRED, + bool = c.XLA_FFI_DataType_PRED, i8 = c.XLA_FFI_DataType_S8, i16 = c.XLA_FFI_DataType_S16, i32 = c.XLA_FFI_DataType_S32, @@ -399,6 +400,10 @@ pub const CallFrame = extern struct { } return false; } + + pub fn stream(call_frame: CallFrame) ?*const Stream { + return call_frame.api.stream(call_frame.ctx); + } }; pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index d8c29166..08ccf217 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -164,7 +164,7 @@ pub const Api = struct { return @ptrCast(ret.context.?); } - pub fn ffi(api: *const Api) ?FFI { + pub fn ffi(api: *const Api) ?Ffi { if (api.lookupExtension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { return .{ .inner = ext }; } @@ -1278,7 +1278,7 @@ pub const NamedValue = extern struct { } }; -pub const FFI = extern struct { +pub const Ffi = extern struct { inner: *const c.PJRT_FFI, pub const UserData = extern struct { @@ -1293,19 +1293,15 @@ pub const FFI = extern struct { } }; - pub const RegisterOptions = struct { - traits: RegisterHandlerTraits = @enumFromInt(0), - }; - // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub fn register( - self: *const FFI, + self: *const Ffi, api: *const Api, target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, - options: RegisterOptions, + traits: ffi.HandlerTraits, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .target_name = target_name.ptr, @@ -1313,7 +1309,7 @@ pub const FFI = extern struct { .handler = @ptrCast(@constCast(func)), .platform_name = platform_name.ptr, .platform_name_size = platform_name.len, - .traits = @intFromEnum(options.traits), + .traits = @bitCast(traits), }); const result = self.inner.register_handler.?(&ret); if (result) |pjrt_c_error| { @@ -1323,7 +1319,7 @@ pub const FFI = extern struct { } } - pub fn registerTypeId(self: *const FFI, api: *const Api, type_name: []const u8) ApiError!ffi.TypeId { + pub fn registerTypeId(self: *const Ffi, api: *const Api, type_name: []const u8) ApiError!ffi.TypeId { var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ .type_name = type_name.ptr, .type_name_size = type_name.len, @@ -1338,7 +1334,7 @@ pub const FFI = extern struct { return .{ .type_id = ret.type_id }; } - pub fn addUserData(self: *const FFI, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { + pub fn addUserData(self: *const Ffi, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_UserData_Add_Args{ .context = @ptrCast(context), .user_data = user_data.toCStruct(), @@ -1351,8 +1347,3 @@ pub const FFI = extern struct { } } }; - -pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { - command_buffer_compatible = c.PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE, - _, -}; diff --git a/zml/custom_call.zig b/zml/custom_call.zig index 0c28a059..8b102fb9 100644 --- a/zml/custom_call.zig +++ b/zml/custom_call.zig @@ -47,9 +47,7 @@ pub fn CustomCallInputsType(CustomOp: type) type { pub const CustomCallOptions = struct { output_operand_aliases: []const i64 = &.{}, copy_inputs_to_host_pinned: bool = false, - register_ffi_options: pjrt.FFI.RegisterOptions = .{ - .traits = @enumFromInt(0), - }, + traits: pjrt.ffi.HandlerTraits, }; /// Insert a user-defined callback into the computation graph. @@ -81,7 +79,7 @@ pub fn custom_call( const ffi_func = customCallProxy(CustomOp, opts); const target_name = "zml$" ++ op_name; - ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.register_ffi_options) catch unreachable; + ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.traits) catch unreachable; log.info("Registered custom call {s} with target name \"{s}\"", .{ op_name, target_name }); const custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; @@ -186,11 +184,6 @@ fn customCallImpl(comptime CustomOp: type, opts: CustomCallOptions, call_frame: pub fn getShape(ffi_buffer: *const ffi.Buffer) Shape { const dt: DataType = switch (ffi_buffer.dtype) { .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), - .pred => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), inline else => |t| @field(DataType, @tagName(t)), }; diff --git a/zml/tensor.zig b/zml/tensor.zig index 04874214..0b7213c7 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -6,6 +6,8 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const CompilationContext = @import("module.zig").CompilationContext; +const custom_call_ = @import("custom_call.zig"); +const custom_call = custom_call_.custom_call; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; @@ -13,8 +15,6 @@ const Memory = @import("buffer.zig").Buffer.Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); -const custom_call_ = @import("custom_call.zig"); -const custom_call = custom_call_.custom_call; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -3826,7 +3826,16 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - return custom_call(custom_call_.Print, .{input}, &[_]Shape{input.shape()}, .{ .output_operand_aliases = &.{0}, .copy_inputs_to_host_pinned = true })[0]; + return custom_call( + custom_call_.Print, + .{input}, + &[_]Shape{input.shape()}, + .{ + .output_operand_aliases = &.{0}, + .copy_inputs_to_host_pinned = true, + .traits = .{ .command_buffer_compatible = 0 }, + }, + )[0]; } }; From 1d930d9f0bfa61b4a6567473d6d86a364a000f12 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 3 Sep 2025 21:46:16 +0200 Subject: [PATCH 09/67] cleanup customCall impl # Conflicts: # zml/custom_call.zig # zml/tensor.zig # zml/zml.zig --- pjrt/ffi.zig | 6 +- zml/custom_call.zig | 153 ++++++++++++++++++++++++++------------------ zml/exe.zig | 6 +- zml/platform.zig | 23 +++++-- zml/tensor.zig | 14 +--- zml/zml.zig | 3 +- 6 files changed, 120 insertions(+), 85 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index f80b11f5..e63d8d27 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -38,11 +38,11 @@ pub const ExtensionBase = extern struct { }; // Based of https://github.com/openxla/xla/blob/145f836bd5175dc5dd262f716a0c59af2b0297a0/xla/ffi/api/c_api.h#L449 -pub const HandlerTraits = packed struct(u32) { +pub const HandlerTraits = packed struct(c_uint) { /// Calls to FFI handler are safe to trace into the command buffer. /// It means that calls to FFI handler always launch exactly the same device operations (can depend on attribute values) /// that can be captured and then replayed. - command_buffer_compatible: u1, + command_buffer_compatible: bool, __unassigned__: u31 = 0, }; @@ -401,7 +401,7 @@ pub const CallFrame = extern struct { return false; } - pub fn stream(call_frame: CallFrame) ?*const Stream { + pub fn stream(call_frame: CallFrame) ?*const pjrt.Stream { return call_frame.api.stream(call_frame.ctx); } }; diff --git a/zml/custom_call.zig b/zml/custom_call.zig index 8b102fb9..593f09e5 100644 --- a/zml/custom_call.zig +++ b/zml/custom_call.zig @@ -3,11 +3,11 @@ const std = @import("std"); const asynk = @import("async"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); +const stablehlo = @import("mlir/dialects").stablehlo; const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const CompilationContext = @import("module.zig").CompilationContext; -const cuda = @import("context.zig").cuda; const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const mlirx = @import("mlirx.zig"); @@ -17,16 +17,8 @@ const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; -}; const log = std.log.scoped(.@"zml/custom_call"); -pub const std_options: std.Options = .{ - .log_level = .info, - .logFn = asynk.logFn(std.log.defaultLog), -}; - pub fn CustomCallInputsType(CustomOp: type) type { const ArgsT = stdx.meta.FnArgs(CustomOp.call); if (@typeInfo(ArgsT) != .@"struct") { @@ -47,19 +39,51 @@ pub fn CustomCallInputsType(CustomOp: type) type { pub const CustomCallOptions = struct { output_operand_aliases: []const i64 = &.{}, copy_inputs_to_host_pinned: bool = false, - traits: pjrt.ffi.HandlerTraits, + handler_traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, }; -/// Insert a user-defined callback into the computation graph. -/// The callback is defined with a struct. +/// Inserts a user-defined callback into the computation graph. +/// The callback is defined with a struct, that store runtime information needed by the callback. +/// +/// ❗Experimental API❗ /// /// ```zig -/// const MyCallback -pub fn custom_call( +/// pub const MyCallback = struct { +/// // a unique type_id will be set by the PJRT plugin during registration. +/// pub var type_id: pjrt.ffi.TypeId = undefined; +/// +/// pub const custom_call_options: CustomCallOptions = .{ +/// // assumption this custom call makes about the input / output buffers +/// }; +/// +/// // Required, this will tell the callback in which env it runs. +/// platform: zml.Platform, +/// // data needed by the callback +/// my_data: []const u8, +/// +/// // storage modified by the runtime to tell the callback where it should write its results. +/// // Normally the callback doesn't need to allocate as the input and output buffers are given. +/// results: [1]Buffer = undefined, +/// +/// pub fn init(my_data: []const u8) !MyCallback { +/// return .{ .my_data = my_data }; +/// } +/// +/// pub fn call(callback: *MyCallback, input: Buffer) !void { +/// // Do something with `input` and `callback.my_data`, write the results inside `callback.results[0]` +/// } +/// }; +/// ``` +/// +/// See eg the implementation of the `zml.Print` callback, for a practical example. +/// +/// Note calling this during the compilation of a module, isn't enough. +/// The generated executable will need to know the specific data needed by `MyCallback` in this executable. +/// This is done through `zm.Exe.bind`. +pub fn customCall( comptime CustomOp: type, inputs: CustomCallInputsType(CustomOp), - res_shapes_: []const Shape, - comptime opts: CustomCallOptions, + output_shapes: []const Shape, ) []Tensor { const op_name = @typeName(CustomOp); stdx.debug.assertComptime(@hasDecl(CustomOp, "call"), "{s} must have a call method", .{op_name}); @@ -70,74 +94,75 @@ pub fn custom_call( const mlir_ctx = ctx.mlirCtx(); const platform = ctx._platform; const pjrt_api = platform.pjrt_api; - const ffi_ = pjrt_api.ffi(); - if (ffi_ == null) { + if (pjrt_api.ffi() == null) { stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); } - const ffi_func = customCallProxy(CustomOp, opts); const target_name = "zml$" ++ op_name; - ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.traits) catch unreachable; - log.info("Registered custom call {s} with target name \"{s}\"", .{ op_name, target_name }); + const output_tensors = allocator.alloc(Tensor, output_shapes.len) catch @panic("OOM"); + // Note: we don't always free output_tensor, because it's returned to the caller. + // It's also why we allocate it first so that it doesn't fragment the arena. + errdefer allocator.free(output_tensors); - const custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; - for (inputs, 0..) |t, i| { - custom_call_inputs[i] = t.value(); + const output_types = allocator.alloc(mlir.Type, output_shapes.len) catch @panic("OOM"); + defer allocator.free(output_types); + for (output_types, output_shapes) |*output_type, output_shape| { + output_type.* = mlirx.tensorType(mlir_ctx, output_shape); + } + const input_values = allocator.alloc(mlir.Value, inputs.len) catch @panic("OOM"); + defer allocator.free(input_values); + for (input_values, inputs) |*input_value, input_tensor| { + input_value.* = input_tensor.value(); } - const res_types = allocator.alloc(mlir.Type, 8) catch unreachable; - for (res_shapes_, 0..) |sh, i| { - res_types[i] = mlirx.tensorType(mlir_ctx, sh); - } - - const frontend_attributes: mlir.Attribute = .dict(mlir_ctx, &.{}); - - const op = dialect.stablehlo.custom_call( + const op = stablehlo.custom_call( mlir_ctx, - custom_call_inputs[0..inputs.len], + input_values, .{ .call_target_name = target_name, .api_version = .typed_ffi, - .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), - .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, + .backend_config = .dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", .dict(mlir_ctx, &.{}) }}, .has_side_effect = true, - .output_operand_aliases = opts.output_operand_aliases, + .output_operand_aliases = CustomOp.custom_call_options.output_operand_aliases, }, - res_types[0..res_shapes_.len], + output_types, mlir_ctx.location(@src()), ); - var custom_call_outputs = allocator.alloc(Tensor, 8) catch unreachable; - - for (custom_call_outputs[0..res_shapes_.len], res_shapes_, 0..) |*t, sh, i| { - t.* = Tensor._result(sh, op.result(i)); + for (output_tensors, output_shapes, 0..) |*output_tensor, output_shape, i| { + output_tensor.* = Tensor._result(output_shape, op.result(i)); } - - return custom_call_outputs[0..res_shapes_.len]; + return output_tensors; } -fn customCallProxy(CustomOp: type, comptime opts: CustomCallOptions) ffi.Handler { +pub fn proxy(CustomOp: type) ffi.Handler { return struct { pub fn cb(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { - return customCallImpl(CustomOp, opts, call_frame); + return customCallImpl(CustomOp, call_frame); } }.cb; } -fn customCallImpl(comptime CustomOp: type, opts: CustomCallOptions, call_frame: *ffi.CallFrame) ?*ffi.Error { +fn customCallImpl(comptime CustomOp: type, call_frame: *ffi.CallFrame) ?*ffi.Error { if (call_frame.registeringHook()) return null; + const opts = CustomOp.custom_call_options; + const execution_context = call_frame.ctx; - // log.info("Custom call {s} called !", .{ @typeName(CustomOp) }); + log.info("Custom call {s} called !", .{@typeName(CustomOp)}); const user_ctx_opaque = execution_context.getContext(CustomOp.type_id, call_frame.api) catch { return .create(call_frame.api, .internal, "failed to fetch user context for custom_call" ++ @typeName(CustomOp)); }; const user_ctx: *CustomOp = @ptrCast(@alignCast(user_ctx_opaque)); + // We actually have one more constraint here, we force the CustomOp to have a platform field, + // and to correctly set it. + // Is this good ? We could also simplify this by registering ourselves the `Platform` type id. const platform: Platform = user_ctx.platform; - // Undocumented hook to get a cuda stream in the callback. + // Hook to get a cuda stream in the callback. if (@hasField(CustomOp, "stream") and platform.target != .cpu) { const stream = call_frame.api.stream(execution_context); user_ctx.stream = stream; @@ -146,19 +171,16 @@ fn customCallImpl(comptime CustomOp: type, opts: CustomCallOptions, call_frame: var callback_args: std.meta.ArgsTuple(@TypeOf(CustomOp.call)) = undefined; callback_args[0] = user_ctx; - inline for (1..callback_args.len) |i| { - const ffi_buffer = call_frame.args.buffers()[i - 1]; - const ffi_buffer_shape = getShape(ffi_buffer); - var zml_buffer: Buffer = undefined; - - // TODO reconsider using PJRT apis at all from pjrt callbacks. - if (platform.target == .cpu) { - zml_buffer = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); - } else { - zml_buffer = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); - } + inline for (1..callback_args.len, call_frame.args.buffers()) |i, ffi_buffer| { + const shape = getShape(ffi_buffer); + var zml_buffer: Buffer = if (platform.target == .cpu) + .asViewOfHostBuffer(platform, .fromBytes(shape, ffi_buffer.data[0..shape.byteSize()])) + else + .asViewOfDeviceBuffer(platform, shape, null, ffi_buffer.data); if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + log.info("Copying argument {} {f} {*} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer() }); zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch unreachable; + log.info("--> {f} {*} ({})", .{ zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), @as(*const f32, @ptrCast(@alignCast(zml_buffer.opaqueDeviceMemoryDataPointer()))).* }); } callback_args[i] = zml_buffer; } @@ -192,12 +214,12 @@ pub fn getShape(ffi_buffer: *const ffi.Buffer) Shape { /// Internal custom calls. /// These are not meant to be used by users, but rather by the library itself. -pub const custom_call_internal_types = [_]type{ +pub const internal_custom_calls = [_]type{ Print, }; pub fn registerInternalCustomCalls(platform: Platform) !void { - inline for (custom_call_internal_types) |custom_call_type| { + inline for (internal_custom_calls) |custom_call_type| { try platform.registerCustomCall(custom_call_type); log.info("Registered internal custom call {s} with type_id {d}", .{ @typeName(custom_call_type), custom_call_type.type_id.type_id }); } @@ -208,6 +230,15 @@ pub const Print = struct { // a unique type_id will be set by the PJRT plugin during registration. pub var type_id: pjrt.ffi.TypeId = undefined; + pub const custom_call_options: CustomCallOptions = .{ + // Print callback pretends to modify the given input buffer, but just returns it unmodified. + .output_operand_aliases = &.{0}, + // It also needs PJRT to copy the data on the host first so it can print it. + .copy_inputs_to_host_pinned = true, + // Print is fairly predictable and can be captured in an execution graph. + .handler_traits = .{ .command_buffer_compatible = true }, + }; + platform: Platform, results: [1]Buffer = undefined, diff --git a/zml/exe.zig b/zml/exe.zig index 265aab7e..54075e2c 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -214,7 +214,7 @@ pub const BaseExe = struct { // Add extra data needed by the ZML provided custom calls. // Atm we don't have a mechanism to detect if the user need this or not. - inline for (custom_call.custom_call_internal_types) |T| { + inline for (custom_call.internal_custom_calls) |T| { const value_ptr = try allocator.create(T); value_ptr.* = try .init(platform); try ffi.addUserData(platform.pjrt_api, execute_context.?, .{ .type_id = T.type_id.type_id, .user_data = @ptrCast(@constCast(value_ptr)) }); @@ -380,6 +380,10 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + /// For a given customCall inside this executable, + /// provide a pointer to runtime data. + /// The caller keeps memory ownership and need to ensure that the value + /// stays alive as long as the executable. pub fn bind(self: Self, comptime T: type, value: *T) !void { try self.inner.bind(T, value); } diff --git a/zml/platform.zig b/zml/platform.zig index 9b67f2db..3c5a979a 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -4,6 +4,7 @@ const runtimes = @import("runtimes"); pub const Target = runtimes.Platform; const stdx = @import("stdx"); +const custom_call = @import("custom_call.zig"); const pjrt = @import("pjrtx.zig"); const log = std.log.scoped(.zml); @@ -22,6 +23,11 @@ pub const Platform = struct { target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, + + // This make the pjrt struct quite fat, but is only used during compilation. + // TODO: Reconsider having it here, and maybe pass explicitly to compile, + // or create an intermediary struct: + // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` compilation_options: CompilationOptions = .{}, pub const MAX_NUM_DEVICES: u8 = 32; @@ -72,14 +78,17 @@ pub const Platform = struct { } pub fn registerCustomCall(self: Platform, comptime CustomOp: type) pjrt.ApiError!void { - if (self.pjrt_api.ffi()) |ffi| { - stdx.debug.assertComptime(@hasDecl(CustomOp, "call"), "{} must have a call method", .{CustomOp}); - stdx.debug.assertComptime(@hasDecl(CustomOp, "type_id") and @TypeOf(CustomOp.type_id) == pjrt.ffi.TypeId, "{} must have a field `pub var type_id: pjrt.ffi.TypeId`", .{CustomOp}); + stdx.debug.assertComptime(@hasDecl(CustomOp, "call"), "{} must have a call method", .{CustomOp}); + stdx.debug.assertComptime(@hasDecl(CustomOp, "type_id") and @TypeOf(CustomOp.type_id) == pjrt.ffi.TypeId, "{} must have a field `pub var type_id: pjrt.ffi.TypeId`", .{CustomOp}); + stdx.debug.assertComptime(@hasDecl(CustomOp, "custom_call_options") and @TypeOf(CustomOp.custom_call_options) == custom_call.CustomCallOptions, "{} must have a field `pub const custom_call_options: zml.CustomCallOptions`", .{CustomOp}); - CustomOp.type_id = try ffi.registerTypeId(self.pjrt_api, @typeName(CustomOp)); - } else { - return error.Unavailable; - } + const ffi = self.pjrt_api.ffi() orelse return error.Unavailable; + const target_name = "zml$" ++ @typeName(CustomOp); + + const proxy = custom_call.proxy(CustomOp); + CustomOp.type_id = try ffi.registerTypeId(self.pjrt_api, @typeName(CustomOp)); + try ffi.register(self.pjrt_api, target_name, @tagName(self.target), &proxy, CustomOp.custom_call_options.handler_traits); + log.info("Registered custom call {} with target name \"{s}\"", .{ CustomOp, target_name }); } pub fn deinit(self: *Platform) void { diff --git a/zml/tensor.zig b/zml/tensor.zig index 0b7213c7..dd330a65 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -6,8 +6,7 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const CompilationContext = @import("module.zig").CompilationContext; -const custom_call_ = @import("custom_call.zig"); -const custom_call = custom_call_.custom_call; +const custom_call = @import("custom_call.zig"); const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; @@ -3826,16 +3825,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - return custom_call( - custom_call_.Print, - .{input}, - &[_]Shape{input.shape()}, - .{ - .output_operand_aliases = &.{0}, - .copy_inputs_to_host_pinned = true, - .traits = .{ .command_buffer_compatible = 0 }, - }, - )[0]; + return custom_call.customCall(custom_call.Print, .{input}, &.{input.shape()})[0]; } }; diff --git a/zml/zml.zig b/zml/zml.zig index 05602e9a..c541d313 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -14,10 +14,11 @@ pub const Bufferized = @import("tensor.zig").Bufferized; pub const CompilationOptions = @import("platform.zig").CompilationOptions; pub const context = @import("context.zig"); pub const Context = @import("context.zig").Context; +pub const customCall = @import("custom_call.zig").customCall; +pub const CustomCallOptions = @import("custom_call.zig").CustomCallOptions; pub const Data = @import("dtype.zig").Data; pub const DataType = @import("dtype.zig").DataType; pub const exe = @import("exe.zig"); -pub const custom_call = @import("custom_call.zig").custom_call; pub const compile = exe.compile; pub const compileWithPrefix = exe.compileWithPrefix; pub const compileFn = exe.compileFn; From ff1847df3d0d2d760675afe8af0edc6795ffb636 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 29 Aug 2025 12:04:51 +0200 Subject: [PATCH 10/67] rewrite examples/custom_call --- examples/custom_call/BUILD.bazel | 2 + examples/custom_call/main.zig | 276 +++++++++++++++++++------------ zml/BUILD.bazel | 2 +- zml/buffer.zig | 5 +- zml/zml.zig | 2 +- 5 files changed, 177 insertions(+), 110 deletions(-) diff --git a/examples/custom_call/BUILD.bazel b/examples/custom_call/BUILD.bazel index a2aa32fb..70bb19d1 100644 --- a/examples/custom_call/BUILD.bazel +++ b/examples/custom_call/BUILD.bazel @@ -7,5 +7,7 @@ zig_binary( deps = [ "@zml//async", "@zml//zml", + "@zml//runtimes", ], + copts = ["-lc"], ) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig index a58a4b11..c837bf42 100644 --- a/examples/custom_call/main.zig +++ b/examples/custom_call/main.zig @@ -1,8 +1,9 @@ const std = @import("std"); -const zml = @import("zml"); -const asynk = @import("async"); -const cuda = zml.context.cuda; +const asynk = @import("async"); +const runtimes = @import("runtimes"); +const zml = @import("zml"); +const cu = zml.platform_specific; pub const std_options: std.Options = .{ .log_level = .info, @@ -11,84 +12,121 @@ pub const std_options: std.Options = .{ const log = std.log.scoped(.@"examples/custom_call"); -fn add_op_host_func(data: *anyopaque) callconv(.c) void { - const ctx = @as(*AddOp, @ptrCast(@alignCast(data))); - - const result = std.mem.bytesAsValue(f32, ctx.result.asHostBuffer().mutBytes()); - result.* = ctx.a.asHostBuffer().item(f32, 0) + ctx.b.asHostBuffer().item(f32, 0); -} - -pub const AddOp = struct { - pub var type_id: i64 = undefined; - const Self = @This(); - - allocator: std.mem.Allocator, +/// Demonstration of the custom_call mechanism. +/// +/// * defines a function to compute the grayscale version of an image. +/// * using simple for loop on CPU +/// * using a custom PTX kernel on Cuda, with manual cuLaunchKernel +/// * Use `zml.customCall` to create an executable calling our callback and our kernel. +pub const GrayScale = struct { + // Mandatory fields to work with ZML custom call api + pub var type_id: zml.pjrt.ffi.TypeId = undefined; + pub const custom_call_options: zml.CustomCallOptions = .{}; platform: zml.Platform, - a: zml.Buffer, - b: zml.Buffer, - result: zml.Buffer, + // Custom field to store our cuda module and function. + cu_data: [2]*anyopaque, + // Set by ZML before `call` is entered results: [1]zml.Buffer = undefined, - stream: *anyopaque = undefined, - - pub fn init( - allocator: std.mem.Allocator, - platform: zml.Platform, - io_shape: zml.Shape, - ) !AddOp { - const host_buffer = try zml.HostBuffer.empty(allocator, io_shape); - defer host_buffer.deinit(allocator); - - const a = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); - const b = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); - const result = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + stream: *zml.pjrt.Stream = undefined, + pub fn init(platform: zml.Platform) !GrayScale { + var cu_data: [2]*anyopaque = undefined; + if (comptime runtimes.isEnabled(.cuda)) { + var module: cu.CUmodule = undefined; + try cuda.check(cuda.moduleLoadData.?(&module, grayscale_ptx)); + log.info("Loaded Grayscale cuda module", .{}); + var function: cu.CUfunction = undefined; + try cuda.check(cuda.moduleGetFunction.?(&function, module, "rgba_to_grayscale")); + log.info("Found Grayscale cuda function", .{}); + cu_data = .{ module.?, function.? }; + } return .{ - .a = a, - .b = b, - .result = result, .platform = platform, - .allocator = allocator, + .cu_data = cu_data, }; } - pub fn deinit(self: *Self) void { - self.a.deinit(); - self.b.deinit(); - self.result.deinit(); + pub fn deinit(self: *GrayScale) void { + if (comptime runtimes.isEnabled(.cuda)) { + const module: cu.CUmodule = @ptrCast(self.cu_data[0]); + _ = cuda.moduleUnload.?(module); + } } - pub fn call(self: *Self, a: zml.Buffer, b: zml.Buffer) !void { - if (self.platform.target == .cuda) { - cuda.memcpyToHostAsync(self.a.asHostBuffer().mutBytes(), a.opaqueDeviceMemoryDataPointer(), self.stream); - cuda.memcpyToHostAsync(self.b.asHostBuffer().mutBytes(), b.opaqueDeviceMemoryDataPointer(), self.stream); - - _ = cuda.cuLaunchHostFunc(self.stream, @ptrCast(&add_op_host_func), @ptrCast(self)); - - cuda.memcpyToDeviceAsync(self.results[0].opaqueDeviceMemoryDataPointer(), self.result.asHostBuffer().bytes(), self.stream); - return; + pub fn call(self: *GrayScale, rgb_d: zml.Buffer) !void { + switch (self.platform.target) { + .cpu => grayScaleCpu(rgb_d, self.results[0]), + // Only try to compile `grayScaleCuda` if we have cuda symbols. + .cuda => if (comptime runtimes.isEnabled(.cuda)) + try self.grayScaleCuda(rgb_d, self.results[0]) + else + unreachable, + else => @panic("Platform not supported"), } - - if (self.platform.target == .cpu) { - const result = std.mem.bytesAsValue(f32, self.results[0].asHostBuffer().mutBytes()); - result.* = a.asHostBuffer().item(f32, 0) + b.asHostBuffer().item(f32, 0); - return; - } - - return error.UnsupportedTarget; } + + pub fn grayScaleCpu(rgb_d: zml.Buffer, gray_d: zml.Buffer) void { + const rgb_h = rgb_d.asHostBuffer().items(u8); + const gray_h = gray_d.asHostBuffer().mutItems(u8); + + for (gray_h, 0..) |*gray, i| { + const px = rgb_h[i * 3 .. i * 3 + 3]; + const R: u32 = @intCast(px[0]); + const G: u32 = @intCast(px[1]); + const B: u32 = @intCast(px[2]); + const gray_u32: u32 = @divFloor(299 * R + 587 * G + 114 * B, 1000); + gray.* = @intCast(gray_u32); + } + } + + pub fn grayScaleCuda(self: GrayScale, rgb_d: zml.Buffer, gray_d: zml.Buffer) !void { + var args: [2][]u8 = .{ + rgb_d.opaqueDeviceMemoryDataPointer()[0..rgb_d.shape().byteSize()], + gray_d.opaqueDeviceMemoryDataPointer()[0..gray_d.shape().byteSize()], + }; + var args_ptr: [2:null]?*anyopaque = .{ @ptrCast(&args[0]), @ptrCast(&args[1]) }; + // This is a naive kernel with one block per pixel. + try cuda.check(cuda.launchKernel.?( + @ptrCast(self.cu_data[1]), // function + @intCast(rgb_d.shape().count() / 3), // num blocks x + 1, // num blocks y + 1, // num blocks z + 1, // num grids x + 1, // num grids y + 1, // num grids z + 0, // shared mem + @ptrCast(self.stream), + &args_ptr, + null, + )); + // Note: no explicit synchronization, we just enqueue work in the stream. + } + + const cuda = struct { + // Here we leverage ZML sandboxing to access cuda symbols and their definitions. + const moduleLoadData = @extern(*const @TypeOf(cu.cuModuleLoadData), .{ .name = "cuModuleLoadData", .linkage = .weak }); + const moduleUnload = @extern(*const @TypeOf(cu.cuModuleUnload), .{ .name = "cuModuleUnload", .linkage = .weak }); + const moduleGetFunction = @extern(*const @TypeOf(cu.cuModuleGetFunction), .{ .name = "cuModuleGetFunction", .linkage = .weak }); + const launchKernel = @extern(*const @TypeOf(cu.cuLaunchKernel), .{ .name = "cuLaunchKernel", .linkage = .weak }); + + pub fn check(result: cu.CUresult) error{CudaError}!void { + if (result == cu.CUDA_SUCCESS) return; + std.log.err("cuda error: {}", .{result}); + return error.CudaError; + } + }; }; -const Layer = struct { - pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { - const result = zml.custom_call(AddOp, .{ a.print(), b.print() }, &[_]zml.Shape{a.shape()}, .{}); - return result[0]; - } -}; +pub fn grayscale(rgb: zml.Tensor) zml.Tensor { + const gray_shape = rgb.shape().setDim(0, @divExact(rgb.dim(0), 3)); + const result = zml.customCall(GrayScale, .{rgb.print()}, &.{gray_shape}); + return result[0]; +} pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); } pub fn asyncMain() !void { @@ -96,59 +134,85 @@ pub fn asyncMain() !void { defer _ = gpa.deinit(); const allocator = gpa.allocator(); - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - var context = try zml.Context.init(); defer context.deinit(); const platform = context.autoPlatform(.{}); context.printAvailablePlatforms(platform); - const shape = zml.Shape.init(.{}, .f32); + // Register our custom call + try platform.registerCustomCall(GrayScale); - const buffers: zml.aio.BufferStore.Buffers = .{}; - const buffer_store: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, - }; + // Compile MLIR code containing our custom call. + const rgb_shape = zml.Shape.init(.{12 * 3}, .u8); + const exe = try zml.compileFn(allocator, grayscale, .{rgb_shape}, platform); + defer exe.deinit(); - const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + // Provide runtime information needed by our custom call. + var gray_op: GrayScale = try .init(platform); + defer gray_op.deinit(); + try exe.bind(GrayScale, &gray_op); - var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + // Load data and run the executable. + const rgb_h: [12][3]u8 = @splat(.{ 0xFF, 0xAA, 0x00 }); + const rgb_d = try zml.Buffer.fromBytes(platform, rgb_shape, @ptrCast(&rgb_h)); + defer rgb_d.deinit(); - var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); + var gray_d: zml.Buffer = exe.call(.{rgb_d}); + defer gray_d.deinit(); - const compiled = try compilation.awaitt(); + // Inspect results + const gray_h = try gray_d.toHostAlloc(allocator); + defer gray_h.deinit(allocator); - var executable = compiled.prepare(model_weights); - defer executable.deinit(); - - executable = try executable.withExecutionContext(); - - try platform.registerFFIType(AddOp); - var add_op: AddOp = try .init(allocator, platform, shape); - defer add_op.deinit(); - try executable.bind(AddOp, &add_op); - - var input_a = [1]f32{1.0}; - var input_buffer_a = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_a), .{}); - defer input_buffer_a.deinit(); - - var input_b = [1]f32{1.0}; - var input_buffer_b = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_b), .{}); - defer input_buffer_b.deinit(); - - var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); - defer result.deinit(); - - const cpu_result = try result.toHostAlloc(arena); - _ = cpu_result; // autofix - - // log.warn( - // "\nThe result of {} + {} = {}\n", - // .{ &input_a, &input_b, cpu_result.items(f32) }, - // ); + std.debug.print("Grayscale conversion of {any} -> {d}\n", .{ rgb_h, gray_h }); + try std.testing.expectEqualSlices(u8, &@as([12]u8, @splat(0xB0)), gray_h.items(u8)); } + +// Compiled with Zig and a little help from `https://github.com/gwenzek/cudaz` +const grayscale_ptx = + \\.version 4.0 + \\.target sm_32 + \\.address_size 64 + \\ + \\.visible .entry rgba_to_grayscale( + \\ .param .align 8 .b8 rgba_to_grayscale_param_0[16], + \\ .param .align 8 .b8 rgba_to_grayscale_param_1[16] + \\) + \\{ + \\ .reg .pred %p<2>; + \\ .reg .b16 %rs<4>; + \\ .reg .b32 %r<13>; + \\ .reg .b64 %rd<9>; + \\ + \\ ld.param.u64 %rd5, [rgba_to_grayscale_param_1+8]; + \\ mov.u32 %r1, %tid.x; + \\ mov.u32 %r3, %ntid.x; + \\ mov.u32 %r2, %ctaid.x; + \\ mad.lo.s32 %r4, %r2, %r3, %r1; + \\ cvt.u64.u32 %rd1, %r4; + \\ setp.gt.u64 %p1, %rd5, %rd1; + \\ @%p1 bra $L__BB0_2; + \\ bra.uni $L__BB0_1; + \\$L__BB0_2: + \\ ld.param.u64 %rd4, [rgba_to_grayscale_param_1]; + \\ ld.param.u64 %rd2, [rgba_to_grayscale_param_0]; + \\ cvt.u32.u64 %r5, %rd1; + \\ mul.lo.s32 %r6, %r5, 3; + \\ cvt.u64.u32 %rd6, %r6; + \\ add.s64 %rd7, %rd2, %rd6; + \\ ld.u8 %rs1, [%rd7]; + \\ ld.u8 %rs2, [%rd7+1]; + \\ ld.u8 %rs3, [%rd7+2]; + \\ mul.wide.u16 %r7, %rs1, 299; + \\ mul.wide.u16 %r8, %rs2, 587; + \\ add.s32 %r9, %r8, %r7; + \\ mul.wide.u16 %r10, %rs3, 114; + \\ add.s32 %r11, %r9, %r10; + \\ mul.hi.u32 %r12, %r11, 4294968; + \\ add.s64 %rd8, %rd4, %rd1; + \\ st.u8 [%rd8], %r12; + \\$L__BB0_1: + \\ ret; + \\} +; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 88876478..f043f08d 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -54,7 +54,7 @@ zig_library( "torch.zig", "zml.zig", ], - copts = ["-lc"], + copts = ["-lc", "-freference-trace=20"], main = "zml.zig", visibility = ["//visibility:public"], deps = [ diff --git a/zml/buffer.zig b/zml/buffer.zig index 3f1ed616..7b09f25e 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -303,9 +303,10 @@ pub const Buffer = struct { }; } - pub fn opaqueDeviceMemoryDataPointer(self: Buffer) *anyopaque { + pub fn opaqueDeviceMemoryDataPointer(self: Buffer) [*]u8 { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); - return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + const opaque_ptr: *anyopaque = self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + return @ptrCast(opaque_ptr); } /// Fetches the content of the given buffer into a stack variable of the given type. diff --git a/zml/zml.zig b/zml/zml.zig index c541d313..ed98d0bb 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -6,6 +6,7 @@ // Namespaces const std = @import("std"); +pub const platform_specific = @import("c"); pub const tokenizer = @import("zml/tokenizer"); pub const aio = @import("aio.zig"); @@ -45,7 +46,6 @@ pub const Tensor = @import("tensor.zig").Tensor; pub const testing = @import("testing.zig"); pub const torch = @import("torch.zig"); -// pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { pub const Tracer = @import("tools/tracer.zig").Tracer; }; From a76635df8f87eff7febcd90291eea14ee8282efc Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 09:08:55 +0200 Subject: [PATCH 11/67] workaround zig build error --- runtimes/cuda/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index 96cc5879..83109cf0 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -5,7 +5,7 @@ zig_shared_library( name = "zmlxcuda", # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. - copts = ["-fno-stack-check"], + copts = ["-fno-stack-check", "-fllvm"], main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", visibility = ["@libpjrt_cuda//:__subpackages__"], From e7df7611760e2d6867520afec02dc0dd08f6f109 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 09:09:15 +0200 Subject: [PATCH 12/67] disable command_buffer_compatible for print only work for toy examples, on complex graph this lead to memory corruption --- zml/custom_call.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/custom_call.zig b/zml/custom_call.zig index 593f09e5..1a779c02 100644 --- a/zml/custom_call.zig +++ b/zml/custom_call.zig @@ -236,7 +236,7 @@ pub const Print = struct { // It also needs PJRT to copy the data on the host first so it can print it. .copy_inputs_to_host_pinned = true, // Print is fairly predictable and can be captured in an execution graph. - .handler_traits = .{ .command_buffer_compatible = true }, + .handler_traits = .{ .command_buffer_compatible = false }, }; platform: Platform, From 49bc087f9f1cc6aca8b010ab0b799d64e6f68627 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 10:17:56 +0200 Subject: [PATCH 13/67] CustomCall -> Callback --- .../{custom_call => callback}/BUILD.bazel | 4 +- examples/{custom_call => callback}/main.zig | 6 +- pjrt/ffi.zig | 2 + zml/BUILD.bazel | 2 +- zml/buffer.zig | 5 +- zml/{custom_call.zig => callback.zig} | 207 +++++++++++------- zml/context.zig | 8 +- zml/exe.zig | 21 +- zml/platform.zig | 18 +- zml/tensor.zig | 4 +- zml/zml.zig | 3 +- 11 files changed, 158 insertions(+), 122 deletions(-) rename examples/{custom_call => callback}/BUILD.bazel (67%) rename examples/{custom_call => callback}/main.zig (97%) rename zml/{custom_call.zig => callback.zig} (52%) diff --git a/examples/custom_call/BUILD.bazel b/examples/callback/BUILD.bazel similarity index 67% rename from examples/custom_call/BUILD.bazel rename to examples/callback/BUILD.bazel index 70bb19d1..3f55040f 100644 --- a/examples/custom_call/BUILD.bazel +++ b/examples/callback/BUILD.bazel @@ -1,13 +1,11 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( - name = "custom_call", - copts = ["-freference-trace=30"], + name = "callback", main = "main.zig", deps = [ "@zml//async", "@zml//zml", "@zml//runtimes", ], - copts = ["-lc"], ) diff --git a/examples/custom_call/main.zig b/examples/callback/main.zig similarity index 97% rename from examples/custom_call/main.zig rename to examples/callback/main.zig index c837bf42..7d43ba54 100644 --- a/examples/custom_call/main.zig +++ b/examples/callback/main.zig @@ -21,7 +21,7 @@ const log = std.log.scoped(.@"examples/custom_call"); pub const GrayScale = struct { // Mandatory fields to work with ZML custom call api pub var type_id: zml.pjrt.ffi.TypeId = undefined; - pub const custom_call_options: zml.CustomCallOptions = .{}; + pub const callback_config: zml.callback.Config = .{}; platform: zml.Platform, // Custom field to store our cuda module and function. @@ -121,7 +121,7 @@ pub const GrayScale = struct { pub fn grayscale(rgb: zml.Tensor) zml.Tensor { const gray_shape = rgb.shape().setDim(0, @divExact(rgb.dim(0), 3)); - const result = zml.customCall(GrayScale, .{rgb.print()}, &.{gray_shape}); + const result = zml.callback.call(GrayScale, .{rgb.print()}, &.{gray_shape}); return result[0]; } @@ -141,7 +141,7 @@ pub fn asyncMain() !void { context.printAvailablePlatforms(platform); // Register our custom call - try platform.registerCustomCall(GrayScale); + try platform.registerCallback(GrayScale); // Compile MLIR code containing our custom call. const rgb_shape = zml.Shape.init(.{12 * 3}, .u8); diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index e63d8d27..2ca639a7 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -450,6 +450,8 @@ pub const Error = opaque { pub const inner = TransmuteMixin(Error, c.XLA_FFI_Error).to; pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; + pub const ok: ?*Error = null; + pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_Create_Args{ .message = message.ptr, diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f043f08d..3d2ae081 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -31,7 +31,7 @@ zig_library( "aio/torch/py.zig", "buffer.zig", "context.zig", - "custom_call.zig", + "callback.zig", "dtype.zig", "exe.zig", "floats.zig", diff --git a/zml/buffer.zig b/zml/buffer.zig index 7b09f25e..f502d703 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -185,8 +185,9 @@ pub const Buffer = struct { } pub fn asHostBuffer(self: Buffer) HostBuffer { - const memory = self.getMemory().kind(self._api); - stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); + // TODO: skip this check on cpu + // const memory = self.getMemory().kind(self._api); + // stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } diff --git a/zml/custom_call.zig b/zml/callback.zig similarity index 52% rename from zml/custom_call.zig rename to zml/callback.zig index 1a779c02..23b66804 100644 --- a/zml/custom_call.zig +++ b/zml/callback.zig @@ -12,35 +12,11 @@ const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const mlirx = @import("mlirx.zig"); const pjrtx = @import("pjrtx.zig"); -const ffi = pjrtx.ffi; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; -const log = std.log.scoped(.@"zml/custom_call"); - -pub fn CustomCallInputsType(CustomOp: type) type { - const ArgsT = stdx.meta.FnArgs(CustomOp.call); - if (@typeInfo(ArgsT) != .@"struct") { - @compileError("Expected struct type"); - } - - const args = @typeInfo(ArgsT).@"struct".fields; - - for (args[1..args.len]) |field| { - if (field.type != Buffer) { - @compileError("Expected all arguments in custom call `call` to be of type zml.Buffer"); - } - } - - return [args.len - 1]Tensor; -} - -pub const CustomCallOptions = struct { - output_operand_aliases: []const i64 = &.{}, - copy_inputs_to_host_pinned: bool = false, - handler_traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, -}; +const log = std.log.scoped(.@"zml/callback"); /// Inserts a user-defined callback into the computation graph. /// The callback is defined with a struct, that store runtime information needed by the callback. @@ -52,7 +28,7 @@ pub const CustomCallOptions = struct { /// // a unique type_id will be set by the PJRT plugin during registration. /// pub var type_id: pjrt.ffi.TypeId = undefined; /// -/// pub const custom_call_options: CustomCallOptions = .{ +/// pub const callback_config: zml.callback.Config = .{ /// // assumption this custom call makes about the input / output buffers /// }; /// @@ -75,19 +51,18 @@ pub const CustomCallOptions = struct { /// }; /// ``` /// -/// See eg the implementation of the `zml.Print` callback, for a practical example. +/// See eg the implementation of the `zml.callback.Print` callback, for a practical example. /// -/// Note calling this during the compilation of a module, isn't enough. -/// The generated executable will need to know the specific data needed by `MyCallback` in this executable. -/// This is done through `zm.Exe.bind`. -pub fn customCall( - comptime CustomOp: type, - inputs: CustomCallInputsType(CustomOp), +/// Note calling this during the compilation of a module, isn't enough: +/// +/// * backend need to be made aware of the callback, see `zml.Platform.registerCallback` +/// * executable need to know the specific data needed by `MyCallback`, see `zml.Exe.bind` +pub fn call( + comptime Callback: type, + inputs: TensorArgs(Callback), output_shapes: []const Shape, ) []Tensor { - const op_name = @typeName(CustomOp); - stdx.debug.assertComptime(@hasDecl(CustomOp, "call"), "{s} must have a call method", .{op_name}); - stdx.debug.assertComptime(@hasDecl(CustomOp, "type_id") and @TypeOf(CustomOp.type_id) == pjrt.ffi.TypeId, "{s} must have a field `pub var type_id: pjrt.ffi.TypeId`", .{op_name}); + checkIsValidCallback(Callback); const ctx = CompilationContext.current(); const allocator = ctx.allocator(); @@ -99,8 +74,6 @@ pub fn customCall( stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); } - const target_name = "zml$" ++ op_name; - const output_tensors = allocator.alloc(Tensor, output_shapes.len) catch @panic("OOM"); // Note: we don't always free output_tensor, because it's returned to the caller. // It's also why we allocate it first so that it doesn't fragment the arena. @@ -117,6 +90,7 @@ pub fn customCall( input_value.* = input_tensor.value(); } + const target_name = "zml$" ++ @typeName(Callback); const op = stablehlo.custom_call( mlir_ctx, input_values, @@ -126,7 +100,7 @@ pub fn customCall( .backend_config = .dict(mlir_ctx, &.{}), .additional_attributes = &.{.{ "mhlo.frontend_attributes", .dict(mlir_ctx, &.{}) }}, .has_side_effect = true, - .output_operand_aliases = CustomOp.custom_call_options.output_operand_aliases, + .output_operand_aliases = Callback.callback_config.output_operand_aliases, }, output_types, mlir_ctx.location(@src()), @@ -138,105 +112,170 @@ pub fn customCall( return output_tensors; } -pub fn proxy(CustomOp: type) ffi.Handler { +/// Describe properties of a callback +/// +/// * output_operand_aliases: the callback reuse input buffer to write the output +/// * copy_inputs_to_host_pinned: the callback need to work on host visible buffers +/// * traits: PJRT specified properties of the callback +pub const Config = struct { + output_operand_aliases: []const i64 = &.{}, + copy_inputs_to_host_pinned: bool = false, + // TODO: document precisely what `command_buffer_compatible` is doing and its limitations. + traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, + // TODO: handle sharded inputs +}; + +/// Compile-time check that a callback has all informations we require. +pub fn checkIsValidCallback(Callback: type) void { + stdx.debug.assertComptime(@hasDecl(Callback, "call"), "Expected callback {} to have a call method", .{Callback}); + const ArgsT = stdx.meta.FnArgs(Callback.call); + inline for (@typeInfo(ArgsT).@"struct".fields[1..]) |field| { + stdx.debug.assertComptime(field.type == Buffer, "Expected callback {}.call arguments to be of type zml.Buffer, got {}", .{ Callback, field.type }); + } + + stdx.debug.assertComptime(@hasDecl(Callback, "type_id") and @TypeOf(Callback.type_id) == pjrt.ffi.TypeId, "Expected callback {} to have a field `pub var type_id: pjrt.ffi.TypeId`", .{Callback}); + stdx.debug.assertComptime(@hasDecl(Callback, "callback_config") and @TypeOf(Callback.callback_config) == Config, "Expected callback {} to have a field `pub const callback_config: zml.CustomCallOptions`", .{Callback}); +} + +pub fn register(Callback: type, platform: Platform) pjrt.ApiError!void { + checkIsValidCallback(Callback); + + const ffi = platform.pjrt_api.ffi() orelse return error.Unavailable; + const target_name = "zml$" ++ @typeName(Callback); + + const proxy_cb = proxy(Callback); + Callback.type_id = try ffi.registerTypeId(platform.pjrt_api, @typeName(Callback)); + try ffi.register(platform.pjrt_api, target_name, @tagName(platform.target), &proxy_cb, Callback.callback_config.traits); + log.debug("Registered custom call {} with target name \"{s}\"", .{ Callback, target_name }); +} + +fn proxy(Callback: type) pjrt.ffi.Handler { return struct { - pub fn cb(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { - return customCallImpl(CustomOp, call_frame); + pub fn cb(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + return CallbackImpl(Callback, call_frame); } }.cb; } -fn customCallImpl(comptime CustomOp: type, call_frame: *ffi.CallFrame) ?*ffi.Error { +fn CallbackImpl(comptime Callback: type, call_frame: *pjrt.ffi.CallFrame) ?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; - const opts = CustomOp.custom_call_options; + const opts = Callback.callback_config; const execution_context = call_frame.ctx; - log.info("Custom call {s} called !", .{@typeName(CustomOp)}); - const user_ctx_opaque = execution_context.getContext(CustomOp.type_id, call_frame.api) catch { - return .create(call_frame.api, .internal, "failed to fetch user context for custom_call" ++ @typeName(CustomOp)); + log.debug("Custom call {s} called !", .{@typeName(Callback)}); + const user_ctx_opaque = execution_context.getContext(Callback.type_id, call_frame.api) catch { + log.err("{} user data was never given for current executable", .{Callback}); + return .create(call_frame.api, .failed_precondition, "failed to fetch user context" ++ @typeName(Callback)); }; - const user_ctx: *CustomOp = @ptrCast(@alignCast(user_ctx_opaque)); - // We actually have one more constraint here, we force the CustomOp to have a platform field, + const user_ctx: *Callback = @ptrCast(@alignCast(user_ctx_opaque)); + // We actually have one more constraint here, we force the Callback to have a platform field, // and to correctly set it. // Is this good ? We could also simplify this by registering ourselves the `Platform` type id. const platform: Platform = user_ctx.platform; // Hook to get a cuda stream in the callback. - if (@hasField(CustomOp, "stream") and platform.target != .cpu) { + if (@hasField(Callback, "stream") and platform.target != .cpu) { const stream = call_frame.api.stream(execution_context); user_ctx.stream = stream; } - var callback_args: std.meta.ArgsTuple(@TypeOf(CustomOp.call)) = undefined; + var callback_args: std.meta.ArgsTuple(@TypeOf(Callback.call)) = undefined; callback_args[0] = user_ctx; inline for (1..callback_args.len, call_frame.args.buffers()) |i, ffi_buffer| { - const shape = getShape(ffi_buffer); + const shape = shapeFromFfi(ffi_buffer); var zml_buffer: Buffer = if (platform.target == .cpu) .asViewOfHostBuffer(platform, .fromBytes(shape, ffi_buffer.data[0..shape.byteSize()])) else .asViewOfDeviceBuffer(platform, shape, null, ffi_buffer.data); if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { - log.info("Copying argument {} {f} {*} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer() }); - zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch unreachable; - log.info("--> {f} {*} ({})", .{ zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), @as(*const f32, @ptrCast(@alignCast(zml_buffer.opaqueDeviceMemoryDataPointer()))).* }); + log.debug("Copying argument {d} {f} {*} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer() }); + zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch |err| { + log.err("Failed to copy input buffer {d} {f} {*} to host_pinned: {}", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), err }); + return .create(call_frame.api, .resource_exhausted, "host pinned OOM"); + }; + log.debug("--> {f} {*} ({})", .{ zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), @as(*const f32, @ptrCast(@alignCast(zml_buffer.opaqueDeviceMemoryDataPointer()))).* }); } callback_args[i] = zml_buffer; } for (0..call_frame.results.len) |i| { const ffi_buffer = call_frame.results.buffers()[i]; - const ffi_buffer_shape = getShape(ffi_buffer); + const ffi_buffer_shape = shapeFromFfi(ffi_buffer); if (platform.target == .cpu) { user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); } else { - user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, shapeFromFfi(ffi_buffer), null, ffi_buffer.data); } } - @call(.auto, CustomOp.call, callback_args) catch |err| { - stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(CustomOp), err }); + @call(.auto, Callback.call, callback_args) catch |err| { + log.err("Callback {} failed with {}", .{ Callback, err }); + return .create(call_frame.api, .internal, "internal callback error"); }; - return null; -} - -pub fn getShape(ffi_buffer: *const ffi.Buffer) Shape { - const dt: DataType = switch (ffi_buffer.dtype) { - .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), - .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), - inline else => |t| @field(DataType, @tagName(t)), - }; - return Shape.init(ffi_buffer.dims(), dt); + return .ok; } /// Internal custom calls. /// These are not meant to be used by users, but rather by the library itself. -pub const internal_custom_calls = [_]type{ +pub const internal_callbacks = [_]type{ Print, }; -pub fn registerInternalCustomCalls(platform: Platform) !void { - inline for (internal_custom_calls) |custom_call_type| { - try platform.registerCustomCall(custom_call_type); - log.info("Registered internal custom call {s} with type_id {d}", .{ @typeName(custom_call_type), custom_call_type.type_id.type_id }); +pub fn registerInternalCallbacks(platform: Platform) !void { + inline for (internal_callbacks) |Callback| { + try register(Callback, platform); + // log.debug("Registered internal custom call {s} with type_id {d}", .{ @typeName(Callback), Callback.type_id.type_id }); } } +/// Allocate user data data needed by the ZML provided custom calls. +pub fn bindInternalCallbacks( + arena: std.mem.Allocator, + platform: Platform, + ffi: pjrt.Ffi, + execute_context: *pjrt.ExecuteContext, +) (std.mem.Allocator.Error || pjrt.ApiError)!void { + // Atm we don't have a mechanism to detect which ZML callbacks the executable needs, + // so we always allocate. + { + // Print + const print_ptr = try arena.create(Print); + print_ptr.* = try .init(platform); + try addUserData(Print, platform.pjrt_api, ffi, execute_context, print_ptr); + } +} + +pub fn addUserData( + Callback: type, + api: *const pjrt.Api, + ffi: pjrt.Ffi, + execute_context: *pjrt.ExecuteContext, + user_data: *Callback, +) pjrt.ApiError!void { + try ffi.addUserData( + api, + execute_context, + .{ .type_id = Callback.type_id.type_id, .user_data = @ptrCast(user_data) }, + ); + log.debug("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(Callback), @intFromPtr(user_data), Callback.type_id.type_id, execute_context }); +} + /// The print callback pub const Print = struct { // a unique type_id will be set by the PJRT plugin during registration. pub var type_id: pjrt.ffi.TypeId = undefined; - pub const custom_call_options: CustomCallOptions = .{ + pub const callback_config: Config = .{ // Print callback pretends to modify the given input buffer, but just returns it unmodified. .output_operand_aliases = &.{0}, // It also needs PJRT to copy the data on the host first so it can print it. .copy_inputs_to_host_pinned = true, // Print is fairly predictable and can be captured in an execution graph. - .handler_traits = .{ .command_buffer_compatible = false }, + .traits = .{ .command_buffer_compatible = false }, }; platform: Platform, @@ -250,3 +289,19 @@ pub const Print = struct { std.log.defaultLog(.info, .zml, "Device buffer: {f}: {d}", .{ input, input.asHostBuffer() }); } }; + +fn shapeFromFfi(ffi_buffer: *const pjrt.ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} + +fn TensorArgs(Callback: type) type { + const ArgsT = stdx.meta.FnArgs(Callback.call); + + const args = @typeInfo(ArgsT).@"struct".fields; + return [args.len - 1]Tensor; +} diff --git a/zml/context.zig b/zml/context.zig index 6a284cfb..3b18646d 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -10,12 +10,12 @@ const stdx = @import("stdx"); const pjrt = @import("pjrtx.zig"); const zml = struct { - pub const Platform = @import("platform.zig").Platform; + pub const callback = @import("callback.zig"); pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; + pub const Platform = @import("platform.zig").Platform; + pub const platform = @import("platform.zig"); pub const Shape = @import("shape.zig").Shape; pub const Target = @import("platform.zig").Target; - pub const custom_call = @import("custom_call.zig"); - pub const platform = @import("platform.zig"); }; const PjrtApiMap = std.EnumArray(zml.Target, ?*const pjrt.Api); @@ -181,7 +181,7 @@ pub const Context = struct { } self.platforms.set(target, p); - try zml.custom_call.registerInternalCustomCalls(p); + try zml.callback.registerInternalCallbacks(p); return p; } diff --git a/zml/exe.zig b/zml/exe.zig index 54075e2c..d4c426b7 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -5,8 +5,8 @@ const stdx = @import("stdx"); const aio = @import("aio.zig"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("tensor.zig").Bufferized; +const callback = @import("callback.zig"); const CompilationContext = @import("module.zig").CompilationContext; -const custom_call = @import("custom_call.zig"); const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; @@ -211,15 +211,7 @@ pub const BaseExe = struct { if (platform.pjrt_api.ffi()) |ffi| { log.info("Created context execution {*} for {*}", .{ execute_context, exe }); execute_context = try platform.pjrt_api.createExecuteContext(); - - // Add extra data needed by the ZML provided custom calls. - // Atm we don't have a mechanism to detect if the user need this or not. - inline for (custom_call.internal_custom_calls) |T| { - const value_ptr = try allocator.create(T); - value_ptr.* = try .init(platform); - try ffi.addUserData(platform.pjrt_api, execute_context.?, .{ .type_id = T.type_id.type_id, .user_data = @ptrCast(@constCast(value_ptr)) }); - log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), @intFromPtr(value_ptr), T.type_id.type_id, execute_context }); - } + try callback.bindInternalCallbacks(allocator, platform, ffi, execute_context.?); } return .{ @@ -303,17 +295,14 @@ pub const BaseExe = struct { stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); } - pub fn bind(exe: BaseExe, comptime CustomOp: type, op: *CustomOp) !void { + pub fn bind(exe: BaseExe, Callback: type, op: *Callback) !void { stdx.debug.assert(exe.execute_context != null, "Exe doesn't have an execution context", .{}); const pjrt_api = exe.platform.pjrt_api; if (pjrt_api.ffi()) |ffi| { - const type_id: i64 = CustomOp.type_id.type_id; - - try ffi.addUserData(pjrt_api, exe.execute_context.?, .{ .type_id = type_id, .user_data = op }); - log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(CustomOp), @intFromPtr(op), type_id, exe.execute_context.? }); + try callback.addUserData(Callback, pjrt_api, ffi, exe.execute_context.?, op); } else { - stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(exe.platform.target)}); + stdx.debug.panic("Callbacks are not supported for target {s}", .{@tagName(exe.platform.target)}); } } diff --git a/zml/platform.zig b/zml/platform.zig index 3c5a979a..1e1d1bfd 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -4,7 +4,9 @@ const runtimes = @import("runtimes"); pub const Target = runtimes.Platform; const stdx = @import("stdx"); -const custom_call = @import("custom_call.zig"); +const zml = struct { + const callback = @import("callback.zig"); +}; const pjrt = @import("pjrtx.zig"); const log = std.log.scoped(.zml); @@ -77,18 +79,8 @@ pub const Platform = struct { return res; } - pub fn registerCustomCall(self: Platform, comptime CustomOp: type) pjrt.ApiError!void { - stdx.debug.assertComptime(@hasDecl(CustomOp, "call"), "{} must have a call method", .{CustomOp}); - stdx.debug.assertComptime(@hasDecl(CustomOp, "type_id") and @TypeOf(CustomOp.type_id) == pjrt.ffi.TypeId, "{} must have a field `pub var type_id: pjrt.ffi.TypeId`", .{CustomOp}); - stdx.debug.assertComptime(@hasDecl(CustomOp, "custom_call_options") and @TypeOf(CustomOp.custom_call_options) == custom_call.CustomCallOptions, "{} must have a field `pub const custom_call_options: zml.CustomCallOptions`", .{CustomOp}); - - const ffi = self.pjrt_api.ffi() orelse return error.Unavailable; - const target_name = "zml$" ++ @typeName(CustomOp); - - const proxy = custom_call.proxy(CustomOp); - CustomOp.type_id = try ffi.registerTypeId(self.pjrt_api, @typeName(CustomOp)); - try ffi.register(self.pjrt_api, target_name, @tagName(self.target), &proxy, CustomOp.custom_call_options.handler_traits); - log.info("Registered custom call {} with target name \"{s}\"", .{ CustomOp, target_name }); + pub fn registerCallback(platform: Platform, Callback: type) pjrt.ApiError!void { + try zml.callback.register(Callback, platform); } pub fn deinit(self: *Platform) void { diff --git a/zml/tensor.zig b/zml/tensor.zig index dd330a65..63ab44d4 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -5,8 +5,8 @@ const mlir = @import("mlir"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; +const callback = @import("callback.zig"); const CompilationContext = @import("module.zig").CompilationContext; -const custom_call = @import("custom_call.zig"); const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; @@ -3825,7 +3825,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - return custom_call.customCall(custom_call.Print, .{input}, &.{input.shape()})[0]; + return callback.call(callback.Print, .{input}, &.{input.shape()})[0]; } }; diff --git a/zml/zml.zig b/zml/zml.zig index ed98d0bb..55559f5c 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -12,11 +12,10 @@ pub const tokenizer = @import("zml/tokenizer"); pub const aio = @import("aio.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const Bufferized = @import("tensor.zig").Bufferized; +pub const callback = @import("callback.zig"); pub const CompilationOptions = @import("platform.zig").CompilationOptions; pub const context = @import("context.zig"); pub const Context = @import("context.zig").Context; -pub const customCall = @import("custom_call.zig").customCall; -pub const CustomCallOptions = @import("custom_call.zig").CustomCallOptions; pub const Data = @import("dtype.zig").Data; pub const DataType = @import("dtype.zig").DataType; pub const exe = @import("exe.zig"); From 4a23125cb7473055e74e6c68f5b83393788bca5a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 10:19:56 +0200 Subject: [PATCH 14/67] remove cuda code from context --- zml/context.zig | 67 ------------------------------------------------- 1 file changed, 67 deletions(-) diff --git a/zml/context.zig b/zml/context.zig index 3b18646d..27d635a0 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -39,7 +39,6 @@ pub const Context = struct { inline for (comptime std.enums.values(runtimes.Platform)) |t| { if (runtimes.load(t)) |api| { Context.apis.set(t, api); - if (t == .cuda) cuda.init(); } else |_| {} } } @@ -216,69 +215,3 @@ pub const Context = struct { } } }; - -pub const cuda = struct { - pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); - pub var cuLaunchHostFunc: CuLaunchHostFunc = @ptrFromInt(0xdeadc00da00); - var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); - var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); - - pub const MemcpyKind = enum(c_int) { - host_to_host = 0, - host_to_device = 1, - device_to_host = 2, - device_to_device = 3, - inferred = 4, - }; - - const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*anyopaque) callconv(.c) c_int; - const MemcpyBlocking = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind) callconv(.c) c_int; - const StreamSynchronize = *const fn (stream: *anyopaque) callconv(.c) c_int; - const CuLaunchHostFunc = *const fn (stream: *anyopaque, host_func: *const fn (user_data: *const anyopaque) callconv(.c) void, user_data: *const anyopaque) callconv(.c) c_int; - - pub fn init() void { - var cudart = std.DynLib.open("libcudart.so.12") catch { - log.err("cudart not found, callback will segfault", .{}); - return; - }; - defer cudart.close(); - - _memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { - @panic("cudaMemcpyAsync not found"); - }; - _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { - @panic("cudaMemcpy not found"); - }; - streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { - @panic("cudaStreamSynchronize not found"); - }; - cuLaunchHostFunc = cudart.lookup(CuLaunchHostFunc, "cudaLaunchHostFunc") orelse { - @panic("cudaLaunchHostFunc not found"); - }; - } - - pub fn memcpyToHostBlocking(dst: []u8, src: *const anyopaque) void { - const err = _memcpyBlocking(dst.ptr, src, dst.len, .device_to_host); - check(err); - } - - pub fn memcpyToDeviceBlocking(dst: *anyopaque, src: []const u8) void { - const err = _memcpyBlocking(dst, src.ptr, src.len, .host_to_device); - check(err); - } - - pub fn memcpyToDeviceAsync(dst: *anyopaque, src: []const u8, stream: ?*anyopaque) void { - const err = _memcpyAsync(dst, src.ptr, src.len, .host_to_device, stream); - check(err); - } - - pub fn memcpyToHostAsync(dst: []u8, src: *const anyopaque, stream: ?*anyopaque) void { - const err = _memcpyAsync(dst.ptr, src, dst.len, .device_to_host, stream); - check(err); - } - - pub fn check(err: c_int) void { - if (err == 0) return; - stdx.debug.panic("CUDA error: {d}", .{err}); - } -}; From 01f84799a14005e4803400e7bd92c2bdd609487d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 10:31:39 +0200 Subject: [PATCH 15/67] cleanup commented code --- zml/pjrtx.zig | 8 -------- 1 file changed, 8 deletions(-) diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index b8ef9b3f..8658aebe 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -279,14 +279,6 @@ pub const LoadedExecutable = opaque { .non_donatable_input_indices = args.non_donatable_input_indices, .context = args.context, }); - // try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ - // .num_args = args.num_args, - // .arguments = @ptrCast(args.arguments), - // .results = @ptrCast(args.results), - // .events = @ptrCast(args.events), - // .non_donatable_input_indices = args.non_donatable_input_indices, - // .context = args.context, - // } }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 87ecbab2f4d9c4ef6a5a414dc6f2b1c0488f6dc2 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 10:46:07 +0200 Subject: [PATCH 16/67] improve hostbuffer printing --- stdx/fmt.zig | 96 ++++++++++++++++++++++++++++++++-------------- zml/callback.zig | 2 +- zml/hostbuffer.zig | 12 +++--- zml/testing.zig | 7 ++-- 4 files changed, 78 insertions(+), 39 deletions(-) diff --git a/stdx/fmt.zig b/stdx/fmt.zig index b3ec64fe..145db768 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -1,77 +1,117 @@ const std = @import("std"); -pub fn formatValue(value: anytype, full: std.fmt.Number, writer: anytype) !void { - return switch (@typeInfo(@TypeOf(value))) { - .comptime_float, .float => try formatFloatValue(value, full, writer), - .comptime_int, .int => try formatIntValue(value, full, writer), - else => try formatAnyValue(value, full, writer), +pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { + return .{ .slice = any_slice }; +} + +fn FmtSlice(T: type) type { + return struct { + slice: []const T, + + pub fn formatNumber(f: @This(), writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + return switch (@typeInfo(T)) { + .comptime_float, .float => try formatFloatSlice(f.slice, n, writer), + .comptime_int, .int => try formatIntSlice(f.slice, n, writer), + .bool => try formatBoolSlice(f.slice, n, writer), + .@"struct" => if (@hasField(T, "re") and @hasField(T, "im")) { + try formatComplexSlice(f.slice, n, writer); + } else if (@hasDecl(T, "toF32")) { + try formatFloatSlice(f.slice, n, writer); + } else { + try formatSliceAny(f.slice, n, writer); + }, + else => @compileError("FmtSlice doesn't support type: " ++ @typeName(T)), + }; + } }; } -pub fn formatFloatValue(value: anytype, full: std.fmt.Number, writer: *std.Io.Writer) !void { +pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), .float => value, - else => @compileError("formatFloatValue expects a float, got: " ++ @typeName(@TypeOf(value))), + else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), }; - return writer.printFloat(x, full); + return writer.printFloat(x, spec); } -pub fn formatIntValue(value: anytype, full: std.fmt.Number, writer: *std.Io.Writer) !void { +pub fn formatInt(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { switch (@typeInfo(@TypeOf(value))) { .int => {}, - else => @compileError("formatIntValue expects an int, got: " ++ @typeName(@TypeOf(value))), + else => @compileError("formatInt expects an int, got: " ++ @typeName(@TypeOf(value))), } - return writer.printInt(value, full.mode.base().?, full.case, .{ .alignment = full.alignment, .fill = full.fill }); + return writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); } -pub fn formatAnyValue(value: anytype, full: std.fmt.Number, writer: *std.Io.Writer) !void { +pub fn formatComplex(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(".{.re="); + try writer.printFloat(value.re, spec); + try writer.writeAll(", .im="); + try writer.printFloat(value.im, spec); + try writer.writeAll("}"); +} + +pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.alignBufferOptions(if (value) "1" else "0", .{ .alignment = spec.alignment, .fill = spec.fill }); +} + +pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; - const s = std.fmt.bufPrint(&buf, "{any}", .{value}) catch blk: { + const T = @TypeOf(value); + const fmt = if (@hasDecl(T, "formatNumber")) "{d}" else "{f}"; + + const s = std.fmt.bufPrint(&buf, fmt, .{value}) catch blk: { buf[45..].* = "...".*; break :blk buf[0..]; }; - return try writer.alignBufferOptions(s, .{ .alignment = full.alignment, .fill = full.fill }); + return try writer.alignBufferOptions(s, .{ .alignment = spec.alignment, .fill = spec.fill }); } -pub fn formatSliceCustom(fmt_func: anytype, values: anytype, full: std.fmt.Number, writer: anytype) !void { - // Write first rows - const num_cols: usize = full.width orelse 12; +pub fn formatSliceCustom(fmt_func: anytype, values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + // use the format "width" for the number of columns instead of individual width. + const num_cols: usize = spec.width orelse 12; + var my_options = spec; + my_options.width = null; const n: usize = values.len; + _ = try writer.write("{"); if (n <= num_cols) { for (values, 0..) |v, i| { // Force inlining so that the switch and the buffer can be done once. - try @call(.always_inline, fmt_func, .{ v, full, writer }); + try @call(.always_inline, fmt_func, .{ v, my_options, writer }); if (i < n - 1) _ = try writer.write(","); } } else { const half = @divFloor(num_cols, 2); for (values[0..half]) |v| { - try @call(.always_inline, fmt_func, .{ v, full, writer }); + try @call(.always_inline, fmt_func, .{ v, my_options, writer }); _ = try writer.write(","); } _ = try writer.write(" ..., "); for (values[n - half ..], 0..) |v, i| { - try @call(.always_inline, fmt_func, .{ v, full, writer }); + try @call(.always_inline, fmt_func, .{ v, my_options, writer }); if (i < half - 1) _ = try writer.write(","); } } _ = try writer.write("}"); } -pub fn formatAny(values: anytype, full: std.fmt.Number, writer: anytype) !void { - return try formatSliceCustom(formatAnyValue, values, full, writer); +pub fn formatSliceAny(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatAny, values, spec, writer); } -pub fn formatFloatSlice(values: anytype, full: std.fmt.Number, writer: anytype) !void { - return try formatSliceCustom(formatFloatValue, values, full, writer); +pub fn formatFloatSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatFloat, values, spec, writer); } -pub fn formatIntSlice(values: anytype, full: std.fmt.Number, writer: anytype) !void { - return try formatSliceCustom(formatIntValue, values, full, writer); +pub fn formatIntSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatInt, values, spec, writer); } -pub fn formatAnySlice(values: anytype, full: std.fmt.Number, writer: anytype) !void { - return try formatSliceCustom(formatAnyValue, values, full, writer); +pub fn formatComplexSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatComplex, values, spec, writer); +} + +pub fn formatBoolSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatBool, values, spec, writer); } diff --git a/zml/callback.zig b/zml/callback.zig index 23b66804..5fa41472 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -286,7 +286,7 @@ pub const Print = struct { } pub fn call(_: *Print, input: Buffer) !void { - std.log.defaultLog(.info, .zml, "Device buffer: {f}: {d}", .{ input, input.asHostBuffer() }); + std.log.defaultLog(.info, .zml, "Device buffer: {f}: {d:20.3}", .{ input, input.asHostBuffer() }); } }; diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 2fc72142..3e65fa16 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -330,8 +330,6 @@ pub const HostBuffer = struct { self: HostBuffer, writer: anytype, ) !void { - // TODO debug option - // try writer.print("HostBuffer(.{f})@0x{x}", .{ self._shape, @intFromPtr(self._data) }); try writer.print("HostBuffer(.{f})", .{self._shape}); } @@ -351,9 +349,10 @@ pub const HostBuffer = struct { const val: dt.toZigType() = self.items(dt.toZigType())[0]; return switch (comptime dt.class()) { // Since we have custom floats, we need to explicitly convert to float32 ourselves. - .float => stdx.fmt.formatFloatValue(floats.floatCast(f32, val), options, writer), - .integer => stdx.fmt.formatIntValue(val, options, writer), - .bool, .complex => stdx.fmt.formatAnyValue(val, options, writer), + .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), + .integer => stdx.fmt.formatInt(val, options, writer), + .bool => stdx.fmt.formatBool(val, options, writer), + .complex => stdx.fmt.formatComplex(val, options, writer), }; }, }; @@ -368,7 +367,8 @@ pub const HostBuffer = struct { switch (comptime dt.class()) { .float => try stdx.fmt.formatFloatSlice(values, options, writer), .integer => try stdx.fmt.formatIntSlice(values, options, writer), - .bool, .complex => try stdx.fmt.formatAnySlice(values, options, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, writer), } }, } diff --git a/zml/testing.zig b/zml/testing.zig index 246f1131..5fec596d 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -51,14 +51,13 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { if (should_free_left) left.deinit(allocator); if (should_free_right) right.deinit(allocator); } - errdefer log.err("\n--> Left: {f}\n--> Right: {f}", .{ left, right }); - + errdefer log.err("\n--> Left: {0f}{0d:24.3}\n--> Right: {1f}{1d:24.3}", .{ left, right }); if (!std.mem.eql(i64, left.shape().dims(), right.shape().dims())) { log.err("left.shape() {f} != right.shape() {f}", .{ left.shape(), right.shape() }); return error.TestUnexpectedResult; } if (left.dtype() != right.dtype() and !(left.dtype() == .f16 and right.dtype() == .bf16)) { - log.err("left.dtype ({}) != right.dtype ({})", .{ left.dtype(), right.dtype() }); + log.err("left.dtype ({f}) != right.dtype ({f})", .{ left.shape(), right.shape() }); return error.TestUnexpectedResult; } switch (left.dtype()) { @@ -89,7 +88,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { const right_data = right.items(R); for (left_data, right_data, 0..) |l, r, i| { if (!approxEq(f32, zml.floats.floatCast(f32, l), zml.floats.floatCast(f32, r), tolerance)) { - log.err("left.data != right_data.\n < {any:.3} \n > {any:.3}\n error at idx {any}: {any:.3} != {any:.3}", .{ center(left_data, i), center(right_data, i), i, left_data[i], right_data[i] }); + log.err("left.data != right_data.\n < {d:40.3} \n > {d:40.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ stdx.fmt.slice(center(left_data, i)), stdx.fmt.slice(center(right_data, i)), i, left_data[i], right_data[i] }); return error.TestUnexpectedResult; } } From 58f4a90f7865377a8e886076755cc22870c4f83d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 10:56:09 +0200 Subject: [PATCH 17/67] remove item --- zml/hostbuffer.zig | 5 ----- 1 file changed, 5 deletions(-) diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 3e65fa16..848fee86 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -176,11 +176,6 @@ pub const HostBuffer = struct { return ptr[0..self._shape.count()]; } - pub fn item(self: HostBuffer, comptime T: type, index: usize) T { - stdx.debug.assert(index < self._shape.count(), "index {} out of bounds for shape {f}", .{ index, self._shape }); - return self.items(T)[index]; - } - pub fn mutItems(self: HostBuffer, comptime T: type) []T { return @constCast(self.items(T)); } From 8d1a6836ac46f64ef3ab0c3587182124d1abf92d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 11:12:13 +0200 Subject: [PATCH 18/67] drop platform.registerCallback --- examples/callback/main.zig | 2 +- zml/platform.zig | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/examples/callback/main.zig b/examples/callback/main.zig index 7d43ba54..be3b3095 100644 --- a/examples/callback/main.zig +++ b/examples/callback/main.zig @@ -141,7 +141,7 @@ pub fn asyncMain() !void { context.printAvailablePlatforms(platform); // Register our custom call - try platform.registerCallback(GrayScale); + try zml.callback.register(GrayScale, platform); // Compile MLIR code containing our custom call. const rgb_shape = zml.Shape.init(.{12 * 3}, .u8); diff --git a/zml/platform.zig b/zml/platform.zig index 1e1d1bfd..0bc792e9 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -4,9 +4,6 @@ const runtimes = @import("runtimes"); pub const Target = runtimes.Platform; const stdx = @import("stdx"); -const zml = struct { - const callback = @import("callback.zig"); -}; const pjrt = @import("pjrtx.zig"); const log = std.log.scoped(.zml); @@ -79,10 +76,6 @@ pub const Platform = struct { return res; } - pub fn registerCallback(platform: Platform, Callback: type) pjrt.ApiError!void { - try zml.callback.register(Callback, platform); - } - pub fn deinit(self: *Platform) void { self.pjrt_client.deinit(self.pjrt_api); } From 2fa09e1d5f5526df369ec8482ec748a003533e52 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 11:30:56 +0200 Subject: [PATCH 19/67] deinit create buffers --- zml/callback.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zml/callback.zig b/zml/callback.zig index 5fa41472..4d6ab90e 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -200,6 +200,12 @@ fn CallbackImpl(comptime Callback: type, call_frame: *pjrt.ffi.CallFrame) ?*pjrt callback_args[i] = zml_buffer; } + defer { + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + inline for (1..callback_args.len) |i| callback_args[i].deinit(); + } + } + for (0..call_frame.results.len) |i| { const ffi_buffer = call_frame.results.buffers()[i]; const ffi_buffer_shape = shapeFromFfi(ffi_buffer); From 8deb98cfd30f584b038660aacfe4bb96fdc98168 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 15 Jul 2025 17:28:27 +0200 Subject: [PATCH 20/67] fix floatCast --- zml/floats.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/zml/floats.zig b/zml/floats.zig index 57bf5150..9344408c 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -334,9 +334,15 @@ test BFloat16 { } pub fn floatCast(T: type, x: anytype) T { - return switch (@TypeOf(x)) { - f64, f32, f16 => @floatCast(x), - else => @floatCast(x.toF32()), + return switch (T) { + f64, f32, f16 => switch (@TypeOf(x)) { + f64, f32, f16 => @floatCast(x), + else => @floatCast(x.toF32()), + }, + else => switch (@TypeOf(x)) { + f64, f32, f16 => .fromF32(x), + else => .fromF32(x.toF32()), + }, }; } From d90e3c9c63857bc23dca225b37b88e7a805817fa Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 16 Jul 2025 06:41:10 +0200 Subject: [PATCH 21/67] drop sdpaMemEfficient --- zml/nn.zig | 332 ----------------------------------------------------- 1 file changed, 332 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 631bce16..3762b654 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -878,338 +878,6 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { return attn.transpose(q.shape()); } -pub const SdpaChunks = struct { q_chunk_size: u32, k_chunk_size: u32 }; - -pub fn sdpaMemEfficient( - q: Tensor, - k: Tensor, - v: Tensor, - sdpa_opts: SdpaOpts, - chunking: SdpaChunks, -) Tensor { - const sdpa_mem_efficient: SdpaMemEfficient = .{ - .q = q, - .k = k, - .v = v, - .sdpa_opts = sdpa_opts, - .chunking = .{ - .q_chunk_size = @intCast(@min(q.dim(.q), chunking.q_chunk_size)), - .k_chunk_size = @intCast(@min(k.dim(.k), chunking.k_chunk_size)), - }, - }; - - return sdpa_mem_efficient.forward(); -} - -const SdpaMemEfficient = struct { - q: Tensor, - k: Tensor, - v: Tensor, - sdpa_opts: SdpaOpts, - chunking: SdpaChunks, - - fn forward(self: SdpaMemEfficient) Tensor { - stdx.debug.assert(@mod(self.q.dim(.q), self.chunking.q_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({f}, {})", .{ self.q, self.chunking }); - stdx.debug.assert(@mod(self.k.dim(.k), self.chunking.k_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({f}, {})", .{ self.k, self.chunking }); - const n_q_chunks: u32 = @intCast(@divExact(self.q.dim(.q), self.chunking.q_chunk_size)); - - const ctx = zml.module.CompilationContext.current(); - const q_chunks = ctx.allocator().alloc(zml.Tensor, n_q_chunks) catch unreachable; - defer ctx.allocator().free(q_chunks); - for (0..n_q_chunks) |i| { - const idx: u32 = @intCast(i); - const q_slice: zml.Tensor.DynSlice = .{ - .start = Tensor.scalar(idx * self.chunking.q_chunk_size, .i32), - .len = self.chunking.q_chunk_size, - }; - const q_chunk = self.q.dynamicSlice(.{ .q = q_slice }); - const attn_chunk = if (self.sdpa_opts.attn_mask) |attn_mask| attn_mask.dynamicSlice(.{ .q = q_slice }) else null; - - var chunk: SdpaMemEfficient = self; - chunk.q = q_chunk; - chunk.sdpa_opts.attn_mask = attn_chunk; - q_chunks[i] = chunk.scanKeyVal(); - } - - const res = zml.Tensor.concatenate(q_chunks, .q); - return res.transpose(self.q.shape()); - } - - fn nextQueriesChunk(self: SdpaMemEfficient, idx: Tensor) Tensor { - const q_slice: zml.Tensor.DynSlice = .{ - .start = idx.scale(self.chunking.q_chunk_size), - .len = self.chunking.q_chunk_size, - }; - const q_chunk = self.q.dynamicSlice(.{ .q = q_slice }); - const attn_chunk = if (self.sdpa_opts.attn_mask) |attn_mask| attn_mask.dynamicSlice(.{ .q = q_slice }) else null; - - var chunk: SdpaMemEfficient = self; - chunk.q = q_chunk; - chunk.sdpa_opts.attn_mask = attn_chunk; - return chunk.scanKeyVal(); - } - - fn scanKeyVal(self: SdpaMemEfficient) Tensor { - const n_chunks = @divExact(self.k.dim(.k), self.chunking.k_chunk_size); - return if (n_chunks <= 4) { - // Unrolled version - var partial_softmax: ?PartialSoftmax = null; - for (0..@intCast(n_chunks)) |idx| { - const next = self.nextKeyValChunk(Tensor.scalar(idx, .i32)); - partial_softmax = if (partial_softmax) |prev| prev.merge(next) else next; - } - return partial_softmax.?.finalize(); - } else { - // stablehlo.while version - const partial_softmax, _ = zml.ops.while_(hasNextKeyValChunk, nextKeyValChunkMerge, self, .{ PartialSoftmax.zeros(self.q.shape(), .f32), Tensor.scalar(0, .i32) }); - return partial_softmax.finalize(); - }; - } - - fn nextKeyValChunkMerge(self: SdpaMemEfficient, prev: PartialSoftmax, idx: Tensor) struct { PartialSoftmax, Tensor } { - const next = self.nextKeyValChunk(idx); - return .{ prev.merge(next), idx.addConstant(1) }; - } - - fn nextKeyValChunk(self: SdpaMemEfficient, idx: Tensor) PartialSoftmax { - const k_slice: zml.Tensor.DynSlice = .{ - .start = idx.scale(self.chunking.k_chunk_size), - .len = self.chunking.k_chunk_size, - }; - - const k_chunk = self.k.dynamicSlice(.{ .k = k_slice }); - const v_chunk = self.v.dynamicSlice(.{ .k = k_slice }); - const attn_chunk = if (self.sdpa_opts.attn_mask) |mask| mask.dynamicSlice(.{ .k = k_slice }) else null; - - return sdpaChunk(self.q, k_chunk, v_chunk, .{ .attn_mask = attn_chunk }); - } - - pub fn hasNextKeyValChunk(self: SdpaMemEfficient, _: PartialSoftmax, idx: Tensor) zml.Tensor { - const n_chunks = @divExact(self.k.dim(.k), self.chunking.k_chunk_size); - return idx.cmp(.LT, Tensor.scalar(n_chunks, idx.dtype())); - } -}; - -pub const PartialSoftmax = struct { - values: Tensor, - exp_sum: Tensor, - max_value: Tensor, - - pub fn zeros(q_shape: Shape, exp_sum_precision: DataType) PartialSoftmax { - return .{ - .values = Tensor.constant(q_shape, q_shape.dtype().zero()), - .exp_sum = Tensor.constant(q_shape.setDim(.hd, 1), exp_sum_precision.zero()), - .max_value = Tensor.constant(q_shape.setDim(.hd, 1), q_shape.dtype().minValue()), - }; - } - - pub fn merge(self: PartialSoftmax, other: PartialSoftmax) PartialSoftmax { - // Rescale self and other using the new global_max. - const global_max = self.max_value.maximum(other.max_value); - const new_self = self.rescale(global_max); - const new_other = other.rescale(global_max); - - // Now that self and other are using the same scale, we can just add them: - return .{ - .max_value = global_max, - .values = new_self.values.add(new_other.values), - .exp_sum = new_self.exp_sum.add(new_other.exp_sum), - }; - } - - /// Update max_value and rescale attn and exp_sum accordingly. - pub fn rescale(self: PartialSoftmax, max_value: Tensor) PartialSoftmax { - const max_diff_exp = self.max_value.sub(max_value).exp(); - const sum_dtype = self.exp_sum.dtype(); - return .{ - .max_value = max_value, - .values = self.values.mul(max_diff_exp.broad(self.values.shape())), - .exp_sum = self.exp_sum.mul(max_diff_exp.convert(sum_dtype)), - }; - } - - /// Divides the intermediary results by the exp_sum to get the proper attention values. - pub fn finalize(self: PartialSoftmax) Tensor { - return self.values.div(self.exp_sum.broad(self.values.shape()).convert(self.values.dtype())); - } -}; - -/// Compute softmax over a chunk. -/// Returns intermediary results to allow aggregating later. -pub fn partialSoftmax(self: Tensor, axis: anytype) PartialSoftmax { - const a = self.axis(axis); - const max_val = self.max(a).maximum(Tensor.scalar(-1e16, self.dtype())); - const out = self.sub(max_val.broad(self.shape())).exp(); - return .{ - .values = out, - .exp_sum = out.convert(.f32).sum(a), - .max_value = max_val, - }; -} - -/// Compute sdpa on a chunk, and computes a partial softmax. -/// q: (B, H, Sq, H_dim) ⊙ k: (B, H, Sk, H_dim) -> qk: (B, H, Sq, Sk) -pub fn sdpaChunk(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) PartialSoftmax { - // this is a dupe of sdpa, but return the PartialSoftmax instead of true Attn. - // Consider implementing sdpa from sdpaChunk. - var q, var k, var v = .{ q_, k_, v_ }; - - const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; - const err_args = .{ q, k, v, opts.attn_mask }; - stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); - stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); - stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - - if (q.dim(.h) != k.dim(.h)) { - stdx.debug.assert(@mod(q.dim(.h), k.dim(.h)) == 0, err_template ++ "Different number of heads for keys and queries, but can't repeat keys.", err_args); - // Note: we don't try to repeat queries. - // Repeating keys is the interesting optimisation cause it reduces KV cache memory usage. - const num_rep: u63 = @intCast(@divExact(q.dim(.h), k.dim(.h))); - k, v = .{ k.repeat1d(.h, num_rep), v.repeat1d(.h, num_rep) }; - } - const attn_mask = if (opts.attn_mask) |m| m else null; - - const dims = helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { - stdx.debug.panic(err_template ++ "Inputs have incompatible shapes.", err_args); - }; - const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); - const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); - k = k.mul(head_scaling.convert(k.dtype())); - - var attn_weights = q.dot(k, .{.hd}); - // log.debug("attn_weights : {}", .{attn_weights}); - // log.debug("attn_mask : {?}", .{attn_mask}); - if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); - - const partial = partialSoftmax(attn_weights, .k); - const attn = partial.values.dot(v, .{.k}).transpose(q.shape()); - - return .{ - .values = attn, - // The renaming is because the above dot projected values.k into .hd, - // do the same thing on the other tensors. - // This work because dot is a linear operation, and commutes with `PartialSoftmax.finalize` - .exp_sum = partial.exp_sum.rename(.{ .k = .hd }).transpose(attn.shape()), - .max_value = partial.max_value.rename(.{ .k = .hd }).transpose(attn.shape()), - }; -} - -test sdpaMemEfficient { - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - - // Note we use small input vectors to have the tests run reasonably fast, - // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 10, 512, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng.deinit(); - - const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng_mask.deinit(); - - // Note: we pass void here, cause Rng.normal doesn't take any runtime inputs. - const q = rng.call({}).withTags(.{ .b, .h, .q, .hd }); - const k = rng.call({}).withTags(.{ .b, .h, .k, .hd }); - const v = rng.call({}).withTags(.{ .b, .h, .k, .hd }); - const mask = rng_mask.call({}).withTags(.{ .q, .k }); - - const ref_res = try zml.testing.compileAndCall( - platform, - sdpa, - .{ q, k, v, .{ .attn_mask = mask, .scale = null } }, - ); - try std.testing.expectEqualSlices(i64, q.shape().dims(), ref_res.shape().dims()); - { - // 4 k_chunks - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = 256, .k_chunk_size = @divExact(512, 4) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 2e-3); - } - { - // 16 k_chunks - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = 256, .k_chunk_size = @divExact(512, 16) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 2e-3); - } -} - -test "sdpaMemEfficient transposed" { - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - - // Note we use small input vectors to have the tests run reasonably fast, - // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 512, 10, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng.deinit(); - - const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng_mask.deinit(); - - // Note: we pass void here, cause Rng.normal doesn't take any runtime inputs. - const q = rng.call({}).withTags(.{ .b, .q, .h, .hd }); - const k = rng.call({}).withTags(.{ .b, .k, .h, .hd }); - const v = rng.call({}).withTags(.{ .b, .k, .h, .hd }); - const mask = rng_mask.call({}).withTags(.{ .q, .k }); - - const ref_res = try zml.testing.compileAndCall( - platform, - sdpa, - .{ q, k, v, .{ .attn_mask = mask, .scale = null } }, - ); - try std.testing.expectEqualSlices(i64, q.shape().dims(), ref_res.shape().dims()); - - { - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = @divExact(512, 2), .k_chunk_size = @divExact(512, 4) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 1e-3); - } - - { - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = 512, .k_chunk_size = @divExact(512, 4) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 1e-3); - } -} - /// Options controlling generation. The default values correspond to greedy decoding. pub const SamplingStrategy = struct { topk: u32 = 1, From 0be25b946c5c4b2e66b3431577ee9855d0121373 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 23 Jul 2025 15:29:20 +0200 Subject: [PATCH 22/67] remove async from zml/tokenizer --- zml/tokenizer/BUILD.bazel | 7 ++++++- zml/tokenizer/sentencepiece/BUILD.bazel | 11 +++++++++++ zml/tokenizer/tokenizer.zig | 7 +++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 9b2f9c71..b25f7070 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "tokenizer", @@ -7,7 +8,6 @@ zig_library( main = "tokenizer.zig", visibility = ["//visibility:public"], deps = [ - "//async", "//ffi:zig", "//stdx", "//zml/tokenizer/hftokenizers", @@ -25,3 +25,8 @@ zig_binary( "//stdx", ], ) + +zig_srcs( + name = "sources", + zig_lib = ":tokenizer", +) diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index dba86ddb..10ac080b 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:swig.bzl", "swig_cc_library") +load("//bazel:zig_srcs.bzl", "zig_srcs") swig_cc_library( name = "sentencepiece_swig", @@ -22,3 +23,13 @@ zig_library( "//stdx", ], ) + +zig_srcs( + name = "sources", + zig_lib = ":sentencepiece", +) + +cc_static_library( + name="sentencepiece_static", + deps = [":sentencepiece_swig"] +) diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 443e2fb8..5109ce31 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,6 +1,5 @@ const std = @import("std"); -const asynk = @import("async"); const hftokenizers = @import("hftokenizers"); const sentencepiece = @import("sentencepiece"); @@ -98,15 +97,15 @@ pub const Tokenizer = union(Tokenizers) { pub fn fromFile(allocator: std.mem.Allocator, model: []const u8) !Tokenizer { if (std.mem.endsWith(u8, model, ".pb")) { - return .{ .sentencepiece = try asynk.callBlocking(sentencepiece.SentencePieceProcessor.fromFile, .{model}) }; + return .{ .sentencepiece = try sentencepiece.SentencePieceProcessor.fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try asynk.callBlocking(hftokenizers.HFTokenizer.fromFile, .{model}) }; + return .{ .hftokenizers = try hftokenizers.HFTokenizer.fromFile(model) }; } if (std.mem.endsWith(u8, model, ".tinyllama")) { const tokenizer = try allocator.create(homemade.Tokenizer); - tokenizer.* = try asynk.callBlocking(homemade.fromTinyLlamaFile, .{ allocator, model, 32000 }); + tokenizer.* = try homemade.fromTinyLlamaFile(allocator, model, 32000); return .{ .homemade = tokenizer }; } From 6268644b0615f496933ef5b78289740f22c741a3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Jul 2025 21:52:26 +0200 Subject: [PATCH 23/67] flatten -> torch.flatten; flattenAll -> flatten fixup! flatten -> torch.flatten; flattenAll -> flatten --- examples/mnist/mnist.zig | 2 +- zml/nn.zig | 31 +++++------ zml/ops.zig | 2 +- zml/tensor.zig | 115 ++++++++++++++++++++++++++++----------- zml/torch.zig | 9 +++ 5 files changed, 107 insertions(+), 52 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 5c85f5b7..241a3a0e 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -27,7 +27,7 @@ const Mnist = struct { /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); - var x = input.flattenAll().convert(.f32); + var x = input.flatten().convert(.f32); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { x = layer.forward(x); diff --git a/zml/nn.zig b/zml/nn.zig index 3762b654..f3e57bb5 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -250,7 +250,7 @@ pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Ten .interleaved => Tensor.concatenate(&.{ x_real.appendAxes(.{.interleaved_real_img}), x_imag.appendAxes(.{.interleaved_real_img}), - }, -1).flatten(-2), + }, -1).reshape(x_imag.shape().setDim(-1, -1)), }; } @@ -332,8 +332,8 @@ test "real/img" { const real, const imag = splitRealImg(x, layout); const y = mergeRealImg(real, imag, layout); const real2, const imag2 = splitRealImg(y, layout); - return real.cmp(.EQ, real2).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, imag2).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, real2).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, imag2).flatten().convert(.i32).sum(-1), ); } @@ -349,8 +349,8 @@ test "real/img" { Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), }, 1); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } @@ -366,8 +366,8 @@ test "real/img" { Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), }, 1); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } @@ -377,8 +377,8 @@ test "real/img" { const x_real = Tensor.arange(.{ .start = 0, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); const x_imag = Tensor.arange(.{ .start = 1, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } }; @@ -853,13 +853,10 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { return cuda.sdpa(q, k, v, opts); } - if (q.dim(.h) != k.dim(.h)) { - stdx.debug.assert(@mod(q.dim(.h), k.dim(.h)) == 0, err_template ++ "Different number of heads for keys and queries, but can't repeat keys.", err_args); - // Note: we don't try to repeat queries. - // Repeating keys is the interesting optimisation cause it reduces KV cache memory usage. - const num_rep: u63 = @intCast(@divExact(q.dim(.h), k.dim(.h))); - k, v = .{ k.repeat1d(.h, num_rep), v.repeat1d(.h, num_rep) }; - } + // Handle different numbers of head by splitting q heads. + // This is a bit error prone in the sense that it depends of the layout of q heads. + // This is the Llama convention though. + q = q.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); const attn_mask = if (opts.attn_mask) |m| m else null; const dims = helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { @@ -875,7 +872,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { attn_weights = attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); var attn = attn_weights.dot(v, .{.k}); - return attn.transpose(q.shape()); + return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); } /// Options controlling generation. The default values correspond to greedy decoding. diff --git a/zml/ops.zig b/zml/ops.zig index ee125086..eadded8e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -462,7 +462,7 @@ pub fn if_( } } - const scalar_pred = if (pred.rank() == 0) pred else pred.flattenAll().squeeze(0); + const scalar_pred = if (pred.rank() == 0) pred else pred.flatten().squeeze(0); const loc = ctx.mlirCtx().location(@src()); const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.if", .{ .operands = &.{scalar_pred.value()}, diff --git a/zml/tensor.zig b/zml/tensor.zig index 63ab44d4..7ea5ac5d 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -712,7 +712,7 @@ pub const Tensor = struct { pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); const rng, const data = rand.gumbel(s); - const flat = data.flattenAll(); + const flat = data.flatten(); const mean_ = flat.mean(0); const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); @@ -1497,27 +1497,7 @@ pub const Tensor = struct { return self.transpose(perm.constSlice()); } - /// Flattens the given axis and the next one, into one new axis. - pub fn flatten(self: Tensor, axis_: anytype) Tensor { - // TODO: move to torch.zig, this is equivalent to merge - const old_shape = self._shape; - const a = self.axis(axis_); - // stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {} on the last axis {}.", .{ self, axis }); - const new_shape = old_shape.remove(a + 1).set(a, old_shape.dim(a) * old_shape.dim(a + 1)); - - const loc = self.getContext().location(@src(), "flatten({f},{})", .{ self, axis_ }); - const reshaped_val = dialect.stablehlo.reshape( - self.getContext().mlirCtx(), - self.value(), - mlirx.tensorType(self.getContext().mlirCtx(), new_shape), - loc, - ); - // log.debug("flatten({d}, {d}) -> {d}", .{ self.dims(), axis_, new_shape[0 .. self.rank() - 1] }); - return _result(new_shape, reshaped_val.result(0)); - } - - pub inline fn flattenAll(self: Tensor) Tensor { - // TODO: rename to just flatten, once flatten is moved to torch + pub inline fn flatten(self: Tensor) Tensor { return self.reshape(.{self.count()}); } @@ -1712,14 +1692,12 @@ pub const Tensor = struct { } const a = self.axis(axis_); - const broadshape = self._shape.insert(a + 1, .{n_rep}); - const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); - var res = self.broadcast(broadshape, repeat_dims.dims()).flatten(a); - // Restor the tag that has been lost by flatten. - res._shape._tags.set(a, self._shape.tag(a)); + const broadshape = self._shape.insert(a, .{n_rep}); + const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a); - return res; + return self.broadcast(broadshape, repeat_dims.dims()).reshape(res_shape); } /// Repeats a Tensor several times along the given axes. @@ -1736,15 +1714,47 @@ pub const Tensor = struct { return res; } + test repeat1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.repeat1d(axis_, n_reps); + } + }; + + { + const inputs: [3]u8 = .{ 1, 2, 3 }; + const expectations: [6]u8 = .{ 1, 2, 3, 1, 2, 3 }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 0, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + { + const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 1, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + } + /// Repeats in line each value along the given axis. /// - /// * stutter1d([0, 1, 2, 3], 0, 2) = [0, 0, 1, 1, 2, 2, 3, 3] - pub fn stutter1d(self: Tensor, axis_: i64, n_rep: u63) Tensor { + /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] + /// This is equivalent to repeat(ax+1) unless ax is the last axis. + pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u63) Tensor { const a = self.axis(axis_); const broadshape = self._shape.insert(a + 1, .{n_rep}); - const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); - return self.broadcast(broadshape, stutter_dims.dims()).flatten(a); + const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); } /// Repeats in line each value along the given axes. @@ -1760,6 +1770,45 @@ pub const Tensor = struct { return res; } + test stutter1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.stutter1d(axis_, n_reps); + } + }; + + { + const inputs: [3]u8 = .{ 1, 2, 3 }; + const expectations: [6]u8 = .{ 1, 1, 2, 2, 3, 3 }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + { + const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + const expectations: [2][6]u8 = .{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 1, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + { + const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + } + /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); @@ -2243,7 +2292,7 @@ pub const Tensor = struct { // Sometimes the backend recognize this pattern, but not always. // So let us handle that. if (indices.count() == 1) { - return self.dynamicSlice1d(coord_axes_.get(0), .{ .start = indices.flattenAll().squeeze(0), .len = 1 }).reshape(res_shape); + return self.dynamicSlice1d(coord_axes_.get(0), .{ .start = indices.flatten().squeeze(0), .len = 1 }).reshape(res_shape); } var slice_dims: Shape.DimsArray = .{}; diff --git a/zml/torch.zig b/zml/torch.zig index e982aaec..43b0339d 100644 --- a/zml/torch.zig +++ b/zml/torch.zig @@ -274,3 +274,12 @@ test meshgrid { ); } } + +/// Flattens the given axis and the next one, into one new axis. +pub fn flatten(self: Tensor, axis_: anytype) Tensor { + const old_shape = self._shape; + const a = self.axis(axis_); + stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {f} on the last axis {}.", .{ self, axis_ }); + const new_shape = old_shape.mergeAxis(a, .{ a, a + 1 }); + return self.reshape(new_shape); +} From ae6905dd7aa9884095eb88a6fc6085acd46c0bdb Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 23 Jul 2025 17:18:29 +0200 Subject: [PATCH 24/67] rewrite gatherValues --- zml/nn.zig | 33 ++++----- zml/ops.zig | 3 +- zml/tensor.zig | 179 ++++++++++++++++++++++++------------------------- zml/torch.zig | 8 +-- 4 files changed, 110 insertions(+), 113 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index f3e57bb5..d0fbf03c 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -42,7 +42,7 @@ pub const TokenEmbedding = struct { pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); - return self.weight.gatherValues(0, idx, .{}); + return self.weight.withTags(.{ .voc, .d }).gather(.{ .voc = idx }, .{}); } }; @@ -484,21 +484,21 @@ pub fn nearest(input: Tensor, scale_factor: []const f64) Tensor { out_shape._dims.set(i + 2, @intFromFloat(@floor(@as(f64, @floatFromInt(out_shape.dim(i + 2))) * sf))); } // TODO(james): remove this implicit two batching dims - var sd: [3]usize = undefined; + var sd: [3]u3 = undefined; var len_sd: usize = 0; for (2..input.rank()) |i| { if (input.dim(i) != out_shape.dim(i)) { - sd[len_sd] = i; + sd[len_sd] = @intCast(i); len_sd += 1; } } - const spatial_dims = sd[0..len_sd]; + const spatial_axes = sd[0..len_sd]; var res = input; - for (spatial_dims) |d| { - const n = out_shape.dim(d); - const ratio = stdx.math.divFloat(f32, input.dim(d), n); + for (spatial_axes) |ax| { + const n = out_shape.dim(ax); + const ratio = stdx.math.divFloat(f32, input.dim(ax), n); const offsets = Tensor.arange(.{ .end = n }, .f32).addConstant(0.5).scale(ratio).floor().convert(.i32); - res = res.gatherValues(d, offsets, .{ .indices_are_sorted = true }); + res = res.gather_(&.{ax}, &.{offsets}, .{ .indices_are_sorted = true }); } return res; } @@ -670,10 +670,11 @@ test resizeBilinear { } pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Tensor { - const res_shape = image.shape().set(axis, new_len); + const ax = image.axis(axis); + const res_shape = image.shape().set(ax, new_len); const dtype = opt.precision orelse if (image.dtype().class() == .integer) .f32 else image.dtype(); - const og_len = opt.original_len orelse Tensor.scalar(image.dim(axis), dtype); + const og_len = opt.original_len orelse Tensor.scalar(image.dim(ax), dtype); const ratio = og_len.convert(dtype).scale(stdx.math.divFloat(f32, 1, new_len)); const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); const left = scaled.floor(); @@ -682,11 +683,11 @@ pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Te // TODO: check that two gather isn't too bad perf wise. // Normally we should use gatherSlices to collect the values 2 by 2, // but gatherSlices messes up with the order of axes. - const left_val = image.gatherValues(axis, left.convert(.i32), .{ .indices_are_sorted = true }).convert(dtype); - const right_val = image.gatherValues(axis, right.convert(.i32), .{ .indices_are_sorted = true }).convert(dtype); + const left_val = image.gather_(&.{ax}, &.{left.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); + const right_val = image.gather_(&.{ax}, &.{right.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); - const left_weight = right.sub(scaled).broadcast(res_shape, &.{axis}); - const right_weight = scaled.sub(left).broadcast(res_shape, &.{axis}); + const left_weight = right.sub(scaled).broadcast(res_shape, &.{ax}); + const right_weight = scaled.sub(left).broadcast(res_shape, &.{ax}); const res = left_val.mul(left_weight).add(right_val.mul(right_weight)); return res.convert(image.dtype()).withTags(image.shape().tags()); @@ -907,7 +908,7 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng // topk_idx is indices into topk.values ! so in the range [0, topk] // Convert for the original indices from the full [0, voc] range. - const next_tokens = topk.indices.gatherValues(.voc, topk_idx.squeeze(.topk), .{}); + const next_tokens = topk.indices.gather(.{ .voc = topk_idx.squeeze(.topk) }, .{}); // log.debug("sampleTokens({}) -> {} -> {} -> {}", .{ activations, topk.indices, topk_idx, next_tokens }); return .{ next_tokens, next_rng }; } @@ -995,7 +996,7 @@ pub fn sampleTokensDynamic(logits: Tensor, opts: DynamicSamplingStrategy, rng: T x = x.add(gumbel_noise); const topk_idx = x.argMax(.topk).indices; - const next_tokens = topk_indices.gatherValues(.voc, topk_idx.squeeze(.topk), .{}); + const next_tokens = topk_indices.gather(.{ .voc = topk_idx.squeeze(.topk) }, .{}); return .{ next_tokens, next_rng }; } diff --git a/zml/ops.zig b/zml/ops.zig index eadded8e..d554e5da 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -462,10 +462,9 @@ pub fn if_( } } - const scalar_pred = if (pred.rank() == 0) pred else pred.flatten().squeeze(0); const loc = ctx.mlirCtx().location(@src()); const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.if", .{ - .operands = &.{scalar_pred.value()}, + .operands = &.{pred.asScalar().value()}, .result_type_inference = true, .blocks = &.{ true_branch_block, false_branch_block }, // We can't verify right away, cause the weights captured by the if haven't been added yet. diff --git a/zml/tensor.zig b/zml/tensor.zig index 7ea5ac5d..549479fd 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -731,7 +731,7 @@ pub const Tensor = struct { break :blk powers; }; const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); - const counts = values.gatherValues(.d, samples, .{}).sum(.n).bitCast(.u16); + const counts = values.gather(.{ .d = samples }, .{}).sum(.n).bitCast(.u16); const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; } @@ -2194,88 +2194,69 @@ pub const Tensor = struct { pub const GatherOpts = struct { indices_are_sorted: bool = false }; - /// For each coordinate in `indices`, - /// `gatherValues` extracts a single value of the given tensor. - /// - /// * axes_ is a single axis, or a tuple of axis: .b, or .{ .b, .c } - /// * indices is an integer tensor - /// * result is a tensor whose shape is similar to the input shape - /// where the gathered axes have been replaced by axes from 'indices'. - /// - /// Some example input for the base case where we work on one axis: - /// - gatherValues(f:[a]->float, .a, ind:[n]->int)[n] == f[ind[n]] - /// - gatherValues(f:[a, b], .a, ind:[n])[n, b] == f[ind[n], b] - /// - gatherValues(f: [a,b,c], .{.b}, ind: [n,m])[a, n, m, c] == f[a, ind[n, m], c] - /// - /// If an axis in common between `self` and `indices`, - /// it is treated as a "batching" axis, meaning that semantically - /// the operator is doing a gatherValues one time per dimension of this axis: - /// - gatherValues(f: [a,b,c], .{.b}, ind: [a,n])[a, n] == f[a, ind[a, n]] - /// - /// It is an error to have an axis present in `self`, `axes_` and `indices`. - /// - /// If several axes are passed, then the last axis of indices is treated as coordinates: - /// - gatherValues(f: [a,b,c], .{.b, .c}, ind: [n,2])[a, n] == f[a, ind[n][0], ind[n][1]] - /// - gatherValues(f: [a,b,c,d], .{.b, .c}, ind: [a, n,2])[a, n, d] == f[a, ind[a, n][0], ind[a, n][1], d] - /// - /// It is possible to use gatherValues without tags, but batching won't be available. - pub fn gatherValues(self: Tensor, coord_axes: anytype, indices: Tensor, opts: GatherOpts) Tensor { - // scoped_log.debug("gatherValues({}, {any}, {})", .{ self, coord_axes, indices }); - const single_coord, const coord_axes_ = _parseGatherCoord(self, coord_axes); + pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { + const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); + var idx_axes: Shape.AxesArray = .{}; + for (idx_tags.slice()) |t| { + idx_axes.appendAssumeCapacity(self.axis(t)); + } - stdx.debug.assert(coord_axes_.len > 0, "gatherValues expects 1 or more axes to operate one, received none. Example: `x.gatherValues(.a, indices, .{{}})`", .{}); - for (coord_axes_.constSlice(), 0..) |a, i| { + // TODO: sort indices following self.shape instead of asking the user to do it. + return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); + } + + pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); + for (idx_axes, 0..) |a, i| { if (i > 0) { - stdx.debug.assert(a == coord_axes_.get(i - 1) + 1, "gatherValues expects 'coord_axes' to be sequential. But {any} aren't sequential in {f}", .{ coord_axes, self }); + stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); } } + var indices_shape = idx_per_axis[0].shape(); + for (idx_per_axis[1..]) |idx| { + if (idx.rank() > indices_shape.rank()) { + indices_shape = idx.shape(); + } + } + for (idx_per_axis) |idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + } + + var idx_batch_axes: Shape.DimsArray = .{}; const AxisKind = enum { batching, offset, collapsed, indices }; - var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; + var self_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const maybe_coord_ax = std.mem.indexOfScalar(u3, coord_axes_.constSlice(), @intCast(self_ax)); - if (indices._shape.hasTag(t)) |id_ax| { + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + if (indices_shape.hasTag(t)) |id_ax| { // tag is both in self and indices -> it's a batching dim // Note: tags are required for batching. - self_kind.appendAssumeCapacity(.batching); - indices_batch_axes.appendAssumeCapacity(id_ax); - stdx.debug.assert(maybe_coord_ax == null, "gatherValues expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'coord_axes_={any}' and in 'indices={f}'", .{ self._shape._tags.get(self_ax), self, coord_axes, indices }); - } else if (maybe_coord_ax) |_| { - // for gatherValues we collapsed all gathered axes - // (contrary to gatherSlices where we collapse none) - self_kind.appendAssumeCapacity(.collapsed); + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { + // we collapsed all gathered axes + self_kind.buffer[self_ax] = .collapsed; + // idx_kind.buffer[id_ax] = .indices; } else { - self_kind.appendAssumeCapacity(.offset); + self_kind.buffer[self_ax] = .offset; } } - // When we receive several coord_axes we need an extra dimension to store - // one index per axis, which makes the coordinates of one value. - // Otherwi se stablehlo uses the "indices.rank()" default value. - const index_coord_axis = if (single_coord) - indices.rank() - else blk: { - const ax = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); - stdx.debug.assert(indices.dim(ax) == coord_axes_.len, "gatherValues with axes={any}, expects indices to be of shape [..., {}], got: {f}", .{ coord_axes, coord_axes_.len, indices }); - break :blk ax; - }; - // compute res shape var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; - for (self_kind.constSlice(), 0..) |kind, ax_usize| { + var res_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{}; + for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); - if (ax == coord_axes_.get(0)) { + if (ax == idx_axes[0]) { // The first val_ax is special cause this is the place where we insert indices axes. - for (indices._shape.tags(), 0..indices.rank()) |t, id_ax| { - if (id_ax == index_coord_axis) continue; - if (std.mem.indexOfScalar(i64, indices_batch_axes.constSlice(), @intCast(id_ax))) |_| { - // batching dim are already in res - continue; - } + for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + // Batching axis is already in self. + if (is_batching_axis) continue; - res_shape = res_shape.appendDim(indices.dim(id_ax), t); + res_shape = res_shape.appendDim(id_inserted_dim, id_axis); res_kind.appendAssumeCapacity(.indices); } } @@ -2291,12 +2272,12 @@ pub const Tensor = struct { // This is not a gather, but a dynamicSlice. // Sometimes the backend recognize this pattern, but not always. // So let us handle that. - if (indices.count() == 1) { - return self.dynamicSlice1d(coord_axes_.get(0), .{ .start = indices.flatten().squeeze(0), .len = 1 }).reshape(res_shape); + if (indices_shape.count() == 1 and idx_axes.len == 1) { + return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); } var slice_dims: Shape.DimsArray = .{}; - for (self_kind.constSlice(), self.dims()) |k, d| { + for (self_kind.slice(), self.dims()) |k, d| { slice_dims.appendAssumeCapacity(switch (k) { .batching, .collapsed => 1, .offset => d, @@ -2304,7 +2285,9 @@ pub const Tensor = struct { }); } - // scoped_log.debug("gatherValues --> {} {any}", .{ res_shape, res_kind.constSlice() }); + // TODO: try changing .last by other axis and see the perf impact. + const indices = Tensor.stack(idx_per_axis, .last, .coord); + // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); const loc = self.getContext().mlirCtx().location(@src()); const gather_op = dialect.stablehlo.gather( self.getContext().mlirCtx(), @@ -2316,22 +2299,30 @@ pub const Tensor = struct { .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), - .start_indices_batching_dims = indices_batch_axes.constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .index_vector_dim = index_coord_axis, + .index_vector_dim = indices.axis(.coord), .indices_are_sorted = opts.indices_are_sorted, }, ); const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); - stdx.debug.assert(mlir_shape.eql(res_shape), "gatherValues expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); return _result(res_shape, gather_op.result(0)); } - test gatherValues { + test gather { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const Local = struct { + pub fn _idx(idx_shape: anytype) Tensor { + return Tensor.constant(idx_shape, .{ .i32 = 0 }); + } + }; + + const idx = Local._idx; + { // Only test shapes var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); @@ -2340,32 +2331,38 @@ pub const Tensor = struct { defer comp.deactivate(); inline for (.{ - .{ .{ .a = 10 }, .a, .{}, .{} }, - .{ .{ .a = 10 }, .a, .{ .n = 8 }, .{ .n = 8 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{}, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{ .n = 8 }, .{ .n = 8, .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, 0, .{ .n = 8 }, .{ .n = 8, .b = 20 } }, + .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, + .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, 0, idx(.{ .n = 8 }), .{ .n = 8, .b = 20 } }, // Favor val shape, instead of indices shape. - .{ .{ .a = 10, .b = 20 }, .b, .{ .n = 8 }, .{ .a = 10, .n = 8 } }, - .{ .{ .a = 10, .b = 20, .c = 30 }, .b, .{ .n = 8 }, .{ .a = 10, .n = 8, .c = 30 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, // batching axes are implicits. - .{ .{ .a = 10, .b = 20 }, .b, .{ .a = 10 }, .{ .a = 10 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{ .b = 20 }, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .b, .{ .a = 10, .n = 8 }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10 }) }, .{ .a = 10 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .b = 20 }) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8 } }, // stablehlo.gather is biased toward indices shape (like gatherSlice). // This make it awkward to use when you have both batching dimension and new indices dimensions. // For now we reject those, and let user explicitly transpose self or indices if needed. - // .{ .{ .a = 10, .b = 20 }, .b, .{ .n = 8, .a = 10 }, .{ .a = 10, .n = 8 } }, + // .{ .{ .a = 10, .b = 20 }, .{.b = idx(.{ .n = 8, .a = 10 })}, .{ .a = 10, .n = 8 } }, // Also handle tuples - .{ .{ .a = 10, .b = 20 }, .{ .a, .b }, .{ .n = 8, ._ = 2 }, .{ .n = 8 } }, - .{ .{ 10, 20 }, .{ -2, -1 }, .{ 8, 2 }, .{8} }, - // and 1-tuple - .{ .{ .a = 10, .b = 20 }, .{.b}, .{ .n = 8, ._ = 1 }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, }) |testcase| { - const x_shape, const tag, const idx_shape, const res_shape = testcase; + const x_shape, const indices, const res_shape = testcase; const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); - const y = gatherValues(x, tag, idx, .{}); + const y = gather(x, indices, .{}); + try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + try std.testing.expect(y.value().owner().verify()); + } + + inline for (.{ + .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, + }) |testcase| { + const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; + const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const y = gather_(x, idx_axes, idx_per_axis, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); } diff --git a/zml/torch.zig b/zml/torch.zig index 43b0339d..09888aee 100644 --- a/zml/torch.zig +++ b/zml/torch.zig @@ -169,7 +169,7 @@ test pixelShuffle { /// /// Note: at the difference of Pytorch, shifts need to be explicitly repeated, even if they are the same for all axes. /// ref: https://pytorch.org/docs/stable/generated/torch.roll.html -pub fn roll(self: Tensor, shifts: []const i64, axes_: []const u8) Tensor { +pub fn roll(self: Tensor, shifts: []const i64, axes_: []const i8) Tensor { // TODO(hugo) accept following syntax: x.roll(.{ .a = 5, .b = 8 }) stdx.debug.assert(self.rank() > 0 and shifts.len == axes_.len, "Shifts length ({d}) and dims length ({d}) are not equal, we expect the same length.", .{ shifts.len, axes_.len }); @@ -180,11 +180,11 @@ pub fn roll(self: Tensor, shifts: []const i64, axes_: []const u8) Tensor { return roll(first_dim_rolled, tail_shifts, tail_dims); } - const a = axes_[0]; + const a = self.axis(axes_[0]); const start = @mod(self.dim(a) - shifts[0], self.dim(a)); const idx = Tensor.arange(.{ .start = start, .end = start + self.dim(a) }, .f32); const divisor: f32 = @floatFromInt(self.dim(a)); - return self.gatherValues(a, idx.fmod(divisor).convert(.i32), .{}); + return self.gather_(&.{a}, &.{idx.fmod(divisor).convert(.i32)}, .{}); } test roll { @@ -194,7 +194,7 @@ test roll { const res = try zml.testing.compileAndCall( platform, roll, - .{ input, &[_]i64{ 2, 1 }, &[_]u8{ 0, 1 } }, + .{ input, &[_]i64{ 2, 1 }, &[_]i8{ 0, 1 } }, ); const expectation = zml.HostBuffer.fromSlice(.{ 4, 2 }, &[_]f32{ 6, 5, 8, 7, 2, 1, 4, 3 }); From 189aad7cb51fe22bf30fcfef36853f8987665797 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 24 Jul 2025 22:33:13 +0200 Subject: [PATCH 25/67] examples/mixtral --- examples/mixtral/BUILD.bazel | 22 ++ examples/mixtral/main.zig | 332 ++++++++++++++++++++++++++ examples/mixtral/mixtral.zig | 443 +++++++++++++++++++++++++++++++++++ zml/aio.zig | 45 +++- 4 files changed, 829 insertions(+), 13 deletions(-) create mode 100644 examples/mixtral/BUILD.bazel create mode 100644 examples/mixtral/main.zig create mode 100644 examples/mixtral/mixtral.zig diff --git a/examples/mixtral/BUILD.bazel b/examples/mixtral/BUILD.bazel new file mode 100644 index 00000000..1bee84ab --- /dev/null +++ b/examples/mixtral/BUILD.bazel @@ -0,0 +1,22 @@ +load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_skylib//rules:native_binary.bzl", "native_test") +load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "mixtral", + srcs = [ + "mixtral.zig", + ], + main = "main.zig", + deps = [ + "@com_github_hejsil_clap//:clap", + "@zml//async", + "@zml//stdx", + "@zml//zml", + ], +) diff --git a/examples/mixtral/main.zig b/examples/mixtral/main.zig new file mode 100644 index 00000000..99c3c1fe --- /dev/null +++ b/examples/mixtral/main.zig @@ -0,0 +1,332 @@ +const std = @import("std"); + +const asynk = @import("async"); +const clap = @import("clap"); +const stdx = @import("stdx"); +const zml = @import("zml"); +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; + +const mixtral = @import("mixtral.zig"); +const MixtralLM = mixtral.MixtralLM; +const Mixtral = mixtral.Llama; +const KvCache = mixtral.KvCache; +const TransformerLayer = mixtral.TransformerLayer; +const SelfAttn = mixtral.SelfAttn; + +const log = std.log.scoped(.mixtral); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: MixtralLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { + var tokens = std.ArrayList(u32).init(allocator); + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + if (skip_llama3_encoding) { + // Copy to the arraylist so the ownership is the same in both branches. + try tokens.appendSlice(try encoder.encode(prompt)); + return tokens.toOwnedSlice(); + } + + const start_header_id = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header_id = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const eot_id = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline_id = (try encoder.encode("\n"))[0]; + + try tokens.append(config.bos_token_id); + + try tokens.append(start_header_id); + try tokens.appendSlice(try encoder.encode("user")); + try tokens.appendSlice(&.{ end_header_id, newline_id }); + + try tokens.appendSlice(try encoder.encode(prompt)); + try tokens.appendSlice(&.{ eot_id, newline_id }); + try tokens.append(start_header_id); + try tokens.appendSlice(try encoder.encode("assistant")); + try tokens.appendSlice(&.{ end_header_id, newline_id }); + + return tokens.toOwnedSlice(); +} + +pub fn generateText( + config: MixtralLM.Config, + llama_: MixtralLM, + mod_prefill: zml.ModuleExe(MixtralLM.forward), + mod_generate: zml.ModuleExe(MixtralLM.forward), + kv_cache_: zml.Bufferized(mixtral.KvCache), + tokenizer: zml.tokenizer.Tokenizer, + allocator: std.mem.Allocator, + seed: u128, + prompt: []const u8, + skip_llama3_encoding: bool, +) ![]const u8 { + const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); + defer allocator.free(prompt_tok); + + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + const platform = mod_generate.platform(); + const max_seq_len = llama_.model.shape().s; + + // init RNG and buffers + var rng = try zml.Tensor.Rng.init(platform, seed); + var generated_token_buffer = [_]u32{undefined}; + + var kv_cache = prefill: { + // prepare device buffers for the prefill tokens and their positions + const prefill_buffer = try allocator.alloc(u32, max_seq_len); + @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); + + var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); + defer prefill_tokens.deinit(); + var prefill_token_pos = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0); + defer prefill_token_pos.deinit(); + + const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); + _ = try prefilled_tokens.toHost(std.mem.sliceAsBytes(prefill_buffer)); + generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; + break :prefill kv_cache; + }; + defer zml.aio.unloadBuffers(&kv_cache); + + // Prepare for token-by-token generation, + // start with the token generated based on the full prompt. + var current_token = try zml.Buffer.fromSlice(platform, .{1}, &generated_token_buffer); + defer current_token.deinit(); + + // Here we collect the generated text + var output = std.ArrayList(u8).init(allocator); + defer output.deinit(); + + const output_tokens_len = max_seq_len - prompt_tok.len - 1; + const start = std.time.microTimestamp(); + + // One token has alreadyh been generated by the prefill. + var num_tokens_generated: usize = 1; + + generation: for (0..output_tokens_len + 1) |i| { + // collect and print generated sequence + num_tokens_generated += 1; + const generated_token = generated_token_buffer[0]; + const chunk = try tokenizer_decoder.next(generated_token) orelse unreachable; + try output.appendSlice(chunk); + std.debug.print("{s}", .{chunk}); + + // check for eos + if (i == output_tokens_len) break :generation; + switch (config.eos_token_id.value) { + .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, + .ints => |eos_list| { + for (eos_list) |eos| { + if (generated_token == @as(u32, @intCast(eos))) break :generation; + } + }, + } + + // current token pos needs to go into a zml.Buffer + const token_pos_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; + const token_pos = try zml.Buffer.fromSlice(platform, .{}, token_pos_buffer); + defer token_pos.deinit(); + + // call to generate the next token + current_token, kv_cache, rng = mod_generate.call(.{ current_token, token_pos, kv_cache, rng }); + + // extract the generated token from the buffer + _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + } + const end = std.time.microTimestamp(); + const duration = stdx.math.divFloat(f64, end - start, std.time.us_per_s); + const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration; + std.debug.print("\n", .{}); + log.info("✅ Generated {d} tokens in {:.3}s: {d:.3}tok/s", .{ num_tokens_generated, duration, speed }); + return output.toOwnedSlice(); +} + +const params = clap.parseParamsComptime( + \\--help print this help + \\--prompt the prompt + \\--hf-model-path path to the directory containing model weights, config and tokenizer + \\--seed random seed (optional) + \\--seq-len sequence length + \\--create-options platform creation options JSON, defaults to {} + \\--no-llama3 skip prompt template + \\--sharding default: true: sharding on or off +); + +pub fn bool_parser(in: []const u8) error{}!bool { + return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + log.info(" mixtral was compiled with {}", .{@import("builtin").mode}); + + const allocator = std.heap.c_allocator; + + const parsers = comptime .{ + .BOOL = bool_parser, + .UINT = clap.parsers.int(usize, 0), + .STRING = clap.parsers.string, + .PATH = clap.parsers.string, + }; + var diag: clap.Diagnostic = .{}; + const stderr = std.io.getStdErr().writer(); + var res = clap.parse(clap.Help, ¶ms, parsers, .{ + .diagnostic = &diag, + .allocator = allocator, + }) catch |err| { + diag.report(stderr, err) catch {}; + stderr.print("usage: ", .{}) catch {}; + clap.usage(stderr, clap.Help, ¶ms) catch {}; + stderr.print("\n", .{}) catch {}; + return; + }; + defer res.deinit(); + + if (res.args.help != 0) { + clap.help(std.io.getStdErr().writer(), clap.Help, ¶ms, .{}) catch {}; + return; + } + + const hf_model_path = res.args.@"hf-model-path" orelse { + log.err("Missing --hf-model-path", .{}); + return; + }; + + const model_config_path = try std.fs.path.join(allocator, &.{ hf_model_path, "config.json" }); + defer allocator.free(model_config_path); + + const model_weights_path = b: { + const simple_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors" }); + if (asynk.File.access(simple_path, .{})) { + break :b simple_path; + } else |_| { + allocator.free(simple_path); + } + + const sharded_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors.index.json" }); + break :b sharded_path; + }; + defer allocator.free(model_weights_path); + + const model_tokenizer_path = try std.fs.path.join(allocator, &.{ hf_model_path, "tokenizer.json" }); + defer allocator.free(model_tokenizer_path); + + const config = blk: { + var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + var reader = std.json.reader(allocator, config_json_file.reader()); + defer reader.deinit(); + const config_obj = try std.json.parseFromTokenSourceLeaky(mixtral.MixtralLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + break :blk config_obj; + }; + + var context = try zml.Context.init(); + defer context.deinit(); + + const compilation_options = zml.CompilationOptions{ + .xla_dump_to = "/tmp/zml/mixtral", + .sharding_enabled = res.args.sharding orelse true, + }; + + // initialize ZML platform with optional create options + // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' + const create_opts_json = res.args.@"create-options" orelse "{}"; + const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); + const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); + create_opts.deinit(); + context.printAvailablePlatforms(platform); + + var ts = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); + defer ts.deinit(); + + var model_arena = std.heap.ArenaAllocator.init(allocator); + var model_instance = try zml.aio.populateModel(mixtral.MixtralLM, model_arena.allocator(), ts); + + const llama_options: mixtral.MixtralLM.Options = .{ + .max_seq_len = @intCast(res.args.@"seq-len" orelse 256), + .sampling_strategy = .{ + .topk = 1, + .temperature = 1.0, + }, + }; + model_instance.init(config, llama_options); + + const dims = model_instance.model.shape(); + const dtype = model_instance.model.embed_tokens.weight.dtype(); + + const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); + const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); + const token_idx_shape = zml.Shape.init(.{}, .u32); + + const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype).withSharding(.{.h}); + + const kv_cache_shape: zml.ShapeOf(mixtral.KvCache) = mixtral.KvCache.initShape(kv_shape); + const rng_shape = zml.Tensor.Rng.shape(); + + var start = try std.time.Timer.start(); + var fut_mod_prefill = try asynk.asyncc(zml.compile, .{ + allocator, mixtral.MixtralLM.forward, .{ config, llama_options }, + .{ + tokens_shape_prefill, + token_idx_shape, + kv_cache_shape, + rng_shape, + }, + ts, + platform, + }); + + var fut_mod = try asynk.asyncc(zml.compile, .{ + allocator, mixtral.MixtralLM.forward, .{ config, llama_options }, + .{ + tokens_shape, + token_idx_shape, + kv_cache_shape, + rng_shape, + }, + ts, + platform, + }); + + log.info("\tLoading mixtral weights from {?s}...", .{model_weights_path}); + var llama_weights = try zml.aio.loadBuffers(mixtral.MixtralLM, .{ config, llama_options }, ts, model_arena.allocator(), platform); + defer zml.aio.unloadBuffers(&llama_weights); + log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); + + var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_weights); + defer llama_module_prefill.deinit(); + var llama_module = (try fut_mod.awaitt()).prepare(llama_weights); + defer llama_module.deinit(); + log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); + + log.info("Creating KvCache", .{}); + const kv_cache = try mixtral.KvCache.initBuffer(kv_shape, platform); + + var tokenizer = blk: { + log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); + var timer = try stdx.time.Timer.start(); + defer log.info("Loaded tokenizer from {s} [{}]", .{ model_tokenizer_path, timer.read() }); + + break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), model_tokenizer_path); + }; + errdefer tokenizer.deinit(); + + const prompt = res.args.prompt orelse "What is the capital of France?"; + log.info("✅\tPrompt: {s}", .{prompt}); + + const seed = res.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); + const skip_llama3_encoding = res.args.@"no-llama3" orelse false; + const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); + // generated text will be printed token by token. + defer allocator.free(generated_text); +} diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig new file mode 100644 index 00000000..21086283 --- /dev/null +++ b/examples/mixtral/mixtral.zig @@ -0,0 +1,443 @@ +const std = @import("std"); +const testing = std.testing; + +const stdx = @import("stdx"); +const zml = @import("zml"); +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; + +const log = std.log.scoped(.Mixtral); + +/// Mixtral architecture, using huggingface transformers naming. +/// Dimensions of activations: {.b, .s, .d} +pub const MixtralLM = struct { + pub const Config = struct { + bos_token_id: u32, + eos_token_id: stdx.json.Union(union(enum) { + int: u32, + ints: []u32, + }), + num_hidden_layers: usize, + num_attention_heads: usize, + num_key_value_heads: usize, + rope_theta: f32, + max_position_embeddings: usize, + rms_norm_eps: f32, + hf_rope_impl: bool = true, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, + }; + + pub const Options = struct { + sampling_strategy: ?zml.nn.SamplingStrategy, + max_seq_len: usize, + }; + + lm_head: ?zml.nn.Linear, + model: Mixtral, + + // Options controlling generation + gen_opts: zml.nn.SamplingStrategy = .{}, + config: Config, + + pub fn init(self: *MixtralLM, config: Config, options: Options) void { + self.config = config; + self.gen_opts = options.sampling_strategy orelse .{}; + self.model.max_seq_len = @intCast(options.max_seq_len); + self.model.num_heads = @intCast(config.num_attention_heads); + self.model.num_kv_heads = @intCast(config.num_key_value_heads); + self.model.rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }; + self.model.norm.eps = config.rms_norm_eps; + for (self.model.layers) |*layer| { + layer.self_attn.num_heads = self.model.num_heads; + layer.self_attn.num_kv_heads = self.model.num_kv_heads; + layer.self_attn.rope_opts = self.model.rope_opts; + layer.input_layernorm.eps = config.rms_norm_eps; + layer.post_attention_layernorm.eps = config.rms_norm_eps; + + // Rewrite to "on_device" layout. + layer.block_sparse_moe.rewrite(); + const mlp: *Mlp = &layer.block_sparse_moe.on_device.mlp_experts; + mlp.up_proj.weight = mlp.up_proj.weight.withSharding(.{0}); + mlp.gate_proj.weight = mlp.gate_proj.weight.withSharding(.{0}); + mlp.down_proj.weight = mlp.down_proj.weight.withSharding(.{1}); + + layer.self_attn.q_proj.weight = layer.self_attn.q_proj.weight.withSharding(.{0}); + layer.self_attn.k_proj.weight = layer.self_attn.k_proj.weight.withSharding(.{0}); + layer.self_attn.v_proj.weight = layer.self_attn.v_proj.weight.withSharding(.{0}); + layer.self_attn.o_proj.weight = layer.self_attn.o_proj.weight.withSharding(.{1}); + } + + // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. + // It currently crashes/compilation fails + if (self.gen_opts.topk == 1 and self.lm_head != null) { + self.lm_head.?.weight = self.lm_head.?.weight.withSharding(.{0}); + } + } + + /// Predicts the token at `token_index` position. + /// Returns: + /// - updated `tokens`, + /// - updated KV cache + /// - a Rng state to allow for probabilistic generation + pub fn forward( + self: MixtralLM, + tokens_: Tensor, + token_index: Tensor, + kv_cache: KvCache, + rng: Tensor.Rng, + ) struct { Tensor, KvCache, Tensor.Rng } { + stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Mixtral ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); + const tokens = tokens_.withPartialTags(.{.s}); + const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + } + + pub fn sampleTokens( + self: MixtralLM, + lm_head_: ?zml.nn.Linear, + out_: Tensor, + rng: Tensor.Rng, + opts: zml.nn.SamplingStrategy, + ) struct { Tensor, Tensor.Rng } { + const out = out_.withPartialTags(.{ .s, .d }); + + var logits = blk: { + if (lm_head_) |lm_head| { + break :blk zml.call(lm_head, .forward, .{out}); + } else { + break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); + return .{ next_tokens, new_rng }; + } + + pub fn increment(_: u8, token_index: Tensor) Tensor { + return token_index.addConstant(1).reuseBuffer(token_index); + } +}; + +pub const Mixtral = struct { + embed_tokens: zml.nn.TokenEmbedding, + norm: RmsNorm, + layers: []TransformerLayer, + + max_seq_len: u32 = 0, + num_heads: i64 = 32, + num_kv_heads: i64 = 32, + rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = 10_000, + }, + + const Shape = struct { + s: u32, + layer: u16, + hd: u16, + nh: u16, + nkvh: u16, + dtype: zml.DataType, + }; + + pub fn shape(self: Mixtral) Shape { + const key_dim = self.layers[0].self_attn.k_proj.weight.dim(0); + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + + return .{ + .s = self.max_seq_len, + .layer = @intCast(self.layers.len), + .hd = @intCast(@divExact(key_dim, num_kv_heads)), + .nh = @intCast(self.num_heads), + .nkvh = @intCast(num_kv_heads), + .dtype = self.embed_tokens.weight.dtype(), + }; + } + + /// Forward one token, using KV cache for previous tokens. + /// Returns result and updated KV cache. + pub fn forward(self: Mixtral, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { + const embeds = embed(self.embed_tokens, tokens); + var hidden = embeds; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden, updated_kv_cache = zml.call(layer, .forward, .{ hidden, token_index, updated_kv_cache.atLayer(i) }); + } + const output = zml.call(self.norm, .forward, .{hidden}); + + return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; + } + + pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { + return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); + } + + fn initKvCache(self: Mixtral, embed_shape: zml.Shape) KvCache { + const dims = self.shape(); + var kv_shape = embed_shape.insert(0, .{ .layer = dims.layer }).rename(.{ .s = .k }).splitAxes(.{ .d = .{ .h = dims.nkvh, .hd = dims.hd } }); + const perm = kv_shape.contiguousPerm(.{ .k, .h, .hd }); + kv_shape = kv_shape.transpose(perm.constSlice()); + return KvCache.init(kv_shape); + } +}; + +pub const TransformerLayer = struct { + input_layernorm: RmsNorm, + self_attn: SelfAttn, + post_attention_layernorm: RmsNorm, + block_sparse_moe: BlockMoe, + + pub fn forward( + self: TransformerLayer, + x0: Tensor, + token_index: Tensor, + kv_cache: KvCache, + ) struct { Tensor, KvCache } { + // Self Attention + //log.debug("TransformerLayer({}) -> {}", .{ x0, self.input_layernorm.forward(x0) }); + stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); + + const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); + const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); + const x1 = x0.add(delta0); + + // Fully Connected + const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); + const x2 = zml.call(self.block_sparse_moe, .forward, .{x1_normalized}).add(x1); + + return .{ x2.reuseBuffer(x0), updated_kv_cache }; + } +}; + +const RmsNorm = struct { + weight: Tensor, + eps: f32 = 1e-5, + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: Tensor) Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + const normalized = zml.nn.rmsNorm(x, .d, self.eps); + return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); + } +}; + +const BlockMoe = union(enum) { + on_disk: struct { + gate: zml.nn.Linear, + experts: []struct { + w1: zml.nn.Linear, + w2: zml.nn.Linear, + w3: zml.nn.Linear, + }, + }, + on_device: struct { + router: zml.nn.Linear, + mlp_experts: Mlp, + }, + + pub fn rewrite(self: *BlockMoe) void { + const on_disk = self.on_disk; + const n = on_disk.experts.len; + var mlp: Mlp = .{ + .up_proj = on_disk.experts[0].w1, + .down_proj = on_disk.experts[0].w2, + .gate_proj = on_disk.experts[0].w3, + }; + mlp.up_proj.weight._shape = mlp.up_proj.weight.shape().insert(0, .{ .expert = n }); + mlp.down_proj.weight._shape = mlp.down_proj.weight.shape().insert(0, .{ .expert = n }); + mlp.gate_proj.weight._shape = mlp.gate_proj.weight.shape().insert(0, .{ .expert = n }); + + var router = on_disk.gate; + router.weight = router.weight.withTags(.{ .expert, .d }); + + self.* = .{ + .on_device = .{ .router = router, .mlp_experts = mlp }, + }; + } + + pub fn forward(self_block: BlockMoe, input: zml.Tensor) zml.Tensor { + const self = self_block.on_device; + const gating = self.router.forward(input).softmax(.expert); + return zml.nn.mixtureOfExperts(Mlp, self.mlp_experts, input, gating, .{ .tokens_per_expert = 2 }); + } +}; + +const Mlp = struct { + up_proj: zml.nn.Linear, // (dim -> hidden_dim) + gate_proj: zml.nn.Linear, // (dim -> hidden_dim) + down_proj: zml.nn.Linear, // (hidden_dim -> dim) + + pub fn forward(self: Mlp, x: Tensor) Tensor { + const proj = zml.call(self.up_proj, .forward, .{x}); + var output = zml.call(self.gate_proj, .forward, .{x}); + output = output.silu().mul(proj); + return zml.call(self.down_proj, .forward, .{output}); + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + + o_proj: zml.nn.Linear, + num_heads: i64 = undefined, + num_kv_heads: i64 = 0, + rope_opts: zml.nn.RopeOpts = undefined, + + /// Self Attention. + /// - If token_index is set, x is assumed to be the representation of one new token, + /// and kv_cache will be read for the previous tokens. + /// - If token_index is not set, x is assumed to be the representation of all tokens + /// since the beginning of the sequence, and kv_cache won't be read. + /// In both case, kv_cache will be updated with the computed key and value. + /// x: {.b, .s, .d } -> .{.b, .s, .d} + pub fn forward( + self: SelfAttn, + x: Tensor, + token_index: Tensor, + kv_cache: KvCache, + ) struct { Tensor, KvCache } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); + var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + + // Generate the attention mask. + const seq_len = kv_cache.k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), null); + + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = x.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); + + // In self-attention, .s axis is used both for keys and queries. + const pos_index = b: { + const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + q = zml.nn.rope(q, pos_index, self.rope_opts); + k = zml.nn.rope(k, pos_index, self.rope_opts); + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + const dtype = q.dtype(); + const new_kv_cache = kv_cache.update(k, v, token_index); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; + } + + fn initKvCache(key_shape: zml.Shape) KvCache { + // When we call initKvCache, we haven't renamed .s to .k yet. + var kv_shape = key_shape.insert(0, .{ .layer = 1 }).rename(.{ .s = .k }); + const perm = kv_shape.contiguousPerm(.{ .h, .k, .hd }); + kv_shape = kv_shape.transpose(perm.constSlice()); + var res = KvCache.init(kv_shape); + res.layer_index = Tensor.scalar(0, .u32); + return res; + } +}; + +pub const KvCache = struct { + k: Tensor, + v: Tensor, + layer_index: Tensor, + + pub fn init(kv_shape: zml.Shape) KvCache { + // The KV-cache is initialized with ones to detect reads of uninitialized memory. + return .{ + .k = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .v = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .layer_index = Tensor.scalar(-1, .u32), + }; + } + + pub fn initShape(kv_shape: zml.Shape) ShapeOf(KvCache) { + return .{ + .k = kv_shape, + .v = kv_shape, + .layer_index = zml.Shape.init(.{}, .u32), + }; + } + + pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .k = try zml.Buffer.constant(platform, kv_shape, 1), + .v = try zml.Buffer.constant(platform, kv_shape, 1), + .layer_index = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0), + }; + } + + pub fn keys(self: KvCache) Tensor { + return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache) Tensor { + return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .k = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .k = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } +}; diff --git a/zml/aio.zig b/zml/aio.zig index 1b926a91..bed45a8a 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -337,12 +337,14 @@ fn _populateStruct( return false; }; } + log.debug("Populating {s} ({s})", .{ prefix, @typeName(T) }); return switch (type_info) { .pointer => |ptr_info| { if (ptr_info.size == .slice) { obj.* = &.{}; + if (!zml.meta.Contains(ptr_info.child, zml.Tensor)) return false; const len = buffer_store.countLayers(prefix); if (len > 0) { obj.* = try allocator.alloc(ptr_info.child, len); @@ -386,19 +388,25 @@ fn _populateStruct( var has_default = false; if (field.default_value_ptr) |_| has_default = true; - const field_found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &@field(obj, field.name), required and !has_default); - partial_struct = partial_struct or field_found; - if (!field_found) { - if (field.default_value_ptr) |v| { - @field(obj, field.name) = @as(*const field.type, @ptrCast(@alignCast(v))).*; - } else { - if (partial_struct) { - log.warn("Incomplete metadata '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); - obj.* = undefined; + + if (zml.meta.Contains(field.type, zml.Tensor)) { + const field_found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &@field(obj, field.name), required and !has_default); + partial_struct = partial_struct or field_found; + if (!field_found) { + if (field.default_value_ptr) |v| { + @field(obj, field.name) = @as(*const field.type, @ptrCast(@alignCast(v))).*; + } else { + if (partial_struct) { + log.warn("Incomplete struct '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); + // if (required) { + // findSimilarBufferKeys(prefix, buffer_store, allocator); + // } + obj.* = undefined; + return false; + } + return false; } - - return false; } } } @@ -415,11 +423,22 @@ fn _populateStruct( return true; }, .float => { - obj.* = std.math.nan(@TypeOf(obj.*)); + // obj.* = std.math.nan(@TypeOf(obj.*)); return true; }, .void => true, - .@"union" => true, + .@"union" => |union_info| { + inline for (union_info.fields) |field| { + obj.* = @unionInit(T, field.name, undefined); + // @compileLog(T, field.name, field.type); + log.warn("Trying to init {s} as a {s}", .{ prefix, @typeName(field.type) }); + const field_found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &@field(obj, field.name), true); + if (field_found) return true; + } + if (required) log.warn("Failed to init {s} as any of {s}", .{ prefix, @typeName(T) }); + obj.* = undefined; + return false; + }, .bool => { obj.* = undefined; return true; From 9fbf45d9c939f9e3d1295b88b3910bc8af643360 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 24 Jul 2025 22:33:31 +0200 Subject: [PATCH 26/67] WIP: mixtureOfExperts --- zml/nn.zig | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/zml/nn.zig b/zml/nn.zig index d0fbf03c..7c7edb58 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -46,6 +46,32 @@ pub const TokenEmbedding = struct { } }; +const MoeOpts = struct { + tokens_per_expert: u32, + // TODO: handle an upper-bound on the number of experts per token. + // There are basically two implementation of MoE. + // The original paper + // experts_per_token: u32, +}; + +/// - input: .{ .s, .d } per-entry vector +/// - gating: .{ .s, .expert } per-entry expert-affinity +/// - experts: .{ .expert, .d_out, .d } expert layer (need to have a .forward method). +/// -> output: .{ .s, .d_out } +pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Tensor, opts: MoeOpts) Tensor { + log.warn("mixtureOfExperts({s}, {}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); + const routing = gating.topK(opts.tokens_per_expert, .expert, .{}); + const routing_score = routing.values.sum(.expert); + + const input_per_expert = input.gather(.{ .s = routing.indices }, .{}); + var output_per_expert = experts.forward(input_per_expert); + output_per_expert = output_per_expert.mul(routing.values.div(routing_score).broad(output_per_expert.shape())); + var output: Tensor = .constant(output_per_expert.shape().drop(.expert), input.dtype().zero()); + output = output.scatterSlices(.{ .expert = routing.indices }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); + + return output; +} + pub const Activation = union(enum) { sigmoid, tanh, From 77e1bf010883f287d14ceabecde49f91209192e0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 25 Jul 2025 11:29:54 +0200 Subject: [PATCH 27/67] fixup MoE --- examples/mixtral/mixtral.zig | 18 ++++---- mlir/dialects/stablehlo/stablehlo.zig | 20 +++++---- zml/nn.zig | 45 ++++++++++++++++--- zml/tensor.zig | 62 +++++++++++++++------------ 4 files changed, 94 insertions(+), 51 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 21086283..cdd8d6ef 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -253,9 +253,9 @@ const BlockMoe = union(enum) { .down_proj = on_disk.experts[0].w2, .gate_proj = on_disk.experts[0].w3, }; - mlp.up_proj.weight._shape = mlp.up_proj.weight.shape().insert(0, .{ .expert = n }); - mlp.down_proj.weight._shape = mlp.down_proj.weight.shape().insert(0, .{ .expert = n }); - mlp.gate_proj.weight._shape = mlp.gate_proj.weight.shape().insert(0, .{ .expert = n }); + mlp.up_proj.weight._shape = mlp.up_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .up, .d }); + mlp.down_proj.weight._shape = mlp.down_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .d, .up }); + mlp.gate_proj.weight._shape = mlp.gate_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .up, .d }); var router = on_disk.gate; router.weight = router.weight.withTags(.{ .expert, .d }); @@ -268,7 +268,7 @@ const BlockMoe = union(enum) { pub fn forward(self_block: BlockMoe, input: zml.Tensor) zml.Tensor { const self = self_block.on_device; const gating = self.router.forward(input).softmax(.expert); - return zml.nn.mixtureOfExperts(Mlp, self.mlp_experts, input, gating, .{ .tokens_per_expert = 2 }); + return zml.nn.mixtureOfExperts(Mlp, self.mlp_experts, input, gating, .{ .experts_per_token = 2 }); } }; @@ -278,10 +278,12 @@ const Mlp = struct { down_proj: zml.nn.Linear, // (hidden_dim -> dim) pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); - output = output.silu().mul(proj); - return zml.call(self.down_proj, .forward, .{output}); + const up = self.up_proj.weight.dot(x, .d); + const gate = self.gate_proj.weight.dot(x, .d).silu(); + + const out = self.down_proj.weight.dot(gate.mul(up), .up); + log.warn("Mlp(x: {f}) -> up: {f} -> gate: {f} -> out: {f}", .{ x, up, gate, out }); + return out; } }; diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 0c60fd01..95b66cf2 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -296,21 +296,23 @@ pub fn select(ctx: mlir.Context, condition: mlir.Value, then: mlir.Value, else_: }); } +pub const GatherArgs = struct { + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, + indices_are_sorted: bool = false, +}; + pub fn gather( ctx: mlir.Context, value: mlir.Value, indices: mlir.Value, slice_sizes: []const i64, location: mlir.Location, - args: struct { - offset_dims: []const i64, - collapsed_slice_dims: []const i64, - operand_batching_dims: []const i64, - start_indices_batching_dims: []const i64, - start_index_map: []const i64, - index_vector_dim: i64, - indices_are_sorted: bool = false, - }, + args: GatherArgs, ) mlir.Operation { return mlir.Operation.make( ctx, diff --git a/zml/nn.zig b/zml/nn.zig index 7c7edb58..d9e1cffb 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -47,27 +47,58 @@ pub const TokenEmbedding = struct { }; const MoeOpts = struct { - tokens_per_expert: u32, - // TODO: handle an upper-bound on the number of experts per token. - // There are basically two implementation of MoE. - // The original paper - // experts_per_token: u32, + experts_per_token: u32, }; +/// We have two algorithms, +/// * one for non-sharded single-stream inference (naive), +/// * one for sharded, batched inference (mixtral). +/// +/// The preferred algorithm is the batched one, +/// it is selected as soon there is enough tokens to guarantee that experts will be active most of the time. +/// /// - input: .{ .s, .d } per-entry vector /// - gating: .{ .s, .expert } per-entry expert-affinity /// - experts: .{ .expert, .d_out, .d } expert layer (need to have a .forward method). /// -> output: .{ .s, .d_out } pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Tensor, opts: MoeOpts) Tensor { log.warn("mixtureOfExperts({s}, {}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); - const routing = gating.topK(opts.tokens_per_expert, .expert, .{}); + const num_tokens = input.dim(.s); + const num_experts = gating.dim(.expert); + const tokens_per_expert = std.math.divCeil(u32, num_tokens * opts.experts_per_token, num_experts) catch unreachable; + + if (num_tokens * opts.experts_per_token < num_experts) { + // Few tokens: some experts will be unused. + // select active experts and compute with that. + const routing = gating.topK(opts.experts_per_token, .expert, .{}); + const routing_score = routing.values.sum(.expert); + const routing_weight = routing.values.div(routing_score.broad(routing.values.shape())); + + var sliced_expert: Expert = undefined; + zml.meta.mapAlloc(struct { + pub fn cb(routing_indices: Tensor, expert_weight: Tensor) Tensor { + return expert_weight.gather(.{ .expert = routing_indices }, .{}); + } + }.cb, undefined, routing.indices, experts, &sliced_expert) catch unreachable; + + const output_per_expert = sliced_expert.forward(input); + return output_per_expert.dot(routing_weight, .expert); + } + + // Lot of tokens, each experts chose their tokens and compute on that. + // It means that not all tokens will receive the same number of experts, + // Each token will get assigned to at least one expert because we normalize gating to sum up to one. + const normed_gating = gating.div(gating.sum(.s).broad(gating.shape())); + const routing = normed_gating.topK(tokens_per_expert, .s, .{}); const routing_score = routing.values.sum(.expert); const input_per_expert = input.gather(.{ .s = routing.indices }, .{}); var output_per_expert = experts.forward(input_per_expert); - output_per_expert = output_per_expert.mul(routing.values.div(routing_score).broad(output_per_expert.shape())); + log.warn(" -> input_per_expert {f} -> output_per_expert {f}", .{ input_per_expert, output_per_expert }); + output_per_expert = output_per_expert.mul(routing.values.div(routing_score.broad(routing.values.shape())).broad(output_per_expert.shape())); var output: Tensor = .constant(output_per_expert.shape().drop(.expert), input.dtype().zero()); output = output.scatterSlices(.{ .expert = routing.indices }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); + log.warn(" -> output {f} ->", .{output}); return output; } diff --git a/zml/tensor.zig b/zml/tensor.zig index 549479fd..a5068ae8 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -126,8 +126,8 @@ pub const Tensor = struct { /// Returns the dimension of axis 'axis_'. /// /// 'axis_' can be an integer or a tag. - pub fn dim(self: Tensor, axis_: anytype) i64 { - return self._shape.dim(axis_); + pub fn dim(self: Tensor, axis_: anytype) u32 { + return @intCast(self._shape.dim(axis_)); } /// Returns the dimensions of a Tensor as a slice. @@ -1078,13 +1078,16 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, comptime contracting: anytype) Tensor { - var contracting_axes: [contracting.len][2]i8 = undefined; - inline for (contracting, 0..) |c, i| { - contracting_axes[i] = .{ lhs.axis(c), rhs.axis(c) }; + var contracting_axes: std.BoundedArray([2]i8, MAX_RANK) = .{}; + if (@TypeOf(contracting) == EnumLiteral) { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(contracting), rhs.axis(contracting) }); + } else { + inline for (contracting) |c| { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(c), rhs.axis(c) }); + } } - var batching_axes: [MAX_RANK][2]i8 = undefined; - var n_batching: u8 = 0; + var batching_axes: std.BoundedArray([2]i8, MAX_RANK) = .{}; for (lhs._shape.tags(), 0..) |l, li| { stdx.debug.assert(l != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, lhs }); @@ -1092,20 +1095,19 @@ pub const Tensor = struct { stdx.debug.assert(r != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, rhs }); if (l == r) { - for (contracting_axes) |ct| { + for (contracting_axes.slice()) |ct| { if (l == lhs._shape.tag(ct[0])) { break; } } else { // tag is both in lhs and rhs but not in contracting -> it's a batching dim. - batching_axes[n_batching] = .{ @intCast(li), @intCast(ri) }; - n_batching += 1; + batching_axes.appendAssumeCapacity(.{ @intCast(li), @intCast(ri) }); } } } } - return dotGeneral(lhs, rhs, contracting_axes[0..], batching_axes[0..n_batching]); + return dotGeneral(lhs, rhs, contracting_axes.slice(), batching_axes.slice()); } test dot { @@ -2206,6 +2208,7 @@ pub const Tensor = struct { } pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + scoped_log.warn("gather({f}, {d}, {f})", .{ self, idx_axes, idx_per_axis }); stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); for (idx_axes, 0..) |a, i| { if (i > 0) { @@ -2228,14 +2231,16 @@ pub const Tensor = struct { var self_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + _ = t; const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); - if (indices_shape.hasTag(t)) |id_ax| { - // tag is both in self and indices -> it's a batching dim - // Note: tags are required for batching. - self_kind.buffer[self_ax] = .batching; - idx_batch_axes.appendAssumeCapacity(id_ax); - stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); - } else if (is_gather_axis) { + // if (indices_shape.hasTag(t)) |id_ax| { + // // tag is both in self and indices -> it's a batching dim + // // Note: tags are required for batching. + // self_kind.buffer[self_ax] = .batching; + // idx_batch_axes.appendAssumeCapacity(id_ax); + // // stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + // } else + if (is_gather_axis) { // we collapsed all gathered axes self_kind.buffer[self_ax] = .collapsed; // idx_kind.buffer[id_ax] = .indices; @@ -2289,21 +2294,24 @@ pub const Tensor = struct { const indices = Tensor.stack(idx_per_axis, .last, .coord); // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); const loc = self.getContext().mlirCtx().location(@src()); + const args: dialect.stablehlo.GatherArgs = .{ + .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), + .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = indices.axis(.coord), + .indices_are_sorted = opts.indices_are_sorted, + }; + scoped_log.warn(" --> stablehlo.gather({f}, {f}, {d}, {})", .{ self, indices, slice_dims.slice(), args }); + const gather_op = dialect.stablehlo.gather( self.getContext().mlirCtx(), self.value(), indices.value(), slice_dims.constSlice(), loc, - .{ - .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), - .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), - .start_indices_batching_dims = idx_batch_axes.constSlice(), - .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .index_vector_dim = indices.axis(.coord), - .indices_are_sorted = opts.indices_are_sorted, - }, + args, ); const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); From 7cdbd02e4c9eab797b668ac7e954af073c512832 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 25 Jul 2025 15:08:57 +0200 Subject: [PATCH 28/67] moe: fix scatter calls --- examples/mixtral/mixtral.zig | 6 +++--- zml/nn.zig | 27 ++++++++++++++++----------- zml/tensor.zig | 9 ++++++++- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index cdd8d6ef..bd45ef8b 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -278,10 +278,10 @@ const Mlp = struct { down_proj: zml.nn.Linear, // (hidden_dim -> dim) pub fn forward(self: Mlp, x: Tensor) Tensor { - const up = self.up_proj.weight.dot(x, .d); - const gate = self.gate_proj.weight.dot(x, .d).silu(); + const up = x.dot(self.up_proj.weight, .d); + const gate = x.dot(self.gate_proj.weight, .d).silu(); - const out = self.down_proj.weight.dot(gate.mul(up), .up); + const out = gate.mul(up).dot(self.down_proj.weight, .up); log.warn("Mlp(x: {f}) -> up: {f} -> gate: {f} -> out: {f}", .{ x, up, gate, out }); return out; } diff --git a/zml/nn.zig b/zml/nn.zig index d9e1cffb..4ebfb95f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -62,7 +62,7 @@ const MoeOpts = struct { /// - experts: .{ .expert, .d_out, .d } expert layer (need to have a .forward method). /// -> output: .{ .s, .d_out } pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Tensor, opts: MoeOpts) Tensor { - log.warn("mixtureOfExperts({s}, {}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); + // log.warn("mixtureOfExperts({s}, {}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); const num_tokens = input.dim(.s); const num_experts = gating.dim(.expert); const tokens_per_expert = std.math.divCeil(u32, num_tokens * opts.experts_per_token, num_experts) catch unreachable; @@ -89,16 +89,21 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te // It means that not all tokens will receive the same number of experts, // Each token will get assigned to at least one expert because we normalize gating to sum up to one. const normed_gating = gating.div(gating.sum(.s).broad(gating.shape())); - const routing = normed_gating.topK(tokens_per_expert, .s, .{}); - const routing_score = routing.values.sum(.expert); - - const input_per_expert = input.gather(.{ .s = routing.indices }, .{}); + // TODO: topK should always rename the operated axis. + const routing = normed_gating.rename(.{ .s = .expert_token }).topK(tokens_per_expert, .expert_token, .{}); + const routing_score = routing.values.div(routing.values.sum(.expert).broad(routing.values.shape())).transpose(.{ .expert, .expert_token }); + const routing_ids = routing.indices.transpose(.{ .expert, .expert_token }); + const input_per_expert = input.gather(.{ .s = routing_ids }, .{}); var output_per_expert = experts.forward(input_per_expert); - log.warn(" -> input_per_expert {f} -> output_per_expert {f}", .{ input_per_expert, output_per_expert }); - output_per_expert = output_per_expert.mul(routing.values.div(routing_score.broad(routing.values.shape())).broad(output_per_expert.shape())); - var output: Tensor = .constant(output_per_expert.shape().drop(.expert), input.dtype().zero()); - output = output.scatterSlices(.{ .expert = routing.indices }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); - log.warn(" -> output {f} ->", .{output}); + // log.warn(" -> input_per_expert {f} -> output_per_expert {f}", .{ input_per_expert, output_per_expert }); + output_per_expert = output_per_expert.mul(routing_score.broad(output_per_expert.shape())); + + // Reverse engineer the normal output shape that one-expert would have produced for all tokens. + // If this provide to not be enough we could use the "sliced_expert" strategy and call forward ourselves. + const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .expert_token = .s }).setDim(.s, num_tokens); + var output: Tensor = .constant(output_shape, input.dtype().zero()); + // log.warn(" -> output_per_expert {f}, routing: {f} -> output {f}", .{ output_per_expert, routing_ids, output }); + output = output.scatterSlices(.{ .s = routing_ids }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); return output; } @@ -244,7 +249,7 @@ pub const RopeOpts = struct { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, undefined, content, .{ .ignore_unknown_fields = true }) }; } else { - log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); + // log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; } } diff --git a/zml/tensor.zig b/zml/tensor.zig index a5068ae8..1dc53b6c 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2204,6 +2204,7 @@ pub const Tensor = struct { } // TODO: sort indices following self.shape instead of asking the user to do it. + // TODO: this doesn't handle batching axes. return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); } @@ -2343,7 +2344,6 @@ pub const Tensor = struct { .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, - // .{ .{ .a = 10, .b = 20 }, 0, idx(.{ .n = 8 }), .{ .n = 8, .b = 20 } }, // Favor val shape, instead of indices shape. .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, @@ -2708,6 +2708,11 @@ pub const Tensor = struct { .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 2 } }, .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }), .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 3, .b = 2 } }, .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .a = 3, .n = 8, .b = 2 } }, + // 2-indices axes, m and n + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8, .m = 9 }) }, .{ .n = 8, .m = 9, .b = 20 } }, + // TODO: here update (m,n) doesn't match indices (n, m). + // We need to either handle this automatically or provide better errors. + // .{ .{. a = 10, .b=20 }, .{ .a = idx(.{.n=8,.m=9}) }, .{.m=9,.n=8,.b=20} }, }) |testcase| { const x_shape, const indices, const updates_shapes = testcase; const x = Tensor.constant(x_shape, .{ .f16 = 0 }); @@ -3005,6 +3010,8 @@ pub const Tensor = struct { /// Returns a Tensor representing the result of Top-K over the given axis. pub fn topK(self: Tensor, k: u32, axis_: anytype, opts: struct { descending: bool = true }) SortRes { + // TODO: ask the user for a new name for the axis, this will simplify tagged code. + // See eg code in `sampleTokens` and `mixtureOfExpert` const a = self.axis(axis_); const result = self.sort(a, .{ .descending = opts.descending }); return .{ From e40805d78cca289b0fa6876c93df36e77bb75426 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 2 Sep 2025 15:51:11 +0200 Subject: [PATCH 29/67] move _unique_id to the BufferStore # Conflicts: # zml/aio/torch.zig # zml/aio/torch/file.zig --- zml/aio.zig | 58 ++++++++++++++++++++++++++++++++--------- zml/aio/safetensors.zig | 4 +-- zml/aio/tinyllama.zig | 4 +-- zml/aio/torch.zig | 2 +- zml/testing.zig | 2 +- 5 files changed, 50 insertions(+), 20 deletions(-) diff --git a/zml/aio.zig b/zml/aio.zig index bed45a8a..8edc2ce4 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -74,17 +74,27 @@ pub fn populateModelWithPrefix(comptime Model: type, allocator: std.mem.Allocato pub const BufferStore = struct { pub const Buffers = std.StringArrayHashMapUnmanaged(HostBuffer); pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); + var _unique_store_id: std.atomic.Value(u64) = .init(0); arena: std.heap.ArenaAllocator, files: []MemoryMappedFile = &.{}, buffers: Buffers = .{}, _metadata: Metadatas = .{}, + _unique_id: u64, - /// Create an empty BufferStore. Takes owneship of the given files. - pub fn init(allocator: std.mem.Allocator, files: []const MemoryMappedFile) error{OutOfMemory}!BufferStore { - var self: zml.aio.BufferStore = .{ + /// Create an empty BufferStore. + /// Takes owneship of the given files. + pub fn init(allocator: std.mem.Allocator) BufferStore { + return .{ .arena = std.heap.ArenaAllocator.init(allocator), + ._unique_id = _unique_store_id.fetchAdd(1024 * 1024 * 1024, .monotonic), }; + } + + /// Create an empty BufferStore. + /// Takes owneship of the given files. + pub fn initWithFiles(allocator: std.mem.Allocator, files: []const MemoryMappedFile) error{OutOfMemory}!BufferStore { + var self: BufferStore = .init(allocator); self.files = try self.arena.allocator().dupe(MemoryMappedFile, files); return self; } @@ -100,6 +110,24 @@ pub const BufferStore = struct { return self.buffers.get(key); } + pub fn getTensor(self: BufferStore, key: []const u8) zml.Tensor { + return self.getTensorOrNull(key) orelse { + findSimilarBufferKeys(key, self, std.heap.smp_allocator); + @panic("Tensor not found"); + }; + } + + pub fn getTensorOrNull(self: BufferStore, key: []const u8) ?zml.Tensor { + return if (self.buffers.getIndex(key)) |entry_idx| + .{ + ._shape = self.buffers.values()[entry_idx].shape(), + ._id = .{ .buffer_id = self._unique_id + entry_idx }, + ._donation = .input_buffer, + } + else + return null; + } + /// Count layers starting with the given prefix. pub fn countLayers(self: BufferStore, prefix: []const u8) usize { // Note: This is kinda inefficient @@ -260,7 +288,7 @@ pub const MemoryMappedFile = struct { /// Helper handling prefix building. /// /// This allows to easily push/pop prefixes and handles the generation of the string with the correct format. -const PrefixBuilder = struct { +pub const PrefixBuilder = struct { /// Stores the computed prefix. data: std.ArrayListUnmanaged(u8) = .{}, /// Stack storing the size of the intermediary prefix. @@ -271,6 +299,17 @@ const PrefixBuilder = struct { self.subprefixes.deinit(allocator); } + pub fn items(self: PrefixBuilder) []const u8 { + return self.data.items; + } + + pub fn concat(self: *PrefixBuilder, allocator: std.mem.Allocator, prefix: []const u8) []const u8 { + self.push(allocator, prefix) catch unreachable; + const res = self.items(); + self.pop(); + return res; + } + pub fn push(self: *PrefixBuilder, allocator: std.mem.Allocator, prefix: []const u8) !void { const old_len: u32 = @intCast(self.data.items.len); try self.subprefixes.append(allocator, old_len); @@ -322,13 +361,8 @@ fn _populateStruct( const prefix = prefix_builder.data.items; if (T == zml.Tensor) { - return if (buffer_store.buffers.getIndex(prefix)) |entry_idx| { - const buffer = buffer_store.get(prefix).?; - obj.* = zml.Tensor{ - ._shape = buffer.shape(), - ._id = .{ .buffer_id = unique_id + entry_idx }, - ._donation = .input_buffer, - }; + return if (buffer_store.getTensorOrNull(prefix)) |tensor| { + obj.* = tensor; return true; } else { if (required) { @@ -468,7 +502,7 @@ test populateModel { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); - var store: BufferStore = .{ .arena = arena_state }; + var store: BufferStore = .init(arena_state.allocator()); try store.buffers.ensureUnusedCapacity(arena_state.allocator(), 16); store.buffers.putAssumeCapacity("a", Model._newHostBuffer(10)); store.buffers.putAssumeCapacity("b.a", Model._newHostBuffer(20)); diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig index 08952b8f..49ba5fca 100644 --- a/zml/aio/safetensors.zig +++ b/zml/aio/safetensors.zig @@ -13,9 +13,7 @@ const StringBuilder = std.ArrayListUnmanaged(u8); const log = std.log.scoped(.@"zml/io"); pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; + var res: zml.aio.BufferStore = .init(allocator); errdefer res.arena.deinit(); const arena = res.arena.allocator(); diff --git a/zml/aio/tinyllama.zig b/zml/aio/tinyllama.zig index 8114c90f..c51319d6 100644 --- a/zml/aio/tinyllama.zig +++ b/zml/aio/tinyllama.zig @@ -25,9 +25,7 @@ const TinyLlamaConfig = extern struct { /// For convenience we use the same layer names /// than the one used by the Llama-3.1 models. pub fn open(allocator: std.mem.Allocator, model_path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; + var res: zml.aio.BufferStore = .init(allocator); errdefer res.arena.deinit(); const arena = res.arena.allocator(); diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig index 79487f64..c14d511f 100644 --- a/zml/aio/torch.zig +++ b/zml/aio/torch.zig @@ -36,7 +36,7 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore const py_values = try eval.evaluate(tmp_alloc, ops, true); // file ownership is transferred to the BufferStore - var res = try zml.aio.BufferStore.init(allocator, &.{torch_file.mmap_file}); + var res = try zml.aio.BufferStore.initWithFiles(allocator, &.{torch_file.buffer_file}); try torch_file.parseModel(py_values, &res); return res; } diff --git a/zml/testing.zig b/zml/testing.zig index 5fec596d..59778eaf 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -298,7 +298,7 @@ test testLayer { }; // create a buffer store containing the activations: - var activations = try zml.aio.BufferStore.init(std.testing.allocator, &.{}); + var activations = zml.aio.BufferStore.init(std.testing.allocator); defer activations.deinit(); { const input = zml.HostBuffer.fromArray(&[2]f32{ 1, -1 }); From 887f0001b510b84ece3277ee73d56e4f64d59917 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 25 Jul 2025 16:15:47 +0200 Subject: [PATCH 30/67] WIP: mixtral custom loading logic --- examples/mixtral/main.zig | 350 ++++++++++++++++++----------------- examples/mixtral/mixtral.zig | 59 ++++-- 2 files changed, 222 insertions(+), 187 deletions(-) diff --git a/examples/mixtral/main.zig b/examples/mixtral/main.zig index 99c3c1fe..7127edb7 100644 --- a/examples/mixtral/main.zig +++ b/examples/mixtral/main.zig @@ -22,6 +22,158 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; +const cli_params = clap.parseParamsComptime( + \\--help print this help + \\--prompt the prompt + \\--hf-model-path path to the directory containing model weights, config and tokenizer + \\--seed random seed (optional) + \\--seq-len sequence length + \\--create-options platform creation options JSON, defaults to {} + \\--no-llama3 skip prompt template + \\--sharding default: true: sharding on or off +); + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + +pub fn asyncMain() !void { + log.info(" mixtral was compiled with {}", .{@import("builtin").mode}); + + const allocator = std.heap.smp_allocator; + const cli = ClapBoilerplate.parseCli(allocator); + defer cli.deinit(); + + const hf_model_path = cli.args.@"hf-model-path" orelse { + log.err("Missing --hf-model-path", .{}); + return; + }; + + const model_config_path = try std.fs.path.join(allocator, &.{ hf_model_path, "config.json" }); + defer allocator.free(model_config_path); + + const model_weights_path = b: { + const simple_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors" }); + if (asynk.File.access(simple_path, .{})) { + break :b simple_path; + } else |_| { + allocator.free(simple_path); + } + + const sharded_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors.index.json" }); + break :b sharded_path; + }; + defer allocator.free(model_weights_path); + + const model_tokenizer_path = try std.fs.path.join(allocator, &.{ hf_model_path, "tokenizer.json" }); + defer allocator.free(model_tokenizer_path); + + const config = blk: { + var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + var reader = std.json.reader(allocator, config_json_file.reader()); + defer reader.deinit(); + const config_obj = try std.json.parseFromTokenSourceLeaky(mixtral.MixtralLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + break :blk config_obj; + }; + + var context = try zml.Context.init(); + defer context.deinit(); + + const compilation_options = zml.CompilationOptions{ + .xla_dump_to = "/tmp/zml/mixtral", + .sharding_enabled = cli.args.sharding orelse true, + }; + + // initialize ZML platform with optional create options + // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' + const create_opts_json = cli.args.@"create-options" orelse "{}"; + const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); + const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); + create_opts.deinit(); + context.printAvailablePlatforms(platform); + + var store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); + defer store.deinit(); + + var model_arena = std.heap.ArenaAllocator.init(allocator); + const llama_options: mixtral.MixtralLM.Options = .{ + .max_seq_len = @intCast(cli.args.@"seq-len" orelse 256), + .sampling_strategy = .{ + .topk = 1, + .temperature = 1.0, + }, + }; + const model_instance = try mixtral.MixtralLM.init(model_arena.allocator(), store, config, llama_options); + + const dims = model_instance.model.shape(); + const dtype = model_instance.model.embed_tokens.weight.dtype(); + + const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); + const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); + const token_idx_shape = zml.Shape.init(.{}, .u32); + + const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype).withSharding(.{.h}); + + const kv_cache_shape: zml.ShapeOf(mixtral.KvCache) = mixtral.KvCache.initShape(kv_shape); + const rng_shape = zml.Tensor.Rng.shape(); + + var start = try std.time.Timer.start(); + var fut_mod_prefill = try asynk.asyncc(zml.compileModel, .{ + allocator, mixtral.MixtralLM.forward, model_instance, + .{ + tokens_shape_prefill, + token_idx_shape, + kv_cache_shape, + rng_shape, + }, + platform, + }); + + var fut_mod = try asynk.asyncc(zml.compileModel, .{ + allocator, mixtral.MixtralLM.forward, model_instance, + .{ + tokens_shape, + token_idx_shape, + kv_cache_shape, + rng_shape, + }, + platform, + }); + + log.info("\tLoading mixtral weights from {?s}...", .{model_weights_path}); + var llama_weights = try zml.aio.loadModelBuffers(mixtral.MixtralLM, model_instance, store, model_arena.allocator(), platform); + defer zml.aio.unloadBuffers(&llama_weights); + log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); + + var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_weights); + defer llama_module_prefill.deinit(); + var llama_module = (try fut_mod.awaitt()).prepare(llama_weights); + defer llama_module.deinit(); + log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); + + log.info("Creating KvCache", .{}); + const kv_cache = try mixtral.KvCache.initBuffer(kv_shape, platform); + + var tokenizer = blk: { + log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); + var timer = try stdx.time.Timer.start(); + defer log.info("Loaded tokenizer from {s} [{}]", .{ model_tokenizer_path, timer.read() }); + + break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), model_tokenizer_path); + }; + errdefer tokenizer.deinit(); + + const prompt = cli.args.prompt orelse "What is the capital of France?"; + log.info("✅\tPrompt: {s}", .{prompt}); + + const seed = cli.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); + const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; + const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); + // generated text will be printed token by token. + defer allocator.free(generated_text); +} + pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: MixtralLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { var tokens = std.ArrayList(u32).init(allocator); var encoder = try tokenizer.encoder(); @@ -148,185 +300,37 @@ pub fn generateText( return output.toOwnedSlice(); } -const params = clap.parseParamsComptime( - \\--help print this help - \\--prompt the prompt - \\--hf-model-path path to the directory containing model weights, config and tokenizer - \\--seed random seed (optional) - \\--seq-len sequence length - \\--create-options platform creation options JSON, defaults to {} - \\--no-llama3 skip prompt template - \\--sharding default: true: sharding on or off -); +const ClapBoilerplate = struct { + pub const Cli = clap.Result(clap.Help, &cli_params, parsers); -pub fn bool_parser(in: []const u8) error{}!bool { - return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; -} + fn bool_parser(in: []const u8) error{}!bool { + return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; + } -pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { - log.info(" mixtral was compiled with {}", .{@import("builtin").mode}); - - const allocator = std.heap.c_allocator; - - const parsers = comptime .{ + const parsers = .{ .BOOL = bool_parser, .UINT = clap.parsers.int(usize, 0), .STRING = clap.parsers.string, .PATH = clap.parsers.string, }; - var diag: clap.Diagnostic = .{}; - const stderr = std.io.getStdErr().writer(); - var res = clap.parse(clap.Help, ¶ms, parsers, .{ - .diagnostic = &diag, - .allocator = allocator, - }) catch |err| { - diag.report(stderr, err) catch {}; - stderr.print("usage: ", .{}) catch {}; - clap.usage(stderr, clap.Help, ¶ms) catch {}; - stderr.print("\n", .{}) catch {}; - return; - }; - defer res.deinit(); - if (res.args.help != 0) { - clap.help(std.io.getStdErr().writer(), clap.Help, ¶ms, .{}) catch {}; - return; - } - - const hf_model_path = res.args.@"hf-model-path" orelse { - log.err("Missing --hf-model-path", .{}); - return; - }; - - const model_config_path = try std.fs.path.join(allocator, &.{ hf_model_path, "config.json" }); - defer allocator.free(model_config_path); - - const model_weights_path = b: { - const simple_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors" }); - if (asynk.File.access(simple_path, .{})) { - break :b simple_path; - } else |_| { - allocator.free(simple_path); + pub fn parseCli(allocator: std.mem.Allocator) Cli { + var diag: clap.Diagnostic = .{}; + const stderr = std.io.getStdErr().writer(); + const cli = clap.parse(clap.Help, &cli_params, parsers, .{ + .diagnostic = &diag, + .allocator = allocator, + }) catch |err| { + diag.report(stderr, err) catch {}; + stderr.print("usage: ", .{}) catch {}; + clap.usage(stderr, clap.Help, &cli_params) catch {}; + stderr.print("\n", .{}) catch {}; + std.process.exit(1); + }; + if (cli.args.help != 0) { + clap.help(std.io.getStdErr().writer(), clap.Help, &cli_params, .{}) catch {}; + std.process.exit(0); } - - const sharded_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors.index.json" }); - break :b sharded_path; - }; - defer allocator.free(model_weights_path); - - const model_tokenizer_path = try std.fs.path.join(allocator, &.{ hf_model_path, "tokenizer.json" }); - defer allocator.free(model_tokenizer_path); - - const config = blk: { - var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); - defer config_json_file.close() catch unreachable; - var reader = std.json.reader(allocator, config_json_file.reader()); - defer reader.deinit(); - const config_obj = try std.json.parseFromTokenSourceLeaky(mixtral.MixtralLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); - break :blk config_obj; - }; - - var context = try zml.Context.init(); - defer context.deinit(); - - const compilation_options = zml.CompilationOptions{ - .xla_dump_to = "/tmp/zml/mixtral", - .sharding_enabled = res.args.sharding orelse true, - }; - - // initialize ZML platform with optional create options - // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' - const create_opts_json = res.args.@"create-options" orelse "{}"; - const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); - const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); - create_opts.deinit(); - context.printAvailablePlatforms(platform); - - var ts = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); - defer ts.deinit(); - - var model_arena = std.heap.ArenaAllocator.init(allocator); - var model_instance = try zml.aio.populateModel(mixtral.MixtralLM, model_arena.allocator(), ts); - - const llama_options: mixtral.MixtralLM.Options = .{ - .max_seq_len = @intCast(res.args.@"seq-len" orelse 256), - .sampling_strategy = .{ - .topk = 1, - .temperature = 1.0, - }, - }; - model_instance.init(config, llama_options); - - const dims = model_instance.model.shape(); - const dtype = model_instance.model.embed_tokens.weight.dtype(); - - const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); - const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); - const token_idx_shape = zml.Shape.init(.{}, .u32); - - const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype).withSharding(.{.h}); - - const kv_cache_shape: zml.ShapeOf(mixtral.KvCache) = mixtral.KvCache.initShape(kv_shape); - const rng_shape = zml.Tensor.Rng.shape(); - - var start = try std.time.Timer.start(); - var fut_mod_prefill = try asynk.asyncc(zml.compile, .{ - allocator, mixtral.MixtralLM.forward, .{ config, llama_options }, - .{ - tokens_shape_prefill, - token_idx_shape, - kv_cache_shape, - rng_shape, - }, - ts, - platform, - }); - - var fut_mod = try asynk.asyncc(zml.compile, .{ - allocator, mixtral.MixtralLM.forward, .{ config, llama_options }, - .{ - tokens_shape, - token_idx_shape, - kv_cache_shape, - rng_shape, - }, - ts, - platform, - }); - - log.info("\tLoading mixtral weights from {?s}...", .{model_weights_path}); - var llama_weights = try zml.aio.loadBuffers(mixtral.MixtralLM, .{ config, llama_options }, ts, model_arena.allocator(), platform); - defer zml.aio.unloadBuffers(&llama_weights); - log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); - - var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_weights); - defer llama_module_prefill.deinit(); - var llama_module = (try fut_mod.awaitt()).prepare(llama_weights); - defer llama_module.deinit(); - log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); - - log.info("Creating KvCache", .{}); - const kv_cache = try mixtral.KvCache.initBuffer(kv_shape, platform); - - var tokenizer = blk: { - log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); - var timer = try stdx.time.Timer.start(); - defer log.info("Loaded tokenizer from {s} [{}]", .{ model_tokenizer_path, timer.read() }); - - break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), model_tokenizer_path); - }; - errdefer tokenizer.deinit(); - - const prompt = res.args.prompt orelse "What is the capital of France?"; - log.info("✅\tPrompt: {s}", .{prompt}); - - const seed = res.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); - const skip_llama3_encoding = res.args.@"no-llama3" orelse false; - const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); - // generated text will be printed token by token. - defer allocator.free(generated_text); -} + return cli; + } +}; diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index bd45ef8b..f28f31b6 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -40,25 +40,54 @@ pub const MixtralLM = struct { gen_opts: zml.nn.SamplingStrategy = .{}, config: Config, - pub fn init(self: *MixtralLM, config: Config, options: Options) void { - self.config = config; - self.gen_opts = options.sampling_strategy orelse .{}; - self.model.max_seq_len = @intCast(options.max_seq_len); - self.model.num_heads = @intCast(config.num_attention_heads); - self.model.num_kv_heads = @intCast(config.num_key_value_heads); - self.model.rope_opts = .{ - .layout = if (config.hf_rope_impl) .sequential else .interleaved, - .freq_base = config.rope_theta, - .scaling = config.rope_scaling, + pub fn init(allocator: std.mem.Allocator, store: zml.aio.BufferStore, config: Config, options: Options) !MixtralLM { + var self: MixtralLM = .{ + .config = config, + .gen_opts = options.sampling_strategy orelse .{}, + .model = .{ + .max_seq_len = @intCast(options.max_seq_len), + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }, + + .embed_tokens = .{ + .weight = store.getTensor("model.embed_tokens.weight"), + }, + .layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers), + .norm = .{ + .weight = store.getTensor("model.norm.weight"), + .eps = config.rms_norm_eps, + }, + }, + .lm_head = .{ .weight = store.getTensor("lm_head.weight") }, }; - self.model.norm.eps = config.rms_norm_eps; - for (self.model.layers) |*layer| { + + var prefix: zml.aio.PrefixBuilder = .{}; + try prefix.push(allocator, "model.layers"); + for (self.model.layers, 0..) |*layer, i| { + try prefix.pushDigit(allocator, i); + defer prefix.pop(); + + // TODO: safer layer init + layer.self_attn = try zml.aio.populateModelWithPrefix(SelfAttn, allocator, store, prefix.concat(allocator, "self_attn")); layer.self_attn.num_heads = self.model.num_heads; layer.self_attn.num_kv_heads = self.model.num_kv_heads; layer.self_attn.rope_opts = self.model.rope_opts; - layer.input_layernorm.eps = config.rms_norm_eps; - layer.post_attention_layernorm.eps = config.rms_norm_eps; + layer.input_layernorm = .{ + .weight = store.getTensor(prefix.concat(allocator, "input_layernorm.weight")), + .eps = config.rms_norm_eps, + }; + layer.post_attention_layernorm = .{ + .weight = store.getTensor(prefix.concat(allocator, "post_attention_layernorm.weight")), + .eps = config.rms_norm_eps, + }; + + // TODO: load moe // Rewrite to "on_device" layout. layer.block_sparse_moe.rewrite(); const mlp: *Mlp = &layer.block_sparse_moe.on_device.mlp_experts; @@ -77,6 +106,8 @@ pub const MixtralLM = struct { if (self.gen_opts.topk == 1 and self.lm_head != null) { self.lm_head.?.weight = self.lm_head.?.weight.withSharding(.{0}); } + + return self; } /// Predicts the token at `token_index` position. From de3e428b47e123fb4a6c90d444d5083b81d275f5 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 28 Jul 2025 14:06:04 +0200 Subject: [PATCH 31/67] zml.aio.BufferStore: loadBufferById --- examples/mixtral/main.zig | 14 +-- examples/mixtral/mixtral.zig | 235 +++++++++++++++++++++-------------- stdx/stdx.zig | 32 ++++- zml/aio.zig | 58 ++++++++- zml/meta.zig | 1 + 5 files changed, 237 insertions(+), 103 deletions(-) diff --git a/examples/mixtral/main.zig b/examples/mixtral/main.zig index 7127edb7..88c9f20c 100644 --- a/examples/mixtral/main.zig +++ b/examples/mixtral/main.zig @@ -142,14 +142,14 @@ pub fn asyncMain() !void { }); log.info("\tLoading mixtral weights from {?s}...", .{model_weights_path}); - var llama_weights = try zml.aio.loadModelBuffers(mixtral.MixtralLM, model_instance, store, model_arena.allocator(), platform); - defer zml.aio.unloadBuffers(&llama_weights); + var mixtral_weights = try model_instance.loadBuffers(model_arena.allocator(), store, platform); + defer zml.aio.unloadBuffers(&mixtral_weights); log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); - var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_weights); - defer llama_module_prefill.deinit(); - var llama_module = (try fut_mod.awaitt()).prepare(llama_weights); - defer llama_module.deinit(); + var mixtral_module_prefill = (try fut_mod_prefill.awaitt()).prepare(mixtral_weights); + defer mixtral_module_prefill.deinit(); + var mixtral_module = (try fut_mod.awaitt()).prepare(mixtral_weights); + defer mixtral_module.deinit(); log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); log.info("Creating KvCache", .{}); @@ -169,7 +169,7 @@ pub fn asyncMain() !void { const seed = cli.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; - const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); + const generated_text = try generateText(config, model_instance, mixtral_module_prefill, mixtral_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); // generated text will be printed token by token. defer allocator.free(generated_text); } diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index f28f31b6..b075b3a5 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -3,9 +3,6 @@ const testing = std.testing; const stdx = @import("stdx"); const zml = @import("zml"); -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; const log = std.log.scoped(.Mixtral); @@ -66,39 +63,44 @@ pub const MixtralLM = struct { .lm_head = .{ .weight = store.getTensor("lm_head.weight") }, }; - var prefix: zml.aio.PrefixBuilder = .{}; - try prefix.push(allocator, "model.layers"); + var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 1024); + try prefix.push(stdx.noalloc, "model.layers"); for (self.model.layers, 0..) |*layer, i| { - try prefix.pushDigit(allocator, i); + try prefix.pushDigit(stdx.noalloc, i); defer prefix.pop(); // TODO: safer layer init - layer.self_attn = try zml.aio.populateModelWithPrefix(SelfAttn, allocator, store, prefix.concat(allocator, "self_attn")); - layer.self_attn.num_heads = self.model.num_heads; - layer.self_attn.num_kv_heads = self.model.num_kv_heads; - layer.self_attn.rope_opts = self.model.rope_opts; + var self_attn = try zml.aio.populateModelWithPrefix(SelfAttn, allocator, store, prefix.concat("self_attn")); + { + self_attn.num_heads = self.model.num_heads; + self_attn.num_kv_heads = self.model.num_kv_heads; + self_attn.rope_opts = self.model.rope_opts; + self_attn.q_proj.weight = self_attn.q_proj.weight.withSharding(.{0}); + self_attn.k_proj.weight = self_attn.k_proj.weight.withSharding(.{0}); + self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); + self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); + } - layer.input_layernorm = .{ - .weight = store.getTensor(prefix.concat(allocator, "input_layernorm.weight")), - .eps = config.rms_norm_eps, + const on_disk_moe = try zml.aio.populateModelWithPrefix(MoE.OnDisk, allocator, store, prefix.concat("block_sparse_moe")); + var moe = on_disk_moe.rewrite(); + { + moe.mlp_experts.up_proj.weight = moe.mlp_experts.up_proj.weight.withSharding(.{.expert}); + moe.mlp_experts.gate_proj.weight = moe.mlp_experts.gate_proj.weight.withSharding(.{.expert}); + moe.mlp_experts.down_proj.weight = moe.mlp_experts.down_proj.weight.withSharding(.{.expert}); + } + + layer.* = .{ + .input_layernorm = .{ + .weight = store.getTensor(prefix.concat("input_layernorm.weight")), + .eps = config.rms_norm_eps, + }, + .post_attention_layernorm = .{ + .weight = store.getTensor(prefix.concat("post_attention_layernorm.weight")), + .eps = config.rms_norm_eps, + }, + .self_attn = self_attn, + .block_sparse_moe = moe, }; - layer.post_attention_layernorm = .{ - .weight = store.getTensor(prefix.concat(allocator, "post_attention_layernorm.weight")), - .eps = config.rms_norm_eps, - }; - - // TODO: load moe - // Rewrite to "on_device" layout. - layer.block_sparse_moe.rewrite(); - const mlp: *Mlp = &layer.block_sparse_moe.on_device.mlp_experts; - mlp.up_proj.weight = mlp.up_proj.weight.withSharding(.{0}); - mlp.gate_proj.weight = mlp.gate_proj.weight.withSharding(.{0}); - mlp.down_proj.weight = mlp.down_proj.weight.withSharding(.{1}); - - layer.self_attn.q_proj.weight = layer.self_attn.q_proj.weight.withSharding(.{0}); - layer.self_attn.k_proj.weight = layer.self_attn.k_proj.weight.withSharding(.{0}); - layer.self_attn.v_proj.weight = layer.self_attn.v_proj.weight.withSharding(.{0}); - layer.self_attn.o_proj.weight = layer.self_attn.o_proj.weight.withSharding(.{1}); } // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. @@ -117,11 +119,11 @@ pub const MixtralLM = struct { /// - a Rng state to allow for probabilistic generation pub fn forward( self: MixtralLM, - tokens_: Tensor, - token_index: Tensor, + tokens_: zml.Tensor, + token_index: zml.Tensor, kv_cache: KvCache, - rng: Tensor.Rng, - ) struct { Tensor, KvCache, Tensor.Rng } { + rng: zml.Tensor.Rng, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Mixtral ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); @@ -132,10 +134,10 @@ pub const MixtralLM = struct { pub fn sampleTokens( self: MixtralLM, lm_head_: ?zml.nn.Linear, - out_: Tensor, - rng: Tensor.Rng, + out_: zml.Tensor, + rng: zml.Tensor.Rng, opts: zml.nn.SamplingStrategy, - ) struct { Tensor, Tensor.Rng } { + ) struct { zml.Tensor, zml.Tensor.Rng } { const out = out_.withPartialTags(.{ .s, .d }); var logits = blk: { @@ -153,8 +155,35 @@ pub const MixtralLM = struct { return .{ next_tokens, new_rng }; } - pub fn increment(_: u8, token_index: Tensor) Tensor { - return token_index.addConstant(1).reuseBuffer(token_index); + pub fn loadBuffers(self: MixtralLM, allocator: std.mem.Allocator, store: zml.aio.BufferStore, platform: zml.Platform) !zml.Bufferized(MixtralLM) { + var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 256); + defer prefix.deinit(allocator); + + const noalloc = stdx.noalloc; + const loaded: zml.Bufferized(MixtralLM) = .{ + .model = .{ + .embed_tokens = try store.loadModelById(zml.nn.TokenEmbedding, noalloc, self.model.embed_tokens, platform), + .layers = try allocator.alloc(zml.Bufferized(TransformerLayer), self.model.layers.len), + .norm = try store.loadModelById(RmsNorm, noalloc, self.model.norm, platform), + }, + .lm_head = try store.loadModelById(?zml.nn.Linear, noalloc, self.lm_head, platform), + }; + + prefix.push(noalloc, "model.layers") catch unreachable; + for (loaded.model.layers, self.model.layers, 0..) |*d_layer, layer, layer_id| { + const ckpt = prefix.checkpoint(); + defer prefix.restore(ckpt); + + prefix.pushDigit(noalloc, layer_id) catch unreachable; + d_layer.* = .{ + .input_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.input_layernorm, platform), + .self_attn = try store.loadModelById(SelfAttn, noalloc, layer.self_attn, platform), + .post_attention_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.post_attention_layernorm, platform), + .block_sparse_moe = try layer.block_sparse_moe.loadSharded(store, &prefix, platform), + }; + } + + return loaded; } }; @@ -196,7 +225,7 @@ pub const Mixtral = struct { /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. - pub fn forward(self: Mixtral, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { + pub fn forward(self: Mixtral, tokens: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache) struct { zml.Tensor, KvCache } { const embeds = embed(self.embed_tokens, tokens); var hidden = embeds; @@ -209,7 +238,7 @@ pub const Mixtral = struct { return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; } - pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { + pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: zml.Tensor) zml.Tensor { return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); } @@ -226,14 +255,14 @@ pub const TransformerLayer = struct { input_layernorm: RmsNorm, self_attn: SelfAttn, post_attention_layernorm: RmsNorm, - block_sparse_moe: BlockMoe, + block_sparse_moe: MoE, pub fn forward( self: TransformerLayer, - x0: Tensor, - token_index: Tensor, + x0: zml.Tensor, + token_index: zml.Tensor, kv_cache: KvCache, - ) struct { Tensor, KvCache } { + ) struct { zml.Tensor, KvCache } { // Self Attention //log.debug("TransformerLayer({}) -> {}", .{ x0, self.input_layernorm.forward(x0) }); stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); @@ -251,55 +280,77 @@ pub const TransformerLayer = struct { }; const RmsNorm = struct { - weight: Tensor, + weight: zml.Tensor, eps: f32 = 1e-5, /// L2 normalization of input tensor along `.d` axis. - pub fn forward(self: RmsNorm, input: Tensor) Tensor { + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); const normalized = zml.nn.rmsNorm(x, .d, self.eps); return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); } }; -const BlockMoe = union(enum) { - on_disk: struct { +const MoE = struct { + router: zml.nn.Linear, + mlp_experts: Mlp, + num_experts: u32, + + pub const OnDisk = struct { gate: zml.nn.Linear, experts: []struct { w1: zml.nn.Linear, w2: zml.nn.Linear, w3: zml.nn.Linear, }, - }, - on_device: struct { - router: zml.nn.Linear, - mlp_experts: Mlp, - }, - pub fn rewrite(self: *BlockMoe) void { - const on_disk = self.on_disk; - const n = on_disk.experts.len; - var mlp: Mlp = .{ - .up_proj = on_disk.experts[0].w1, - .down_proj = on_disk.experts[0].w2, - .gate_proj = on_disk.experts[0].w3, - }; - mlp.up_proj.weight._shape = mlp.up_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .up, .d }); - mlp.down_proj.weight._shape = mlp.down_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .d, .up }); - mlp.gate_proj.weight._shape = mlp.gate_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .up, .d }); + pub fn rewrite(on_disk: OnDisk) MoE { + const n = on_disk.experts.len; + var mlp: Mlp = .{ + .up_proj = on_disk.experts[0].w1, + .down_proj = on_disk.experts[0].w2, + .gate_proj = on_disk.experts[0].w3, + }; + mlp.up_proj.weight._shape = mlp.up_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .up, .d }); + mlp.down_proj.weight._shape = mlp.down_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .d, .up }); + mlp.gate_proj.weight._shape = mlp.gate_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .up, .d }); - var router = on_disk.gate; - router.weight = router.weight.withTags(.{ .expert, .d }); + var router = on_disk.gate; + router.weight = router.weight.withTags(.{ .expert, .d }); - self.* = .{ - .on_device = .{ .router = router, .mlp_experts = mlp }, + return .{ .router = router, .mlp_experts = mlp, .num_experts = n }; + } + }; + + pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { + const gating = self.router.forward(input).softmax(.expert); + return zml.nn.mixtureOfExperts(Mlp, self.mlp_experts, input, gating, .{ .experts_per_token = 2 }); + } + + pub fn loadSharded(self: MoE, store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, platform: zml.Platform) !zml.Bufferized(MoE) { + prefix.push(stdx.noalloc, "block_sparse_moe.experts") catch unreachable; + var experts_buffers_buf: [16]zml.HostBuffer = undefined; + const experts_buffers = experts_buffers_buf[0..self.num_experts]; + + return .{ + .router = try store.loadModelById(zml.nn.Linear, stdx.noalloc, self.router, platform), + .mlp_experts = .{ + .up_proj = .{ .weight = try loadShardedWeight(store, prefix, platform, self.mlp_experts.up_proj.weight, experts_buffers) }, + }, }; } - pub fn forward(self_block: BlockMoe, input: zml.Tensor) zml.Tensor { - const self = self_block.on_device; - const gating = self.router.forward(input).softmax(.expert); - return zml.nn.mixtureOfExperts(Mlp, self.mlp_experts, input, gating, .{ .experts_per_token = 2 }); + fn loadShardedWeight(store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, name: []const u8, platform: zml.Platform, weight: Tensor, experts_buffers: []zml.HostBuffer) !zml.Buffer { + // "model.layers.0.block_sparse_moe.experts.0.w1.weight" ... "model.layers.0.block_sparse_moe.experts.7.w1.weight" + for (0.., experts_buffers) |i, *expert_host_buffer| { + const ckpt = prefix.checkpoint(); + defer prefix.restore(ckpt); + try prefix.pushDigit(stdx.noalloc, i); + try prefix.push(name); + try prefix.push("weight"); + + expert_host_buffer.* = store.get(prefix.items()) orelse return error.BufferNotFound; + } } }; @@ -308,7 +359,7 @@ const Mlp = struct { gate_proj: zml.nn.Linear, // (dim -> hidden_dim) down_proj: zml.nn.Linear, // (hidden_dim -> dim) - pub fn forward(self: Mlp, x: Tensor) Tensor { + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const up = x.dot(self.up_proj.weight, .d); const gate = x.dot(self.gate_proj.weight, .d).silu(); @@ -337,10 +388,10 @@ pub const SelfAttn = struct { /// x: {.b, .s, .d } -> .{.b, .s, .d} pub fn forward( self: SelfAttn, - x: Tensor, - token_index: Tensor, + x: zml.Tensor, + token_index: zml.Tensor, kv_cache: KvCache, - ) struct { Tensor, KvCache } { + ) struct { zml.Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); @@ -356,7 +407,7 @@ pub const SelfAttn = struct { // In self-attention, .s axis is used both for keys and queries. const pos_index = b: { - const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + const temp = zml.Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); break :b temp.add(token_index.broad(temp.shape())); }; @@ -383,26 +434,26 @@ pub const SelfAttn = struct { const perm = kv_shape.contiguousPerm(.{ .h, .k, .hd }); kv_shape = kv_shape.transpose(perm.constSlice()); var res = KvCache.init(kv_shape); - res.layer_index = Tensor.scalar(0, .u32); + res.layer_index = .scalar(0, .u32); return res; } }; pub const KvCache = struct { - k: Tensor, - v: Tensor, - layer_index: Tensor, + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, pub fn init(kv_shape: zml.Shape) KvCache { // The KV-cache is initialized with ones to detect reads of uninitialized memory. return .{ - .k = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), - .v = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), - .layer_index = Tensor.scalar(-1, .u32), + .k = .constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .v = .constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .layer_index = .scalar(-1, .u32), }; } - pub fn initShape(kv_shape: zml.Shape) ShapeOf(KvCache) { + pub fn initShape(kv_shape: zml.Shape) zml.ShapeOf(KvCache) { return .{ .k = kv_shape, .v = kv_shape, @@ -418,15 +469,15 @@ pub const KvCache = struct { }; } - pub fn keys(self: KvCache) Tensor { - return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + pub fn keys(self: KvCache) zml.Tensor { + return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); } - pub fn values(self: KvCache) Tensor { - return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + pub fn values(self: KvCache) zml.Tensor { + return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); } - pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) KvCache { const k_shape = self.k.shape().drop(.layer); var layer = self.layer_index; layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; @@ -462,7 +513,7 @@ pub const KvCache = struct { return .{ .k = self.k, .v = self.v, - .layer_index = Tensor.scalar(layer_index, .u32), + .layer_index = .scalar(layer_index, .u32), }; } diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 9bfa532a..6d2c28bf 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -10,8 +10,10 @@ pub const meta = @import("meta.zig"); pub const queue = @import("queue.zig"); pub const time = @import("time.zig"); +const std = @import("std"); +const builtin = @import("builtin"); + test { - const std = @import("std"); std.testing.refAllDecls(@This()); } @@ -20,3 +22,31 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { var storage: [max_len]T = undefined; return storage[0..len]; } + +pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; + +pub const mem = struct { + pub fn groupedAlloc(SliceTuple: type, allocator: std.mem.Allocator, len: [@typeInfo(SliceTuple).@"struct".fields.len]usize) error{OutOfMemory}!SliceTuple { + var res: SliceTuple = undefined; + var full_alloc_len: usize = 0; + + const SliceFields = @typeInfo(SliceTuple).@"struct".fields; + inline for (SliceFields, 0.., &res) |field, i, *slice| { + // needed because full_alloc_len starts at 0. + @setRuntimeSafety(false); + + const T = std.meta.Child(field.type); + full_alloc_len = std.mem.alignForward(usize, full_alloc_len, @alignOf(T)); + slice.ptr = @ptrFromInt(full_alloc_len); + slice.len = len[i]; + full_alloc_len += @sizeOf(T) * len[i]; + } + + const bytes = try allocator.alignedAlloc(u8, @alignOf(SliceFields[0].type), full_alloc_len); + inline for (&res) |*slice| { + slice.ptr = @ptrFromInt(@intFromPtr(bytes.ptr) + @intFromPtr(slice.ptr)); + } + + return res; + } +}; diff --git a/zml/aio.zig b/zml/aio.zig index 8edc2ce4..f5538a1c 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -75,6 +75,7 @@ pub const BufferStore = struct { pub const Buffers = std.StringArrayHashMapUnmanaged(HostBuffer); pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); var _unique_store_id: std.atomic.Value(u64) = .init(0); + const _store_id_range: u64 = 1024 * 1024 * 1024; arena: std.heap.ArenaAllocator, files: []MemoryMappedFile = &.{}, @@ -87,7 +88,7 @@ pub const BufferStore = struct { pub fn init(allocator: std.mem.Allocator) BufferStore { return .{ .arena = std.heap.ArenaAllocator.init(allocator), - ._unique_id = _unique_store_id.fetchAdd(1024 * 1024 * 1024, .monotonic), + ._unique_id = _unique_store_id.fetchAdd(_store_id_range, .monotonic), }; } @@ -110,6 +111,41 @@ pub const BufferStore = struct { return self.buffers.get(key); } + pub fn loadBufferById(self: BufferStore, x: zml.Tensor, platform: zml.Platform) !zml.Buffer { + const host_buffer: zml.HostBuffer = switch (x._id) { + .buffer_id => |id| hb: { + if (id < self._unique_id or self._unique_id + _store_id_range <= id) { + @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`, using the same store object."); + } + break :hb self.buffers.values()[id - self._unique_id]; + }, + else => @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`"), + }; + + // TODO use x shape + return try host_buffer.toDevice(platform); + } + + /// Creates a bufferized version of a model from the given BufferStore. + /// + /// This will represent the weights of the model, loaded on a specific platform. + /// It can be used with a `module.Exe` (a compiled version of the same Model), to make a + /// `module.ExeWithWeights` ready to be called. + pub fn loadModelById(self: BufferStore, Model: type, allocator: std.mem.Allocator, model: Model, platform: zml.Platform) !zml.Bufferized(Model) { + const Ctx = struct { + platform: *const zml.Platform, + store: *const BufferStore, + + pub fn cb(ctx: @This(), x: zml.Tensor) zml.Buffer { + return ctx.store.loadBufferById(x, ctx.platform.*) catch @panic("Failed to load buffer to device"); + } + }; + + var res: zml.Bufferized(Model) = undefined; + try zml.meta.mapAlloc(Ctx.cb, allocator, .{ .platform = &platform, .store = &self }, model, &res); + return res; + } + pub fn getTensor(self: BufferStore, key: []const u8) zml.Tensor { return self.getTensorOrNull(key) orelse { findSimilarBufferKeys(key, self, std.heap.smp_allocator); @@ -294,6 +330,14 @@ pub const PrefixBuilder = struct { /// Stack storing the size of the intermediary prefix. subprefixes: std.ArrayListUnmanaged(u32) = .{}, + pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) !PrefixBuilder { + const data, const subprefixes = try stdx.mem.groupedAlloc(struct { []u8, []u32 }, allocator, .{ capacity, @divFloor(capacity, 4) }); + return .{ + .data = .initBuffer(data), + .subprefixes = .initBuffer(subprefixes), + }; + } + pub fn deinit(self: *PrefixBuilder, allocator: std.mem.Allocator) void { self.data.deinit(allocator); self.subprefixes.deinit(allocator); @@ -303,8 +347,8 @@ pub const PrefixBuilder = struct { return self.data.items; } - pub fn concat(self: *PrefixBuilder, allocator: std.mem.Allocator, prefix: []const u8) []const u8 { - self.push(allocator, prefix) catch unreachable; + pub fn concat(self: *PrefixBuilder, prefix: []const u8) []const u8 { + self.push(stdx.noalloc, prefix) catch unreachable; const res = self.items(); self.pop(); return res; @@ -340,6 +384,14 @@ pub const PrefixBuilder = struct { const last_prefix_len = self.subprefixes.pop() orelse unreachable; self.data.shrinkRetainingCapacity(last_prefix_len); } + + pub fn checkpoint(self: PrefixBuilder) [2]usize { + return .{ self.data.items.len, self.subprefixes.items.len }; + } + + pub fn restore(self: *PrefixBuilder, ckpt: [2]usize) void { + self.data.items.len, self.subprefixes.items.len = ckpt; + } }; fn _populateStruct( diff --git a/zml/meta.zig b/zml/meta.zig index ee2fbc68..855318a5 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -141,6 +141,7 @@ test MapType { /// Any `To` struct inside `from` will be copied over to the target. pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam(cb, 0), from: anytype, to: anytype) !void { // TODO: handle tuple to slice conversion + // TODO: handle error bubbling up const From = FnParam(cb, 1); const To = stdx.meta.FnResult(cb); const FromStruct = @TypeOf(from); From 268b6abf779151c9f5c83269e3414f8efb51d2c0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 28 Jul 2025 16:12:54 +0200 Subject: [PATCH 32/67] add CPU device count create option --- zml/platform.zig | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/zml/platform.zig b/zml/platform.zig index 0bc792e9..6e13d7f0 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -82,9 +82,7 @@ pub const Platform = struct { }; const _CreateOptions = struct { - // XLA CPU client doesn't read options - // https://github.com/openxla/xla/blob/42496a28c374bd35f493cc5dbde74805407245dc/xla/pjrt/c/pjrt_c_api_cpu_internal.cc#L33-L46 - cpu: struct {} = .{}, + cpu: Cpu = .{ .device_count = 4 }, // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the Cuda driver @@ -94,6 +92,14 @@ const _CreateOptions = struct { tpu: struct {} = .{}, neuron: struct {} = .{}, + pub const Cpu = struct { + device_count: u32, + + fn writeNamedValues(self: Cpu, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.from("cpu_device_count", @as(i64, self.device_count))); + } + }; + pub const Cuda = struct { allocator: Allocator = .{ .bfc = .{} }, // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 @@ -118,7 +124,7 @@ const _CreateOptions = struct { }; }; - pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); @@ -142,6 +148,7 @@ const _CreateOptions = struct { var values = std.ArrayListUnmanaged(pjrt.NamedValue).fromOwnedSlice(out); values.shrinkRetainingCapacity(0); switch (target) { + .cpu => self.cpu.writeNamedValues(&values), .cuda => self.cuda.writeNamedValues(&values), inline else => |t| { stdx.debug.assertComptime(@hasField(_CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); From 3760650532cad36a4cd625772e3a691f4928d97d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 28 Jul 2025 16:13:29 +0200 Subject: [PATCH 33/67] load sharded experts --- examples/mixtral/main.zig | 7 ++- examples/mixtral/mixtral.zig | 55 ++++++++++++++--- zml/aio.zig | 114 ++++++++++++++++++----------------- 3 files changed, 109 insertions(+), 67 deletions(-) diff --git a/examples/mixtral/main.zig b/examples/mixtral/main.zig index 88c9f20c..390af7c6 100644 --- a/examples/mixtral/main.zig +++ b/examples/mixtral/main.zig @@ -87,8 +87,13 @@ pub fn asyncMain() !void { // initialize ZML platform with optional create options // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' + // or: --create-options='{"cpu":{"device_count":8}}' const create_opts_json = cli.args.@"create-options" orelse "{}"; - const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); + const create_opts = std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}) catch |err| { + log.err("Failed to parse --create-options as json ({}): {s}", .{ err, create_opts_json }); + return err; + }; + const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); create_opts.deinit(); context.printAvailablePlatforms(platform); diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index b075b3a5..87436e70 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -294,7 +294,6 @@ const RmsNorm = struct { const MoE = struct { router: zml.nn.Linear, mlp_experts: Mlp, - num_experts: u32, pub const OnDisk = struct { gate: zml.nn.Linear, @@ -318,7 +317,7 @@ const MoE = struct { var router = on_disk.gate; router.weight = router.weight.withTags(.{ .expert, .d }); - return .{ .router = router, .mlp_experts = mlp, .num_experts = n }; + return .{ .router = router, .mlp_experts = mlp }; } }; @@ -329,28 +328,64 @@ const MoE = struct { pub fn loadSharded(self: MoE, store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, platform: zml.Platform) !zml.Bufferized(MoE) { prefix.push(stdx.noalloc, "block_sparse_moe.experts") catch unreachable; - var experts_buffers_buf: [16]zml.HostBuffer = undefined; - const experts_buffers = experts_buffers_buf[0..self.num_experts]; return .{ .router = try store.loadModelById(zml.nn.Linear, stdx.noalloc, self.router, platform), .mlp_experts = .{ - .up_proj = .{ .weight = try loadShardedWeight(store, prefix, platform, self.mlp_experts.up_proj.weight, experts_buffers) }, + .up_proj = .{ .weight = try loadShardedWeight(store, prefix, "w1", platform, self.mlp_experts.up_proj.weight) }, + .down_proj = .{ .weight = try loadShardedWeight(store, prefix, "w2", platform, self.mlp_experts.down_proj.weight) }, + .gate_proj = .{ .weight = try loadShardedWeight(store, prefix, "w3", platform, self.mlp_experts.gate_proj.weight) }, }, }; } - fn loadShardedWeight(store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, name: []const u8, platform: zml.Platform, weight: Tensor, experts_buffers: []zml.HostBuffer) !zml.Buffer { + fn loadShardedWeight(store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, name: []const u8, platform: zml.Platform, weight: zml.Tensor) !zml.Buffer { + const devices = platform.getDevices(); + const num_experts = weight.dim(.expert); + const num_shards = platform.sharding().num_partitions; + std.debug.assert(num_shards == num_experts); + std.debug.assert(devices.len == num_experts); + // Note: this assumes num_devices == num_partitions, it's incorrect. + + const buffer_type: zml.pjrt.BufferType = switch (weight.shape().dtype()) { + inline else => |tag| @field(zml.pjrt.BufferType, @tagName(tag)), + }; + + var res: zml.Buffer = .{ + ._shape = weight.shape(), + ._api = platform.pjrt_api, + ._shards = .{}, + }; + // "model.layers.0.block_sparse_moe.experts.0.w1.weight" ... "model.layers.0.block_sparse_moe.experts.7.w1.weight" - for (0.., experts_buffers) |i, *expert_host_buffer| { + for (0.., devices) |i, dev| { const ckpt = prefix.checkpoint(); defer prefix.restore(ckpt); try prefix.pushDigit(stdx.noalloc, i); - try prefix.push(name); - try prefix.push("weight"); + try prefix.push(stdx.noalloc, name); + try prefix.push(stdx.noalloc, "weight"); - expert_host_buffer.* = store.get(prefix.items()) orelse return error.BufferNotFound; + const expert = store.get(prefix.items()) orelse { + log.err("Buffer not found: {s}", .{prefix.items()}); + store.findSimilarBufferKeys(std.heap.smp_allocator, prefix.items()); + @panic("Buffer not found"); + }; + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = expert._data, + .buffer_type = buffer_type, + .dims = expert.shape().dims(), + .byte_strides = expert.strides(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .device = dev, + }); + + res._shards.appendAssumeCapacity(pjrt_buffer); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } } + + return res; } }; diff --git a/zml/aio.zig b/zml/aio.zig index f5538a1c..a08e1ae2 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -112,7 +112,7 @@ pub const BufferStore = struct { } pub fn loadBufferById(self: BufferStore, x: zml.Tensor, platform: zml.Platform) !zml.Buffer { - const host_buffer: zml.HostBuffer = switch (x._id) { + var host_buffer: zml.HostBuffer = switch (x._id) { .buffer_id => |id| hb: { if (id < self._unique_id or self._unique_id + _store_id_range <= id) { @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`, using the same store object."); @@ -122,7 +122,9 @@ pub const BufferStore = struct { else => @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`"), }; - // TODO use x shape + stdx.debug.assert(x.shape().eql(host_buffer.shape()), "Can't load buffer {f} for tensor {f}: shape mismatch", .{ host_buffer, x }); + // Copy sharding info + host_buffer._shape = x._shape; return try host_buffer.toDevice(platform); } @@ -148,7 +150,7 @@ pub const BufferStore = struct { pub fn getTensor(self: BufferStore, key: []const u8) zml.Tensor { return self.getTensorOrNull(key) orelse { - findSimilarBufferKeys(key, self, std.heap.smp_allocator); + self.findSimilarBufferKeys(std.heap.smp_allocator, key); @panic("Tensor not found"); }; } @@ -204,6 +206,58 @@ pub const BufferStore = struct { return null; } + + /// Assists in debuggigng `BufferNotFound` error + /// This is useful when a buffer key is not found and you want to identify possible alternatives (or typos) + pub fn findSimilarBufferKeys(store: BufferStore, tmp_alloc: std.mem.Allocator, original_key: []const u8) void { + const suffixes = [_][]const u8{ "", ".weight", ".bias" }; + var shown_keys = std.StringHashMap(void).init(tmp_alloc); + defer shown_keys.deinit(); + + // remove suffix .weight and .bias + var base_key = original_key; + for (suffixes) |suffix| { + if (std.mem.endsWith(u8, original_key, suffix)) { + base_key = original_key[0 .. original_key.len - suffix.len]; + break; + } + } + + // first test: look for exact matches + var matches: usize = 0; + var it = store.buffers.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + if (std.mem.startsWith(u8, key, base_key)) { + if (matches == 0) log.warn("Similar buffers found:", .{}); + if (!shown_keys.contains(key)) { + log.warn(" - {s}: {}", .{ key, entry.value_ptr.*.shape() }); + shown_keys.put(key, {}) catch continue; + matches += 1; + } + } + } + + // second test: progressive partial matches + if (matches == 0) { + var components = std.mem.splitScalar(u8, base_key, '.'); + while (components.next()) |component| { + matches = 0; + it = store.buffers.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + if (std.mem.indexOf(u8, key, component) != null and !shown_keys.contains(key)) { + if (matches == 0) log.warn("Partial matches for '{s}':", .{component}); + log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); + shown_keys.put(key, {}) catch continue; + matches += 1; + if (matches >= 5) break; + } + } + if (matches > 0) break; + } + } + } }; pub const Metadata = union(enum) { @@ -695,58 +749,6 @@ pub fn unloadBuffers(model: anytype) void { }).cb, {}, model); } -/// Assists in debuggigng `BufferNotFound` error -/// This is useful when a buffer key is not found and you want to identify possible alternatives (or typos) -fn findSimilarBufferKeys(original_key: []const u8, store: BufferStore, temp_allocator: std.mem.Allocator) void { - const suffixes = [_][]const u8{ "", ".weight", ".bias" }; - var shown_keys = std.StringHashMap(void).init(temp_allocator); - defer shown_keys.deinit(); - - // remove suffix .weight and .bias - var base_key = original_key; - for (suffixes) |suffix| { - if (std.mem.endsWith(u8, original_key, suffix)) { - base_key = original_key[0 .. original_key.len - suffix.len]; - break; - } - } - - // first test: look for exact matches - var matches: usize = 0; - var it = store.buffers.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.startsWith(u8, key, base_key)) { - if (matches == 0) log.warn("Similar buffers found:", .{}); - if (!shown_keys.contains(key)) { - log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); - shown_keys.put(key, {}) catch continue; - matches += 1; - } - } - } - - // second test: progressive partial matches - if (matches == 0) { - var components = std.mem.splitScalar(u8, base_key, '.'); - while (components.next()) |component| { - matches = 0; - it = store.buffers.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.indexOf(u8, key, component) != null and !shown_keys.contains(key)) { - if (matches == 0) log.warn("Partial matches for '{s}':", .{component}); - log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); - shown_keys.put(key, {}) catch continue; - matches += 1; - if (matches >= 5) break; - } - } - if (matches > 0) break; - } - } -} - /// deinit all buffers in the given struct pub fn awaitAll(buffers: anytype) !void { zml.meta.visit((struct { @@ -778,7 +780,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi } else { log.err("Buffer not found: {s}", .{prefix}); - findSimilarBufferKeys(prefix, buffer_store, allocator); + buffer_store.findSimilarBufferKeys(allocator, prefix); return error.BufferNotFound; }; From 7f762bc394c66462c29013d9924f9371a48403a4 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 28 Jul 2025 17:27:07 +0200 Subject: [PATCH 34/67] use buffer.uninitialized --- examples/mixtral/mixtral.zig | 82 +++++++++++++++--------------------- 1 file changed, 34 insertions(+), 48 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 87436e70..210f4666 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -52,7 +52,7 @@ pub const MixtralLM = struct { }, .embed_tokens = .{ - .weight = store.getTensor("model.embed_tokens.weight"), + .weight = store.getTensor("model.embed_tokens.weight").withSharding(.{1}), }, .layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers), .norm = .{ @@ -60,7 +60,7 @@ pub const MixtralLM = struct { .eps = config.rms_norm_eps, }, }, - .lm_head = .{ .weight = store.getTensor("lm_head.weight") }, + .lm_head = .{ .weight = store.getTensor("lm_head.weight").withSharding(.{0}) }, }; var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 1024); @@ -241,14 +241,6 @@ pub const Mixtral = struct { pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: zml.Tensor) zml.Tensor { return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); } - - fn initKvCache(self: Mixtral, embed_shape: zml.Shape) KvCache { - const dims = self.shape(); - var kv_shape = embed_shape.insert(0, .{ .layer = dims.layer }).rename(.{ .s = .k }).splitAxes(.{ .d = .{ .h = dims.nkvh, .hd = dims.hd } }); - const perm = kv_shape.contiguousPerm(.{ .k, .h, .hd }); - kv_shape = kv_shape.transpose(perm.constSlice()); - return KvCache.init(kv_shape); - } }; pub const TransformerLayer = struct { @@ -343,8 +335,6 @@ const MoE = struct { const devices = platform.getDevices(); const num_experts = weight.dim(.expert); const num_shards = platform.sharding().num_partitions; - std.debug.assert(num_shards == num_experts); - std.debug.assert(devices.len == num_experts); // Note: this assumes num_devices == num_partitions, it's incorrect. const buffer_type: zml.pjrt.BufferType = switch (weight.shape().dtype()) { @@ -357,32 +347,38 @@ const MoE = struct { ._shards = .{}, }; - // "model.layers.0.block_sparse_moe.experts.0.w1.weight" ... "model.layers.0.block_sparse_moe.experts.7.w1.weight" - for (0.., devices) |i, dev| { - const ckpt = prefix.checkpoint(); - defer prefix.restore(ckpt); - try prefix.pushDigit(stdx.noalloc, i); - try prefix.push(stdx.noalloc, name); - try prefix.push(stdx.noalloc, "weight"); + if (devices.len == num_experts and num_shards == num_experts) { + // "model.layers.0.block_sparse_moe.experts.0.w1.weight" ... "model.layers.0.block_sparse_moe.experts.7.w1.weight" + for (0.., devices) |i, dev| { + const ckpt = prefix.checkpoint(); + defer prefix.restore(ckpt); + try prefix.pushDigit(stdx.noalloc, i); + try prefix.push(stdx.noalloc, name); + try prefix.push(stdx.noalloc, "weight"); - const expert = store.get(prefix.items()) orelse { - log.err("Buffer not found: {s}", .{prefix.items()}); - store.findSimilarBufferKeys(std.heap.smp_allocator, prefix.items()); - @panic("Buffer not found"); - }; - const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ - .data = expert._data, - .buffer_type = buffer_type, - .dims = expert.shape().dims(), - .byte_strides = expert.strides(), - .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .device = dev, - }); + const expert = store.get(prefix.items()) orelse { + log.err("Buffer not found: {s}", .{prefix.items()}); + store.findSimilarBufferKeys(std.heap.smp_allocator, prefix.items()); + @panic("Buffer not found"); + }; + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = expert._data, + .buffer_type = buffer_type, + .dims = expert.shape().dims(), + .byte_strides = expert.strides(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .device = dev, + }); - res._shards.appendAssumeCapacity(pjrt_buffer); - if (event) |ev| { - ev.deinit(platform.pjrt_api); + res._shards.appendAssumeCapacity(pjrt_buffer); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } } + } else if (num_shards == 1) { + @panic("TODO"); + } else { + @panic("TODO"); } return res; @@ -462,16 +458,6 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; } - - fn initKvCache(key_shape: zml.Shape) KvCache { - // When we call initKvCache, we haven't renamed .s to .k yet. - var kv_shape = key_shape.insert(0, .{ .layer = 1 }).rename(.{ .s = .k }); - const perm = kv_shape.contiguousPerm(.{ .h, .k, .hd }); - kv_shape = kv_shape.transpose(perm.constSlice()); - var res = KvCache.init(kv_shape); - res.layer_index = .scalar(0, .u32); - return res; - } }; pub const KvCache = struct { @@ -498,9 +484,9 @@ pub const KvCache = struct { pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { return .{ - .k = try zml.Buffer.constant(platform, kv_shape, 1), - .v = try zml.Buffer.constant(platform, kv_shape, 1), - .layer_index = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0), + .k = try zml.Buffer.uninitialized(platform, kv_shape, .{}), + .v = try zml.Buffer.uninitialized(platform, kv_shape, .{}), + .layer_index = try zml.Buffer.uninitialized(platform, .scalar(.u32), .{}), }; } From 564dba43aaf193977126ae8d451986bdd2f64857 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 29 Jul 2025 09:40:31 +0200 Subject: [PATCH 35/67] move bufferTypeFromDtype to pjrtx.zig --- examples/mixtral/mixtral.zig | 7 +++-- zml/buffer.zig | 32 ++----------------- zml/nn.zig | 2 +- zml/pjrtx.zig | 59 +++++++++++++++++++++++++++--------- 4 files changed, 54 insertions(+), 46 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 210f4666..9eabafa7 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -349,6 +349,8 @@ const MoE = struct { if (devices.len == num_experts and num_shards == num_experts) { // "model.layers.0.block_sparse_moe.experts.0.w1.weight" ... "model.layers.0.block_sparse_moe.experts.7.w1.weight" + const individual_shape = weight.shape().setDim(.expert, 1); + const individual_strides = individual_shape.computeStrides(); for (0.., devices) |i, dev| { const ckpt = prefix.checkpoint(); defer prefix.restore(ckpt); @@ -361,11 +363,12 @@ const MoE = struct { store.findSimilarBufferKeys(std.heap.smp_allocator, prefix.items()); @panic("Buffer not found"); }; + std.debug.assert(individual_shape.drop(.expert).eql(expert.shape())); const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ .data = expert._data, .buffer_type = buffer_type, - .dims = expert.shape().dims(), - .byte_strides = expert.strides(), + .dims = individual_shape.dims(), + .byte_strides = individual_strides.slice(), .host_buffer_semantics = .ImmutableUntilTransferCompletes, .device = dev, }); diff --git a/zml/buffer.zig b/zml/buffer.zig index f502d703..9f261e6e 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -71,7 +71,7 @@ pub const Buffer = struct { break :cs @divExact(host_buffer.dim(ax), n_partitions); } else 0; - const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); + const buffer_type = pjrt.bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides(); const devices = platform.getDevices(); @@ -281,7 +281,7 @@ pub const Buffer = struct { const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, - .element_type = bufferTypeFromDtype(shape_.dtype()), + .element_type = pjrt.bufferTypeFromDtype(shape_.dtype()), .dims = shape_.dims(), // TODO: exposes sharding in the API. .device = platform.getDevices()[0], @@ -462,7 +462,7 @@ pub const Buffer = struct { break :s shard_shape; } else shape_; - const buffer_type = bufferTypeFromDtype(shape_.dtype()); + const buffer_type = pjrt.bufferTypeFromDtype(shape_.dtype()); const devices = platform.getDevices(); for (0..n_partitions) |i| { var args = pjrt.Client.CreateUninitializedBufferArgs{ @@ -494,29 +494,3 @@ pub const Buffer = struct { return res; } }; - -pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { - return switch (dt) { - inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), - }; -} - -pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { - return switch (pjrt_type) { - .invalid => @panic("Found an invalid pjrt buffer"), - inline else => |tag| @field(DataType, @tagName(tag)), - }; -} - -test bufferTypeFromDtype { - inline for (@typeInfo(DataType).@"enum".fields) |field| { - const dt: DataType = @enumFromInt(field.value); - try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); - } - - inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { - const dt: pjrt.BufferType = @enumFromInt(field.value); - if (dt == .invalid) continue; - try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); - } -} diff --git a/zml/nn.zig b/zml/nn.zig index 4ebfb95f..b652bc42 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -98,7 +98,7 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te // log.warn(" -> input_per_expert {f} -> output_per_expert {f}", .{ input_per_expert, output_per_expert }); output_per_expert = output_per_expert.mul(routing_score.broad(output_per_expert.shape())); - // Reverse engineer the normal output shape that one-expert would have produced for all tokens. + // Reverse engineer the normal output shape that one expert would have produced for all tokens. // If this provide to not be enough we could use the "sliced_expert" strategy and call forward ourselves. const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .expert_token = .s }).setDim(.s, num_tokens); var output: Tensor = .constant(output_shape, input.dtype().zero()); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 8658aebe..476bcf4a 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -5,25 +5,30 @@ const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); pub const ffi = pjrt.ffi; -pub const ApiError = pjrt.ApiError; -pub const ErrorCode = pjrt.ErrorCode; -pub const ExecuteContext = pjrt.ExecuteContext; -pub const BufferType = pjrt.BufferType; -pub const Device = pjrt.Device; -pub const MemoryStats = pjrt.MemoryStats; -pub const DeviceDescription = pjrt.DeviceDescription; pub const Api = pjrt.Api; -pub const NamedValue = pjrt.NamedValue; +pub const ApiError = pjrt.ApiError; +pub const BufferType = pjrt.BufferType; pub const ClientInitError = pjrt.ClientInitError; -pub const Error = pjrt.Error; -pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; -pub const SerializeResult = pjrt.SerializeResult; -pub const Executable = pjrt.Executable; pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; +pub const Device = pjrt.Device; +pub const DeviceDescription = pjrt.DeviceDescription; +pub const Error = pjrt.Error; +pub const ErrorCode = pjrt.ErrorCode; +pub const Executable = pjrt.Executable; +pub const ExecuteContext = pjrt.ExecuteContext; pub const ExecuteError = ApiError; +pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; pub const Memory = pjrt.Memory; +pub const MemoryStats = pjrt.MemoryStats; +pub const NamedValue = pjrt.NamedValue; +pub const Profiler = pjrt.Profiler; +pub const SerializeResult = pjrt.SerializeResult; pub const Stream = pjrt.Stream; +const zml = struct { + pub const DataType = @import("dtype.zig").DataType; +}; + const log = std.log.scoped(.zml); pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || ApiError; @@ -159,8 +164,8 @@ pub const Buffer = opaque { return @ptrCast(try self.inner().toHostBuffer(api, dst)); } - pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { - return self.inner().getElementType(api); + pub fn getElementType(self: *const Buffer, api: *const Api) zml.DataType { + return dtypeFromBufferType(self.inner().getElementType(api)); } pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { @@ -321,3 +326,29 @@ pub const AsyncHostToDeviceTransferManager = opaque { return self.inner().addMetadata(api, transfer_metadata); } }; + +pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), + }; +} + +pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) zml.DataType { + return switch (pjrt_type) { + .invalid => @panic("Found an invalid pjrt buffer"), + inline else => |tag| @field(zml.DataType, @tagName(tag)), + }; +} + +test bufferTypeFromDtype { + inline for (@typeInfo(zml.DataType).@"enum".fields) |field| { + const dt: zml.DataType = @enumFromInt(field.value); + try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); + } + + inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { + const dt: pjrt.BufferType = @enumFromInt(field.value); + if (dt == .invalid) continue; + try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); + } +} From 60c15b92ca565d00678f9c257e8879941980e21f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 29 Jul 2025 10:14:38 +0200 Subject: [PATCH 36/67] sort imports --- examples/mixtral/mixtral.zig | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 9eabafa7..45f6c98a 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -337,9 +337,7 @@ const MoE = struct { const num_shards = platform.sharding().num_partitions; // Note: this assumes num_devices == num_partitions, it's incorrect. - const buffer_type: zml.pjrt.BufferType = switch (weight.shape().dtype()) { - inline else => |tag| @field(zml.pjrt.BufferType, @tagName(tag)), - }; + const buffer_type = zml.pjrt.bufferTypeFromDtype(weight.shape().dtype()); var res: zml.Buffer = .{ ._shape = weight.shape(), From 0adfd2ead69b55879785da6b6f4c3e8df51225b1 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 29 Jul 2025 16:24:11 +0200 Subject: [PATCH 37/67] mixtral: use transferData for MoE layers --- examples/mixtral/mixtral.zig | 49 ++++++++++++++---------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 45f6c98a..4587c01e 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -335,24 +335,23 @@ const MoE = struct { const devices = platform.getDevices(); const num_experts = weight.dim(.expert); const num_shards = platform.sharding().num_partitions; - // Note: this assumes num_devices == num_partitions, it's incorrect. - const buffer_type = zml.pjrt.bufferTypeFromDtype(weight.shape().dtype()); + // Note: this requires num_devices == num_partitions, this is overly restrictive. + if (devices.len == num_shards and num_experts % num_shards == 0 and num_experts >= num_shards) { + const num_experts_per_shard = @divExact(num_experts, num_shards); + const tmp_buf: []u8 = try std.heap.smp_allocator.alloc(u8, 4096); + defer std.heap.smp_allocator.free(tmp_buf); + var fba: std.heap.FixedBufferAllocator = .init(tmp_buf); - var res: zml.Buffer = .{ - ._shape = weight.shape(), - ._api = platform.pjrt_api, - ._shards = .{}, - }; + const allocator = fba.allocator(); + const transfer = try platform.batchedTransfer(allocator, &.{weight.shape()}, .device); - if (devices.len == num_experts and num_shards == num_experts) { - // "model.layers.0.block_sparse_moe.experts.0.w1.weight" ... "model.layers.0.block_sparse_moe.experts.7.w1.weight" - const individual_shape = weight.shape().setDim(.expert, 1); - const individual_strides = individual_shape.computeStrides(); - for (0.., devices) |i, dev| { + for (0..num_experts) |expert_id| { + const part_id = expert_id % num_experts_per_shard; + const shard_id = @divFloor(expert_id, num_experts_per_shard); const ckpt = prefix.checkpoint(); defer prefix.restore(ckpt); - try prefix.pushDigit(stdx.noalloc, i); + try prefix.pushDigit(stdx.noalloc, expert_id); try prefix.push(stdx.noalloc, name); try prefix.push(stdx.noalloc, "weight"); @@ -361,28 +360,18 @@ const MoE = struct { store.findSimilarBufferKeys(std.heap.smp_allocator, prefix.items()); @panic("Buffer not found"); }; - std.debug.assert(individual_shape.drop(.expert).eql(expert.shape())); - const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ - .data = expert._data, - .buffer_type = buffer_type, - .dims = individual_shape.dims(), - .byte_strides = individual_strides.slice(), - .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .device = dev, - }); - res._shards.appendAssumeCapacity(pjrt_buffer); - if (event) |ev| { - ev.deinit(platform.pjrt_api); - } + const ev = transfer.transferData( + .{ .buffer_id = 0, .device_id = @intCast(shard_id), .offset = expert.bytes().len * part_id }, + expert.bytes(), + part_id + 1 == num_experts_per_shard, + ); + _ = ev; } - } else if (num_shards == 1) { - @panic("TODO"); + return transfer.buffers[0]; } else { @panic("TODO"); } - - return res; } }; From 1204928cf817a6c3046c986de1a1ef0ed6742313 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 29 Jul 2025 23:06:24 +0200 Subject: [PATCH 38/67] topK should rename the axis by default --- zml/nn.zig | 22 +++++++++++----------- zml/tensor.zig | 39 ++++++++++++++++++++++++--------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index b652bc42..5873a5f8 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -70,10 +70,10 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te if (num_tokens * opts.experts_per_token < num_experts) { // Few tokens: some experts will be unused. // select active experts and compute with that. - const routing = gating.topK(opts.experts_per_token, .expert, .{}); - const routing_score = routing.values.sum(.expert); - const routing_weight = routing.values.div(routing_score.broad(routing.values.shape())); + const routing = gating.topK(opts.experts_per_token, .expert, .{ .descending = true }); + const routing_score = routing.values.div(routing.values.sum(.expert).broad(routing.values.shape())); + // TODO consider unrolling this on num_tokens * opts.experts_per_token to avoid gather of weights. var sliced_expert: Expert = undefined; zml.meta.mapAlloc(struct { pub fn cb(routing_indices: Tensor, expert_weight: Tensor) Tensor { @@ -82,7 +82,8 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te }.cb, undefined, routing.indices, experts, &sliced_expert) catch unreachable; const output_per_expert = sliced_expert.forward(input); - return output_per_expert.dot(routing_weight, .expert); + // log.warn(" -> routing {f} -> sliced_expert {any} -> output_per_expert {f}", .{ routing_score, sliced_expert, output_per_expert }); + return output_per_expert.dot(routing_score, .expert); } // Lot of tokens, each experts chose their tokens and compute on that. @@ -95,14 +96,13 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te const routing_ids = routing.indices.transpose(.{ .expert, .expert_token }); const input_per_expert = input.gather(.{ .s = routing_ids }, .{}); var output_per_expert = experts.forward(input_per_expert); - // log.warn(" -> input_per_expert {f} -> output_per_expert {f}", .{ input_per_expert, output_per_expert }); output_per_expert = output_per_expert.mul(routing_score.broad(output_per_expert.shape())); // Reverse engineer the normal output shape that one expert would have produced for all tokens. // If this provide to not be enough we could use the "sliced_expert" strategy and call forward ourselves. const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .expert_token = .s }).setDim(.s, num_tokens); var output: Tensor = .constant(output_shape, input.dtype().zero()); - // log.warn(" -> output_per_expert {f}, routing: {f} -> output {f}", .{ output_per_expert, routing_ids, output }); + // log.warn(" -> input_per_expert {f} -> output_per_expert {f}, routing: {f} -> output {f}", .{ input_per_expert, output_per_expert, routing_ids, output }); output = output.scatterSlices(.{ .s = routing_ids }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); return output; @@ -953,9 +953,9 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng return .{ next_tokens, rng }; } - const topk = activations.topK(opts.topk, .voc, .{}); - // After the topk, we don't have .voc values, anymore, only topk. - var x = topk.values.rename(.{ .voc = .topk }); + const topk = activations.topK(.{ .topk = .voc }, opts.topk, .{}); + // After the topk, we don't have .voc values, anymore, only .topk. + var x = topk.values; if (opts.temperature != 1.0) { x = x.scale(1 / opts.temperature); } @@ -970,7 +970,7 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng // topk_idx is indices into topk.values ! so in the range [0, topk] // Convert for the original indices from the full [0, voc] range. - const next_tokens = topk.indices.gather(.{ .voc = topk_idx.squeeze(.topk) }, .{}); + const next_tokens = topk.indices.gather(.{ .topk = topk_idx.squeeze(.topk) }, .{}); // log.debug("sampleTokens({}) -> {} -> {} -> {}", .{ activations, topk.indices, topk_idx, next_tokens }); return .{ next_tokens, next_rng }; } @@ -1067,7 +1067,7 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { // First reduce the vocab size to a reasonable sub set of candidate. const full_topk = if (opts.max_top_k > 0) - logits.topK(opts.max_top_k, .voc, .{ .descending = true }) + logits.topK(.{ .voc = .voc }, opts.max_top_k, .{ .descending = true }) else logits.sort(.voc, .{ .descending = true }); diff --git a/zml/tensor.zig b/zml/tensor.zig index 1dc53b6c..b55f2514 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2232,16 +2232,14 @@ pub const Tensor = struct { var self_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - _ = t; const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); - // if (indices_shape.hasTag(t)) |id_ax| { - // // tag is both in self and indices -> it's a batching dim - // // Note: tags are required for batching. - // self_kind.buffer[self_ax] = .batching; - // idx_batch_axes.appendAssumeCapacity(id_ax); - // // stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); - // } else - if (is_gather_axis) { + if (indices_shape.hasTag(t)) |id_ax| { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { // we collapsed all gathered axes self_kind.buffer[self_ax] = .collapsed; // idx_kind.buffer[id_ax] = .indices; @@ -3009,15 +3007,26 @@ pub const Tensor = struct { } /// Returns a Tensor representing the result of Top-K over the given axis. - pub fn topK(self: Tensor, k: u32, axis_: anytype, opts: struct { descending: bool = true }) SortRes { + pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: struct { descending: bool = true }) SortRes { // TODO: ask the user for a new name for the axis, this will simplify tagged code. // See eg code in `sampleTokens` and `mixtureOfExpert` - const a = self.axis(axis_); - const result = self.sort(a, .{ .descending = opts.descending }); - return .{ - .values = result.values.slice1d(a, .{ .end = k }), - .indices = result.indices.slice1d(a, .{ .end = k }), + const err_msg = "topK named axis should be an integer or a named axis, eg `x.topK(.{{ .best_token = .token }}, 16)` or `x.topK(-1, 16)`"; + const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { + .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, + .@"struct" => |info| blk: { + stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); + break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; + }, + else => stdx.debug.compileError(err_msg, .{}), }; + var result = self.sort(a, .{ .descending = opts.descending }); + result.values = result.values.slice1d(a, .{ .end = k }); + result.indices = result.indices.slice1d(a, .{ .end = k }); + if (has_name) |new_name| { + result.values._shape._tags.set(a, new_name.ptr); + result.indices._shape._tags.set(a, new_name.ptr); + } + return result; } pub const MaxPoolRes = ArgMaxRes; From 3de51d55acd754126247849e3b8ac7eb3649cedc Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 30 Jul 2025 15:58:30 +0200 Subject: [PATCH 39/67] binaryOp does some broadcasting --- zml/aio.zig | 2 +- zml/shape.zig | 8 ++++++++ zml/tensor.zig | 44 +++++++++++++++++++++++--------------------- zml/testing.zig | 2 +- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/zml/aio.zig b/zml/aio.zig index a08e1ae2..48100979 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -231,7 +231,7 @@ pub const BufferStore = struct { if (std.mem.startsWith(u8, key, base_key)) { if (matches == 0) log.warn("Similar buffers found:", .{}); if (!shown_keys.contains(key)) { - log.warn(" - {s}: {}", .{ key, entry.value_ptr.*.shape() }); + log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); shown_keys.put(key, {}) catch continue; matches += 1; } diff --git a/zml/shape.zig b/zml/shape.zig index bd5f2a22..efb3d808 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -419,6 +419,13 @@ pub const Shape = struct { // Already the right shape if (std.mem.eql(i64, self.dims(), other.dims())) return true; + if (std.mem.eql(Tag, self.tags(), other.tags())) { + for (0..self.rank()) |i| { + if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; + } + return true; + } + // Non ambiguous broadcasting // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } @@ -430,6 +437,7 @@ pub const Shape = struct { } for (self.dims(), self.tags()) |d, t| { + // TODO this is also wrong when the axes are in different order in the two shapes. const other_ax = other.hasTag(t) orelse return false; if (d != 1 and d != other.dim(other_ax)) return false; } diff --git a/zml/tensor.zig b/zml/tensor.zig index b55f2514..68a7ad2b 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1078,7 +1078,7 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, comptime contracting: anytype) Tensor { - var contracting_axes: std.BoundedArray([2]i8, MAX_RANK) = .{}; + var contracting_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; if (@TypeOf(contracting) == EnumLiteral) { contracting_axes.appendAssumeCapacity(.{ lhs.axis(contracting), rhs.axis(contracting) }); } else { @@ -1087,7 +1087,7 @@ pub const Tensor = struct { } } - var batching_axes: std.BoundedArray([2]i8, MAX_RANK) = .{}; + var batching_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; for (lhs._shape.tags(), 0..) |l, li| { stdx.debug.assert(l != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, lhs }); @@ -2209,7 +2209,7 @@ pub const Tensor = struct { } pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - scoped_log.warn("gather({f}, {d}, {f})", .{ self, idx_axes, idx_per_axis }); + scoped_log.warn("gather({f}, {any}, {any})", .{ self, idx_axes, idx_per_axis }); stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); for (idx_axes, 0..) |a, i| { if (i > 0) { @@ -2229,7 +2229,7 @@ pub const Tensor = struct { var idx_batch_axes: Shape.DimsArray = .{}; const AxisKind = enum { batching, offset, collapsed, indices }; - var self_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); @@ -2250,7 +2250,7 @@ pub const Tensor = struct { // compute res shape var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{}; + var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); if (ax == idx_axes[0]) { @@ -2302,7 +2302,7 @@ pub const Tensor = struct { .index_vector_dim = indices.axis(.coord), .indices_are_sorted = opts.indices_are_sorted, }; - scoped_log.warn(" --> stablehlo.gather({f}, {f}, {d}, {})", .{ self, indices, slice_dims.slice(), args }); + scoped_log.warn(" --> stablehlo.gather({f}, {f}, {any}, {})", .{ self, indices, slice_dims.slice(), args }); const gather_op = dialect.stablehlo.gather( self.getContext().mlirCtx(), @@ -3881,11 +3881,20 @@ pub const Tensor = struct { return binaryOpHelper(self, other.broad(self._shape)); } - stdx.debug.assert(self._shape.eql(other._shape), "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); + var other_ = other; + var same_shape = self._shape.eql(other._shape); + if (!same_shape and std.mem.eql(Shape.Tag, self._shape.tags(), other._shape.tags()) and other._shape.canBroadcastTo(self._shape)) { + // Only a restrictive version of broadcasting is allowed here, where all the tags matches already. + // Typical use case: `x.div(x.sum(.a))` + same_shape = true; + other_ = other.broad(self._shape); + } + + stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); const ctx = self.getContext(); - const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other }); - const ret = @call(.auto, op_fn, .{ ctx.mlirCtx(), self.value(), other.value(), location }); + const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other_ }); + const ret = @call(.auto, op_fn, .{ ctx.mlirCtx(), self.value(), other_.value(), location }); return _result(self._shape, ret.result(0)); } }.binaryOpHelper; @@ -3958,21 +3967,14 @@ test "Tensor.maxPool1d" { const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5 }, &data); const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2 }, .f32), result.values.shape()); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); - const buffer = result.values.getValue([2][2][2]f32); + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); try std.testing.expectEqualDeep( [2][2][2]f32{ - [2][2]f32{ - [2]f32{ 2, 4 }, - [2]f32{ 7, 9 }, - }, - [2][2]f32{ - [2]f32{ 12, 14 }, - [2]f32{ 17, 19 }, - }, + .{ .{ 2, 4 }, .{ 7, 9 } }, + .{ .{ 12, 14 }, .{ 17, 19 } }, }, - buffer, + result.values.getValue([2][2][2]f32), ); } diff --git a/zml/testing.zig b/zml/testing.zig index 59778eaf..0e2959bb 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -33,7 +33,7 @@ pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool return closeRel or closeAbs; } -/// Testing utility. Accepts both Tensor and HostBuffer but Tensor will be copied to the +/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the /// host for comparison ! pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; From 97af4005136090ca8c643c7a5423c54da75750d4 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 30 Jul 2025 15:58:48 +0200 Subject: [PATCH 40/67] fix mixtral weight loading --- examples/mixtral/mixtral.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 4587c01e..6db61586 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -314,7 +314,7 @@ const MoE = struct { }; pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { - const gating = self.router.forward(input).softmax(.expert); + const gating = self.router.forward(input).convert(.f32).softmax(.expert); return zml.nn.mixtureOfExperts(Mlp, self.mlp_experts, input, gating, .{ .experts_per_token = 2 }); } @@ -324,9 +324,9 @@ const MoE = struct { return .{ .router = try store.loadModelById(zml.nn.Linear, stdx.noalloc, self.router, platform), .mlp_experts = .{ - .up_proj = .{ .weight = try loadShardedWeight(store, prefix, "w1", platform, self.mlp_experts.up_proj.weight) }, + .gate_proj = .{ .weight = try loadShardedWeight(store, prefix, "w1", platform, self.mlp_experts.gate_proj.weight) }, .down_proj = .{ .weight = try loadShardedWeight(store, prefix, "w2", platform, self.mlp_experts.down_proj.weight) }, - .gate_proj = .{ .weight = try loadShardedWeight(store, prefix, "w3", platform, self.mlp_experts.gate_proj.weight) }, + .up_proj = .{ .weight = try loadShardedWeight(store, prefix, "w3", platform, self.mlp_experts.up_proj.weight) }, }, }; } @@ -381,8 +381,8 @@ const Mlp = struct { down_proj: zml.nn.Linear, // (hidden_dim -> dim) pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const up = x.dot(self.up_proj.weight, .d); - const gate = x.dot(self.gate_proj.weight, .d).silu(); + const up = self.up_proj.weight.dot(x, .d); + const gate = self.gate_proj.weight.dot(x, .d).silu(); const out = gate.mul(up).dot(self.down_proj.weight, .up); log.warn("Mlp(x: {f}) -> up: {f} -> gate: {f} -> out: {f}", .{ x, up, gate, out }); From 0d02e0e37ef64faff997ce2858d8f4c516bd8c1d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 30 Jul 2025 15:58:59 +0200 Subject: [PATCH 41/67] add moe tests --- zml/nn.zig | 153 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 118 insertions(+), 35 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 5873a5f8..04af6b85 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -48,6 +48,7 @@ pub const TokenEmbedding = struct { const MoeOpts = struct { experts_per_token: u32, + tokens_per_expert_ratio: ?u32 = null, }; /// We have two algorithms, @@ -62,50 +63,132 @@ const MoeOpts = struct { /// - experts: .{ .expert, .d_out, .d } expert layer (need to have a .forward method). /// -> output: .{ .s, .d_out } pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Tensor, opts: MoeOpts) Tensor { - // log.warn("mixtureOfExperts({s}, {}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); + log.warn("mixtureOfExperts({s}, {}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); const num_tokens = input.dim(.s); const num_experts = gating.dim(.expert); - const tokens_per_expert = std.math.divCeil(u32, num_tokens * opts.experts_per_token, num_experts) catch unreachable; + // const tokens_per_expert = std.math.divCeil(u32, num_tokens * opts.experts_per_token, num_experts) catch unreachable; - if (num_tokens * opts.experts_per_token < num_experts) { - // Few tokens: some experts will be unused. - // select active experts and compute with that. - const routing = gating.topK(opts.experts_per_token, .expert, .{ .descending = true }); - const routing_score = routing.values.div(routing.values.sum(.expert).broad(routing.values.shape())); + // if (num_tokens == 1) { + // // Few tokens: some experts will be unused. + // // select active experts and compute with that. + // const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); + // const per_token_score = routing.values.div(routing.values.sum(.top_expert)); - // TODO consider unrolling this on num_tokens * opts.experts_per_token to avoid gather of weights. - var sliced_expert: Expert = undefined; - zml.meta.mapAlloc(struct { - pub fn cb(routing_indices: Tensor, expert_weight: Tensor) Tensor { - return expert_weight.gather(.{ .expert = routing_indices }, .{}); - } - }.cb, undefined, routing.indices, experts, &sliced_expert) catch unreachable; + // // TODO unroll this on num_experts to avoid gather of weights. + // for (0..num_experts) |expert| { + // _ = expert; // autofix + // var sliced_expert: Expert = undefined; + // zml.meta.mapAlloc(struct { + // pub fn cb(routing_indices: Tensor, expert_weight: Tensor) Tensor { + // return expert_weight.gather(.{ .expert = routing_indices }, .{}); + // } + // }.cb, undefined, routing.indices, experts, &sliced_expert) catch unreachable; + // } - const output_per_expert = sliced_expert.forward(input); - // log.warn(" -> routing {f} -> sliced_expert {any} -> output_per_expert {f}", .{ routing_score, sliced_expert, output_per_expert }); - return output_per_expert.dot(routing_score, .expert); + // const output_per_expert = sliced_expert.forward(input); + // // log.warn(" -> routing {f} -> sliced_expert {any} -> output_per_expert {f}", .{ routing_score, sliced_expert, output_per_expert }); + // return output_per_expert.dot(per_token_score.convert(input.dtype()), .top_expert); + // } + + const tokens_per_expert: u32 = if (opts.tokens_per_expert_ratio) |tokens_per_expert_ratio| + std.math.divCeil(u32, num_tokens * opts.experts_per_token * tokens_per_expert_ratio, num_experts) catch unreachable + else + num_tokens; + + if (tokens_per_expert * 2 < num_tokens) { + // Lot of tokens, each experts chose their tokens and compute on that. + // It means that not all tokens will receive the same number of experts, + // Each token will get assigned to at least one expert because we normalize gating to sum up to one. + const normed_gating = gating.div(gating.sum(.expert)); + // TODO the routing score is wrong here. + const routing = normed_gating.topK(.{ .token_expert = .s }, tokens_per_expert, .{}); + const routing_score = routing.values.div(routing.values.sum(.expert)).transpose(.{ .expert, .token_expert }); + const routing_ids = routing.indices.transpose(.{ .expert, .token_expert }); + const input_per_expert = input.gather(.{ .s = routing_ids }, .{}); + var output_per_expert = experts.forward(input_per_expert); + output_per_expert = output_per_expert.mul(routing_score.broad(output_per_expert.shape())); + + // Reverse engineer the normal output shape that one expert would have produced for all tokens. + // If this provide to not be enough we could use the "sliced_expert" strategy and call forward ourselves. + const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .token_expert = .s }).setDim(.s, num_tokens); + var output: Tensor = .constant(output_shape, input.dtype().zero()); + log.warn(" -> input_per_expert {f} -> output_per_expert {f}, routing: {f} -> output {f}", .{ input_per_expert, output_per_expert, routing_ids, output }); + output = output.scatterSlices(.{ .s = routing_ids }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); + + @panic("FIXME"); + } else { + // We send all tokens to all experts, then discard result from non-topk experts using hard_gating. + const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); + const per_token_score = routing.values.div(routing.values.sum(.top_expert)); + + // hard_gating is eql to gating where only the top-2 experts have a non-zero score. + var hard_gating = zml.Tensor.constant(gating.shape(), gating.dtype().zero()); + hard_gating = hard_gating.scatterSlices(.{ .expert = routing.indices }, per_token_score, .{ .indices_are_unique = true }).withSharding(.{.expert}); + + const output_per_expert = experts.forward(input); + return output_per_expert.dot(hard_gating.convert(input.dtype()), .expert); + } +} + +test mixtureOfExperts { + const asynk = @import("async"); + + const MoE = struct { + router: Linear, + experts: Linear, + + pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { + return mixtureOfExperts(Linear, self.experts, x, self.router.forward(x).convert(.f32).softmax(.expert), .{ .experts_per_token = 2 }); + } + }; + + const platform = zml.testing.env(); + const store = try zml.aio.detectFormatAndOpen(std.testing.allocator, "zml/test_data/moe.activations.pt"); + defer store.deinit(); + + var model_arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer model_arena.deinit(); + var moe_model = try zml.aio.populateModel(MoE, model_arena.allocator(), store); + moe_model.router.weight = moe_model.router.weight.withTags(.{ .expert, .d }); + moe_model.experts.weight = moe_model.experts.weight.withTags(.{ .expert, ._, .d }); + + var compilation_prefill = try asynk.asyncc(zml.compileModel, .{ + std.testing.allocator, + MoE.forward, + moe_model, + .{zml.Shape.init(.{ .s = 24, .d = 64 }, .bf16)}, + platform, + }); + var compilation_gen = try asynk.asyncc(zml.compileModel, .{ + std.testing.allocator, + MoE.forward, + moe_model, + .{zml.Shape.init(.{ .s = 1, .d = 64 }, .bf16)}, + platform, + }); + + var moe_weights = try zml.aio.loadModelBuffers(MoE, moe_model, store, std.testing.allocator, platform); + defer zml.aio.unloadBuffers(&moe_weights); + + { + const moe_prefill = (try compilation_prefill.awaitt()).prepare(moe_weights); + defer moe_prefill.deinit(); + + const x = try store.get("in").?.toDevice(platform); + const actual = moe_prefill.call(.{x}); + + try zml.testing.expectClose(store.get("out").?, actual, 1e-2); } - // Lot of tokens, each experts chose their tokens and compute on that. - // It means that not all tokens will receive the same number of experts, - // Each token will get assigned to at least one expert because we normalize gating to sum up to one. - const normed_gating = gating.div(gating.sum(.s).broad(gating.shape())); - // TODO: topK should always rename the operated axis. - const routing = normed_gating.rename(.{ .s = .expert_token }).topK(tokens_per_expert, .expert_token, .{}); - const routing_score = routing.values.div(routing.values.sum(.expert).broad(routing.values.shape())).transpose(.{ .expert, .expert_token }); - const routing_ids = routing.indices.transpose(.{ .expert, .expert_token }); - const input_per_expert = input.gather(.{ .s = routing_ids }, .{}); - var output_per_expert = experts.forward(input_per_expert); - output_per_expert = output_per_expert.mul(routing_score.broad(output_per_expert.shape())); + { + const moe_gen = (try compilation_gen.awaitt()).prepare(moe_weights); + defer moe_gen.deinit(); - // Reverse engineer the normal output shape that one expert would have produced for all tokens. - // If this provide to not be enough we could use the "sliced_expert" strategy and call forward ourselves. - const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .expert_token = .s }).setDim(.s, num_tokens); - var output: Tensor = .constant(output_shape, input.dtype().zero()); - // log.warn(" -> input_per_expert {f} -> output_per_expert {f}, routing: {f} -> output {f}", .{ input_per_expert, output_per_expert, routing_ids, output }); - output = output.scatterSlices(.{ .s = routing_ids }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); + const x = try store.get("in").?.slice1d(0, .{ .start = 12, .end = 13 }).toDevice(platform); + const actual = moe_gen.call(.{x}); - return output; + try zml.testing.expectClose(store.get("out").?.slice1d(0, .{ .start = 12, .end = 13 }), actual, 1e-2); + } } pub const Activation = union(enum) { From 4e6a0fc65ce9eff399348d1ecc9a9f0bab2848d7 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 31 Jul 2025 10:30:42 +0200 Subject: [PATCH 42/67] Moe: wip batched strategy --- zml/nn.zig | 126 +++++++++++++++++++++++++++++++------------------ zml/tensor.zig | 7 ++- 2 files changed, 87 insertions(+), 46 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 04af6b85..197243f9 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -48,12 +48,19 @@ pub const TokenEmbedding = struct { const MoeOpts = struct { experts_per_token: u32, - tokens_per_expert_ratio: ?u32 = null, + tokens_per_expert_ratio: ?f32 = 0.0, }; -/// We have two algorithms, -/// * one for non-sharded single-stream inference (naive), -/// * one for sharded, batched inference (mixtral). +/// We have three algorithms, +/// * one for single-stream inference (naive), +/// * one for small batch sized with exact precision that sends all tokens to all experts. +/// this isn't too costly as long as the batch size is small and the experts are IO bound. +/// * one for big batch size that assign a fixed compute budget per expert and +/// experts chose the tokens they want to handle. This introduces noise since it's possible +/// a token doesn't get their requested expert. +/// The parameter `tokens_per_expert_ratio` control how much compute budget is granted: +/// expert_budget = ratio * (num_tokens * experts_per_token / num_experts). +/// Bigger values of ratio will ensure it's rare a token doesn't get examined. /// /// The preferred algorithm is the batched one, /// it is selected as soon there is enough tokens to guarantee that experts will be active most of the time. @@ -66,56 +73,75 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te log.warn("mixtureOfExperts({s}, {}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); const num_tokens = input.dim(.s); const num_experts = gating.dim(.expert); + stdx.debug.assert(opts.experts_per_token > 0, "mixtureOfExperts expects opts.experts_per_token > 0, got {}", .{opts}); // const tokens_per_expert = std.math.divCeil(u32, num_tokens * opts.experts_per_token, num_experts) catch unreachable; - // if (num_tokens == 1) { - // // Few tokens: some experts will be unused. - // // select active experts and compute with that. - // const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); - // const per_token_score = routing.values.div(routing.values.sum(.top_expert)); + if (num_tokens == 1) { + // Only one tokens: most experts are unused. + // select active experts and compute with that. + const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); + const per_token_score = routing.values.div(routing.values.sum(.top_expert)); - // // TODO unroll this on num_experts to avoid gather of weights. - // for (0..num_experts) |expert| { - // _ = expert; // autofix - // var sliced_expert: Expert = undefined; - // zml.meta.mapAlloc(struct { - // pub fn cb(routing_indices: Tensor, expert_weight: Tensor) Tensor { - // return expert_weight.gather(.{ .expert = routing_indices }, .{}); - // } - // }.cb, undefined, routing.indices, experts, &sliced_expert) catch unreachable; - // } + var output: ?Tensor = null; + for (0..opts.experts_per_token) |expert_rank| { + const expert_id = routing.indices.choose(.{ .top_expert = expert_rank }).asScalar(); + const expert_score = per_token_score.choose(.{ .top_expert = expert_rank }).asScalar(); - // const output_per_expert = sliced_expert.forward(input); - // // log.warn(" -> routing {f} -> sliced_expert {any} -> output_per_expert {f}", .{ routing_score, sliced_expert, output_per_expert }); - // return output_per_expert.dot(per_token_score.convert(input.dtype()), .top_expert); - // } + var sliced_expert: Expert = undefined; + zml.meta.mapAlloc(struct { + pub fn cb(expert_id_: Tensor, expert_weight: Tensor) Tensor { + return expert_weight.gather(.{ .expert = expert_id_ }, .{}); + } + }.cb, stdx.noalloc, expert_id, experts, &sliced_expert) catch unreachable; - const tokens_per_expert: u32 = if (opts.tokens_per_expert_ratio) |tokens_per_expert_ratio| - std.math.divCeil(u32, num_tokens * opts.experts_per_token * tokens_per_expert_ratio, num_experts) catch unreachable - else - num_tokens; + // TODO how does this work when the two experts are on different gpus? + // does the compute overlap ? + const expert_output = sliced_expert.forward(input).mul(expert_score.convert(input.dtype())); + output = if (output) |o| o.add(expert_output) else expert_output; + } - if (tokens_per_expert * 2 < num_tokens) { + log.warn("mixtureOfExperts({s}, {}, {f}, {f}) -> single-stream impl", .{ @typeName(Expert), experts, input, gating }); + return output.?; + } + + const tokens_per_expert: u32 = if (opts.tokens_per_expert_ratio) |ratio| tpe: { + const compute_budget = ratio * @as(f32, @floatFromInt(num_tokens * opts.experts_per_token)); + var tpe: u32 = @intFromFloat(stdx.math.divFloat(f32, compute_budget, num_experts)); + // Round to next multiple of 8 to avoid weird shapes. + tpe = tpe + 8 - (tpe % 8); + break :tpe tpe; + } else num_tokens; + + if (3 * tokens_per_expert <= 2 * num_tokens) { // Lot of tokens, each experts chose their tokens and compute on that. - // It means that not all tokens will receive the same number of experts, + // It means that some tokens may have only one expert assigned. // Each token will get assigned to at least one expert because we normalize gating to sum up to one. - const normed_gating = gating.div(gating.sum(.expert)); - // TODO the routing score is wrong here. - const routing = normed_gating.topK(.{ .token_expert = .s }, tokens_per_expert, .{}); - const routing_score = routing.values.div(routing.values.sum(.expert)).transpose(.{ .expert, .token_expert }); - const routing_ids = routing.indices.transpose(.{ .expert, .token_expert }); - const input_per_expert = input.gather(.{ .s = routing_ids }, .{}); + const expert_routing = gating.powByConst(2).topK(.{ .top_token = .s }, tokens_per_expert, .{}); + const scores_per_expert = expert_routing.values.transpose(.{ .expert, .top_token }); + const ids_per_expert = expert_routing.indices.transpose(.{ .expert, .top_token }); + const input_per_expert = input.gather(.{ .s = ids_per_expert }, .{}); var output_per_expert = experts.forward(input_per_expert); - output_per_expert = output_per_expert.mul(routing_score.broad(output_per_expert.shape())); + output_per_expert = output_per_expert.mul(scores_per_expert.convert(output_per_expert.dtype()).broad(output_per_expert.shape())); // Reverse engineer the normal output shape that one expert would have produced for all tokens. // If this provide to not be enough we could use the "sliced_expert" strategy and call forward ourselves. - const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .token_expert = .s }).setDim(.s, num_tokens); + const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .top_token = .s }).setDim(.s, num_tokens); var output: Tensor = .constant(output_shape, input.dtype().zero()); - log.warn(" -> input_per_expert {f} -> output_per_expert {f}, routing: {f} -> output {f}", .{ input_per_expert, output_per_expert, routing_ids, output }); - output = output.scatterSlices(.{ .s = routing_ids }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); + // log.warn(" -> input_per_expert {f} -> output_per_expert {f}, routing: {f} -> output {f}", .{ input_per_expert, output_per_expert, ids_per_expert, output }); + output = output.scatterSlices(.{ .s = ids_per_expert }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); - @panic("FIXME"); + // Now we need to normalize each output by the sum of the scores of the expert that looked at it. + var routing_score: Tensor = .constant(.{ .s = input.dim(.s) }, gating.dtype().zero()); + routing_score = routing_score.scatterSlices(.{ .s = ids_per_expert }, scores_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); + output = output.div(routing_score.convert(output.dtype()).broad(output_shape)); + + var expert_allocation = Tensor.constant(gating.shape().drop(.expert), .{ .u32 = 0 }); + expert_allocation = expert_allocation.scatterSlices(.{ .s = ids_per_expert.print() }, .constant(ids_per_expert.shape(), .{ .u32 = 1 }), .{}); + expert_allocation = expert_allocation.print(); + output = output.add(expert_allocation.sum(.s).divByConst(num_tokens).convert(output.dtype()).asScalar()); + + log.warn("mixtureOfExperts({s}, {}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), experts, input, gating, tokens_per_expert, num_tokens }); + return output; } else { // We send all tokens to all experts, then discard result from non-topk experts using hard_gating. const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); @@ -125,6 +151,7 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te var hard_gating = zml.Tensor.constant(gating.shape(), gating.dtype().zero()); hard_gating = hard_gating.scatterSlices(.{ .expert = routing.indices }, per_token_score, .{ .indices_are_unique = true }).withSharding(.{.expert}); + log.warn("mixtureOfExperts({s}, {}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), experts, input, gating }); const output_per_expert = experts.forward(input); return output_per_expert.dot(hard_gating.convert(input.dtype()), .expert); } @@ -133,12 +160,21 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te test mixtureOfExperts { const asynk = @import("async"); - const MoE = struct { - router: Linear, - experts: Linear, + const Expert = struct { + weight: Tensor, pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { - return mixtureOfExperts(Linear, self.experts, x, self.router.forward(x).convert(.f32).softmax(.expert), .{ .experts_per_token = 2 }); + return self.weight.dot(x, .{.d}).rename(.{ .d_out = .d }); + } + }; + + const MoE = struct { + router: Linear, + experts: Expert, + + pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { + const routing = self.router.forward(x).convert(.f32).softmax(.expert); + return mixtureOfExperts(Expert, self.experts, x, routing, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 1 }); } }; @@ -150,7 +186,7 @@ test mixtureOfExperts { defer model_arena.deinit(); var moe_model = try zml.aio.populateModel(MoE, model_arena.allocator(), store); moe_model.router.weight = moe_model.router.weight.withTags(.{ .expert, .d }); - moe_model.experts.weight = moe_model.experts.weight.withTags(.{ .expert, ._, .d }); + moe_model.experts.weight = moe_model.experts.weight.withTags(.{ .expert, .d_out, .d }); var compilation_prefill = try asynk.asyncc(zml.compileModel, .{ std.testing.allocator, diff --git a/zml/tensor.zig b/zml/tensor.zig index 68a7ad2b..4a456610 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1014,10 +1014,15 @@ pub const Tensor = struct { } /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. - pub inline fn scale(self: Tensor, val: anytype) Tensor { + pub fn scale(self: Tensor, val: anytype) Tensor { return self.mul(Tensor.scalar(val, self.dtype())); } + /// Returns a Tensor containing the element-wise power of the input Tensor by a constant. + pub fn powByConst(self: Tensor, val: anytype) Tensor { + return self.pow(Tensor.scalar(val, self.dtype())); + } + pub const LogicalOp = enum { OR, XOR, AND }; /// Returns a Tensor containing the element-wise logical operation of the input Tensors. From 3a2f7d5fa1fe4ee8864df0bfd431d7a1a494bd94 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 31 Jul 2025 16:24:06 +0200 Subject: [PATCH 43/67] mixtureOfExperts: improve the routing schema to avoid droping top-1 experts --- examples/mixtral/mixtral.zig | 2 +- examples/modernbert/modernbert.zig | 2 +- zml/BUILD.bazel | 2 +- zml/dtype.zig | 2 +- zml/nn.zig | 101 +++++++++++++++++++---------- zml/ops.zig | 6 +- zml/tensor.zig | 19 ++++-- 7 files changed, 87 insertions(+), 47 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 6db61586..a4fbfeeb 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -315,7 +315,7 @@ const MoE = struct { pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { const gating = self.router.forward(input).convert(.f32).softmax(.expert); - return zml.nn.mixtureOfExperts(Mlp, self.mlp_experts, input, gating, .{ .experts_per_token = 2 }); + return zml.nn.mixtureOfExperts(Mlp, self.mlp_experts, input, gating, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 1.5 }); } pub fn loadSharded(self: MoE, store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, platform: zml.Platform) !zml.Bufferized(MoE) { diff --git a/examples/modernbert/modernbert.zig b/examples/modernbert/modernbert.zig index 79585f14..7894d5a8 100644 --- a/examples/modernbert/modernbert.zig +++ b/examples/modernbert/modernbert.zig @@ -43,7 +43,7 @@ pub const ModernBertForMaskedLM = struct { const biased_logits = logits.add(self.decoder.bias.withTags(.{.voc}).broad(logits.shape())); const probabilities = biased_logits.softmax(.voc); - return probabilities.topK(5, .voc, .{ .descending = true }); + return probabilities.topK(.{ .best_token = .voc }, 5, .{ .descending = true }); } }; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 3d2ae081..b917f9ec 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -81,7 +81,7 @@ zig_test( data = [ "aio/torch/simple.pt", "aio/torch/simple_test_4.pickle", - ], + ] + glob(["test_data/**"]), test_runner = ":test_runner", deps = [":zml"], ) diff --git a/zml/dtype.zig b/zml/dtype.zig index b0deb8f4..81750da2 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -260,7 +260,7 @@ pub const Data = union(DataType) { if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { break :blk @unionInit(Data, @tagName(tag), @intCast(value)); } else { - @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + std.debug.panic("Could not create Data of type {s} from value: {d}", .{ @tagName(tag), value }); } }, .int => @unionInit(Data, @tagName(tag), @intCast(value)), diff --git a/zml/nn.zig b/zml/nn.zig index 197243f9..9f29502a 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -60,7 +60,7 @@ const MoeOpts = struct { /// a token doesn't get their requested expert. /// The parameter `tokens_per_expert_ratio` control how much compute budget is granted: /// expert_budget = ratio * (num_tokens * experts_per_token / num_experts). -/// Bigger values of ratio will ensure it's rare a token doesn't get examined. +/// Bigger values of ratio will ensure it's rare a token doesn't get it's top 2 tokens. /// /// The preferred algorithm is the batched one, /// it is selected as soon there is enough tokens to guarantee that experts will be active most of the time. @@ -74,7 +74,6 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te const num_tokens = input.dim(.s); const num_experts = gating.dim(.expert); stdx.debug.assert(opts.experts_per_token > 0, "mixtureOfExperts expects opts.experts_per_token > 0, got {}", .{opts}); - // const tokens_per_expert = std.math.divCeil(u32, num_tokens * opts.experts_per_token, num_experts) catch unreachable; if (num_tokens == 1) { // Only one tokens: most experts are unused. @@ -100,7 +99,7 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te output = if (output) |o| o.add(expert_output) else expert_output; } - log.warn("mixtureOfExperts({s}, {}, {f}, {f}) -> single-stream impl", .{ @typeName(Expert), experts, input, gating }); + log.debug("mixtureOfExperts({s}, {}, {f}, {f}) -> single-stream impl", .{ @typeName(Expert), experts, input, gating }); return output.?; } @@ -115,43 +114,69 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te if (3 * tokens_per_expert <= 2 * num_tokens) { // Lot of tokens, each experts chose their tokens and compute on that. // It means that some tokens may have only one expert assigned. - // Each token will get assigned to at least one expert because we normalize gating to sum up to one. - const expert_routing = gating.powByConst(2).topK(.{ .top_token = .s }, tokens_per_expert, .{}); - const scores_per_expert = expert_routing.values.transpose(.{ .expert, .top_token }); - const ids_per_expert = expert_routing.indices.transpose(.{ .expert, .top_token }); - const input_per_expert = input.gather(.{ .s = ids_per_expert }, .{}); + // Each token will get assigned to at least one expert IIF the input gating is sums up to 1 (typically softmax output). + const token_pref = gating.argsort(.expert, .{ .descending = true }); + var expert_rank: zml.Tensor = .scatterSlices( + .zeros(gating.shape().withDtype(.i32)), + .{ .expert = token_pref }, + .addConstant(.iota(gating.shape(), .expert), 1), + .{ .indices_are_unique = true }, + ); + // The pow(expert_rank) here means that we strongly favor top 1 over top 2 and top 2 over top 3. + // expert_routing: (expert, top_token) -> token + const expert_routing = gating.pow(expert_rank.convert(gating.dtype())).topK(.{ .top_token = .s }, tokens_per_expert, .{}); + const scores_per_expert = gating.gather(.{ .s = expert_routing.indices }, .{}); + + // Update the gating coefficient to account for the expert routing. + // Each (token, expert) which can't be computed within the given budget is left to 0. + const gating_v2: zml.Tensor = .scatterSlices( + .zeros(gating.shape()), + .{ .s = expert_routing.indices }, + scores_per_expert, + .{ .indices_are_unique = true, .update_fn = Tensor.ScatterOpts.override }, + ); + // Now set to zero the scores (token, expert) for tokens that have been assigned more than experts_per_token. + const lowest_experts = gating_v2.topK(.{ .top_expert = .expert }, num_experts - opts.experts_per_token, .{ .descending = false }); + var gating_v3: zml.Tensor = .scatterSlices( + gating_v2, + .{ .expert = lowest_experts.indices }, + .zeros(lowest_experts.values.shape()), + .{ .indices_are_unique = true, .update_fn = Tensor.ScatterOpts.override }, + ); + // Then normalize so the sum of experts scores for one token sums up to 1. + gating_v3 = gating_v3.div(gating_v3.sum(.expert)); + // Recompute scores_per_expert using the updated gating. + const tokens_ids_per_expert = expert_routing.indices.transpose(.{ .expert, .top_token }); + const scores_per_expert_v2 = gating_v3.gather(.{ .s = expert_routing.indices }, .{}).transpose(.{ .expert, .top_token }); + const input_per_expert = input.gather(.{ .s = tokens_ids_per_expert }, .{}); var output_per_expert = experts.forward(input_per_expert); - output_per_expert = output_per_expert.mul(scores_per_expert.convert(output_per_expert.dtype()).broad(output_per_expert.shape())); + output_per_expert = output_per_expert.mul(scores_per_expert_v2.convert(output_per_expert.dtype()).broad(output_per_expert.shape())); // Reverse engineer the normal output shape that one expert would have produced for all tokens. - // If this provide to not be enough we could use the "sliced_expert" strategy and call forward ourselves. + // If this fall short, we could use the "sliced_expert" strategy and call forward ourselves. const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .top_token = .s }).setDim(.s, num_tokens); - var output: Tensor = .constant(output_shape, input.dtype().zero()); - // log.warn(" -> input_per_expert {f} -> output_per_expert {f}, routing: {f} -> output {f}", .{ input_per_expert, output_per_expert, ids_per_expert, output }); - output = output.scatterSlices(.{ .s = ids_per_expert }, output_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); + const output = zml.Tensor.scatterSlices( + .zeros(output_shape), + .{ .s = tokens_ids_per_expert }, + output_per_expert, + .{ .update_fn = zml.Tensor.ScatterOpts.increment }, + ); - // Now we need to normalize each output by the sum of the scores of the expert that looked at it. - var routing_score: Tensor = .constant(.{ .s = input.dim(.s) }, gating.dtype().zero()); - routing_score = routing_score.scatterSlices(.{ .s = ids_per_expert }, scores_per_expert, .{ .update_fn = Tensor.ScatterOpts.increment }); - output = output.div(routing_score.convert(output.dtype()).broad(output_shape)); - - var expert_allocation = Tensor.constant(gating.shape().drop(.expert), .{ .u32 = 0 }); - expert_allocation = expert_allocation.scatterSlices(.{ .s = ids_per_expert.print() }, .constant(ids_per_expert.shape(), .{ .u32 = 1 }), .{}); - expert_allocation = expert_allocation.print(); - output = output.add(expert_allocation.sum(.s).divByConst(num_tokens).convert(output.dtype()).asScalar()); - - log.warn("mixtureOfExperts({s}, {}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), experts, input, gating, tokens_per_expert, num_tokens }); + log.debug("mixtureOfExperts({s}, {}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), experts, input, gating, tokens_per_expert, num_tokens }); return output; } else { // We send all tokens to all experts, then discard result from non-topk experts using hard_gating. const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); - const per_token_score = routing.values.div(routing.values.sum(.top_expert)); + const per_token_score = routing.values.div(routing.values.sum(.top_expert)).convert(input.dtype()); // hard_gating is eql to gating where only the top-2 experts have a non-zero score. - var hard_gating = zml.Tensor.constant(gating.shape(), gating.dtype().zero()); - hard_gating = hard_gating.scatterSlices(.{ .expert = routing.indices }, per_token_score, .{ .indices_are_unique = true }).withSharding(.{.expert}); - - log.warn("mixtureOfExperts({s}, {}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), experts, input, gating }); + const hard_gating = zml.Tensor.scatterSlices( + .zeros(gating.shape()), + .{ .expert = routing.indices }, + per_token_score, + .{ .indices_are_unique = true }, + ).withSharding(.{.expert}); + log.debug("mixtureOfExperts({s}, {}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), experts, input, gating }); const output_per_expert = experts.forward(input); return output_per_expert.dot(hard_gating.convert(input.dtype()), .expert); } @@ -164,7 +189,8 @@ test mixtureOfExperts { weight: Tensor, pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { - return self.weight.dot(x, .{.d}).rename(.{ .d_out = .d }); + // I'm always dumb founded by the weird output layout of `dot`. + return self.weight.dot(x, .{.d}).rename(.{ .d_out = .d }).contiguous(.{.d}); } }; @@ -174,12 +200,15 @@ test mixtureOfExperts { pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { const routing = self.router.forward(x).convert(.f32).softmax(.expert); - return mixtureOfExperts(Expert, self.experts, x, routing, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 1 }); + return mixtureOfExperts(Expert, self.experts, x, routing, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 2 }); } }; const platform = zml.testing.env(); - const store = try zml.aio.detectFormatAndOpen(std.testing.allocator, "zml/test_data/moe.activations.pt"); + const store = zml.aio.detectFormatAndOpen(std.testing.allocator, "zml/test_data/moe.activations.pt") catch |err| switch (err) { + error.FileNotFound => return error.SkipZigTest, + else => return err, + }; defer store.deinit(); var model_arena = std.heap.ArenaAllocator.init(std.testing.allocator); @@ -213,7 +242,7 @@ test mixtureOfExperts { const x = try store.get("in").?.toDevice(platform); const actual = moe_prefill.call(.{x}); - try zml.testing.expectClose(store.get("out").?, actual, 1e-2); + try zml.testing.expectClose(store.get("out").?, actual, 5e-2); } { @@ -223,7 +252,7 @@ test mixtureOfExperts { const x = try store.get("in").?.slice1d(0, .{ .start = 12, .end = 13 }).toDevice(platform); const actual = moe_gen.call(.{x}); - try zml.testing.expectClose(store.get("out").?.slice1d(0, .{ .start = 12, .end = 13 }), actual, 1e-2); + try zml.testing.expectClose(store.get("out").?.slice1d(0, .{ .start = 12, .end = 13 }), actual, 5e-2); } } @@ -316,7 +345,7 @@ pub fn normalizeVariance(x: Tensor, eps: f32) Tensor { // ref: https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html // Implementation equivalent to `nn.functional.normalize(tensor, dim=-1)` call pub fn normalizeL2(input: Tensor, eps: f32) Tensor { - const inv_norm = input.pow(Tensor.scalar(2, input.dtype())).sum(-1).addConstant(eps).rsqrt(); + const inv_norm = input.powByConst(2).sum(-1).addConstant(eps).rsqrt(); return input.mul(inv_norm.broad(input.shape())); } @@ -925,7 +954,7 @@ pub fn resizeCubic1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Ten Tensor.constant(t.shape(), dtype.one()), t, t.mul(t), - t.pow(Tensor.scalar(3, dtype)), + t.powByConst(3), }, .last, ._interpolated); std.debug.assert(pos.dim(0) == new_len); diff --git a/zml/ops.zig b/zml/ops.zig index d554e5da..d475a86b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -909,7 +909,6 @@ pub fn scatter( // Note: I was a bit lazy here, and I only look at tags on the first tensor. // we probably should check all of them. const self = meta.first(Tensor, inputs); - const update = meta.first(Tensor, updates); var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); if (indices_per_axis.len == 0) return inputs; @@ -929,6 +928,7 @@ pub fn scatter( } break :blk higher_rank; }; + stdx.debug.assert(indices_shape.dtype().isInteger(), "zml.ops.scatter expects all indices tensor to have an integer dtype, got {f}", .{indices_shape}); for (indices_per_axis.slice()) |*idx| { stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); @@ -940,6 +940,9 @@ pub fn scatter( return self.dynamicUpdateSlice(index_tensors, updates); } + var update = meta.first(Tensor, updates); + if (update.rank() == 0) update = update.broadcast(indices_shape, &.{}); + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); @@ -1021,6 +1024,7 @@ fn scatterConfig( var indices_batch_axes: Shape.DimsArray = .{}; var scatter_to_operand_axes: Shape.DimsArray = .{}; var updates_transpose: Shape.AxesArray = .{}; + // FIXME: updates_transpose is not correctly computed, so we can't transpose updates, and the user has to do it. const tagged_api = indices_axes.len > 0; const indices = indices_per_axis.get(0).shape(); diff --git a/zml/tensor.zig b/zml/tensor.zig index 4a456610..4ae2beba 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2003,6 +2003,10 @@ pub const Tensor = struct { return _result(sh, constant_op.result(0)).convert(val.dtype()); } + pub fn zeros(sh: Shape) Tensor { + return constant(sh, sh.dtype().zero()); + } + /// Embeds a buffer with concrete values into an Mlir program. pub fn constantTensor(val: HostBuffer) Tensor { const ctx = CompilationContext.current().mlirCtx(); @@ -2214,7 +2218,7 @@ pub const Tensor = struct { } pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - scoped_log.warn("gather({f}, {any}, {any})", .{ self, idx_axes, idx_per_axis }); + // scoped_log.warn("gather({f}, {any}, {any})", .{ self, idx_axes, idx_per_axis }); stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); for (idx_axes, 0..) |a, i| { if (i > 0) { @@ -2307,7 +2311,7 @@ pub const Tensor = struct { .index_vector_dim = indices.axis(.coord), .indices_are_sorted = opts.indices_are_sorted, }; - scoped_log.warn(" --> stablehlo.gather({f}, {f}, {any}, {})", .{ self, indices, slice_dims.slice(), args }); + // scoped_log.warn(" --> stablehlo.gather({f}, {f}, {any}, {})", .{ self, indices, slice_dims.slice(), args }); const gather_op = dialect.stablehlo.gather( self.getContext().mlirCtx(), @@ -2319,6 +2323,7 @@ pub const Tensor = struct { ); const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); + // Note: this is midly annoying, I'm inclined to automatically transpose the indices. stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); return _result(res_shape, gather_op.result(0)); } @@ -2716,6 +2721,10 @@ pub const Tensor = struct { // TODO: here update (m,n) doesn't match indices (n, m). // We need to either handle this automatically or provide better errors. // .{ .{. a = 10, .b=20 }, .{ .a = idx(.{.n=8,.m=9}) }, .{.m=9,.n=8,.b=20} }, + // Ideally this should be refused: + // .{.{ .expert = 8, .top_token=8}, .{ .top_token = idx(.{.s=24, .top_expert=2}) }, .{} } + // But it implies having an extra tag on indices tensor generated by topk and argmax. Here the problem is that the indices point into a [0..token] range, + // but are intepreted to point into a [0..top_token] range. }) |testcase| { const x_shape, const indices, const updates_shapes = testcase; const x = Tensor.constant(x_shape, .{ .f16 = 0 }); @@ -3575,11 +3584,10 @@ pub const Tensor = struct { res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); - const zeros = Tensor.constant(res_shape, self.dtype().zero()); const x = Tensor.iota(res_shape, a); const y = Tensor.iota(res_shape, a + 1); - var res = x.cmp(.EQ, y).select(values, zeros); + var res = x.cmp(.EQ, y).select(values, .zeros(res_shape)); res._shape = res_shape; return res; } @@ -3630,8 +3638,7 @@ pub const Tensor = struct { const x = Tensor.iota(self.shape(), _axes.get(0)); const y = Tensor.iota(self.shape(), _axes.get(1)); - const zeros = Tensor.constant(self.shape(), self.dtype().zero()); - return x.addConstant(num_diagonals).cmp(.GE, y).select(self, zeros); + return x.addConstant(num_diagonals).cmp(.GE, y).select(self, .zeros(self._shape)); } test triangular { From 7b5f8e887de858de8590da07687cd946f413c4d6 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 1 Aug 2025 11:59:44 +0200 Subject: [PATCH 44/67] refactor examples/simple_layer --- examples/simple_layer/main.zig | 116 ++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 54 deletions(-) diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig index a84ab056..cde53778 100644 --- a/examples/simple_layer/main.zig +++ b/examples/simple_layer/main.zig @@ -1,3 +1,8 @@ +//! This examples show how to specify a model, +//! compile it, and execute it with specific weights. +//! +//! This code is very explicit and doesn't use zml.aio helper functions +//! in order to show what is happening behind the scene. const std = @import("std"); const asynk = @import("async"); @@ -17,83 +22,86 @@ const Layer = struct { } }; -pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); + var gpa_state = std.heap.DebugAllocator(.{}){}; + defer _ = gpa_state.deinit(); + const gpa = gpa_state.allocator(); + // Start ZML, detect available devices and chose a platform. var context = try zml.Context.init(); defer context.deinit(); - const platform = context.autoPlatform(.{}); context.printAvailablePlatforms(platform); - // Our weights and bias to use - var weights = [4]f32{ 2.0, 2.0, 2.0, 2.0 }; - var bias = [4]f32{ 1.0, 2.0, 3.0, 4.0 }; - const input_shape = zml.Shape.init(.{4}, .f32); - - // We manually produce a BufferStore. You would not normally do that. - // A BufferStore is usually created by loading model data from a file. - var buffers: zml.aio.BufferStore.Buffers = .{}; - try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); - try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - - // the actual BufferStore - const buffer_store: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, + // Create a skeleton for our model with only the shapes. + // The `buffer_id` are typically not set manually + // but instead set by a zml.BufferStore + // to map abstract Tensors to actual part of the model weight file. + const model_shapes: Layer = .{ + .bias = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 0 }, + }, + .weight = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 1 }, + }, }; - // A clone of our model, consisting of shapes. We only need shapes for compiling. - // We use the BufferStore to infer the shapes. - var model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); - model_shapes.weight = model_shapes.weight.withSharding(.{-1}); - model_shapes.bias = model_shapes.bias.?.withSharding(.{-1}); + // Start compilation in a different thread. + // We already specified the shape of the model weights, + // but we still need to specifiy the shape of the input tensor. + const input_shape = zml.Shape.init(.{4}, .f16); + var compilation = try asynk.asyncc( + zml.compileModel, + .{ std.heap.page_allocator, Layer.forward, model_shapes, .{input_shape}, platform }, + ); - // Start compiling. This uses the inferred shapes from the BufferStore. - // The shape of the input tensor, we have to pass in manually. - var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); + // Now we need to create a model instance with actual weights. + const weights = [4]f16{ 2.0, -2.0, 1.0, -1.0 }; + const bias = [4]f16{ 1.0, 2.0, 3.0, 4.0 }; - // Produce a bufferized weights struct from the fake BufferStore. - // This is like the inferred shapes, but with actual values. - // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); // for good practice + var model_weights: zml.Bufferized(Layer) = .{ + .bias = try zml.Buffer.fromSlice(platform, .{4}, &bias), + .weight = try zml.Buffer.fromSlice(platform, .{4}, &weights), + }; + defer zml.aio.unloadBuffers(&model_weights); // Wait for compilation to finish const compiled = try compilation.awaitt(); + defer compiled.deinit(); - // pass the model weights to the compiled module to create an executable module - var executable = compiled.prepare(model_weights); - defer executable.deinit(); + // pass the model weights to the compiled module to create an executable module. + // This is where the shapes of the buffers will be compared to the + // shape expected by the executable. + // Note: we don't call `executable.deinit()` since it uses the same memory than `compiled`. + const executable = compiled.prepare(model_weights); - // prepare an input buffer - // Here, we use zml.HostBuffer.fromSlice to show how you would create a HostBuffer - // with a specific shape from an array. - // For situations where e.g. you have an [4]f32 array but need a .{2, 2} input shape. - var input = [4]f32{ 5.0, 5.0, 5.0, 5.0 }; - var input_buffer = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(input_shape, &input), .{}); + // prepare the input buffer + const input = [4]f16{ 5.0, 5.0, 5.0, 5.0 }; + var input_buffer = try zml.Buffer.fromSlice(platform, input_shape, &input); defer input_buffer.deinit(); - // call our executable module + // call our executable module, the result is still on the device. var result: zml.Buffer = executable.call(.{input_buffer}); defer result.deinit(); - // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); + // copy the result to CPU memory + const cpu_result: zml.HostBuffer = try result.toHostAlloc(gpa); + defer cpu_result.deinit(gpa); std.debug.print( "\nThe result of {any} * {any} + {any} = {any}\n", .{ &weights, &input, &bias, cpu_result.items(f32) }, ); } + +const log = std.log.scoped(.simple_layer); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} From f6279991cc523b646aaa37ca756d15d93fe2d1cc Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 1 Aug 2025 12:00:38 +0200 Subject: [PATCH 45/67] BaseExe: use groupedAlloc --- stdx/stdx.zig | 44 +++++++++++++++++++++------- zml/aio.zig | 3 +- zml/exe.zig | 77 ++++++++++++++++++++++++++++++------------------- zml/module.zig | 2 +- zml/testing.zig | 6 ++-- 5 files changed, 87 insertions(+), 45 deletions(-) diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 6d2c28bf..ea14b0f1 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -1,3 +1,6 @@ +const std = @import("std"); +const builtin = @import("builtin"); + pub const BoundedArray = @import("bounded_array.zig").BoundedArray; pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned; pub const debug = @import("debug.zig"); @@ -10,9 +13,6 @@ pub const meta = @import("meta.zig"); pub const queue = @import("queue.zig"); pub const time = @import("time.zig"); -const std = @import("std"); -const builtin = @import("builtin"); - test { std.testing.refAllDecls(@This()); } @@ -26,27 +26,49 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; pub const mem = struct { - pub fn groupedAlloc(SliceTuple: type, allocator: std.mem.Allocator, len: [@typeInfo(SliceTuple).@"struct".fields.len]usize) error{OutOfMemory}!SliceTuple { + pub fn groupedAlloc( + SliceTuple: type, + allocator: std.mem.Allocator, + len: [@typeInfo(SliceTuple).@"struct".fields.len]usize, + ) error{OutOfMemory}!SliceTuple { var res: SliceTuple = undefined; var full_alloc_len: usize = 0; const SliceFields = @typeInfo(SliceTuple).@"struct".fields; - inline for (SliceFields, 0.., &res) |field, i, *slice| { + inline for (SliceFields, len) |field, l| { // needed because full_alloc_len starts at 0. @setRuntimeSafety(false); const T = std.meta.Child(field.type); full_alloc_len = std.mem.alignForward(usize, full_alloc_len, @alignOf(T)); - slice.ptr = @ptrFromInt(full_alloc_len); - slice.len = len[i]; - full_alloc_len += @sizeOf(T) * len[i]; + @field(res, field.name).ptr = @ptrFromInt(full_alloc_len); + @field(res, field.name).len = l; + full_alloc_len += @sizeOf(T) * l; } - const bytes = try allocator.alignedAlloc(u8, @alignOf(SliceFields[0].type), full_alloc_len); - inline for (&res) |*slice| { - slice.ptr = @ptrFromInt(@intFromPtr(bytes.ptr) + @intFromPtr(slice.ptr)); + const bytes = try allocator.alignedAlloc(u8, .of(SliceFields[0].type), full_alloc_len); + // std.log.warn("groupedAlloc({s}) -> {*}[0..{d}] align({})", .{ @typeName(SliceTuple), bytes.ptr, bytes.len, alignment }); + + inline for (SliceFields) |field| { + @field(res, field.name).ptr = @ptrFromInt(@intFromPtr(bytes.ptr) + @intFromPtr(@field(res, field.name).ptr)); } return res; } + + pub fn groupedFree(SliceTuple: type, allocator: std.mem.Allocator, slice_tuple: SliceTuple) void { + const SliceFields = @typeInfo(SliceTuple).@"struct".fields; + + const slice_start: [*]u8 = @ptrCast(slice_tuple[0].ptr); + const N = SliceFields.len; + const FirstT = std.meta.Child(SliceFields[0].type); + const LastT = std.meta.Child(SliceFields[N - 1].type); + + const last_slice = slice_tuple[N - 1]; + const slice_end: usize = @intFromPtr(last_slice.ptr) + @sizeOf(LastT) * last_slice.len; + const slice_len = slice_end - @intFromPtr(slice_start); + + // std.log.warn("groupedFree({s}) -> {*}[0..{d}] align({})", .{ @typeName(SliceTuple), slice_start, slice_len, @alignOf(FirstT) }); + allocator.rawFree(slice_start[0..slice_len], .fromByteUnits(@alignOf(FirstT)), @returnAddress()); + } }; diff --git a/zml/aio.zig b/zml/aio.zig index 48100979..c0e2b5d0 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -32,7 +32,8 @@ pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) // else if (std.mem.endsWith(u8, model_path, ".tinyllama")) // try tinyllama.open(allocator, model_path) else { - std.debug.panic("File extension not recognized: {s}", .{model_path}); + log.err("File extension not recognized: {s}", .{model_path}); + return error.NotSupported; }; } diff --git a/zml/exe.zig b/zml/exe.zig index d4c426b7..fd728fd8 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -167,10 +167,10 @@ pub const BaseExe = struct { ready_buffer_count: u32, /// Total number of buffers needed by this executable. - input_buffer_count: u32, + num_inputs: u32, input_shapes: []Shape, - result_shapes: []Shape, + output_shapes: []Shape, /// Num devices used (>1 for sharded executable) num_devices: u8, @@ -182,30 +182,44 @@ pub const BaseExe = struct { parent_allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, - args: struct { input_shapes: []const Shape, result_shapes: []const Shape, n_devices: u8 }, + args: struct { input_shapes: []const Shape, output_shapes: []const Shape, n_devices: u8 }, ) !BaseExe { + // TODO: considere if we still need the arena since we only allocate once. var arena = std.heap.ArenaAllocator.init(parent_allocator); errdefer arena.deinit(); const allocator = arena.allocator(); + const n_in = args.input_shapes.len; - const n_out = args.result_shapes.len; + const n_out = args.output_shapes.len; const n_devices = args.n_devices; - // Allocate once for all the *pjrt.Buffer we need to store ... - const all_buffers = try allocator.alloc(*pjrt.Buffer, (n_in + n_out) * n_devices); - const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ n_in * n_devices, n_out * n_devices }); - // ... and once for all the [*]*pjrt.Buffer. - const all_per_device = try allocator.alloc([*]*pjrt.Buffer, 2 * n_devices); - const input_per_device, const output_per_device = splitBuffer([*]*pjrt.Buffer, all_per_device, .{ n_devices, n_devices }); + const Slices = struct { + all_input_buffers: []*pjrt.Buffer, + all_output_buffers: []*pjrt.Buffer, + input_per_device: [][*]*pjrt.Buffer, + output_per_device: [][*]*pjrt.Buffer, + input_shapes: []Shape, + output_shapes: []Shape, + }; + const slices = try stdx.mem.groupedAlloc(Slices, allocator, .{ + n_in * n_devices, + n_out * n_devices, + n_devices, + n_devices, + n_in, + n_out, + }); + + // Init the slices. + // all_input_buffers is left uninitialized, will be fed by `prepare` and `call` + // all_output_buffers will be written to by pjrt.execute for (0..n_devices) |i| { - input_per_device[i] = all_input_buffers[i * n_in ..].ptr; - output_per_device[i] = all_output_buffers[i * n_out ..].ptr; + slices.input_per_device[i] = slices.all_input_buffers[i * n_in ..].ptr; + slices.output_per_device[i] = slices.all_output_buffers[i * n_out ..].ptr; } - - const all_shapes = try allocator.alloc(Shape, n_in + n_out); - @memcpy(all_shapes[0..n_in], args.input_shapes); - @memcpy(all_shapes[n_in..], args.result_shapes); + @memcpy(slices.input_shapes, args.input_shapes); + @memcpy(slices.output_shapes, args.output_shapes); var execute_context: ?*pjrt.ExecuteContext = null; if (platform.pjrt_api.ffi()) |ffi| { @@ -219,12 +233,12 @@ pub const BaseExe = struct { .exe = exe, .execute_context = execute_context, .ready_buffer_count = 0, - .input_buffer_count = @intCast(n_in), + .num_inputs = @intCast(n_in), .num_devices = args.n_devices, - .input_per_device = input_per_device, - .output_per_device = output_per_device, - .input_shapes = all_shapes[0..n_in], - .result_shapes = all_shapes[n_in..], + .input_per_device = slices.input_per_device, + .output_per_device = slices.output_per_device, + .input_shapes = slices.input_shapes, + .output_shapes = slices.output_shapes, ._arena = arena, }; } @@ -237,7 +251,7 @@ pub const BaseExe = struct { } pub fn call(self: BaseExe) void { - stdx.debug.assert(self.input_buffer_count == self.ready_buffer_count, "BaseExe isn't ready to be called, expected {} buffer inputs got {}", .{ self.input_buffer_count, self.ready_buffer_count }); + stdx.debug.assert(self.num_inputs == self.ready_buffer_count, "BaseExe isn't ready to be called, expected {} buffer inputs got {}", .{ self.num_inputs, self.ready_buffer_count }); return self._unsafeCall(); } @@ -247,7 +261,7 @@ pub const BaseExe = struct { self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, - .num_args = self.input_buffer_count, + .num_args = self.num_inputs, .results = self.output_per_device, .events = events[0..sharding.num_partitions], // this allows to tell a specific buffer shouldn't be donated, @@ -277,7 +291,7 @@ pub const BaseExe = struct { .index = 0, .platform = self.platform, .outputs = self.output_per_device, - .output_shapes = self.result_shapes, + .output_shapes = self.output_shapes, }; meta.visit((struct { fn cb(ctx: *LocalContext, buffer: *Buffer) void { @@ -292,7 +306,7 @@ pub const BaseExe = struct { buffer.* = Buffer.fromPjrtBuffers(ctx.platform, ctx.output_shapes[i], shards.constSlice()); } }).cb, &local_ctx, result); - stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); + stdx.debug.internalAssert(local_ctx.index == self.output_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); } pub fn bind(exe: BaseExe, Callback: type, op: *Callback) !void { @@ -330,13 +344,13 @@ pub const BaseExe = struct { shards.appendAssumeCapacity(dev_out[i]); } - return Buffer.fromPjrtBuffers(self.platform, self.result_shapes[i], shards.constSlice()); + return Buffer.fromPjrtBuffers(self.platform, self.output_shapes[i], shards.constSlice()); } - pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { + pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) error{OutOfMemory}!BaseExe { var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ .input_shapes = self.input_shapes, - .result_shapes = self.result_shapes, + .output_shapes = self.output_shapes, .n_devices = self.num_devices, }); exe.execute_context = self.execute_context; @@ -363,6 +377,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { /// /// **Warning:** the new Exe reuses the underlying memory of the previous one. /// The caller is responsible to come up with a strategy to call `deinit` exactly once. + /// If needed, new executable with independent memory can be created with `clone`. pub fn prepare(self: Self, first_arg: Bufferized(stdx.meta.Head(ArgsT))) Exe(stdx.meta.Tail(ArgsT), ReturnT) { var new: Exe(stdx.meta.Tail(ArgsT), ReturnT) = .{ .inner = self.inner }; new.inner.prepare(first_arg); @@ -387,12 +402,16 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { pub fn call(self: Self, args: Bufferized(ArgsT)) Bufferized(ReturnT) { const total_ready = fillBuffers(&args, self.inner.input_shapes, self.inner.input_per_device, self.inner.ready_buffer_count); - std.debug.assert(total_ready == self.inner.input_buffer_count); + std.debug.assert(total_ready == self.inner.num_inputs); self.inner._unsafeCall(); var result: Bufferized(ReturnT) = undefined; self.inner._unsafeAssignResults(Bufferized(ReturnT), &result); return result; } + + pub fn clone(self: Self, allocator: std.mem.Allocator) error{OutOfMemory}!Self { + return .{ .inner = try self.inner.clone(allocator) }; + } }; } diff --git a/zml/module.zig b/zml/module.zig index 6263d3fd..30852ba8 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -234,7 +234,7 @@ pub const CompilationContext = struct { loaded_executable, .{ .input_shapes = f.args_shapes, - .result_shapes = f.res_shapes, + .output_shapes = f.res_shapes, .n_devices = sharding.num_replicas * sharding.num_partitions, }, ); diff --git a/zml/testing.zig b/zml/testing.zig index 0e2959bb..eabe3791 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -221,8 +221,8 @@ pub fn testLayerOut( const exe = try zml.compileModel(alloc, fwd, layer, input_shapes, platform); const n_out_exp = activations.countLayers(out_name); - if (exe.inner.result_shapes.len != n_out_exp) { - log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ n_out_exp, exe.inner.result_shapes.len }); + if (exe.inner.output_shapes.len != n_out_exp) { + log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ n_out_exp, exe.inner.output_shapes.len }); } const mod = exe.prepare(layer_weights); @@ -263,7 +263,7 @@ pub fn testLayerOut( var buf: [1024]u8 = undefined; var failed: bool = false; - for (0..mod.inner.result_shapes.len) |i| { + for (0..mod.inner.output_shapes.len) |i| { const full_name = std.fmt.bufPrint(&buf, "{s}.{d}", .{ out_name, i }) catch unreachable; const expected_out = activations.get(full_name) orelse { log.warn("Output buffer not found: {s}", .{full_name}); From ab73e1c516ae66b5384570a643c4c61b5756c53a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 1 Aug 2025 12:09:07 +0200 Subject: [PATCH 46/67] remove ununsed Tensor._reserveIdRange --- zml/aio.zig | 60 ++++++++++++++++++++++++-------------------------- zml/tensor.zig | 12 ---------- 2 files changed, 29 insertions(+), 43 deletions(-) diff --git a/zml/aio.zig b/zml/aio.zig index c0e2b5d0..e87640f6 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -44,8 +44,8 @@ pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) /// whose shape is read from the "a.b" tensor. /// * If `Model` contains a list of layers, then the field: /// `Model.layers[2].a.b` will be populated from the "layers.2.a.b" tensor. -pub fn populateModel(comptime Model: type, allocator: std.mem.Allocator, buffer_store: BufferStore) !Model { - return populateModelWithPrefix(Model, allocator, buffer_store, ""); +pub fn populateModel(comptime Model: type, allocator: std.mem.Allocator, store: BufferStore) !Model { + return populateModelWithPrefix(Model, allocator, store, ""); } /// Creates a Model struct with tensor shapes read from the given TensorStore, @@ -63,8 +63,7 @@ pub fn populateModelWithPrefix(comptime Model: type, allocator: std.mem.Allocato try prefix_builder.push(allocator, prefix); defer prefix_builder.deinit(allocator); - const unique_id = zml.Tensor._reserveIdRange(@intCast(store.buffers.count())); - const ok = _populateStruct(allocator, &prefix_builder, unique_id, store, &model, true) catch |err| { + const ok = _populateStruct(allocator, &prefix_builder, store, &model, true) catch |err| { std.debug.panic("Can't populate model of type {s}: {s}", .{ @typeName(type), @errorName(err) }); }; if (!ok) return error.TensorNotFound; @@ -452,8 +451,7 @@ pub const PrefixBuilder = struct { fn _populateStruct( allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, - unique_id: u64, - buffer_store: BufferStore, + store: BufferStore, obj: anytype, required: bool, ) !bool { @@ -468,12 +466,12 @@ fn _populateStruct( const prefix = prefix_builder.data.items; if (T == zml.Tensor) { - return if (buffer_store.getTensorOrNull(prefix)) |tensor| { + return if (store.getTensorOrNull(prefix)) |tensor| { obj.* = tensor; return true; } else { if (required) { - log.err("Tensor not found: {s} ({d})", .{ prefix, buffer_store.buffers.count() }); + log.err("Tensor not found: {s} ({d})", .{ prefix, store.buffers.count() }); } return false; }; @@ -486,14 +484,14 @@ fn _populateStruct( obj.* = &.{}; if (!zml.meta.Contains(ptr_info.child, zml.Tensor)) return false; - const len = buffer_store.countLayers(prefix); + const len = store.countLayers(prefix); if (len > 0) { obj.* = try allocator.alloc(ptr_info.child, len); for (obj.*, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, value, required); + const found = try _populateStruct(allocator, prefix_builder, store, value, required); if (!found) { log.err("Not able to load {s} as {s}", .{ prefix_builder.data.items, @typeName(ptr_info.child) }); return false; @@ -512,7 +510,7 @@ fn _populateStruct( for (obj, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, value, required); + const found = try _populateStruct(allocator, prefix_builder, store, value, required); if (!found) { log.err("Not able to load {s} as {s}", .{ prefix_builder.data.items, @typeName(arr_info.child) }); return false; @@ -531,7 +529,7 @@ fn _populateStruct( if (field.default_value_ptr) |_| has_default = true; if (zml.meta.Contains(field.type, zml.Tensor)) { - const field_found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &@field(obj, field.name), required and !has_default); + const field_found = try _populateStruct(allocator, prefix_builder, store, &@field(obj, field.name), required and !has_default); partial_struct = partial_struct or field_found; if (!field_found) { if (field.default_value_ptr) |v| { @@ -540,7 +538,7 @@ fn _populateStruct( if (partial_struct) { log.warn("Incomplete struct '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); // if (required) { - // findSimilarBufferKeys(prefix, buffer_store, allocator); + // findSimilarBufferKeys(prefix, store, allocator); // } obj.* = undefined; return false; @@ -555,7 +553,7 @@ fn _populateStruct( }, .optional => |opt_info| { obj.* = @as(opt_info.child, undefined); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &(obj.*.?), false); + const found = try _populateStruct(allocator, prefix_builder, store, &(obj.*.?), false); if (!found) obj.* = null; return true; }, @@ -573,7 +571,7 @@ fn _populateStruct( obj.* = @unionInit(T, field.name, undefined); // @compileLog(T, field.name, field.type); log.warn("Trying to init {s} as a {s}", .{ prefix, @typeName(field.type) }); - const field_found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &@field(obj, field.name), true); + const field_found = try _populateStruct(allocator, prefix_builder, store, &@field(obj, field.name), true); if (field_found) return true; } if (required) log.warn("Failed to init {s} as any of {s}", .{ prefix, @typeName(T) }); @@ -654,11 +652,11 @@ test populateModel { pub fn loadBuffers( comptime Model: type, init_args: if (@hasDecl(Model, "init")) stdx.meta.Tail(stdx.meta.FnArgs(Model.init)) else void, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, ) !zml.Bufferized(Model) { - return loadBuffersWithPrefix(Model, init_args, buffer_store, allocator, platform, ""); + return loadBuffersWithPrefix(Model, init_args, store, allocator, platform, ""); } /// Creates a bufferized version of a Model from the given BufferStore with a specified prefix. @@ -672,7 +670,7 @@ pub fn loadBuffers( pub fn loadBuffersWithPrefix( comptime Model: type, init_args: if (@hasDecl(Model, "init")) stdx.meta.Tail(stdx.meta.FnArgs(Model.init)) else void, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, prefix: []const u8, @@ -682,14 +680,14 @@ pub fn loadBuffersWithPrefix( const arena = arena_state.allocator(); // Get model structure with tensor shapes from the buffer store with prefix - var model: Model = try zml.aio.populateModelWithPrefix(Model, arena, buffer_store, prefix); + var model: Model = try zml.aio.populateModelWithPrefix(Model, arena, store, prefix); // If the Model has a "init" function, call it with the given parameters. if (@hasDecl(Model, "init")) { @call(.auto, Model.init, .{&model} ++ init_args); } - return loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, prefix); + return loadModelBuffersWithPrefix(Model, model, store, allocator, platform, prefix); } /// Creates a bufferized version of a Model from the given BufferStore. For details about @@ -701,11 +699,11 @@ pub fn loadBuffersWithPrefix( pub fn loadModelBuffers( comptime Model: type, model: Model, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, ) !zml.Bufferized(Model) { - return try loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, ""); + return try loadModelBuffersWithPrefix(Model, model, store, allocator, platform, ""); } /// Creates a bufferized version of a Model from the given BufferStore and the given prefix. @@ -717,7 +715,7 @@ pub fn loadModelBuffers( pub fn loadModelBuffersWithPrefix( comptime Model: type, model: Model, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, prefix: []const u8, @@ -736,7 +734,7 @@ pub fn loadModelBuffersWithPrefix( try prefix_builder.push(allocator, prefix); defer prefix_builder.deinit(allocator); - try visitStructAndLoadBuffer(allocator, &prefix_builder, buffer_store, &res, platform); + try visitStructAndLoadBuffer(allocator, &prefix_builder, store, &res, platform); return res; } @@ -759,7 +757,7 @@ pub fn awaitAll(buffers: anytype) !void { }).cb, {}, buffers); } -fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, buffer_store: BufferStore, obj: anytype, platform: zml.Platform) !void { +fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, store: BufferStore, obj: anytype, platform: zml.Platform) !void { const err_msg = "visitStructAndLoadBuffer must be called with a pointer to type. Received "; const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { .pointer => |ptr_info| switch (ptr_info.size) { @@ -771,7 +769,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi const prefix = prefix_builder.data.items; if (T == zml.Buffer) { - return if (buffer_store.get(prefix)) |host_buffer| { + return if (store.get(prefix)) |host_buffer| { // obj._shape has been set inside `loadModelBuffersWithPrefix`, before calling us. var buf_with_metadata = host_buffer; log.debug("Loading buffer {s} ({f})", .{ prefix, obj._shape }); @@ -781,7 +779,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi } else { log.err("Buffer not found: {s}", .{prefix}); - buffer_store.findSimilarBufferKeys(allocator, prefix); + store.findSimilarBufferKeys(allocator, prefix); return error.BufferNotFound; }; @@ -794,7 +792,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, value, platform); } } else stdx.debug.compileError("type not supported by visitStructAndLoadBuffer: {}", .{T}); }, @@ -802,7 +800,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi for (obj, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, value, platform); } }, @@ -812,12 +810,12 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi try prefix_builder.push(allocator, field.name); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, &@field(obj, field.name), platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, &@field(obj, field.name), platform); } }, .optional => { if (obj.*) |*obj_val| { - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, obj_val, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, obj_val, platform); } }, else => {}, diff --git a/zml/tensor.zig b/zml/tensor.zig index 4ae2beba..3cdc099a 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -269,18 +269,6 @@ pub const Tensor = struct { return res; } - var _global_tensor_counter: u64 = 0; - - /// Internal use - pub fn _reserveIdRange(len: u32) u64 { - return @atomicRmw(u64, &_global_tensor_counter, .Add, len, .seq_cst); - } - - /// Internal use - pub fn setUniqueId(self: *Tensor) void { - self._id = .{ .buffer_id = _reserveIdRange(1) }; - } - /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); From 6804b571fb40f3d38e63ab8041f05190adcec0c9 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 1 Aug 2025 14:22:41 +0200 Subject: [PATCH 47/67] update write_first_model.md --- docs/tutorials/write_first_model.md | 368 +++++++++++++--------------- examples/simple_layer/main.zig | 22 +- 2 files changed, 179 insertions(+), 211 deletions(-) diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 7cea735c..5a6f91df 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -65,14 +65,10 @@ Module. For compilation, only the _Shapes_ of all tensors must be known. No actual tensor data is needed at this step. This is important for large models: we can compile them while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _BufferStore_. The _BufferStore_ knows how to -only load shapes and when to load actual tensor data. In our example, we will -fake the _BufferStore_ a bit: we won't load from disk; we'll use float arrays -instead. - -After compilation is done (and the _BufferStore_ has finished loading weights), -we can send the weights from the _BufferStore_ to our computation device. That -produces an _executable_ module which we can call with different _inputs_. +To accomplish this, most ZML code uses a _BufferStore_. +The _BufferStore_ knows how to only load shapes and when to load actual tensor data. +In our example, we wont' use _BufferStore_ at all, +and manually fill out the shapes first then the weights. In our example, we then copy the result from the computation device to CPU memory and print it. @@ -80,11 +76,10 @@ memory and print it. **So the steps for us are:** - describe the computation as ZML _Module_, using tensor operations -- create a _BufferStore_ that provides _Shapes_ and data of weights and bias - (ca. 5 lines of code). +- describe the shapes of our model - compile the _Module_ **asynchronously** -- make the compiled _Module_ send the weights (and bias) to the computation - device utilizing the _BufferStore_, producing an _executable_ module +- send the weights of the model to the computation device +- bind the model weights to the _Module_ producing an _executable_ module - prepare input tensor and call the _executable_ module. - get the result back to CPU memory and print it @@ -100,9 +95,6 @@ Let's start by writing some Zig code, importing ZML and often-used modules: const std = @import("std"); const zml = @import("zml"); const asynk = @import("async"); - -// shortcut to the asyncc function in the asynk module -const asyncc = asynk.asyncc; ``` You will use above lines probably in all ZML projects. Also, note that **ZML is @@ -110,11 +102,10 @@ async** and comes with its own async runtime, thanks to [zigcoro](https://github.com/rsepassi/zigcoro). - ### Defining our Model -We will start with a very simple "Model". One that resembles a "multiply and -add" operation. +We will start with a very simple "Model". +One that resembles a "multiply and add" operation. ```zig /// Model definition @@ -151,6 +142,15 @@ ZML code is async. Hence, We need to provide an async main function. It works like this: ```zig +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); @@ -170,166 +170,143 @@ So, let's start with the async main function: ```zig pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); + var gpa_state = std.heap.DebugAllocator(.{}){}; + defer _ = gpa_state.deinit(); + const gpa = gpa_state.allocator(); - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context + // Start ZML, detect available devices and chose a platform. var context = try zml.Context.init(); defer context.deinit(); - const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + ... } ``` -This is boilerplate code that provides a general-purpose allocator and, for -convenience, an arena allocator that we will use later. The advantage of arena -allocators is that you don't need to deallocate individual allocations; you -simply call `.deinit()` to deinitialize the entire arena instead! - -We also initialize the ZML context `context` and get our CPU `platform` -automatically. +This is boilerplate code that provides a debug allocator, +initialize the ZML context `context`, discover available device +and chose a `platform` to use. +A `platform` is an homogenous set of devices that can work together, +eg 2 Nivdia 4090 GPUs. -### The BufferStore +### Compilation -Next, we need to set up the concrete weight and bias tensors for our model. -Typically, we would load them from disk. But since our example works without -stored weights, we are going to create a BufferStore manually, containing -_HostBuffers_ (buffers on the CPU) for both the `weight` and the `bias` tensor. +We already described the model logic, but we didn't specify any sizes. +To compile an executable specialized to our usecase we are going to provide +the shapes of the individual tensors in the model as well as input shapes. -A BufferStore basically contains a dictionary with string keys that match the -name of the struct fields of our `Layer` struct. So, let's create this -dictionary: +To do that we need to instantiate the `Layer` struct. +Note that `zml.Shape` always takes the data type associated with the tensor. +In our example, we will use half-precision floats `f16`, expressed as the enum value `.f16`. + +We also set give a `.buffer_id` to each `Tensor` to let the compiler know those are different objects despite the same shape. ```zig -// Our weights and bias to use -var weights = [3]f16{ 2.0, 2.0, 2.0 }; -var bias = [3]f16{ 1.0, 2.0, 3.0 }; -const input_shape = zml.Shape.init(.{3}, .f16); + const model_shapes: Layer = .{ + .bias = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 0 }, + }, + .weight = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 1 }, + }, + }; -// We manually produce a BufferStore. You would not normally do that. -// A BufferStore is usually created by loading model data from a file. -var buffers: zml.aio.BufferStore.Buffers = .{}; -try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); -try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - -// the actual BufferStore -const bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, -}; + const input_shape = zml.Shape.init(.{4}, .f16); ``` -Our weights are `{2.0, 2.0, 2.0}`, and our bias is just `{1.0, 2.0, 3.0}`. The -shape of the weight and bias tensors is `{3}`, and because of that, the **shape -of the input tensor** is also going to be `{3}`! +Note: for real models the shape are generally loaded from a config file or directly from the +metadata of the weight file using `zml.aio.BufferStore`. -Note that `zml.Shape` always takes the data type associated with the tensor. In -our example, that is `f16`, expressed as the enum value `.f16`. - - - -### Compiling our Module for the accelerator - -We're only going to use the CPU for our simple model, but we need to compile the -`forward()` function nonetheless. This compilation is usually done -asynchronously. That means, we can continue doing other things while the module -is compiling: +Then we can compile the model asynchronously so we can do other things in parallel, +typically loading the weights on the device. ```zig -// A clone of our model, consisting of shapes. We only need shapes for compiling. -// We use the BufferStore to infer the shapes. -const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); - -// Start compiling. This uses the inferred shapes from the BufferStore. -// The shape of the input tensor, we have to pass in manually. -var compilation = try asyncc( +var compilation = try asynk.asyncc( zml.compileModel, - .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }, + .{ gpa, Layer.forward, model_shapes, .{input_shape}, platform }, ); - -// Produce a bufferized weights struct from the fake BufferStore. -// This is like the inferred shapes, but with actual values. -// We will need to send those to the computation device later. -var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); -defer zml.aio.unloadBuffers(&model_weights); // for good practice - -// Wait for compilation to finish -const compiled = try compilation.awaitt(); ``` Compiling is happening in the background via the `asyncc` function. We call `asyncc` with the `zml.compileModel` function and its arguments -separately. The arguments themselves are basically the shapes of the weights in -the BufferStore, the `.forward` function name in order to compile -`Layer.forward`, the shape of the input tensor(s), and the platform for which to -compile (we used auto platform). +separately. +The arguments themselves are: +- an allocator to allocate memory needed for the returned executable. +- `Layer.forward`: the function to be compiled +- the shapes of the model +- the shape of the input tensors (`Layer.forward` only got one argument, so it's a 1-tuple) +- the platform for which to compile +### Loading the weights -### Creating the Executable Model +Now the model is compiling, we can load the model data to the GPU. +To represent the loaded weights, we are going to instantiate a `zml.Bufferized(Layer)` struct. +This is a struct that looks like our `Layer` struct, +but where all `Tensor`s have been replaced by the `zml.Buffer` type, +and where all non-`Tensor` fields have been straight up removed. -Now that we have compiled the module utilizing the shapes, we turn it into an -executable. +``` + // Now we need to create a model instance with actual weights. +const weights = [4]f16{ 2.0, -2.0, 1.0, -1.0 }; +const bias = [4]f16{ 1.0, 2.0, 3.0, 4.0 }; + +var model_weights: zml.Bufferized(Layer) = .{ + .bias = try zml.Buffer.fromSlice(platform, .{4}, &bias), + .weight = try zml.Buffer.fromSlice(platform, .{4}, &weights), +}; +defer zml.aio.unloadBuffers(&model_weights); +``` + +Since `zml.Bufferized(Layer)` is a normal struct we have type checking enabled, +and the Zig compiler will tell us if we forget to load a struct. +Here we hard-coded the weights in the code, +but typically weights would be read from disk at this point using `BufferStore`. + +### Putting it all together + +Now we want to bind the weights we just loaded, +with the executable we started to compile earlier on. ```zig -// pass the model weights to the compiled module to create an executable module -// all required memory has been allocated in `compile`. -var executable = compiled.prepare(model_weights); -defer executable.deinit(); +// Wait for compilation to finish +const compiled = try compilation.awaitt(); +defer compiled.deinit(); + +// pass the model weights to the compiled module to create an executable module. +const executable = compiled.prepare(model_weights); ``` +`prepare` will check that the shapes of the buffers in `model_weights` match +with the shapes given to the compiler. +Also note we don't call `executable.deinit()` since it re-uses the memory of `compiled`. + ### Calling / running the Model The executable can now be invoked with an input of our choice. -To create the `input`, we directly use `zml.Buffer` by calling -`zml.Buffer.fromArray()`. It's important to note that `Buffer`s reside in -_accelerator_ (or _device_) memory, which is precisely where the input needs to -be for the executable to process it on the device. - -For clarity, let's recap the distinction: `HostBuffer`s are located in standard -_host_ memory, which is accessible by the CPU. When we initialized the weights, -we used `HostBuffers` to set up the `BufferStore`. This is because the -`BufferStore` typically loads weights from disk into `HostBuffer`s, and then -converts them into `Buffer`s when we call `loadBuffers()`. - -However, for inputs, we bypass the `BufferStore` and create `Buffer`s directly -in device memory. - - ```zig -// prepare an input buffer -// Here, we use zml.HostBuffer.fromSlice to show how you would create a -// HostBuffer with a specific shape from an array. -// For situations where e.g. you have an [4]f16 array but need a .{2, 2} input -// shape. -var input = [3]f16{ 5.0, 5.0, 5.0 }; -var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), -); -defer input_buffer.deinit(); +// prepare the input buffer + const input = [4]f16{ 5.0, 5.0, 5.0, 5.0 }; + var input_buffer = try zml.Buffer.fromSlice(platform, input_shape, &input); + defer input_buffer.deinit(); -// call our executable module -var result: zml.Buffer = executable.call(.{input_buffer}); -defer result.deinit(); + // call our executable module, the result is still on the device. + var result: zml.Buffer = executable.call(.{input_buffer}); + defer result.deinit(); -// fetch the result buffer to CPU memory -const cpu_result = try result.toHostAlloc(arena); -std.debug.print( - "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, -); + // copy the result to CPU memory + const cpu_result: zml.HostBuffer = try result.toHostAlloc(gpa); + defer cpu_result.deinit(gpa); + std.debug.print( + "\nThe result of {d} * {d} + {d} = {d}\n", + .{ &weights, &input, &bias, cpu_result.items(f16) }, + ); ``` Note that the result of a computation is usually residing in the memory of the @@ -344,8 +321,6 @@ And that's it! Now, let's have a look at building and actually running this example! - - ## Building it As mentioned already, ZML uses Bazel; so to build our model, we just need to @@ -417,10 +392,9 @@ You can access the complete source code of this walkthrough here: ```zig const std = @import("std"); -const zml = @import("zml"); -const asynk = @import("async"); -const asyncc = asynk.asyncc; +const asynk = @import("async"); +const zml = @import("zml"); /// Model definition const Layer = struct { @@ -436,88 +410,84 @@ const Layer = struct { } }; +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain); + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); } pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); + var gpa_state = std.heap.DebugAllocator(.{}){}; + defer _ = gpa_state.deinit(); + const gpa = gpa_state.allocator(); - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context + // Start ZML, detect available devices and chose a platform. var context = try zml.Context.init(); defer context.deinit(); - const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); - // Our weights and bias to use - var weights = [3]f16{ 2.0, 2.0, 2.0 }; - var bias = [3]f16{ 1.0, 2.0, 3.0 }; - const input_shape = zml.Shape.init(.{3}, .f16); - - // We manually produce a BufferStore. You would not normally do that. - // A BufferStore is usually created by loading model data from a file. - var buffers: zml.aio.BufferStore.Buffers = .{}; - try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); - try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - - // the actual BufferStore - const bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, + // Create a skeleton for our model with only the shapes. + // The `buffer_id` are typically not set manually + // but instead set by a zml.BufferStore + // to map abstract Tensors to actual part of the model weight file. + const model_shapes: Layer = .{ + .bias = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 0 }, + }, + .weight = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 1 }, + }, }; - // A clone of our model, consisting of shapes. We only need shapes for - // compiling. We use the BufferStore to infer the shapes. - const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); + // Start compilation in a different thread. + // We already specified the shape of the model weights, + // but we still need to specifiy the shape of the input tensor. + const input_shape = zml.Shape.init(.{4}, .f16); + var compilation = try asynk.asyncc( + zml.compileModel, + .{ gpa, Layer.forward, model_shapes, .{input_shape}, platform }, + ); - // Start compiling. This uses the inferred shapes from the BufferStore. - // The shape of the input tensor, we have to pass in manually. - var compilation = try asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); + // Now we need to create a model instance with actual weights. + const weights = [4]f16{ 2.0, -2.0, 1.0, -1.0 }; + const bias = [4]f16{ 1.0, 2.0, 3.0, 4.0 }; - // Produce a bufferized weights struct from the fake BufferStore. - // This is like the inferred shapes, but with actual values. - // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); // for good practice + var model_weights: zml.Bufferized(Layer) = .{ + .bias = try zml.Buffer.fromSlice(platform, .{4}, &bias), + .weight = try zml.Buffer.fromSlice(platform, .{4}, &weights), + }; + defer zml.aio.unloadBuffers(&model_weights); // Wait for compilation to finish const compiled = try compilation.awaitt(); + defer compiled.deinit(); - // pass the model weights to the compiled module to create an executable - // module - var executable = compiled.prepare(model_weights); - defer executable.deinit(); + // pass the model weights to the compiled module to create an executable module. + // This is where the shapes of the buffers will be compared to the + // shape expected by the executable. + // Note: we don't call `executable.deinit()` since it uses the same memory than `compiled`. + const executable = compiled.prepare(model_weights); - // prepare an input buffer - // Here, we use zml.HostBuffer.fromSlice to show how you would create a - // HostBuffer with a specific shape from an array. - // For situations where e.g. you have an [4]f16 array but need a .{2, 2} - // input shape. - var input = [3]f16{ 5.0, 5.0, 5.0 }; - var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), - ); + // prepare the input buffer + const input = [4]f16{ 5.0, 5.0, 5.0, 5.0 }; + var input_buffer = try zml.Buffer.fromSlice(platform, input_shape, &input); defer input_buffer.deinit(); - // call our executable module + // call our executable module, the result is still on the device. var result: zml.Buffer = executable.call(.{input_buffer}); defer result.deinit(); - // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); + // copy the result to CPU memory + const cpu_result: zml.HostBuffer = try result.toHostAlloc(gpa); + defer cpu_result.deinit(gpa); std.debug.print( - "\n\nThe result of {d} * {d} + {d} = {d}\n", + "\nThe result of {d} * {d} + {d} = {d}\n", .{ &weights, &input, &bias, cpu_result.items(f16) }, ); } diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig index cde53778..1621fd6b 100644 --- a/examples/simple_layer/main.zig +++ b/examples/simple_layer/main.zig @@ -22,6 +22,15 @@ const Layer = struct { } }; +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + pub fn asyncMain() !void { var gpa_state = std.heap.DebugAllocator(.{}){}; defer _ = gpa_state.deinit(); @@ -54,7 +63,7 @@ pub fn asyncMain() !void { const input_shape = zml.Shape.init(.{4}, .f16); var compilation = try asynk.asyncc( zml.compileModel, - .{ std.heap.page_allocator, Layer.forward, model_shapes, .{input_shape}, platform }, + .{ gpa, Layer.forward, model_shapes, .{input_shape}, platform }, ); // Now we need to create a model instance with actual weights. @@ -94,14 +103,3 @@ pub fn asyncMain() !void { .{ &weights, &input, &bias, cpu_result.items(f32) }, ); } - -const log = std.log.scoped(.simple_layer); - -pub const std_options: std.Options = .{ - .log_level = .info, - .logFn = asynk.logFn(std.log.defaultLog), -}; - -pub fn main() !void { - try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); -} From 789bb5c4ecb071d53f15f015db4358dacc52ea71 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 1 Aug 2025 16:21:40 +0200 Subject: [PATCH 48/67] Update gather docs --- zml/tensor.zig | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index 3cdc099a..5ad46d75 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2193,6 +2193,35 @@ pub const Tensor = struct { pub const GatherOpts = struct { indices_are_sorted: bool = false }; + /// `gather` extracts slices from the given tensor at the specified offsets. + /// example: `values.gather(.{ .a = idx }, .{})` + /// + /// * indices is a named list of integer tensors: eg `.{ .a = idx }`. + /// Each names specify a gathering axis, it must refer to an axis of self, + /// and the corresponding idx Tensor must contains valid indices into axis .a. + /// All indices must have the same shape or broadcast to the same shape. + /// + /// * result is a tensor whose shape is similar to the input shape + /// where the gathered axes have been replaced by axes from 'indices'. + /// + /// Some example input for the base case where we work on one axis: + /// - gather(f:[a], .{ .a = idx:[n]})[n] == f[idx[n]] + /// - gather(f:[a, b], .a, idx:[n])[n, b] == f[idx[n], b] + /// - gather(f:[a,b,c], .{.b = idx:[n,m]})[a, n, m, c] == f[a, idx[n, m], c] + /// + /// If an axis in common between `self` and `indices`, + /// it is treated as a "batching" axis, meaning that semantically + /// the operator is doing a gather one time per dimension of this axis: + /// - gather(f: [a,b,c], .{.b=idx: [a,n]})[a, n] == f[a, idx[a, n]] + /// + /// It's possible to pass several indices: + /// - gather(f: [a,b,c], .{.b=idx_b[n], .c=idx_c[n]})[a, n] == f[a, idx_b[n], idx_c[n]] + /// - gather(f: [a,b,c,d], .{.b=idx_b[a,n], .c=idx_c[a, n]})[a, n, d] == f[a, idx_b[a, n], idx_c[a, n], d] + /// + /// If `self` isn't tagged, you can use `gather_` to specify gathered axis by their position but batching won't be available. + /// + /// For performance it's better to have batching and gathering axes of `self` be the first one, + /// so that gather can pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); var idx_axes: Shape.AxesArray = .{}; @@ -2201,7 +2230,6 @@ pub const Tensor = struct { } // TODO: sort indices following self.shape instead of asking the user to do it. - // TODO: this doesn't handle batching axes. return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); } From fbc8763f4f8961248f2a8e9a049ab85317024d10 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 5 Aug 2025 14:01:18 +0200 Subject: [PATCH 49/67] modernize code --- zml/nn.zig | 10 +++++----- zml/quantization.zig | 38 +++++++------------------------------- zml/tensor.zig | 2 +- 3 files changed, 13 insertions(+), 37 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 9f29502a..b579a10a 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -32,7 +32,7 @@ pub const Linear = struct { } // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); - return if (self.bias) |bias| y.add(bias.broadcastLeft(y.shape())) else y; + return if (self.bias) |bias| y.add(bias.broadcast(y.shape(), &.{y.axis(-1)})) else y; } }; @@ -335,18 +335,18 @@ pub fn normalizeVariance(x: Tensor, eps: f32) Tensor { // Upcast to improve precision const xf32 = x.convert(.f32); const mean = xf32.sum(-1).scale(1.0 / N); - const mean_dev = xf32.sub(mean.broadcastRight(xf32.shape())); - const variance = mean_dev.mul(mean_dev).sum(-1).scale(1.0 / N); + const mean_dev = xf32.sub(mean); + const variance = mean_dev.mul(mean_dev).sum(-1).divByConst(N); const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); - return mean_dev.mul(rsqrt.broadcastRight(mean_dev.shape())).convert(x.dtype()); + return mean_dev.mul(rsqrt).convert(x.dtype()); } // ref: https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html // Implementation equivalent to `nn.functional.normalize(tensor, dim=-1)` call pub fn normalizeL2(input: Tensor, eps: f32) Tensor { const inv_norm = input.powByConst(2).sum(-1).addConstant(eps).rsqrt(); - return input.mul(inv_norm.broad(input.shape())); + return input.mul(inv_norm); } test normalizeL2 { diff --git a/zml/quantization.zig b/zml/quantization.zig index 2a850a4d..90a35bda 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -1,9 +1,11 @@ const std = @import("std"); -const zml = @import("zml.zig"); - const Allocator = std.mem.Allocator; + +const zml = @import("zml.zig"); const module = zml.module; +// TODO add tests, use modern zml + pub fn Q4_0(comptime dtype: zml.DataType) type { return struct { const Self = @This(); @@ -38,38 +40,12 @@ pub fn Q4_0(comptime dtype: zml.DataType) type { const scales = extractScales(block_count, input); const weights = extractWeights(block_count, input); - return scales.reshape(.{ block_count, 1 }) - .broadcastLeft(zml.Shape.init(.{ block_count, 32 }, .f32)) + return scales.broadcast(weights.shape(), &.{0}) .mul(weights) .convert(dtype) - .reshape(.{block_count * 32}) .reshape(shape); } - pub fn scaleIndices(block_count: u63) zml.Tensor { - // indices1 is the offsets of the scale bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 0, .end = 2 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each scale byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 2 }, .i32)).reshape(.{2 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - return indices1.add(indices2); - } - - pub fn weightIndices(block_count: u63) zml.Tensor { - // indices1 is the offsets of the data bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 2, .end = 18 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each data byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 16 }, .i32)).reshape(.{16 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - return indices1.add(indices2); - } - pub fn extractScales(block_count: u63, input: zml.Tensor) zml.Tensor { // The goal here is to get the first two bytes of every 18-bytes block in the input. For that, // we generate a list of indices that we will use to gather from the input. @@ -85,7 +61,7 @@ pub fn Q4_0(comptime dtype: zml.DataType) type { const indices = indices1.add(indices2); // We select the values we are interested in with the indices, group them by pair and bitcast them to f16, then convert them to f32. - const scales = input.gatherValues(0, indices, .{ .indices_are_sorted = true }).reshape(.{ block_count, 2 }).bitCast(.f16).convert(.f32); + const scales = input.gather_(&.{0}, &.{indices}, .{ .indices_are_sorted = true }).reshape(.{ block_count, 2 }).bitCast(.f16).convert(.f32); return scales; } @@ -107,7 +83,7 @@ pub fn Q4_0(comptime dtype: zml.DataType) type { // NOTE(Corendos): i4 is not supported by bitcast convert, so we need the following workaround. // We select the values we are interested in with the indices, these are our quantized_weights. - const quantized_weights = input.gatherValues(0, indices, .{ .indices_are_sorted = true }); + const quantized_weights = input.gather_(&.{0}, &.{indices}, .{ .indices_are_sorted = true }); const lb_weights = quantized_weights .logical(.And, zml.Tensor.constant(.{16 * block_count}, zml.Data.init(.u8, 0xf))) .bitCast(.i8); diff --git a/zml/tensor.zig b/zml/tensor.zig index 5ad46d75..7037fae5 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1298,7 +1298,7 @@ pub const Tensor = struct { const row_mask = max_val.cmp(.GT, Tensor.scalar(-std.math.inf(f64), self.dtype())); const exp_diff_max = self.sub(self.max(a).broad(self._shape)).exp(); - const res = exp_diff_max.div(exp_diff_max.sum(a).broad(self._shape)); + const res = exp_diff_max.div(exp_diff_max.sum(a)); // If a row is full -inf return full 0 instead of full nan, // this fix attention when mask hides a full row. From 6650422ce90593331e768e638de9f5610d26d04e Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 26 Aug 2025 12:21:43 +0200 Subject: [PATCH 50/67] update build.bazel --- examples/mixtral/BUILD.bazel | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/examples/mixtral/BUILD.bazel b/examples/mixtral/BUILD.bazel index 1bee84ab..b7160096 100644 --- a/examples/mixtral/BUILD.bazel +++ b/examples/mixtral/BUILD.bazel @@ -1,13 +1,6 @@ -load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:native_binary.bzl", "native_test") -load("@bazel_skylib//rules:write_file.bzl", "write_file") -load("@rules_cc//cc:cc_binary.bzl", "cc_binary") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "mixtral", srcs = [ "mixtral.zig", From 06304d641d147648cf7a59880b452499755c22f1 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 26 Aug 2025 12:24:05 +0200 Subject: [PATCH 51/67] add Yarn rope variant --- zml/nn.zig | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index b579a10a..2d13b86a 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -375,12 +375,22 @@ pub const RopeOpts = struct { default: void, custom: []const f32, llama3: Llama3, + yarn: Yarn, pub const Llama3 = struct { factor: f32, high_freq_factor: f32, low_freq_factor: f32, original_max_position_embeddings: u32, + truncate: bool = true, + }; + + pub const Yarn = struct { + beta_fast: f32 = 32.0, + beta_slow: f32 = 1.0, + factor: f32, + truncate: bool = true, + original_max_position_embeddings: u32, }; /// Read a Rope scaling config from HF config.json format. @@ -396,11 +406,20 @@ pub const RopeOpts = struct { if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, undefined, content, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "yarn")) { + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, undefined, content, .{ .ignore_unknown_fields = true }) }; } else { - // log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); + log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; } } + + pub fn attentionScaling(scaling: Scaling) f32 { + return switch (scaling) { + .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, + else => 1.0, + }; + } }; }; @@ -428,8 +447,9 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { // compute sin and cos in f32 before downcasting to x type. const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); - const cos = inv_freq_pos.cos().convert(x.dtype()).broad(x_real.shape()); - const sin = inv_freq_pos.sin().convert(x.dtype()).broad(x_real.shape()); + const scaling = opts.scaling.attentionScaling(); + const cos = inv_freq_pos.cos().scale(scaling).convert(x.dtype()).broad(x_real.shape()); + const sin = inv_freq_pos.sin().scale(scaling).convert(x.dtype()).broad(x_real.shape()); // apply rotation const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); @@ -469,6 +489,8 @@ pub fn invFreq(N: i64, opts: RopeOpts) Tensor { const allocator = zml.module.CompilationContext.current().allocator(); const N_half: usize = @intCast(@divExact(N, 2)); const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); + defer allocator.free(inv_freq); + _invFreq(opts, inv_freq); return zml.Tensor.constantTensor(.fromSlice(.{@divExact(N, 2)}, inv_freq)); } @@ -509,10 +531,38 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { } } }, + .yarn => |s| { + const N_f: f64 = @floatFromInt(inv_freq.len); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.beta_fast * (2 * std.math.pi) / M; + const f_low = s.beta_slow * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + // This isn't a typo: low n have a high frequency, high n have a low frequency. + var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; + var n_high: f64 = -@log(f_low) / @log(opts.freq_base) * N_f; + if (s.truncate) { + n_high = std.math.ceil(n_high); + n_low = std.math.floor(n_low); + } + std.debug.assert(n_high > n_low); + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Yarn use lerp too but not in the frequency space, in the time space. + const lerp: f64 = (n_high - @as(f64, @floatFromInt(n))) / (n_high - n_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, } } -test invFreq { +test "invFreq Llama3" { // Llama 3.2-1B config const llama_conf: RopeOpts = .{ .freq_base = 500_000, @@ -533,6 +583,28 @@ test invFreq { } } +test "invFreq Yarn" { + const yarn_conf: RopeOpts = .{ + .freq_base = 150_000, + .scaling = .{ .yarn = .{ + .factor = 32.0, + .beta_fast = 32.0, + .beta_slow = 1.0, + .original_max_position_embeddings = 4096, + .truncate = true, + } }, + }; + const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; + + var inv_freq: @TypeOf(yarn_freq) = undefined; + _invFreq(yarn_conf, &inv_freq); + for (yarn_freq, inv_freq, 0..) |expected, actual, i| { + errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, yarn_freq, inv_freq }); + try std.testing.expectApproxEqRel(expected, actual, 1e-5); + } + try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); +} + test "real/img" { const platform = zml.testing.env(); From 295fef0c10979bb99e0f5024c5d5228d78734c63 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 26 Aug 2025 12:46:40 +0200 Subject: [PATCH 52/67] mixtral -> gpt_oss --- examples/mixtral/mixtral.zig | 74 ++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index a4fbfeeb..9e941f47 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -10,7 +10,7 @@ const log = std.log.scoped(.Mixtral); /// Dimensions of activations: {.b, .s, .d} pub const MixtralLM = struct { pub const Config = struct { - bos_token_id: u32, + bos_token_id: u32 = 0, eos_token_id: stdx.json.Union(union(enum) { int: u32, ints: []u32, @@ -81,12 +81,11 @@ pub const MixtralLM = struct { self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); } - const on_disk_moe = try zml.aio.populateModelWithPrefix(MoE.OnDisk, allocator, store, prefix.concat("block_sparse_moe")); + const on_disk_moe = try zml.aio.populateModelWithPrefix(MoE.OnDisk, allocator, store, prefix.concat("mlp")); var moe = on_disk_moe.rewrite(); { - moe.mlp_experts.up_proj.weight = moe.mlp_experts.up_proj.weight.withSharding(.{.expert}); - moe.mlp_experts.gate_proj.weight = moe.mlp_experts.gate_proj.weight.withSharding(.{.expert}); - moe.mlp_experts.down_proj.weight = moe.mlp_experts.down_proj.weight.withSharding(.{.expert}); + moe.experts.gate_up_proj.weight = moe.experts.gate_up_proj.weight.withSharding(.{.expert}); + moe.experts.down_proj.weight = moe.experts.down_proj.weight.withSharding(.{.expert}); } layer.* = .{ @@ -99,7 +98,7 @@ pub const MixtralLM = struct { .eps = config.rms_norm_eps, }, .self_attn = self_attn, - .block_sparse_moe = moe, + .mlp = moe, }; } @@ -179,7 +178,7 @@ pub const MixtralLM = struct { .input_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.input_layernorm, platform), .self_attn = try store.loadModelById(SelfAttn, noalloc, layer.self_attn, platform), .post_attention_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.post_attention_layernorm, platform), - .block_sparse_moe = try layer.block_sparse_moe.loadSharded(store, &prefix, platform), + .mlp = try layer.mlp.loadSharded(store, &prefix, platform), }; } @@ -247,7 +246,7 @@ pub const TransformerLayer = struct { input_layernorm: RmsNorm, self_attn: SelfAttn, post_attention_layernorm: RmsNorm, - block_sparse_moe: MoE, + mlp: MoE, pub fn forward( self: TransformerLayer, @@ -265,7 +264,7 @@ pub const TransformerLayer = struct { // Fully Connected const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - const x2 = zml.call(self.block_sparse_moe, .forward, .{x1_normalized}).add(x1); + const x2 = zml.call(self.mlp, .forward, .{x1_normalized}).add(x1); return .{ x2.reuseBuffer(x0), updated_kv_cache }; } @@ -285,37 +284,47 @@ const RmsNorm = struct { const MoE = struct { router: zml.nn.Linear, - mlp_experts: Mlp, + experts: Mlp, pub const OnDisk = struct { - gate: zml.nn.Linear, - experts: []struct { - w1: zml.nn.Linear, - w2: zml.nn.Linear, - w3: zml.nn.Linear, + router: zml.nn.Linear, + experts: struct { + down_proj_bias: zml.Tensor, + down_proj_blocks: zml.Tensor, + down_proj_scales: zml.Tensor, + gate_up_proj_bias: zml.Tensor, + gate_up_proj_blocks: zml.Tensor, + gate_up_proj_scales: zml.Tensor, }, pub fn rewrite(on_disk: OnDisk) MoE { - const n = on_disk.experts.len; var mlp: Mlp = .{ - .up_proj = on_disk.experts[0].w1, - .down_proj = on_disk.experts[0].w2, - .gate_proj = on_disk.experts[0].w3, + .gate_up_proj = .{ + .weight = on_disk.experts.gate_up_proj_blocks, + .bias = on_disk.experts.gate_up_proj_bias, + }, + .down_proj = .{ + .weight = on_disk.experts.down_proj_blocks, + .bias = on_disk.experts.down_proj_bias, + }, }; - mlp.up_proj.weight._shape = mlp.up_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .up, .d }); - mlp.down_proj.weight._shape = mlp.down_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .d, .up }); - mlp.gate_proj.weight._shape = mlp.gate_proj.weight.shape().insert(0, .{ .expert = n }).withTags(.{ .expert, .up, .d }); + mlp.gate_up_proj.weight._shape = mlp.gate_up_proj.weight.shape().withTags(.{ .expert, .up, .blocks, .d }); + if (mlp.gate_up_proj.bias) |*bias| { + bias._shape = bias.shape().withTags(.{ .expert, .up }); + } + mlp.down_proj.weight._shape = mlp.down_proj.weight.shape().withTags(.{ .expert, .d, .blocks, .up }); + if (mlp.down_proj.bias) |*bias| bias._shape = bias.shape().withTags(.{ .expert, .d }); - var router = on_disk.gate; + var router = on_disk.router; router.weight = router.weight.withTags(.{ .expert, .d }); - return .{ .router = router, .mlp_experts = mlp }; + return .{ .router = router, .experts = mlp }; } }; pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { const gating = self.router.forward(input).convert(.f32).softmax(.expert); - return zml.nn.mixtureOfExperts(Mlp, self.mlp_experts, input, gating, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 1.5 }); + return zml.nn.mixtureOfExperts(Mlp, self.experts, input, gating, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 1.5 }); } pub fn loadSharded(self: MoE, store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, platform: zml.Platform) !zml.Bufferized(MoE) { @@ -323,10 +332,9 @@ const MoE = struct { return .{ .router = try store.loadModelById(zml.nn.Linear, stdx.noalloc, self.router, platform), - .mlp_experts = .{ - .gate_proj = .{ .weight = try loadShardedWeight(store, prefix, "w1", platform, self.mlp_experts.gate_proj.weight) }, - .down_proj = .{ .weight = try loadShardedWeight(store, prefix, "w2", platform, self.mlp_experts.down_proj.weight) }, - .up_proj = .{ .weight = try loadShardedWeight(store, prefix, "w3", platform, self.mlp_experts.up_proj.weight) }, + .experts = .{ + .gate_up_proj = .{ .weight = try loadShardedWeight(store, prefix, "w1", platform, self.experts.gate_up_proj.weight) }, + .down_proj = .{ .weight = try loadShardedWeight(store, prefix, "w2", platform, self.experts.down_proj.weight) }, }, }; } @@ -376,13 +384,11 @@ const MoE = struct { }; const Mlp = struct { - up_proj: zml.nn.Linear, // (dim -> hidden_dim) - gate_proj: zml.nn.Linear, // (dim -> hidden_dim) + gate_up_proj: zml.nn.Linear, // (dim -> hidden_dim * 2) down_proj: zml.nn.Linear, // (hidden_dim -> dim) pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const up = self.up_proj.weight.dot(x, .d); - const gate = self.gate_proj.weight.dot(x, .d).silu(); + const gate, const up = self.gate_up_proj.weight.dot(x, .d).silu().chunkExact(.up, 2); const out = gate.mul(up).dot(self.down_proj.weight, .up); log.warn("Mlp(x: {f}) -> up: {f} -> gate: {f} -> out: {f}", .{ x, up, gate, out }); @@ -394,8 +400,10 @@ pub const SelfAttn = struct { q_proj: zml.nn.Linear, k_proj: zml.nn.Linear, v_proj: zml.nn.Linear, + sinks: zml.Tensor, o_proj: zml.nn.Linear, + num_heads: i64 = undefined, num_kv_heads: i64 = 0, rope_opts: zml.nn.RopeOpts = undefined, From 7923c40b735dc155cc1dfd5d5016179cb8a79451 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Sun, 10 Aug 2025 21:38:25 +0200 Subject: [PATCH 53/67] attempt at generating mxfp4 code in stablehlo fusion doesn't work as expected, even when enabling triton --- examples/benchmark/main.zig | 51 ++++++++++++++++++++++++++----------- zml/floats.zig | 13 ++++++++++ 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 6428eda9..cfd5fcd5 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -1,8 +1,9 @@ const std = @import("std"); -const zml = @import("zml"); -const stdx = @import("stdx"); + const asynk = @import("async"); +const stdx = @import("stdx"); const flags = stdx.flags; +const zml = @import("zml"); // set log level to debug to print the generated IR pub const std_options: std.Options = .{ @@ -10,12 +11,27 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; -pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); +pub fn benchmark(x: zml.Tensor, q_w: zml.Tensor, q_scale: zml.Tensor) zml.Tensor { + return x.dot(dequantize(q_w, q_scale), .{.d}); +} + +fn dequantize(quantized_weight: zml.Tensor, q_scale: zml.Tensor) zml.Tensor { + var fp4_encoding_buf: [16]zml.floats.BFloat16 = undefined; + for (0..16) |i| { + const i_u4: u4 = @truncate(i); + const f4: zml.floats.Float4E2M1 = @bitCast(i_u4); + fp4_encoding_buf[i] = .fromF32(f4.toF32()); + } + std.log.warn("fp4: {d}", .{fp4_encoding_buf}); + const fp4_encoding: zml.Tensor = .constantTensor(.fromSlice(.{ .fp4 = 16 }, &fp4_encoding_buf)); + + const scale: zml.Tensor = .pow(.constant(q_scale.shape(), .init(.bf16, 2)), q_scale.convert(.bf16)); + const actual_weight = fp4_encoding.gatherValues(.fp4, quantized_weight, .{}); + return actual_weight.mul(scale.broad(actual_weight.shape())).merge(.{ .d = .{ .block, .d } }); } pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); } pub fn asyncMain() !void { @@ -38,22 +54,25 @@ pub fn asyncMain() !void { // Auto-select platform const platform = context.autoPlatform(.{}).withCompilationOptions(.{ .sharding_enabled = true, + .xla_dump_to = "/tmp/zml/benchmark", + .xla_dump_fusion_visualization = true, }); context.printAvailablePlatforms(platform); var args = std.process.args(); const cli_args = flags.parse(&args, CliArgs); - const a_shape = zml.Shape.init(.{ cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .m, .k }).withSharding(.{.k}); - const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); - var timer = try std.time.Timer.start(); + const x_shape: zml.Shape = .init(.{ .b = 16, .d = cli_args.size }, .bf16); + const w_shape: zml.Shape = .init(.{ .d_out = cli_args.size, .block = @divExact(cli_args.size, 32), .d = 32 }, .i4); + const s_shape: zml.Shape = .init(.{ .d_out = cli_args.size, .block = @divExact(cli_args.size, 32) }, .i8); + var timer = try std.time.Timer.start(); std.debug.print("\nCompiling model to MLIR....\n", .{}); std.debug.print("-" ** 160 ++ "\n", .{}); // Start compiling. // The shape of the input tensor, we have to pass in manually. timer.reset(); - var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, benchmark, .{ a_shape, b_shape }, platform }); + var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, benchmark, .{ x_shape, w_shape, s_shape }, platform }); // Wait for compilation to finish const executable = try compilation.awaitt(); @@ -65,22 +84,24 @@ pub fn asyncMain() !void { var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); - var a_buffer = try createRandomBuffer(allocator, platform, a_shape, random); - defer a_buffer.deinit(); - var b_buffer = try createRandomBuffer(allocator, platform, b_shape, random); - defer b_buffer.deinit(); + var x = try createRandomBuffer(allocator, platform, x_shape, random); + defer x.deinit(); + var w = try createRandomBuffer(allocator, platform, w_shape, random); + defer w.deinit(); + var s = try createRandomBuffer(allocator, platform, s_shape, random); + defer s.deinit(); std.debug.print("\nRunning benchmark....\n", .{}); // Ignore first run { - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + var result: zml.Buffer = executable.call(.{ x, w, s }); defer result.deinit(); } // call our executable module timer.reset(); - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + var result: zml.Buffer = executable.call(.{ x, w, s }); defer result.deinit(); const elapsed_ns = timer.lap(); const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; diff --git a/zml/floats.zig b/zml/floats.zig index 9344408c..fd9a4ab8 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -333,6 +333,19 @@ test BFloat16 { }); } +pub const Float4E2M1 = packed struct(u4) { + mantissa: u1, + exponent: u2, + sign: u1, + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const format = Helpers.format; +}; + pub fn floatCast(T: type, x: anytype) T { return switch (T) { f64, f32, f16 => switch (@TypeOf(x)) { From 305d6729ad6b4eb341c0ce808f2eae5b7c323a97 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 19 Aug 2025 08:16:56 +0200 Subject: [PATCH 54/67] expose more PJRT types --- mlir/mlir.zig | 16 ++++++- pjrt/pjrt.zig | 6 +++ zml/dtype.zig | 45 ++++++++++++++++--- zml/floats.zig | 112 ++++++++++++++++++++++++++++++++++++++++-------- zml/mlirx.zig | 6 +++ zml/tensor.zig | 10 ++--- zml/testing.zig | 6 ++- 7 files changed, 168 insertions(+), 33 deletions(-) diff --git a/mlir/mlir.zig b/mlir/mlir.zig index a6721914..1940c4c9 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -416,9 +416,9 @@ pub const ArrayAttribute = struct { pub fn IntegerAttribute(comptime it: IntegerTypes) type { const ZigType, const getter = comptime switch (it) { - .i1, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, + .i1, .i2, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, .si4, .si8, .si16, .si32, .si64 => .{ i64, c.mlirIntegerAttrGetValueSInt }, - .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, + .u2, .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, .unknown => @compileError("IntegerAttribute(unknown)"), }; @@ -1289,6 +1289,7 @@ pub const IndexType = struct { pub const IntegerTypes = enum { i1, + i2, i4, i8, i16, @@ -1299,6 +1300,7 @@ pub const IntegerTypes = enum { si16, si32, si64, + u2, u4, u8, u16, @@ -1311,6 +1313,7 @@ pub const IntegerTypes = enum { pub fn IntegerType(comptime it: IntegerTypes) type { const Config = switch (it) { .i1 => .{ 1, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, + .i2 => .{ 2, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, .i4 => .{ 4, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, .i8 => .{ 8, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, .i16 => .{ 16, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, @@ -1321,6 +1324,7 @@ pub fn IntegerType(comptime it: IntegerTypes) type { .si16 => .{ 16, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, .si32 => .{ 32, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, .si64 => .{ 64, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, + .u2 => .{ 2, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, .u4 => .{ 4, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, .u8 => .{ 8, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, .u16 => .{ 16, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, @@ -1360,11 +1364,15 @@ pub fn IntegerType(comptime it: IntegerTypes) type { } pub const FloatTypes = enum { + f4e2m1fn, + f8e3m4, + f8e4m3, f8e4m3b11fnuz, f8e4m3fn, f8e4m3fnuz, f8e5m2, f8e5m2fnuz, + f8e8m0fnu, bf16, f16, f32, @@ -1379,11 +1387,15 @@ pub const FloatTypes = enum { pub fn FloatType(comptime ft: FloatTypes) type { const Config = switch (ft) { + .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, + .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, + .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, + .f8e8m0fnu => .{ .cmlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 08ccf217..e57b4575 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -793,11 +793,13 @@ pub const LoadedExecutable = opaque { pub const BufferType = enum(c.PJRT_Buffer_Type) { invalid = c.PJRT_Buffer_Type_INVALID, bool = c.PJRT_Buffer_Type_PRED, + i2 = c.PJRT_Buffer_Type_S2, i4 = c.PJRT_Buffer_Type_S4, i8 = c.PJRT_Buffer_Type_S8, i16 = c.PJRT_Buffer_Type_S16, i32 = c.PJRT_Buffer_Type_S32, i64 = c.PJRT_Buffer_Type_S64, + u2 = c.PJRT_Buffer_Type_U2, u4 = c.PJRT_Buffer_Type_U4, u8 = c.PJRT_Buffer_Type_U8, u16 = c.PJRT_Buffer_Type_U16, @@ -814,6 +816,10 @@ pub const BufferType = enum(c.PJRT_Buffer_Type) { f8e4m3b11fnuz = c.PJRT_Buffer_Type_F8E4M3B11FNUZ, f8e5m2fnuz = c.PJRT_Buffer_Type_F8E5M2FNUZ, f8e4m3fnuz = c.PJRT_Buffer_Type_F8E4M3FNUZ, + f8e4m3 = c.PJRT_Buffer_Type_F8E4M3, + f8e3m4 = c.PJRT_Buffer_Type_F8E3M4, + f8e8m0 = c.PJRT_Buffer_Type_F8E8M0FNU, + f4e2m1 = c.PJRT_Buffer_Type_F4E2M1FN, }; pub const MemoryLayoutType = enum(c.PJRT_Buffer_MemoryLayout_Type) { diff --git a/zml/dtype.zig b/zml/dtype.zig index 81750da2..00fa8112 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -12,20 +12,26 @@ test { pub const DataType = enum(u8) { bool, // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. + f4e2m1, + f8e3m4, + f8e4m3, f8e4m3b11fnuz, f8e4m3fn, f8e4m3fnuz, f8e5m2, f8e5m2fnuz, + f8e8m0, bf16, f16, f32, f64, + i2, i4, i8, i16, i32, i64, + u2, u4, u8, u16, @@ -50,8 +56,21 @@ pub const DataType = enum(u8) { pub fn class(self: DataType) Class { return switch (self) { .bool => .bool, - .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .float, - .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => .integer, + .f4e2m1, + .f8e3m4, + .f8e4m3, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + .f8e8m0, + .bf16, + .f16, + .f32, + .f64, + => .float, + .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => .integer, .c64, .c128 => .complex, }; } @@ -70,21 +89,27 @@ pub const DataType = enum(u8) { pub fn fromZigType(comptime T: type) DataType { return switch (T) { + floats.Float4E2M1 => .f4e2m1, + floats.Float8E3M4 => .f8e3m4, + floats.Float8E4M3 => .f8e4m3, floats.Float8E4M3B11FNUZ => .f8e4m3b11fnuz, floats.Float8E4M3FN => .f8e4m3fn, floats.Float8E4M3FNUZ => .f8e4m3fnuz, floats.Float8E5M2 => .f8e5m2, floats.Float8E5M2FNUZ => .f8e5m2fnuz, + floats.Float8E8M0 => .f8e8m0, floats.BFloat16 => .bf16, f16 => .f16, f32 => .f32, f64 => .f64, bool => .bool, + i2 => .i2, i4 => .i4, i8 => .i8, i16 => .i16, i32 => .i32, i64 => .i64, + u2 => .u2, u4 => .u4, u8 => .u8, u16 => .u16, @@ -192,10 +217,10 @@ pub const DataType = enum(u8) { pub fn maxValue(dtype: DataType) Data { return switch (dtype) { .bool => .{ .bool = true }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), + inline .f4e2m1, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), + inline .f8e3m4, .f8e4m3, .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), }; } @@ -207,20 +232,26 @@ pub const DataType = enum(u8) { pub const Data = union(DataType) { bool: bool, + f4e2m1: floats.Float4E2M1, + f8e3m4: floats.Float8E3M4, + f8e4m3: floats.Float8E4M3, f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, f8e4m3fn: floats.Float8E4M3FN, f8e4m3fnuz: floats.Float8E4M3FNUZ, f8e5m2: floats.Float8E5M2, f8e5m2fnuz: floats.Float8E5M2FNUZ, + f8e8m0: floats.Float8E8M0, bf16: floats.BFloat16, f16: f16, f32: f32, f64: f64, + i2: i2, i4: i4, i8: i8, i16: i16, i32: i32, i64: i64, + u2: u2, u4: u4, u8: u8, u16: u16, @@ -244,7 +275,7 @@ pub const Data = union(DataType) { .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |tag| switch (Ti) { + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), @@ -254,7 +285,7 @@ pub const Data = union(DataType) { .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), }, - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { .comptime_int => blk: { const OutT = @FieldType(Data, @tagName(tag)); if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { diff --git a/zml/floats.zig b/zml/floats.zig index fd9a4ab8..642f1d6a 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -129,11 +129,7 @@ pub const Float8E4M3B11FNUZ = packed struct(u8) { exponent: u4, sign: u1, - pub const nan: Float8E4M3B11FNUZ = .{ - .sign = 1, - .exponent = 0, - .mantissa = 0, - }; + pub const nan: Float8E4M3B11FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; pub fn isNan(self: Float8E4M3B11FNUZ) bool { return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; @@ -155,7 +151,7 @@ pub const Float8E4M3FN = packed struct(u8) { pub const nan: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; pub fn isNan(self: Float8E4M3FN) bool { - return allBitsOne(self.exponent) and allBitsOne(self.mantissa); + return self.exponent == nan.exponent and self.mantissa == nan.mantissa; } const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; @@ -170,11 +166,7 @@ pub const Float8E4M3FNUZ = packed struct(u8) { exponent: u4, sign: u1, - pub const nan: Float8E4M3FNUZ = .{ - .sign = 1, - .exponent = 0, - .mantissa = 0, - }; + pub const nan: Float8E4M3FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; pub fn isNan(self: Float8E4M3FNUZ) bool { return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; @@ -209,14 +201,10 @@ pub const Float8E5M2 = packed struct(u8) { exponent: u5, sign: u1, - pub const nan: Float8E5M2 = .{ - .sign = 0, - .exponent = std.math.maxInt(u5), - .mantissa = 1, - }; + pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 1 }; pub fn isNan(self: Float8E5M2) bool { - return allBitsOne(self.exponent) and self.mantissa != 0; + return self.exponent == nan.exponent and self.mantissa != 0; } pub const minus_inf: Float8E5M2 = .{ @@ -232,7 +220,7 @@ pub const Float8E5M2 = packed struct(u8) { }; pub fn isInf(self: Float8E5M2) bool { - return allBitsOne(self.exponent) and self.mantissa == 0; + return self == inf or self == minus_inf; } const Helpers = FloatHelpers(@This()); @@ -333,11 +321,99 @@ test BFloat16 { }); } +pub const Float8E4M3 = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3 = @bitCast(0xFF); + + pub fn isNan(self: Float8E4M3) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const minus_inf: Float8E4M3 = .{ + .sign = 1, + .exponent = std.math.maxInt(u4), + .mantissa = 0, + }; + + pub const inf: Float8E4M3 = .{ + .sign = 0, + .exponent = std.math.maxInt(u4), + .mantissa = 0, + }; + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const format = Helpers.format; +}; + +pub const Float8E3M4 = packed struct(u8) { + mantissa: u4, + exponent: u3, + sign: u1, + + pub const nan: Float8E3M4 = @bitCast(0xFF); + + pub fn isNan(self: Float8E3M4) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const minus_inf: Float8E3M4 = .{ + .sign = 1, + .exponent = std.math.maxInt(u3), + .mantissa = 0, + }; + + pub const inf: Float8E3M4 = .{ + .sign = 0, + .exponent = std.math.maxInt(u3), + .mantissa = 0, + }; + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const format = Helpers.format; +}; + +pub const Float8E8M0 = packed struct(u8) { + mantissa: u0, + exponent: u8, + sign: u0, + + /// Lossy conversion from f32, similar to @floatCast + pub fn fromF32(f: f32) Float8E8M0 { + const vf32: Float32 = @bitCast(f); + return .{ .mantissa = 0, .exponent = @intCast(vf32.exponent), .sign = 0 }; + } + + /// Lossless conversion to f32. + pub fn toF32(x: Float8E8M0) f32 { + const vf32: Float32 = .{ + .sign = 1, + .exponent = x.exponent, + .mantissa = 0, + }; + return @bitCast(vf32); + } + + const Helpers = FloatHelpers(@This()); + pub const format = Helpers.format; +}; + pub const Float4E2M1 = packed struct(u4) { mantissa: u1, exponent: u2, sign: u1, + pub const nan: Float4E2M1 = @bitCast(@as(u4, 0xF)); const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 28f40383..6aea3a79 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -35,20 +35,26 @@ pub const Type = struct { pub fn fromDType(ctx: mlir.Context, dt: dtype.DataType) mlir.Type { return switch (dt) { .bool => .int(ctx, .i1), + .f4e2m1 => .float(ctx, .f4e2m1fn), + .f8e3m4 => .float(ctx, .f8e3m4), + .f8e4m3 => .float(ctx, .f8e4m3), .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), .f8e4m3fn => .float(ctx, .f8e4m3fn), .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), .f8e5m2 => .float(ctx, .f8e5m2), .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), + .f8e8m0 => .float(ctx, .f8e8m0fnu), .bf16 => .float(ctx, .bf16), .f16 => .float(ctx, .f16), .f32 => .float(ctx, .f32), .f64 => .float(ctx, .f64), + .i2 => .int(ctx, .i2), .i4 => .int(ctx, .i4), .i8 => .int(ctx, .i8), .i16 => .int(ctx, .i16), .i32 => .int(ctx, .i32), .i64 => .int(ctx, .i64), + .u2 => .int(ctx, .u2), .u4 => .int(ctx, .u4), .u8 => .int(ctx, .u8), .u16 => .int(ctx, .u16), diff --git a/zml/tensor.zig b/zml/tensor.zig index 7037fae5..f14b67e7 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -3969,11 +3969,11 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Te } fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialect.stablehlo.CompareType { - return dialect.stablehlo.CompareType.init(ctx, switch (dtype) { - .i4, .i8, .i16, .i32, .i64 => .SIGNED, - .bool, .u4, .u8, .u16, .u32, .u64 => .UNSIGNED, - .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .FLOAT, - .c64, .c128 => @panic("Can't compare complex numbers"), + return dialect.stablehlo.CompareType.init(ctx, switch (dtype.class()) { + .bool => .UNSIGNED, + .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, + .float => .FLOAT, + .complex => @panic("Can't compare complex numbers"), }); } diff --git a/zml/testing.zig b/zml/testing.zig index eabe3791..a4b91718 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -65,11 +65,15 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { .f16, .f32, .f64, + .f4e2m1, + .f8e3m4, + .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, + .f8e8m0, => |t| { const L = t.toZigType(); const left_data = left.items(L); @@ -96,7 +100,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { else => unreachable, } }, - inline .bool, .u4, .u8, .u16, .u32, .u64, .i4, .i8, .i16, .i32, .i64 => |t| { + inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { const T = t.toZigType(); return std.testing.expectEqualSlices(T, left.items(T), right.items(T)); }, From 9c71941e54aeb9f1eef131ed5d0cdfae651ef91f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 26 Aug 2025 16:48:13 +0200 Subject: [PATCH 55/67] fix mlir mixed precisions --- mlir/dialects/stablehlo/stablehlo.zig | 1 + mlir/mlir.zig | 2 +- zml/mlirx.zig | 9 ++++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 95b66cf2..e06a5617 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -744,6 +744,7 @@ pub const CustomCallOpts = struct { api_version: ApiVersion, }; +// TODO move the call_target_name to second argument pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { const MAX_OPERANDS = 64; const MAX_RESULTS = 16; diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 1940c4c9..8dc63606 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1395,7 +1395,7 @@ pub fn FloatType(comptime ft: FloatTypes) type { .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, - .f8e8m0fnu => .{ .cmlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, + .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 6aea3a79..7002a664 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -68,23 +68,28 @@ pub const Type = struct { pub fn toDType(mlir_type: mlir.Type) dtype.DataType { const mapping = .{ .{ .bool, mlir.IntegerType(.i1) }, - + .{ .f4e2m1, mlir.FloatType(.f4e2m1fn) }, + .{ .f8e3m4, mlir.FloatType(.f8e3m4) }, + .{ .f8e4m3, mlir.FloatType(.f8e4m3) }, .{ .f8e4m3b11fnuz, mlir.FloatType(.f8e4m3b11fnuz) }, .{ .f8e4m3fn, mlir.FloatType(.f8e4m3fn) }, .{ .f8e4m3fnuz, mlir.FloatType(.f8e4m3fnuz) }, .{ .f8e5m2, mlir.FloatType(.f8e5m2) }, .{ .f8e5m2fnuz, mlir.FloatType(.f8e5m2fnuz) }, + .{ .f8e8m0, mlir.FloatType(.f8e8m0fnu) }, .{ .bf16, mlir.FloatType(.bf16) }, .{ .f16, mlir.FloatType(.f16) }, .{ .f32, mlir.FloatType(.f32) }, .{ .f64, mlir.FloatType(.f64) }, + .{ .i2, mlir.IntegerType(.i2) }, .{ .i4, mlir.IntegerType(.i4) }, .{ .i8, mlir.IntegerType(.i8) }, .{ .i16, mlir.IntegerType(.i16) }, .{ .i32, mlir.IntegerType(.i32) }, .{ .i64, mlir.IntegerType(.i64) }, + .{ .u2, mlir.IntegerType(.u2) }, .{ .u4, mlir.IntegerType(.u4) }, .{ .u8, mlir.IntegerType(.u8) }, .{ .u16, mlir.IntegerType(.u16) }, @@ -95,6 +100,8 @@ pub const Type = struct { .{ .c128, mlir.ComplexType(.c128) }, }; + // TODO: this seems quite slow to have all of those functions calls. + // Maybe we should memoize the ptr of a set of mlir types when creating the context. inline for (mapping) |entry| { const dt, const mlirT = entry; if (mlirT.is_a_fn(mlir_type._inner)) { From 036fb8981bf6c59a9bfb47c327c128083154eef5 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 27 Aug 2025 16:08:12 +0200 Subject: [PATCH 56/67] BlockScaledLinear, attention sinks fixup! BlockScaledLinear, attention sinks fixup! BlockScaledLinear, attention sinks --- examples/llama/llama.zig | 4 +- examples/mixtral/main.zig | 36 +++++++------ examples/mixtral/mixtral.zig | 62 ++++++++++++---------- zml/aio.zig | 2 +- zml/buffer.zig | 26 ++++++++++ zml/floats.zig | 32 +++++++++++- zml/hostbuffer.zig | 7 ++- zml/nn.zig | 75 ++++++++++++++++++++++++--- zml/shape.zig | 99 ++++++++++++++++++++++++++++++++++++ zml/tensor.zig | 68 ++++++++++++++++++------- 10 files changed, 338 insertions(+), 73 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index da487bb3..efe7f220 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -234,8 +234,8 @@ const Mlp = struct { pub fn forward(self: Mlp, x: Tensor) Tensor { const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); - output = output.silu().mul(proj); + const gate = zml.call(self.gate_proj, .forward, .{x}); + const output = gate.silu().mul(proj); return zml.call(self.down_proj, .forward, .{output}); } }; diff --git a/examples/mixtral/main.zig b/examples/mixtral/main.zig index 390af7c6..dccd7c9f 100644 --- a/examples/mixtral/main.zig +++ b/examples/mixtral/main.zig @@ -180,6 +180,7 @@ pub fn asyncMain() !void { } pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: MixtralLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { + _ = config; // autofix var tokens = std.ArrayList(u32).init(allocator); var encoder = try tokenizer.encoder(); defer encoder.deinit(); @@ -190,22 +191,23 @@ pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok return tokens.toOwnedSlice(); } - const start_header_id = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header_id = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const eot_id = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline_id = (try encoder.encode("\n"))[0]; + const start_header = tokenizer.tokenToId("<|start|>") orelse return error.NoSuchToken; + const end_header_start_message = tokenizer.tokenToId("<|message|>") orelse return error.NoSuchToken; + const end_message = tokenizer.tokenToId("<|end|>") orelse return error.NoSuchToken; - try tokens.append(config.bos_token_id); + { + try tokens.appendSlice(&.{ start_header, tokenizer.tokenToId("system").?, end_header_start_message }); + try tokens.appendSlice(try encoder.encode("You are ChatGPT, a large language model trained by OpenAI.\n")); + try tokens.append(end_message); + } - try tokens.append(start_header_id); - try tokens.appendSlice(try encoder.encode("user")); - try tokens.appendSlice(&.{ end_header_id, newline_id }); + { + try tokens.appendSlice(&.{ start_header, tokenizer.tokenToId("user").?, end_header_start_message }); + try tokens.appendSlice(try encoder.encode(prompt)); + try tokens.append(end_message); + } - try tokens.appendSlice(try encoder.encode(prompt)); - try tokens.appendSlice(&.{ eot_id, newline_id }); - try tokens.append(start_header_id); - try tokens.appendSlice(try encoder.encode("assistant")); - try tokens.appendSlice(&.{ end_header_id, newline_id }); + try tokens.appendSlice(&.{ start_header, tokenizer.tokenToId("assistant").?, end_header_start_message }); return tokens.toOwnedSlice(); } @@ -223,6 +225,7 @@ pub fn generateText( skip_llama3_encoding: bool, ) ![]const u8 { const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); + log.info("\t Tokenized prompt: {d}", .{prompt_tok}); defer allocator.free(prompt_tok); var tokenizer_decoder = try tokenizer.decoder(); @@ -271,9 +274,10 @@ pub fn generateText( // collect and print generated sequence num_tokens_generated += 1; const generated_token = generated_token_buffer[0]; - const chunk = try tokenizer_decoder.next(generated_token) orelse unreachable; - try output.appendSlice(chunk); - std.debug.print("{s}", .{chunk}); + if (try tokenizer_decoder.next(generated_token)) |chunk| { + try output.appendSlice(chunk); + std.debug.print("{s}", .{chunk}); + } // check for eos if (i == output_tokens_len) break :generation; diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 9e941f47..909f109a 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -10,7 +10,7 @@ const log = std.log.scoped(.Mixtral); /// Dimensions of activations: {.b, .s, .d} pub const MixtralLM = struct { pub const Config = struct { - bos_token_id: u32 = 0, + bos_token_id: u32 = 199998, eos_token_id: stdx.json.Union(union(enum) { int: u32, ints: []u32, @@ -84,8 +84,8 @@ pub const MixtralLM = struct { const on_disk_moe = try zml.aio.populateModelWithPrefix(MoE.OnDisk, allocator, store, prefix.concat("mlp")); var moe = on_disk_moe.rewrite(); { - moe.experts.gate_up_proj.weight = moe.experts.gate_up_proj.weight.withSharding(.{.expert}); - moe.experts.down_proj.weight = moe.experts.down_proj.weight.withSharding(.{.expert}); + moe.experts.gate_up_proj.blocks = moe.experts.gate_up_proj.blocks.withSharding(.{.expert}); + moe.experts.down_proj.blocks = moe.experts.down_proj.blocks.withSharding(.{.expert}); } layer.* = .{ @@ -178,7 +178,7 @@ pub const MixtralLM = struct { .input_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.input_layernorm, platform), .self_attn = try store.loadModelById(SelfAttn, noalloc, layer.self_attn, platform), .post_attention_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.post_attention_layernorm, platform), - .mlp = try layer.mlp.loadSharded(store, &prefix, platform), + .mlp = try store.loadModelById(MoE, noalloc, layer.mlp, platform), }; } @@ -272,13 +272,14 @@ pub const TransformerLayer = struct { const RmsNorm = struct { weight: zml.Tensor, - eps: f32 = 1e-5, + eps: f32 = 1e-6, /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); - const normalized = zml.nn.rmsNorm(x, .d, self.eps); - return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); + // Note: contrary to Llama here the full layer is done in .f32, not just the variance computation. + const normalized = zml.nn.rmsNorm(x.convert(.f32), .d, self.eps); + return normalized.mul(self.weight.convert(.f32).withTags(.{.d}).broad(x.shape())).convert(input.dtype()); } }; @@ -298,27 +299,28 @@ const MoE = struct { }, pub fn rewrite(on_disk: OnDisk) MoE { - var mlp: Mlp = .{ + const e = on_disk.experts; + const experts: Mlp = .{ .gate_up_proj = .{ - .weight = on_disk.experts.gate_up_proj_blocks, - .bias = on_disk.experts.gate_up_proj_bias, + // We need to bitcast the scale cause safetensors doesn't encode f8 types correctly + .scale = e.gate_up_proj_scales.bitCast(.f8e8m0).withTags(.{ .expert, .out, .d }), + // We don't bitcast here because PJRT doesn't handle packed host buffers + .blocks = e.gate_up_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), + .blocks_dtype = .f4e2m1, + .bias = e.gate_up_proj_bias.withTags(.{ .expert, .d }), }, .down_proj = .{ - .weight = on_disk.experts.down_proj_blocks, - .bias = on_disk.experts.down_proj_bias, + .blocks = e.down_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), + .blocks_dtype = .f4e2m1, + .scale = e.down_proj_scales.bitCast(.f8e8m0).withTags(.{ .expert, .out, .d }), + .bias = e.down_proj_bias.withTags(.{ .expert, .d }), }, }; - mlp.gate_up_proj.weight._shape = mlp.gate_up_proj.weight.shape().withTags(.{ .expert, .up, .blocks, .d }); - if (mlp.gate_up_proj.bias) |*bias| { - bias._shape = bias.shape().withTags(.{ .expert, .up }); - } - mlp.down_proj.weight._shape = mlp.down_proj.weight.shape().withTags(.{ .expert, .d, .blocks, .up }); - if (mlp.down_proj.bias) |*bias| bias._shape = bias.shape().withTags(.{ .expert, .d }); var router = on_disk.router; router.weight = router.weight.withTags(.{ .expert, .d }); - return .{ .router = router, .experts = mlp }; + return .{ .router = router, .experts = experts }; } }; @@ -333,8 +335,8 @@ const MoE = struct { return .{ .router = try store.loadModelById(zml.nn.Linear, stdx.noalloc, self.router, platform), .experts = .{ - .gate_up_proj = .{ .weight = try loadShardedWeight(store, prefix, "w1", platform, self.experts.gate_up_proj.weight) }, - .down_proj = .{ .weight = try loadShardedWeight(store, prefix, "w2", platform, self.experts.down_proj.weight) }, + .gate_up_proj = .{ .blocks = try loadShardedWeight(store, prefix, "w1", platform, self.experts.gate_up_proj.blocks) }, + .down_proj = .{ .blocks = try loadShardedWeight(store, prefix, "w2", platform, self.experts.down_proj.blocks) }, }, }; } @@ -384,15 +386,17 @@ const MoE = struct { }; const Mlp = struct { - gate_up_proj: zml.nn.Linear, // (dim -> hidden_dim * 2) - down_proj: zml.nn.Linear, // (hidden_dim -> dim) + gate_up_proj: zml.nn.BlockScaledLinear, // {.out = intermediate_size * 2, .d = hidden_size / block_size, .d_block = block_size } + down_proj: zml.nn.BlockScaledLinear, // {.out = hidden_size * 2, .d = intermediate_size / block_size, .d_block = block_size } pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const gate, const up = self.gate_up_proj.weight.dot(x, .d).silu().chunkExact(.up, 2); + const dt = x.dtype(); + var gate, var up = zml.nn.splitRealImg(self.gate_up_proj.forward(x), .interleaved); + gate = .minimum(gate, .scalar(7, dt)); + up = .clamp(up, .scalar(-7, dt), .scalar(7, dt)); - const out = gate.mul(up).dot(self.down_proj.weight, .up); - log.warn("Mlp(x: {f}) -> up: {f} -> gate: {f} -> out: {f}", .{ x, up, gate, out }); - return out; + const out = gate.quickGelu().mul(up.addConstant(1)); + return zml.call(self.down_proj, .forward, .{out}); } }; @@ -451,7 +455,9 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + const softmax_bias = q.dot(self.sinks.withTags(.{.hd}), .hd); + std.log.warn("q: {}, sinks: {} -> softmax_bias: {}", .{ q, self.sinks, softmax_bias }); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true, .softmax_bias = softmax_bias }); // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; diff --git a/zml/aio.zig b/zml/aio.zig index e87640f6..8d13585a 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -122,7 +122,7 @@ pub const BufferStore = struct { else => @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`"), }; - stdx.debug.assert(x.shape().eql(host_buffer.shape()), "Can't load buffer {f} for tensor {f}: shape mismatch", .{ host_buffer, x }); + stdx.debug.assert(x.byteSize() == host_buffer.shape().byteSize(), "Can't load buffer {f} for tensor {f}: shape mismatch", .{ host_buffer, x }); // Copy sharding info host_buffer._shape = x._shape; return try host_buffer.toDevice(platform); diff --git a/zml/buffer.zig b/zml/buffer.zig index 9f261e6e..8d202127 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -157,6 +157,32 @@ pub const Buffer = struct { return from(platform, HostBuffer.fromBytes(sh, data), .{}); } + test fromBytes { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + { + const data: [8]u32 = .{ 0, 9, 2, 11, 4, 13, 6, 15 }; + const data_bytes: []const u8 = @ptrCast(&data); + try std.testing.expectEqual(@sizeOf(@TypeOf(data)), data_bytes.len); + const data_d = try zml.Buffer.fromBytes(platform, .init(.{8}, .u32), data_bytes); + const data_read = try data_d.toHostAlloc(std.testing.allocator); + defer data_read.deinit(std.testing.allocator); + try std.testing.expectEqualSlices(u32, &data, data_read.items(u32)); + } + + { + // We use u4 so that the bits are packed. + const data: [8]u4 = .{ 0, 9, 2, 11, 4, 13, 6, 15 }; + const data_bytes: []const u8 = @ptrCast(&data); + try std.testing.expectEqual(@sizeOf(@TypeOf(data)), data_bytes.len); + const expect: [8]u4 = .{ 0, 9, 2, 11, 4, 13, 6, 15 }; + const data_d = try zml.Buffer.fromBytes(platform, .init(.{8}, .u4), data_bytes); + const data_read = try data_d.toHostAlloc(std.testing.allocator); + defer data_read.deinit(std.testing.allocator); + try std.testing.expectEqualSlices(u8, std.mem.asBytes(&expect), data_read.bytes()); + } + } + /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { diff --git a/zml/floats.zig b/zml/floats.zig index 642f1d6a..0d2ef25d 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -48,6 +48,9 @@ fn FloatHelpers(Float: type) type { return if (vf32.sign == 0) Float.inf else Float.minus_inf; } else Float.nan; } + // TODO: This is too lossy. + // If exponent underflow, and the mantissa is zero we drop to 0, + // while we should set the mantissa to it's max value. return .{ .sign = vf32.sign, .exponent = @intCast(exponent), @@ -418,8 +421,35 @@ pub const Float4E2M1 = packed struct(u4) { pub const zero = Helpers.zero; pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; pub const format = Helpers.format; + + pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + + pub fn toF32(x: Float4E2M1) f32 { + // the baseline toF32 doesn't work correctly: + // 0b0001 and 0b1001 shoud map to ±0.5, but are mapped to ±epsilon + return values[@as(u4, @bitCast(x))]; + } + + test toF32 { + var to_f32_res: [16]f32 = undefined; + for (&to_f32_res, 0..) |*r, i| { + const x_f4: Float4E2M1 = @bitCast(@as(u4, @intCast(i))); + r.* = x_f4.toF32(); + } + try std.testing.expectEqualSlices(f32, &Float4E2M1.values, &to_f32_res); + } + + test fromF32 { + // the baseline fromF32 doesn't work correctly: + // ±0.5 should map to 0b0001/0b1001 but are map to ±0.0 instead. + // TODO: it probably affects other types. + var from_f32_res: [16]Float4E2M1 = undefined; + for (&from_f32_res, 0..) |*r, i| { + r.* = .fromF32(Float4E2M1.values[i]); + } + try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); + } }; pub fn floatCast(T: type, x: anytype) T { diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 848fee86..91406bd6 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -40,7 +40,7 @@ pub const HostBuffer = struct { /// The returned HostBuffer doesn't take ownership of the slice /// that will still need to be deallocated. pub fn fromBytes(shape_: Shape, data_: []const u8) HostBuffer { - stdx.debug.assert(shape_.byteSize() == data_.len, "shape {} and data {} don't match", .{ shape_.byteSize(), data_.len }); + stdx.debug.assert(shape_.byteSize() == data_.len, "shape ({} bytes) and data ({} bytes) don't match", .{ shape_.byteSize(), data_.len }); return .{ ._shape = shape_, ._strides = shape_.computeStrides().buffer, @@ -96,7 +96,10 @@ pub const HostBuffer = struct { pub fn fromArray(arr_ptr: anytype) HostBuffer { const T = @TypeOf(arr_ptr.*); const sh = parseArrayInfo(T); - std.debug.assert(sh.byteSize() == @sizeOf(T)); + if (sh.dtype().bitSizeOf() < 8) { + // Zig isn't correctly packing the sub byte types in arrays. + stdx.debug.assert(sh.byteSize() == @sizeOf(T), "fromArray doesn't work on {}, cause the pjrt layout ({d} bytes) doesn't match the Zig layout ({d} bytes)", .{ T, sh.byteSize(), @sizeOf(T) }); + } return .{ ._shape = sh, ._strides = sh.computeStrides().buffer, diff --git a/zml/nn.zig b/zml/nn.zig index 2d13b86a..4cbbe3e6 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -3,11 +3,13 @@ const std = @import("std"); const assert = std.debug.assert; const testing = std.testing; +const dialect = @import("mlir/dialects"); const stdx = @import("stdx"); const DataType = @import("dtype.zig").DataType; const helpers = @import("helpers.zig"); const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); const cuda = @import("nn/cuda.zig"); const ops = @import("ops.zig"); const Shape = @import("shape.zig").Shape; @@ -36,6 +38,61 @@ pub const Linear = struct { } }; +pub const BlockScaledLinear = struct { + blocks: Tensor, + scale: Tensor, + bias: ?Tensor = null, + blocks_dtype: DataType, + + pub fn forward(self: BlockScaledLinear, x: Tensor) Tensor { + const ctx = x.getContext(); + const res_shape = x.shape().setDim(-1, self.blocks.dim(-3)); + + // Bitcast to our actual type. This allows to load weights in a packed layout. + const blocks = self.blocks.bitCast(self.blocks_dtype).merge(.{ .d_block = .{ .d_block, .bitcast } }); + + // log.warn("BlockScaledLinear({}): {f} -> {f}", .{ self, x, res_shape }); + const y = switch (ctx._platform.target) { + .cuda, .rocm, .tpu => y: { + // Merge the .d and .d_block axes, the custom call takes care of dequant. + const weight = blocks.merge(.{ .d = .{ .d, .d_block } }); + const scaled_dot_res_shape = weight.shape().dot(x.shape(), .{.d}).rename(.{ .out = .d }).withDtype(x.dtype()); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + &.{ + weight.value(), + x.value(), + self.scale.value(), + }, + .{ + .call_target_name = "__op$block_scaled_dot", + .backend_config = null, + .has_side_effect = false, + .api_version = .original, + }, + &.{mlirx.tensorType(mlir_ctx, scaled_dot_res_shape)}, + loc, + ); + break :y Tensor._result(scaled_dot_res_shape, op.result(0)).transpose(res_shape); + }, + else => y: { + const dequantized_weight: Tensor = .mul( + blocks.convert(x.dtype()), + self.scale.convert(x.dtype()).appendAxes(.{.d_block}), + ); + var y = x.dot(dequantized_weight.merge(.{ .d = .{ .d, .d_block } }), .{.d}); + std.log.warn("output shape: {_}", .{y}); + std.debug.assert(y.shape().eql(res_shape)); + y._shape = res_shape; + break :y y; + }, + }; + return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; + } +}; + pub const TokenEmbedding = struct { weight: Tensor, @@ -320,9 +377,8 @@ pub const LayerNorm = struct { pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { const ax = x.axis(axis); // upcast to improve precision - const xf32 = x.convert(.f32); - const mean = xf32.mul(xf32).mean(ax); - const rsqrt = Tensor.rsqrt(mean.addConstant(eps)).convert(x.dtype()); + const variance = x.convert(.f32).powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); return x.mul(rsqrt.broad(x.shape())); } @@ -1104,6 +1160,7 @@ pub fn causalAttnMask( pub const SdpaOpts = struct { attn_mask: ?Tensor = null, scale: ?Tensor = null, + softmax_bias: ?Tensor = null, allow_cudnn: bool = true, // TODO: put a callback instead of all this field, // so that @@ -1132,7 +1189,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape())) { + if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { return cuda.sdpa(q, k, v, opts); } @@ -1150,9 +1207,15 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { k = k.mul(head_scaling.convert(k.dtype())); var attn_weights = q.dot(k, .{.hd}); - // log.debug("attn_weights : {}, attn_mask : {?}", .{ attn_weights, attn_mask }); if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); - attn_weights = attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); + log.warn("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); + attn_weights = attn_weights.convert(.f32); + attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { + // The split is needed because we also split q ourselves, + // but the contiguous seems brittle, what we want is a kind of "broadTranspose". + const bias = softmax_bias.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }).contiguous(.{ .h, .q, .hq }); + break :attn attn_weights.convert(.f32).softmaxBiased(.k, bias).convert(q.dtype()); + } else attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); var attn = attn_weights.dot(v, .{.k}); return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); diff --git a/zml/shape.zig b/zml/shape.zig index efb3d808..dc7ea9ec 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -444,6 +444,15 @@ pub const Shape = struct { return true; } + test canBroadcastTo { + try std.testing.expect(canBroadcastTo(.scalar(.i32), .init(.{32}, .bf16))); + try std.testing.expect(canBroadcastTo(.scalar(.i32), .init(.{ 40, 16 }, .u16))); + try std.testing.expect(canBroadcastTo(.init(.{ 1, 16 }, .i32), .init(.{ 40, 16 }, .u16))); + try std.testing.expect(canBroadcastTo(.init(.{ .a = 16 }, .i32), .init(.{ .a = 16, .b = 32 }, .u16))); + try std.testing.expect(canBroadcastTo(.init(.{ .b = 32 }, .i32), .init(.{ .a = 16, .b = 32 }, .u16))); + try std.testing.expect(!canBroadcastTo(.init(.{ .b = 20 }, .i32), .init(.{ .a = 16, .b = 32 }, .u16))); + } + pub fn reshape(self: Shape, new_shape_: anytype) Shape { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); @@ -1100,4 +1109,94 @@ pub const Shape = struct { } return res_shape; } + + pub fn dot(lhs: Shape, rhs: Shape, comptime contracting: anytype) Shape { + var contracting_axes: std.BoundedArray([2]i8, MAX_RANK) = .{}; + if (@TypeOf(contracting) == EnumLiteral) { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(contracting), rhs.axis(contracting) }); + } else { + inline for (contracting) |c| { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(c), rhs.axis(c) }); + } + } + + var batching_axes: std.BoundedArray([2]i8, MAX_RANK) = .{}; + for (lhs.tags(), 0..) |l, li| { + stdx.debug.assert(l != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, lhs }); + + for (rhs.tags(), 0..) |r, ri| { + stdx.debug.assert(r != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, rhs }); + + if (l == r) { + for (contracting_axes.slice()) |ct| { + if (l == lhs.tag(ct[0])) { + break; + } + } else { + // tag is both in lhs and rhs but not in contracting -> it's a batching dim. + batching_axes.appendAssumeCapacity(.{ @intCast(li), @intCast(ri) }); + } + } + } + } + + return dot_(lhs, rhs, contracting_axes.slice(), batching_axes.slice()); + } + + pub fn dot_( + lhs: Shape, + rhs: Shape, + contracting_axes: []const [2]i8, + batching_axes: []const [2]i8, + ) Shape { + const Axes = std.BoundedArray(u8, MAX_RANK); + + var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + // Validate batching axes + var lhs_batching_axes: Axes = .{}; + var rhs_batching_axes: Axes = .{}; + for (batching_axes) |b_axes| { + const l, const r = b_axes; + stdx.debug.assert(lhs.dim(l) == rhs.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {} and {}", .{ l, r, lhs, rhs }); + var t = lhs.tag(l); + if (t == Shape.TagUnknown) t = rhs.tag(r); + res_shape = res_shape.appendDim(lhs.dim(l), t); + lhs_batching_axes.appendAssumeCapacity(lhs.axis(l)); + rhs_batching_axes.appendAssumeCapacity(rhs.axis(r)); + } + + // Validate contracting axes + var lhs_contracting_axes: Axes = .{}; + var rhs_contracting_axes: Axes = .{}; + for (contracting_axes) |c_axes| { + const l, const r = c_axes; + stdx.debug.assert(lhs.dim(l) == rhs.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {} and {}", .{ l, r, lhs, rhs }); + lhs_contracting_axes.appendAssumeCapacity(lhs.axis(l)); + rhs_contracting_axes.appendAssumeCapacity(rhs.axis(r)); + } + + // Result shape is obtained by concatenating batching dimensions, (already done) + // then dimensions from lhs axes that aren't contracting nor batching, + // then dimensions from rhs axes that aren't contracting nor batching. + for (0..lhs.rank()) |l| { + if (std.mem.indexOfScalar(u8, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { + continue; + } + if (std.mem.indexOfScalar(u8, lhs_batching_axes.constSlice(), @intCast(l))) |_| { + continue; + } + res_shape = res_shape.appendDim(lhs.dim(l), lhs.tag(l)); + } + for (0..rhs.rank()) |r| { + if (std.mem.indexOfScalar(u8, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { + continue; + } + if (std.mem.indexOfScalar(u8, rhs_batching_axes.constSlice(), @intCast(r))) |_| { + continue; + } + res_shape = res_shape.appendDim(rhs.dim(r), rhs.tag(r)); + } + + return res_shape; + } }; diff --git a/zml/tensor.zig b/zml/tensor.zig index f14b67e7..3eb2ec2e 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -303,6 +303,13 @@ pub const Tensor = struct { res_shape = res_shape.withDtype(dt); + if (self._id == .buffer_id) { + // Allow to use bitcast outside of compilation. + var res = self; + res._shape = res_shape; + return res; + } + const loc = self.getContext().location(@src(), "bitCast({s})", .{@tagName(dt)}); const op = dialect.stablehlo.bitcast_convert( self.getContext().mlirCtx(), @@ -720,7 +727,7 @@ pub const Tensor = struct { }; const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); const counts = values.gather(.{ .d = samples }, .{}).sum(.n).bitCast(.u16); - const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); + const actual_dist = counts.convert(target_dist.dtype()).divByConst(s.dim(.n)); return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; } }; @@ -1047,6 +1054,21 @@ pub const Tensor = struct { return _result(self._shape.withDtype(to), op.result(0)); } + test convert { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + { + // We use u4 so that the bits of Float4E2M1 are correctly packed. + const input: [8]u4 = .{ 0, 9, 2, 11, 4, 13, 6, 15 }; + const expect: [8]f32 = .{ 0.0, -0.5, 1.0, -1.5, 2.0, -3.0, 4.0, -6.0 }; + const input_d = try zml.Buffer.fromBytes(platform, .init(.{8}, .f4e2m1), std.mem.asBytes(&input)); + const output_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ input_d, .f32 }); + const output = try output_d.getValue([8]f32); + try std.testing.expectEqualSlices(f32, &expect, &output); + } + } + /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); @@ -1280,7 +1302,7 @@ pub const Tensor = struct { /// /// It's an even more crude approximation than gelu. pub fn quickGelu(x: Tensor) Tensor { - return x.scale(1.702).sigmoid().mul(x); + return x.mul(.sigmoid(x.scale(1.702))); } /// Returns a Tensor containing the Sigmoid Linear Unit (SiLU) activation function applied to each element of the input Tensor. @@ -1288,21 +1310,38 @@ pub const Tensor = struct { /// silu(x) = x σ(x) /// https://paperswithcode.com/method/silu pub fn silu(x: Tensor) Tensor { - return x.mul(x.sigmoid()); + return x.mul(.sigmoid(x)); } - /// Returns a Tensor containing the softmax function applied to each element of the input Tensor. + /// Computes softmax along the given axis. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) pub fn softmax(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); const max_val = self.max(a); - const row_mask = max_val.cmp(.GT, Tensor.scalar(-std.math.inf(f64), self.dtype())); + const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), self.dtype())); - const exp_diff_max = self.sub(self.max(a).broad(self._shape)).exp(); + const exp_diff_max = self.sub(max_val).exp(); const res = exp_diff_max.div(exp_diff_max.sum(a)); // If a row is full -inf return full 0 instead of full nan, // this fix attention when mask hides a full row. - return row_mask.broad(self.shape()).select(res, Tensor.scalar(0, self.dtype())); + return row_mask.broad(self.shape()).select(res, .scalar(0, self.dtype())); + } + + /// Computes softmax, but adds a bias to the sum of exponentiel. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) + pub fn softmaxBiased(self: Tensor, axis_: anytype, bias: ?Tensor) Tensor { + const a = self.axis(axis_); + + if (bias == null) return self.softmax(axis_); + const b = bias.?.convert(self.dtype()).broad(self.shape().setDim(a, 1)); + const max_val: Tensor = maximum(self.max(a), b); + const exp_diff_max = self.sub(max_val).exp(); + const bias_diff_max = b.mul(.exp(.negate(max_val))); + const res = exp_diff_max.div(exp_diff_max.sum(a).add(bias_diff_max)); + + // The bias means that denominator won't be 0, we don't need to handle that case. + return res; } /// Returns a Tensor containing the log of the sum of exponential over the given axis. @@ -2094,6 +2133,11 @@ pub const Tensor = struct { /// Reshapes the input Tensor with the given shape. pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); + if (self._id == .buffer_id) { + var res = self; + res._shape = output_shape; + return res; + } const tensor_type = mlirx.tensorType(self.getContext().mlirCtx(), output_shape); const loc = self.getContext().location(@src(), "reshape({f})", .{output_shape}); const reshape_value = dialect.stablehlo.reshape(self.getContext().mlirCtx(), self.value(), tensor_type, loc); @@ -4167,16 +4211,6 @@ fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedAr return .{ axes_is_scalar, coord_axes }; } -fn parseArrayInfo(T: type) Shape { - return switch (@typeInfo(T)) { - .Array => |arr| { - const s = parseArrayInfo(arr.child); - return s.insert(0, .{arr.len}); - }, - else => .{ ._dtype = DataType.fromZigType(T) }, - }; -} - inline fn toI64(values: anytype) []i64 { var res: [Tensor.MAX_RANK]i64 = undefined; for (values, 0..) |val, i| res[i] = @intCast(val); From f0445456408b2ac4fa50f11e60f517f0b2b02796 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 29 Aug 2025 14:29:53 +0200 Subject: [PATCH 57/67] update to 0.15 --- examples/mixtral/main.zig | 49 ++++++++++++++++++------------------ examples/mixtral/mixtral.zig | 6 ++--- zml/nn.zig | 2 +- zml/shape.zig | 10 ++++---- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/examples/mixtral/main.zig b/examples/mixtral/main.zig index dccd7c9f..f521d8bb 100644 --- a/examples/mixtral/main.zig +++ b/examples/mixtral/main.zig @@ -71,7 +71,9 @@ pub fn asyncMain() !void { const config = blk: { var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); defer config_json_file.close() catch unreachable; - var reader = std.json.reader(allocator, config_json_file.reader()); + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(&config_json_buffer); + var reader = std.json.Reader.init(allocator, &config_reader.interface); defer reader.deinit(); const config_obj = try std.json.parseFromTokenSourceLeaky(mixtral.MixtralLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); break :blk config_obj; @@ -146,16 +148,16 @@ pub fn asyncMain() !void { platform, }); - log.info("\tLoading mixtral weights from {?s}...", .{model_weights_path}); + log.info("\tLoading mixtral weights from {s}...", .{model_weights_path}); var mixtral_weights = try model_instance.loadBuffers(model_arena.allocator(), store, platform); defer zml.aio.unloadBuffers(&mixtral_weights); - log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); + log.info("✅\tLoaded weights in {D}", .{start.read()}); var mixtral_module_prefill = (try fut_mod_prefill.awaitt()).prepare(mixtral_weights); defer mixtral_module_prefill.deinit(); var mixtral_module = (try fut_mod.awaitt()).prepare(mixtral_weights); defer mixtral_module.deinit(); - log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); + log.info("✅\tCompiled model in {D}", .{start.read()}); log.info("Creating KvCache", .{}); const kv_cache = try mixtral.KvCache.initBuffer(kv_shape, platform); @@ -163,7 +165,7 @@ pub fn asyncMain() !void { var tokenizer = blk: { log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); var timer = try stdx.time.Timer.start(); - defer log.info("Loaded tokenizer from {s} [{}]", .{ model_tokenizer_path, timer.read() }); + defer log.info("Loaded tokenizer from {s} [{f}]", .{ model_tokenizer_path, timer.read() }); break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), model_tokenizer_path); }; @@ -174,14 +176,17 @@ pub fn asyncMain() !void { const seed = cli.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; - const generated_text = try generateText(config, model_instance, mixtral_module_prefill, mixtral_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); + + // Unbuffered writing of the tokens to stdout. + var output = std.fs.File.stdout().writer(&.{}); + + try generateText(config, model_instance, mixtral_module_prefill, mixtral_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding, &output.interface); // generated text will be printed token by token. - defer allocator.free(generated_text); } pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: MixtralLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { _ = config; // autofix - var tokens = std.ArrayList(u32).init(allocator); + var tokens = std.array_list.Managed(u32).init(allocator); var encoder = try tokenizer.encoder(); defer encoder.deinit(); @@ -223,9 +228,10 @@ pub fn generateText( seed: u128, prompt: []const u8, skip_llama3_encoding: bool, -) ![]const u8 { + output: *std.Io.Writer, +) !void { const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); - log.info("\t Tokenized prompt: {d}", .{prompt_tok}); + log.info("\t Tokenized prompt: {any}", .{prompt_tok}); defer allocator.free(prompt_tok); var tokenizer_decoder = try tokenizer.decoder(); @@ -260,10 +266,6 @@ pub fn generateText( var current_token = try zml.Buffer.fromSlice(platform, .{1}, &generated_token_buffer); defer current_token.deinit(); - // Here we collect the generated text - var output = std.ArrayList(u8).init(allocator); - defer output.deinit(); - const output_tokens_len = max_seq_len - prompt_tok.len - 1; const start = std.time.microTimestamp(); @@ -275,8 +277,7 @@ pub fn generateText( num_tokens_generated += 1; const generated_token = generated_token_buffer[0]; if (try tokenizer_decoder.next(generated_token)) |chunk| { - try output.appendSlice(chunk); - std.debug.print("{s}", .{chunk}); + try output.writeAll(chunk); } // check for eos @@ -304,9 +305,8 @@ pub fn generateText( const end = std.time.microTimestamp(); const duration = stdx.math.divFloat(f64, end - start, std.time.us_per_s); const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration; - std.debug.print("\n", .{}); + log.info("✅ Generated {d} tokens in {:.3}s: {d:.3}tok/s", .{ num_tokens_generated, duration, speed }); - return output.toOwnedSlice(); } const ClapBoilerplate = struct { @@ -325,19 +325,20 @@ const ClapBoilerplate = struct { pub fn parseCli(allocator: std.mem.Allocator) Cli { var diag: clap.Diagnostic = .{}; - const stderr = std.io.getStdErr().writer(); + var stderr_buffer: [1024]u8 = undefined; + var stderr = std.fs.File.stderr().writer(&stderr_buffer); const cli = clap.parse(clap.Help, &cli_params, parsers, .{ .diagnostic = &diag, .allocator = allocator, }) catch |err| { - diag.report(stderr, err) catch {}; - stderr.print("usage: ", .{}) catch {}; - clap.usage(stderr, clap.Help, &cli_params) catch {}; - stderr.print("\n", .{}) catch {}; + diag.report(&stderr.interface, err) catch {}; + stderr.interface.print("usage: ", .{}) catch {}; + clap.usage(&stderr.interface, clap.Help, &cli_params) catch {}; + stderr.interface.print("\n", .{}) catch {}; std.process.exit(1); }; if (cli.args.help != 0) { - clap.help(std.io.getStdErr().writer(), clap.Help, &cli_params, .{}) catch {}; + clap.help(&stderr.interface, clap.Help, &cli_params, .{}) catch {}; std.process.exit(0); } return cli; diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 909f109a..9b230da1 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -123,7 +123,7 @@ pub const MixtralLM = struct { kv_cache: KvCache, rng: zml.Tensor.Rng, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { - stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Mixtral ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); + stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Mixtral ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); @@ -256,7 +256,7 @@ pub const TransformerLayer = struct { ) struct { zml.Tensor, KvCache } { // Self Attention //log.debug("TransformerLayer({}) -> {}", .{ x0, self.input_layernorm.forward(x0) }); - stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); + stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); @@ -456,7 +456,7 @@ pub const SelfAttn = struct { v = new_kv_cache.values().convert(dtype); const softmax_bias = q.dot(self.sinks.withTags(.{.hd}), .hd); - std.log.warn("q: {}, sinks: {} -> softmax_bias: {}", .{ q, self.sinks, softmax_bias }); + std.log.warn("q: {f}, sinks: {f} -> softmax_bias: {f}", .{ q, self.sinks, softmax_bias }); const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true, .softmax_bias = softmax_bias }); // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); diff --git a/zml/nn.zig b/zml/nn.zig index 4cbbe3e6..57adfc9e 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -83,7 +83,7 @@ pub const BlockScaledLinear = struct { self.scale.convert(x.dtype()).appendAxes(.{.d_block}), ); var y = x.dot(dequantized_weight.merge(.{ .d = .{ .d, .d_block } }), .{.d}); - std.log.warn("output shape: {_}", .{y}); + std.log.warn("output shape: {f}", .{y}); std.debug.assert(y.shape().eql(res_shape)); y._shape = res_shape; break :y y; diff --git a/zml/shape.zig b/zml/shape.zig index dc7ea9ec..b60cce8a 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -1111,7 +1111,7 @@ pub const Shape = struct { } pub fn dot(lhs: Shape, rhs: Shape, comptime contracting: anytype) Shape { - var contracting_axes: std.BoundedArray([2]i8, MAX_RANK) = .{}; + var contracting_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; if (@TypeOf(contracting) == EnumLiteral) { contracting_axes.appendAssumeCapacity(.{ lhs.axis(contracting), rhs.axis(contracting) }); } else { @@ -1120,7 +1120,7 @@ pub const Shape = struct { } } - var batching_axes: std.BoundedArray([2]i8, MAX_RANK) = .{}; + var batching_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; for (lhs.tags(), 0..) |l, li| { stdx.debug.assert(l != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, lhs }); @@ -1149,7 +1149,7 @@ pub const Shape = struct { contracting_axes: []const [2]i8, batching_axes: []const [2]i8, ) Shape { - const Axes = std.BoundedArray(u8, MAX_RANK); + const Axes = stdx.BoundedArray(u8, MAX_RANK); var res_shape: Shape = .{ ._dtype = lhs.dtype() }; // Validate batching axes @@ -1157,7 +1157,7 @@ pub const Shape = struct { var rhs_batching_axes: Axes = .{}; for (batching_axes) |b_axes| { const l, const r = b_axes; - stdx.debug.assert(lhs.dim(l) == rhs.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {} and {}", .{ l, r, lhs, rhs }); + stdx.debug.assert(lhs.dim(l) == rhs.dim(r), "dotGeneral expects batching dimensions to be equal, got {d} and {d} in {f} and {f}", .{ l, r, lhs, rhs }); var t = lhs.tag(l); if (t == Shape.TagUnknown) t = rhs.tag(r); res_shape = res_shape.appendDim(lhs.dim(l), t); @@ -1170,7 +1170,7 @@ pub const Shape = struct { var rhs_contracting_axes: Axes = .{}; for (contracting_axes) |c_axes| { const l, const r = c_axes; - stdx.debug.assert(lhs.dim(l) == rhs.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {} and {}", .{ l, r, lhs, rhs }); + stdx.debug.assert(lhs.dim(l) == rhs.dim(r), "dotGeneral expects contracting dimensions to be equal, got {d} and {d} in {f} and {f}", .{ l, r, lhs, rhs }); lhs_contracting_axes.appendAssumeCapacity(lhs.axis(l)); rhs_contracting_axes.appendAssumeCapacity(rhs.axis(r)); } From 0bec0cfb08e66bf53f4555502bc5b4377ba2b843 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 29 Aug 2025 14:50:15 +0200 Subject: [PATCH 58/67] mixtra.Mixtral -> Mixtral.zig --- examples/mixtral/BUILD.bazel | 2 +- examples/mixtral/main.zig | 33 ++-- examples/mixtral/mixtral.zig | 362 +++++++++++++++++------------------ 3 files changed, 195 insertions(+), 202 deletions(-) diff --git a/examples/mixtral/BUILD.bazel b/examples/mixtral/BUILD.bazel index b7160096..b0fee69c 100644 --- a/examples/mixtral/BUILD.bazel +++ b/examples/mixtral/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( name = "mixtral", srcs = [ - "mixtral.zig", + "Mixtral.zig", ], main = "main.zig", deps = [ diff --git a/examples/mixtral/main.zig b/examples/mixtral/main.zig index f521d8bb..333fc76e 100644 --- a/examples/mixtral/main.zig +++ b/examples/mixtral/main.zig @@ -8,12 +8,7 @@ const Buffer = zml.Buffer; const Tensor = zml.Tensor; const ShapeOf = zml.ShapeOf; -const mixtral = @import("mixtral.zig"); -const MixtralLM = mixtral.MixtralLM; -const Mixtral = mixtral.Llama; -const KvCache = mixtral.KvCache; -const TransformerLayer = mixtral.TransformerLayer; -const SelfAttn = mixtral.SelfAttn; +const Mixtral = @import("Mixtral.zig"); const log = std.log.scoped(.mixtral); @@ -75,7 +70,7 @@ pub fn asyncMain() !void { var config_reader = config_json_file.reader(&config_json_buffer); var reader = std.json.Reader.init(allocator, &config_reader.interface); defer reader.deinit(); - const config_obj = try std.json.parseFromTokenSourceLeaky(mixtral.MixtralLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + const config_obj = try std.json.parseFromTokenSourceLeaky(Mixtral.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); break :blk config_obj; }; @@ -104,14 +99,14 @@ pub fn asyncMain() !void { defer store.deinit(); var model_arena = std.heap.ArenaAllocator.init(allocator); - const llama_options: mixtral.MixtralLM.Options = .{ + const llama_options: Mixtral.Options = .{ .max_seq_len = @intCast(cli.args.@"seq-len" orelse 256), .sampling_strategy = .{ .topk = 1, .temperature = 1.0, }, }; - const model_instance = try mixtral.MixtralLM.init(model_arena.allocator(), store, config, llama_options); + const model_instance = try Mixtral.init(model_arena.allocator(), store, config, llama_options); const dims = model_instance.model.shape(); const dtype = model_instance.model.embed_tokens.weight.dtype(); @@ -122,12 +117,12 @@ pub fn asyncMain() !void { const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype).withSharding(.{.h}); - const kv_cache_shape: zml.ShapeOf(mixtral.KvCache) = mixtral.KvCache.initShape(kv_shape); + const kv_cache_shape: zml.ShapeOf(Mixtral.KvCache) = Mixtral.KvCache.initShape(kv_shape); const rng_shape = zml.Tensor.Rng.shape(); var start = try std.time.Timer.start(); var fut_mod_prefill = try asynk.asyncc(zml.compileModel, .{ - allocator, mixtral.MixtralLM.forward, model_instance, + allocator, Mixtral.forward, model_instance, .{ tokens_shape_prefill, token_idx_shape, @@ -138,7 +133,7 @@ pub fn asyncMain() !void { }); var fut_mod = try asynk.asyncc(zml.compileModel, .{ - allocator, mixtral.MixtralLM.forward, model_instance, + allocator, Mixtral.forward, model_instance, .{ tokens_shape, token_idx_shape, @@ -160,7 +155,7 @@ pub fn asyncMain() !void { log.info("✅\tCompiled model in {D}", .{start.read()}); log.info("Creating KvCache", .{}); - const kv_cache = try mixtral.KvCache.initBuffer(kv_shape, platform); + const kv_cache = try Mixtral.KvCache.initBuffer(kv_shape, platform); var tokenizer = blk: { log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); @@ -184,7 +179,7 @@ pub fn asyncMain() !void { // generated text will be printed token by token. } -pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: MixtralLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { +pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: Mixtral.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { _ = config; // autofix var tokens = std.array_list.Managed(u32).init(allocator); var encoder = try tokenizer.encoder(); @@ -218,11 +213,11 @@ pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok } pub fn generateText( - config: MixtralLM.Config, - llama_: MixtralLM, - mod_prefill: zml.ModuleExe(MixtralLM.forward), - mod_generate: zml.ModuleExe(MixtralLM.forward), - kv_cache_: zml.Bufferized(mixtral.KvCache), + config: Mixtral.Config, + llama_: Mixtral, + mod_prefill: zml.ModuleExe(Mixtral.forward), + mod_generate: zml.ModuleExe(Mixtral.forward), + kv_cache_: zml.Bufferized(Mixtral.KvCache), tokenizer: zml.tokenizer.Tokenizer, allocator: std.mem.Allocator, seed: u128, diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 9b230da1..a84a84e7 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -1,192 +1,190 @@ +///! Mixtral architecture, using huggingface transformers naming. +///! Dimensions of activations: {.b, .s, .d} const std = @import("std"); -const testing = std.testing; const stdx = @import("stdx"); const zml = @import("zml"); +const Mixtral = @This(); const log = std.log.scoped(.Mixtral); -/// Mixtral architecture, using huggingface transformers naming. -/// Dimensions of activations: {.b, .s, .d} -pub const MixtralLM = struct { - pub const Config = struct { - bos_token_id: u32 = 199998, - eos_token_id: stdx.json.Union(union(enum) { - int: u32, - ints: []u32, - }), - num_hidden_layers: usize, - num_attention_heads: usize, - num_key_value_heads: usize, - rope_theta: f32, - max_position_embeddings: usize, - rms_norm_eps: f32, - hf_rope_impl: bool = true, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, - }; - - pub const Options = struct { - sampling_strategy: ?zml.nn.SamplingStrategy, - max_seq_len: usize, - }; - - lm_head: ?zml.nn.Linear, - model: Mixtral, - - // Options controlling generation - gen_opts: zml.nn.SamplingStrategy = .{}, - config: Config, - - pub fn init(allocator: std.mem.Allocator, store: zml.aio.BufferStore, config: Config, options: Options) !MixtralLM { - var self: MixtralLM = .{ - .config = config, - .gen_opts = options.sampling_strategy orelse .{}, - .model = .{ - .max_seq_len = @intCast(options.max_seq_len), - .num_heads = @intCast(config.num_attention_heads), - .num_kv_heads = @intCast(config.num_key_value_heads), - .rope_opts = .{ - .layout = if (config.hf_rope_impl) .sequential else .interleaved, - .freq_base = config.rope_theta, - .scaling = config.rope_scaling, - }, - - .embed_tokens = .{ - .weight = store.getTensor("model.embed_tokens.weight").withSharding(.{1}), - }, - .layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers), - .norm = .{ - .weight = store.getTensor("model.norm.weight"), - .eps = config.rms_norm_eps, - }, - }, - .lm_head = .{ .weight = store.getTensor("lm_head.weight").withSharding(.{0}) }, - }; - - var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 1024); - try prefix.push(stdx.noalloc, "model.layers"); - for (self.model.layers, 0..) |*layer, i| { - try prefix.pushDigit(stdx.noalloc, i); - defer prefix.pop(); - - // TODO: safer layer init - var self_attn = try zml.aio.populateModelWithPrefix(SelfAttn, allocator, store, prefix.concat("self_attn")); - { - self_attn.num_heads = self.model.num_heads; - self_attn.num_kv_heads = self.model.num_kv_heads; - self_attn.rope_opts = self.model.rope_opts; - self_attn.q_proj.weight = self_attn.q_proj.weight.withSharding(.{0}); - self_attn.k_proj.weight = self_attn.k_proj.weight.withSharding(.{0}); - self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); - self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); - } - - const on_disk_moe = try zml.aio.populateModelWithPrefix(MoE.OnDisk, allocator, store, prefix.concat("mlp")); - var moe = on_disk_moe.rewrite(); - { - moe.experts.gate_up_proj.blocks = moe.experts.gate_up_proj.blocks.withSharding(.{.expert}); - moe.experts.down_proj.blocks = moe.experts.down_proj.blocks.withSharding(.{.expert}); - } - - layer.* = .{ - .input_layernorm = .{ - .weight = store.getTensor(prefix.concat("input_layernorm.weight")), - .eps = config.rms_norm_eps, - }, - .post_attention_layernorm = .{ - .weight = store.getTensor(prefix.concat("post_attention_layernorm.weight")), - .eps = config.rms_norm_eps, - }, - .self_attn = self_attn, - .mlp = moe, - }; - } - - // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. - // It currently crashes/compilation fails - if (self.gen_opts.topk == 1 and self.lm_head != null) { - self.lm_head.?.weight = self.lm_head.?.weight.withSharding(.{0}); - } - - return self; - } - - /// Predicts the token at `token_index` position. - /// Returns: - /// - updated `tokens`, - /// - updated KV cache - /// - a Rng state to allow for probabilistic generation - pub fn forward( - self: MixtralLM, - tokens_: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache, - rng: zml.Tensor.Rng, - ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { - stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Mixtral ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); - const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); - const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; - } - - pub fn sampleTokens( - self: MixtralLM, - lm_head_: ?zml.nn.Linear, - out_: zml.Tensor, - rng: zml.Tensor.Rng, - opts: zml.nn.SamplingStrategy, - ) struct { zml.Tensor, zml.Tensor.Rng } { - const out = out_.withPartialTags(.{ .s, .d }); - - var logits = blk: { - if (lm_head_) |lm_head| { - break :blk zml.call(lm_head, .forward, .{out}); - } else { - break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); - } - }; - - if (logits.shape().hasTag(.voc) == null) - logits = logits.rename(.{ .d = .voc }); - - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); - return .{ next_tokens, new_rng }; - } - - pub fn loadBuffers(self: MixtralLM, allocator: std.mem.Allocator, store: zml.aio.BufferStore, platform: zml.Platform) !zml.Bufferized(MixtralLM) { - var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 256); - defer prefix.deinit(allocator); - - const noalloc = stdx.noalloc; - const loaded: zml.Bufferized(MixtralLM) = .{ - .model = .{ - .embed_tokens = try store.loadModelById(zml.nn.TokenEmbedding, noalloc, self.model.embed_tokens, platform), - .layers = try allocator.alloc(zml.Bufferized(TransformerLayer), self.model.layers.len), - .norm = try store.loadModelById(RmsNorm, noalloc, self.model.norm, platform), - }, - .lm_head = try store.loadModelById(?zml.nn.Linear, noalloc, self.lm_head, platform), - }; - - prefix.push(noalloc, "model.layers") catch unreachable; - for (loaded.model.layers, self.model.layers, 0..) |*d_layer, layer, layer_id| { - const ckpt = prefix.checkpoint(); - defer prefix.restore(ckpt); - - prefix.pushDigit(noalloc, layer_id) catch unreachable; - d_layer.* = .{ - .input_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.input_layernorm, platform), - .self_attn = try store.loadModelById(SelfAttn, noalloc, layer.self_attn, platform), - .post_attention_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.post_attention_layernorm, platform), - .mlp = try store.loadModelById(MoE, noalloc, layer.mlp, platform), - }; - } - - return loaded; - } +pub const Config = struct { + bos_token_id: u32 = 199998, + eos_token_id: stdx.json.Union(union(enum) { + int: u32, + ints: []u32, + }), + num_hidden_layers: usize, + num_attention_heads: usize, + num_key_value_heads: usize, + rope_theta: f32, + max_position_embeddings: usize, + rms_norm_eps: f32, + hf_rope_impl: bool = true, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, }; -pub const Mixtral = struct { +pub const Options = struct { + sampling_strategy: ?zml.nn.SamplingStrategy, + max_seq_len: usize, +}; + +lm_head: ?zml.nn.Linear, +model: Model, + +// Options controlling generation +gen_opts: zml.nn.SamplingStrategy = .{}, +config: Config, + +pub fn init(allocator: std.mem.Allocator, store: zml.aio.BufferStore, config: Config, options: Options) !Mixtral { + var self: Mixtral = .{ + .config = config, + .gen_opts = options.sampling_strategy orelse .{}, + .model = .{ + .max_seq_len = @intCast(options.max_seq_len), + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }, + + .embed_tokens = .{ + .weight = store.getTensor("model.embed_tokens.weight").withSharding(.{1}), + }, + .layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers), + .norm = .{ + .weight = store.getTensor("model.norm.weight"), + .eps = config.rms_norm_eps, + }, + }, + .lm_head = .{ .weight = store.getTensor("lm_head.weight").withSharding(.{0}) }, + }; + + var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 1024); + try prefix.push(stdx.noalloc, "model.layers"); + for (self.model.layers, 0..) |*layer, i| { + try prefix.pushDigit(stdx.noalloc, i); + defer prefix.pop(); + + // TODO: safer layer init + var self_attn = try zml.aio.populateModelWithPrefix(SelfAttn, allocator, store, prefix.concat("self_attn")); + { + self_attn.num_heads = self.model.num_heads; + self_attn.num_kv_heads = self.model.num_kv_heads; + self_attn.rope_opts = self.model.rope_opts; + self_attn.q_proj.weight = self_attn.q_proj.weight.withSharding(.{0}); + self_attn.k_proj.weight = self_attn.k_proj.weight.withSharding(.{0}); + self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); + self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); + } + + const on_disk_moe = try zml.aio.populateModelWithPrefix(MoE.OnDisk, allocator, store, prefix.concat("mlp")); + var moe = on_disk_moe.rewrite(); + { + moe.experts.gate_up_proj.blocks = moe.experts.gate_up_proj.blocks.withSharding(.{.expert}); + moe.experts.down_proj.blocks = moe.experts.down_proj.blocks.withSharding(.{.expert}); + } + + layer.* = .{ + .input_layernorm = .{ + .weight = store.getTensor(prefix.concat("input_layernorm.weight")), + .eps = config.rms_norm_eps, + }, + .post_attention_layernorm = .{ + .weight = store.getTensor(prefix.concat("post_attention_layernorm.weight")), + .eps = config.rms_norm_eps, + }, + .self_attn = self_attn, + .mlp = moe, + }; + } + + // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. + // It currently crashes/compilation fails + if (self.gen_opts.topk == 1 and self.lm_head != null) { + self.lm_head.?.weight = self.lm_head.?.weight.withSharding(.{0}); + } + + return self; +} + +/// Predicts the token at `token_index` position. +/// Returns: +/// - updated `tokens`, +/// - updated KV cache +/// - a Rng state to allow for probabilistic generation +pub fn forward( + self: Mixtral, + tokens_: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, +) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Mixtral ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); + const tokens = tokens_.withPartialTags(.{.s}); + const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; +} + +pub fn sampleTokens( + self: Mixtral, + lm_head_: ?zml.nn.Linear, + out_: zml.Tensor, + rng: zml.Tensor.Rng, + opts: zml.nn.SamplingStrategy, +) struct { zml.Tensor, zml.Tensor.Rng } { + const out = out_.withPartialTags(.{ .s, .d }); + + var logits = blk: { + if (lm_head_) |lm_head| { + break :blk zml.call(lm_head, .forward, .{out}); + } else { + break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); + return .{ next_tokens, new_rng }; +} + +pub fn loadBuffers(self: Mixtral, allocator: std.mem.Allocator, store: zml.aio.BufferStore, platform: zml.Platform) !zml.Bufferized(Mixtral) { + var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 256); + defer prefix.deinit(allocator); + + const noalloc = stdx.noalloc; + const loaded: zml.Bufferized(Mixtral) = .{ + .model = .{ + .embed_tokens = try store.loadModelById(zml.nn.TokenEmbedding, noalloc, self.model.embed_tokens, platform), + .layers = try allocator.alloc(zml.Bufferized(TransformerLayer), self.model.layers.len), + .norm = try store.loadModelById(RmsNorm, noalloc, self.model.norm, platform), + }, + .lm_head = try store.loadModelById(?zml.nn.Linear, noalloc, self.lm_head, platform), + }; + + prefix.push(noalloc, "model.layers") catch unreachable; + for (loaded.model.layers, self.model.layers, 0..) |*d_layer, layer, layer_id| { + const ckpt = prefix.checkpoint(); + defer prefix.restore(ckpt); + + prefix.pushDigit(noalloc, layer_id) catch unreachable; + d_layer.* = .{ + .input_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.input_layernorm, platform), + .self_attn = try store.loadModelById(SelfAttn, noalloc, layer.self_attn, platform), + .post_attention_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.post_attention_layernorm, platform), + .mlp = try store.loadModelById(MoE, noalloc, layer.mlp, platform), + }; + } + + return loaded; +} + +pub const Model = struct { embed_tokens: zml.nn.TokenEmbedding, norm: RmsNorm, layers: []TransformerLayer, @@ -208,7 +206,7 @@ pub const Mixtral = struct { dtype: zml.DataType, }; - pub fn shape(self: Mixtral) Shape { + pub fn shape(self: Model) Shape { const key_dim = self.layers[0].self_attn.k_proj.weight.dim(0); const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; @@ -224,7 +222,7 @@ pub const Mixtral = struct { /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. - pub fn forward(self: Mixtral, tokens: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache) struct { zml.Tensor, KvCache } { + pub fn forward(self: Model, tokens: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache) struct { zml.Tensor, KvCache } { const embeds = embed(self.embed_tokens, tokens); var hidden = embeds; From 82fe2cd9cbc36b110468491e016b56f599393f4e Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 1 Sep 2025 10:08:31 +0200 Subject: [PATCH 59/67] mixtral test --- examples/mixtral/BUILD.bazel | 11 ++++ examples/mixtral/gpt_oss.py | 39 +++++++++++ examples/mixtral/test.zig | 124 +++++++++++++++++++++++++++++++++++ zml/tensor.zig | 2 +- 4 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 examples/mixtral/gpt_oss.py create mode 100644 examples/mixtral/test.zig diff --git a/examples/mixtral/BUILD.bazel b/examples/mixtral/BUILD.bazel index b0fee69c..b014de4b 100644 --- a/examples/mixtral/BUILD.bazel +++ b/examples/mixtral/BUILD.bazel @@ -13,3 +13,14 @@ zig_binary( "@zml//zml", ], ) + +zig_binary( + name = "test", + srcs = ["Mixtral.zig"], + main = "test.zig", + deps = [ + "//async", + "//stdx", + "//zml", + ], + ) diff --git a/examples/mixtral/gpt_oss.py b/examples/mixtral/gpt_oss.py new file mode 100644 index 00000000..a4860809 --- /dev/null +++ b/examples/mixtral/gpt_oss.py @@ -0,0 +1,39 @@ +import torch +import transformers +import sys +import safetensors + +sys.path.append("/Users/guw/Documents/zml/tools") +import zml_utils + +model_path = "/Users/guw/models/openai/gpt-oss-20b" + +pipeline = transformers.pipeline( + "text-generation", + model=model_path, + # model_kwargs={"torch_dtype": torch.bfloat16}, + # device="cuda", + device="cpu", +) +model, tokenizer = pipeline.model, pipeline.tokenizer + +prompt = "What is the largest animal?" +# Wrap the pipeline, and extract activations. +# Activations files can be huge for big models, +# so let's stop collecting after 1000 layers. +pipeline = zml_utils.ActivationCollector(pipeline, max_layers=1000, stop_after_first_step=True) +output, activations = pipeline(prompt) + +# `output` can be `None` if activations collection +# has stopped before the end of the inference +if output: + print(output) + +# Save activations to a file. +filename = model_path.split("/")[-1] + ".activations.pt" +torch.save(activations, filename) +print(f"Saved {len(activations)} activations to {filename}") + +filename = model_path.split("/")[-1] + ".activations.safetensors" +safetensors.torch.save_file(activations, filename) +print(f"Saved {len(activations)} activations to {filename}") diff --git a/examples/mixtral/test.zig b/examples/mixtral/test.zig new file mode 100644 index 00000000..8d859be4 --- /dev/null +++ b/examples/mixtral/test.zig @@ -0,0 +1,124 @@ +const std = @import("std"); + +const asynk = @import("async"); +const stdx = @import("stdx"); +const flags = stdx.flags; +const zml = @import("zml"); + +const Mixtral = @import("./Mixtral.zig"); + +const log = std.log.scoped(.mixtral); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + +pub fn asyncMain() !void { + const CliArgs = struct { + pub const help = + \\ test-implementation --model-path=mixtral/ --config=config.json --reference=activation.pt + ; + model_path: []const u8, + activations: []const u8, + }; + var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Create ZML context + var context = try zml.Context.init(); + defer context.deinit(); + + // Select platform + const platform = context.autoPlatform(.{}); + + // Resolve model files + var args = std.process.args(); + const cli = flags.parse(&args, CliArgs); + + const model_tokenizer_path = try std.fs.path.join(allocator, &.{ cli.model_path, "tokenizer.json" }); + defer allocator.free(model_tokenizer_path); + + // Memory arena dedicated to model shapes and weights + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const model_arena = arena_state.allocator(); + + // Parse model config + const config: Mixtral.Config = cfg: { + const model_config_path = try std.fs.path.join(allocator, &.{ cli.model_path, "config.json" }); + defer allocator.free(model_config_path); + var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(&config_json_buffer); + var reader = std.json.Reader.init(model_arena, &config_reader.interface); + defer reader.deinit(); + const cfg = try std.json.parseFromTokenSourceLeaky(Mixtral.Config, model_arena, &reader, .{ .ignore_unknown_fields = true }); + + std.log.info("Parsed mixtral config: {}", .{cfg}); + break :cfg cfg; + }; + + // Read model shapes. + var store: zml.aio.BufferStore = st: { + const simple_path = try std.fs.path.join(allocator, &.{ cli.model_path, "model.safetensors" }); + defer allocator.free(simple_path); + + if (asynk.File.access(simple_path, .{})) { + log.info("\tLoading mixtral weights from {s}...", .{simple_path}); + break :st try zml.aio.detectFormatAndOpen(allocator, simple_path); + } else |_| {} + + const sharded_path = try std.fs.path.join(allocator, &.{ cli.model_path, "model.safetensors.index.json" }); + defer allocator.free(sharded_path); + log.info("\tLoading mixtral weights from {s}...", .{sharded_path}); + break :st try zml.aio.detectFormatAndOpen(allocator, sharded_path); + }; + defer store.deinit(); + + // Create the model and configure it. + const options: Mixtral.Options = .{ + .max_seq_len = 256, + .sampling_strategy = .{ + .topk = 1, + .temperature = 1.0, + }, + }; + const mixtral = try Mixtral.init(model_arena, store, config, options); + + // Load the weights. + var mixtral_weights = try mixtral.loadBuffers(model_arena, store, platform); + defer zml.aio.unloadBuffers(&mixtral_weights); + + // Load the activations. + var activation_store = try zml.aio.detectFormatAndOpen(allocator, cli.activations); + defer activation_store.deinit(); + // for (activation_store.buffers.keys(), activation_store.buffers.values()) |k, v| { + // log.info("Loaded activations: {s} -> {f}", .{k, v}); + // } + + // Test implementation + try testImplementation(platform, mixtral, mixtral_weights, activation_store); +} + +fn testImplementation( + platform: zml.Platform, + mixtral: Mixtral, + mixtral_weights: zml.Bufferized(Mixtral), + store: zml.aio.BufferStore, +) !void { + try zml.testing.testLayer(platform, store, "model.model.embed_tokens", mixtral.model.embed_tokens, mixtral_weights.model.embed_tokens, 1e-3); + try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.v_proj", mixtral.model.layers[0].self_attn.v_proj, mixtral_weights.model.layers[0].self_attn.v_proj, 1e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.q_proj", mixtral.model.layers[0].self_attn.q_proj, mixtral_weights.model.layers[0].self_attn.q_proj, 2e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.k_proj", mixtral.model.layers[0].self_attn.k_proj, mixtral_weights.model.layers[0].self_attn.k_proj, 2e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.o_proj", mixtral.model.layers[0].self_attn.o_proj, mixtral_weights.model.layers[0].self_attn.o_proj, 2e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.mlp", mixtral.model.layers[0].mlp, mixtral_weights.model.layers[0].mlp, 1e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.input_layernorm", mixtral.model.layers[0].input_layernorm, mixtral_weights.model.layers[0].input_layernorm, 1e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.post_attention_layernorm", mixtral.model.layers[0].post_attention_layernorm, mixtral_weights.model.layers[0].post_attention_layernorm, 1e-2); +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 3eb2ec2e..e49e2e08 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2716,7 +2716,7 @@ pub const Tensor = struct { /// that requires host<->device synchronization. /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { - scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); + scoped_log.debug("scatterSlices({f}, {any}, {f})", .{ self, indices, updates }); const UpdateType = @TypeOf(ScatterOpts.increment); From ebcd411365e1c493b2e20000b6415d57f96e897f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 1 Sep 2025 15:21:24 +0200 Subject: [PATCH 60/67] more std.format workaround --- zml/floats.zig | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/zml/floats.zig b/zml/floats.zig index 0d2ef25d..3e19dff8 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -352,7 +352,7 @@ pub const Float8E4M3 = packed struct(u8) { pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; pub const toF32 = Helpers.toF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; pub const Float8E3M4 = packed struct(u8) { @@ -383,34 +383,47 @@ pub const Float8E3M4 = packed struct(u8) { pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; pub const toF32 = Helpers.toF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; pub const Float8E8M0 = packed struct(u8) { - mantissa: u0, + mantissa: u0 = 0, exponent: u8, - sign: u0, + sign: u0 = 0, /// Lossy conversion from f32, similar to @floatCast pub fn fromF32(f: f32) Float8E8M0 { const vf32: Float32 = @bitCast(f); - return .{ .mantissa = 0, .exponent = @intCast(vf32.exponent), .sign = 0 }; + return .{ .exponent = @intCast(vf32.exponent) }; } /// Lossless conversion to f32. pub fn toF32(x: Float8E8M0) f32 { + if (x.exponent == 0) return min_scale; const vf32: Float32 = .{ - .sign = 1, + .sign = 0, .exponent = x.exponent, .mantissa = 0, }; return @bitCast(vf32); } + pub const min_scale: f32 = Float32.toF32(.{ .sign = 0, .exponent = 0, .mantissa = 0b1 << 22 }); + const Helpers = FloatHelpers(@This()); - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; +test Float8E8M0 { + try std.testing.expectEqual(Float8E8M0{ .exponent = 127 }, Float8E8M0.fromF32(1.0)); + // try std.testing.expectEqual(5.877472e-39, Float8E8M0.toF32(.{ .exponent = 0})); + + try testCustomFloat(Float8E8M0, .{ + .lossless = &[_]f32{ Float8E8M0.min_scale, 1.0, 64.0, 1.0 / 128.0, std.math.pow(f32, 2.0, 127) }, + .lossy = &[_]f32{1.00001}, + }); +} + pub const Float4E2M1 = packed struct(u4) { mantissa: u1, exponent: u2, @@ -421,7 +434,7 @@ pub const Float4E2M1 = packed struct(u4) { pub const zero = Helpers.zero; pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; From 2e0cc3f4da19d06829fa9448c7bb0708eef27b26 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 2 Sep 2025 11:13:40 +0200 Subject: [PATCH 61/67] add mixtral to zls --- BUILD.bazel | 2 ++ examples/mixtral/BUILD.bazel | 2 ++ 2 files changed, 4 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index aa1656d4..84d72ab8 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -12,6 +12,8 @@ zls_completion( deps = [ "//async", "//examples/llama", + "//examples/mixtral", + "//examples/mixtral:test", "//stdx", "//zml", ], diff --git a/examples/mixtral/BUILD.bazel b/examples/mixtral/BUILD.bazel index b014de4b..ebe38f94 100644 --- a/examples/mixtral/BUILD.bazel +++ b/examples/mixtral/BUILD.bazel @@ -12,6 +12,7 @@ zig_binary( "@zml//stdx", "@zml//zml", ], + visibility = ["//visibility:public"], ) zig_binary( @@ -23,4 +24,5 @@ zig_binary( "//stdx", "//zml", ], + visibility = ["//visibility:public"], ) From 96d7add5ec90d95161666586855e217f511be5ab Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 2 Sep 2025 11:15:59 +0200 Subject: [PATCH 62/67] move bitcast to inference code --- examples/mixtral/mixtral.zig | 9 ++++- examples/mixtral/test.zig | 72 ++++++++++++++++++++++++++++++++---- zml/nn.zig | 34 +++++++++++------ zml/tensor.zig | 24 ++++++------ 4 files changed, 106 insertions(+), 33 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index a84a84e7..0baf81f7 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -301,7 +301,7 @@ const MoE = struct { const experts: Mlp = .{ .gate_up_proj = .{ // We need to bitcast the scale cause safetensors doesn't encode f8 types correctly - .scale = e.gate_up_proj_scales.bitCast(.f8e8m0).withTags(.{ .expert, .out, .d }), + .scale = e.gate_up_proj_scales.withTags(.{ .expert, .out, .d }), // We don't bitcast here because PJRT doesn't handle packed host buffers .blocks = e.gate_up_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), .blocks_dtype = .f4e2m1, @@ -310,7 +310,7 @@ const MoE = struct { .down_proj = .{ .blocks = e.down_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), .blocks_dtype = .f4e2m1, - .scale = e.down_proj_scales.bitCast(.f8e8m0).withTags(.{ .expert, .out, .d }), + .scale = e.down_proj_scales.withTags(.{ .expert, .out, .d }), .bias = e.down_proj_bias.withTags(.{ .expert, .d }), }, }; @@ -323,6 +323,7 @@ const MoE = struct { }; pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { + log.warn("compiling moe with {f}", .{input}); const gating = self.router.forward(input).convert(.f32).softmax(.expert); return zml.nn.mixtureOfExperts(Mlp, self.experts, input, gating, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 1.5 }); } @@ -396,6 +397,10 @@ const Mlp = struct { const out = gate.quickGelu().mul(up.addConstant(1)); return zml.call(self.down_proj, .forward, .{out}); } + + pub fn format(self: Mlp, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Mlp(gate_up_proj=.{f}, down_proj=.{f})", .{ self.gate_up_proj, self.down_proj }); + } }; pub const SelfAttn = struct { diff --git a/examples/mixtral/test.zig b/examples/mixtral/test.zig index 8d859be4..dbe32816 100644 --- a/examples/mixtral/test.zig +++ b/examples/mixtral/test.zig @@ -35,7 +35,10 @@ pub fn asyncMain() !void { defer context.deinit(); // Select platform - const platform = context.autoPlatform(.{}); + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .xla_dump_to = "/tmp/zml/mixtral.test", + .sharding_enabled = false, + }); // Resolve model files var args = std.process.args(); @@ -99,14 +102,33 @@ pub fn asyncMain() !void { // Load the activations. var activation_store = try zml.aio.detectFormatAndOpen(allocator, cli.activations); defer activation_store.deinit(); - // for (activation_store.buffers.keys(), activation_store.buffers.values()) |k, v| { - // log.info("Loaded activations: {s} -> {f}", .{k, v}); - // } + for (activation_store.buffers.keys(), activation_store.buffers.values()) |k, *v| { + if (std.mem.endsWith(u8, k, ".mlp.in.0") or std.mem.endsWith(u8, k, ".mlp.out.0")) { + v.* = v.squeeze(0).withTags(.{ .s, .d }); + } + log.info("Loaded activations: {s} -> {f}", .{ k, v }); + // if (std.mem.endsWith(u8, k, "layers.23.mlp.experts.in.1")) log.info("Routing {s}: {d:32.2}", .{k, v.*}); + // if (std.mem.endsWith(u8, k, "layers.23.mlp.experts.in.2")) log.info("Routing weights {s}: {d:32.2}", .{k, v.*}); + } // Test implementation try testImplementation(platform, mixtral, mixtral_weights, activation_store); } +pub fn dequantize(self: zml.nn.BlockScaledLinear) zml.Tensor { + // Bitcast to our actual type. This allows to load weights in a packed layout. + const blocks_0 = self.blocks.bitCast(self.blocks_dtype); + const blocks = blocks_0.merge(.{ .d_block = .{ .d_block, .bitcast } }); + + const scale = self.scale.bitCast(.f8e8m0); + + const dequantized_weight: zml.Tensor = .mul( + blocks.convert(.bf16), + scale.convert(.bf16).appendAxes(.{.d_block}), + ); + return dequantized_weight.merge(.{ .d = .{ .d, .d_block } }).contiguous(.{ .d, .out }); +} + fn testImplementation( platform: zml.Platform, mixtral: Mixtral, @@ -114,11 +136,47 @@ fn testImplementation( store: zml.aio.BufferStore, ) !void { try zml.testing.testLayer(platform, store, "model.model.embed_tokens", mixtral.model.embed_tokens, mixtral_weights.model.embed_tokens, 1e-3); - try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.v_proj", mixtral.model.layers[0].self_attn.v_proj, mixtral_weights.model.layers[0].self_attn.v_proj, 1e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.v_proj", mixtral.model.layers[0].self_attn.v_proj, mixtral_weights.model.layers[0].self_attn.v_proj, 2e-2); try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.q_proj", mixtral.model.layers[0].self_attn.q_proj, mixtral_weights.model.layers[0].self_attn.q_proj, 2e-2); try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.k_proj", mixtral.model.layers[0].self_attn.k_proj, mixtral_weights.model.layers[0].self_attn.k_proj, 2e-2); try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.o_proj", mixtral.model.layers[0].self_attn.o_proj, mixtral_weights.model.layers[0].self_attn.o_proj, 2e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.input_layernorm", mixtral.model.layers[0].input_layernorm, mixtral_weights.model.layers[0].input_layernorm, 1e-3); + try zml.testing.testLayer(platform, store, "model.model.layers.0.post_attention_layernorm", mixtral.model.layers[0].post_attention_layernorm, mixtral_weights.model.layers[0].post_attention_layernorm, 1e-3); + + const allocator = std.heap.smp_allocator; + { + const w = mixtral.model.layers[22].mlp.experts.gate_up_proj; + const dequant_mod = try zml.compileFn(allocator, dequantize, .{.{ + .blocks = w.blocks.shape(), + .scale = w.scale.shape(), + .bias = w.bias.?.shape(), + .blocks_dtype = w.blocks_dtype, + }}, platform); + defer dequant_mod.deinit(); + + const dequant_weight = dequant_mod.call(.{mixtral_weights.model.layers[22].mlp.experts.gate_up_proj}); + const expected = store.buffers.get("model.model.layers.22.mlp.experts.gate_up_proj.bf16").?; + + try zml.testing.expectClose(expected, dequant_weight, 1e-4); + log.info("All good for dequantize gate_up_proj", .{}); + } + + { + const w = mixtral.model.layers[22].mlp.experts.down_proj; + const dequant_mod = try zml.compileFn(allocator, dequantize, .{.{ + .blocks = w.blocks.shape(), + .scale = w.scale.shape(), + .bias = w.bias.?.shape(), + .blocks_dtype = w.blocks_dtype, + }}, platform); + defer dequant_mod.deinit(); + + const dequant_weight = dequant_mod.call(.{mixtral_weights.model.layers[22].mlp.experts.down_proj}); + const expected = store.buffers.get("model.model.layers.22.mlp.experts.down_proj.bf16").?; + + try zml.testing.expectClose(expected, dequant_weight, 1e-4); + log.info("All good for dequantize down_proj", .{}); + } + try zml.testing.testLayer(platform, store, "model.model.layers.0.mlp", mixtral.model.layers[0].mlp, mixtral_weights.model.layers[0].mlp, 1e-2); - try zml.testing.testLayer(platform, store, "model.model.layers.0.input_layernorm", mixtral.model.layers[0].input_layernorm, mixtral_weights.model.layers[0].input_layernorm, 1e-2); - try zml.testing.testLayer(platform, store, "model.model.layers.0.post_attention_layernorm", mixtral.model.layers[0].post_attention_layernorm, mixtral_weights.model.layers[0].post_attention_layernorm, 1e-2); } diff --git a/zml/nn.zig b/zml/nn.zig index 57adfc9e..5082b195 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -47,15 +47,20 @@ pub const BlockScaledLinear = struct { pub fn forward(self: BlockScaledLinear, x: Tensor) Tensor { const ctx = x.getContext(); const res_shape = x.shape().setDim(-1, self.blocks.dim(-3)); + log.warn("BlockScaledLinear({f} ({t}), {f}, {f}) -> {f}", .{ self.blocks, self.blocks_dtype, self.scale, x, res_shape }); // Bitcast to our actual type. This allows to load weights in a packed layout. - const blocks = self.blocks.bitCast(self.blocks_dtype).merge(.{ .d_block = .{ .d_block, .bitcast } }); + const blocks_0 = self.blocks.bitCast(self.blocks_dtype); + const blocks = blocks_0.merge(.{ .d_block = .{ .d_block, .bitcast } }); + + const scale = self.scale.bitCast(.f8e8m0); // log.warn("BlockScaledLinear({}): {f} -> {f}", .{ self, x, res_shape }); const y = switch (ctx._platform.target) { - .cuda, .rocm, .tpu => y: { + .rocm, .tpu => y: { // Merge the .d and .d_block axes, the custom call takes care of dequant. const weight = blocks.merge(.{ .d = .{ .d, .d_block } }); + log.warn("BlockScaledLinear: {f} -> {f} -> {f}", .{ self.blocks, blocks, weight }); const scaled_dot_res_shape = weight.shape().dot(x.shape(), .{.d}).rename(.{ .out = .d }).withDtype(x.dtype()); const mlir_ctx = ctx.mlirCtx(); const loc = mlir_ctx.location(@src()); @@ -64,7 +69,7 @@ pub const BlockScaledLinear = struct { &.{ weight.value(), x.value(), - self.scale.value(), + scale.value(), }, .{ .call_target_name = "__op$block_scaled_dot", @@ -78,9 +83,9 @@ pub const BlockScaledLinear = struct { break :y Tensor._result(scaled_dot_res_shape, op.result(0)).transpose(res_shape); }, else => y: { - const dequantized_weight: Tensor = .mul( + var dequantized_weight: Tensor = .mul( blocks.convert(x.dtype()), - self.scale.convert(x.dtype()).appendAxes(.{.d_block}), + scale.convert(x.dtype()).appendAxes(.{.d_block}), ); var y = x.dot(dequantized_weight.merge(.{ .d = .{ .d, .d_block } }), .{.d}); std.log.warn("output shape: {f}", .{y}); @@ -91,6 +96,10 @@ pub const BlockScaledLinear = struct { }; return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } + + pub fn format(self: BlockScaledLinear, writer: *std.Io.Writer) !void { + try writer.print("BlockScaledLinear(blocks={f}, scale={f}, bias={?f}, dt={t})", .{ self.blocks, self.scale, self.bias, self.blocks_dtype }); + } }; pub const TokenEmbedding = struct { @@ -127,7 +136,7 @@ const MoeOpts = struct { /// - experts: .{ .expert, .d_out, .d } expert layer (need to have a .forward method). /// -> output: .{ .s, .d_out } pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Tensor, opts: MoeOpts) Tensor { - log.warn("mixtureOfExperts({s}, {}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); + log.warn("mixtureOfExperts({s}, {f}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); const num_tokens = input.dim(.s); const num_experts = gating.dim(.expert); stdx.debug.assert(opts.experts_per_token > 0, "mixtureOfExperts expects opts.experts_per_token > 0, got {}", .{opts}); @@ -156,7 +165,7 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te output = if (output) |o| o.add(expert_output) else expert_output; } - log.debug("mixtureOfExperts({s}, {}, {f}, {f}) -> single-stream impl", .{ @typeName(Expert), experts, input, gating }); + log.warn("mixtureOfExperts({s}, {f}, {f}) -> single-stream impl", .{ @typeName(Expert), input, gating }); return output.?; } @@ -168,7 +177,8 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te break :tpe tpe; } else num_tokens; - if (3 * tokens_per_expert <= 2 * num_tokens) { + // if (3 * tokens_per_expert <= 2 * num_tokens) { + if (false) { // Lot of tokens, each experts chose their tokens and compute on that. // It means that some tokens may have only one expert assigned. // Each token will get assigned to at least one expert IIF the input gating is sums up to 1 (typically softmax output). @@ -219,12 +229,12 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te .{ .update_fn = zml.Tensor.ScatterOpts.increment }, ); - log.debug("mixtureOfExperts({s}, {}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), experts, input, gating, tokens_per_expert, num_tokens }); + log.warn("mixtureOfExperts({s}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), input, gating, tokens_per_expert, num_tokens }); return output; } else { // We send all tokens to all experts, then discard result from non-topk experts using hard_gating. const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); - const per_token_score = routing.values.div(routing.values.sum(.top_expert)).convert(input.dtype()); + const per_token_score = routing.values.div(routing.values.sum(.top_expert)).convert(gating.dtype()); // hard_gating is eql to gating where only the top-2 experts have a non-zero score. const hard_gating = zml.Tensor.scatterSlices( @@ -233,8 +243,8 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te per_token_score, .{ .indices_are_unique = true }, ).withSharding(.{.expert}); - log.debug("mixtureOfExperts({s}, {}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), experts, input, gating }); - const output_per_expert = experts.forward(input); + log.warn("mixtureOfExperts({s}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), input, gating }); + const output_per_expert = experts.forward(input.insertAxes(0, .{.expert}).repeat1d(.expert, num_experts)); return output_per_expert.dot(hard_gating.convert(input.dtype()), .expert); } } diff --git a/zml/tensor.zig b/zml/tensor.zig index e49e2e08..caa2eb13 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -303,12 +303,13 @@ pub const Tensor = struct { res_shape = res_shape.withDtype(dt); - if (self._id == .buffer_id) { - // Allow to use bitcast outside of compilation. - var res = self; - res._shape = res_shape; - return res; - } + // TODO: it would be nice to do the bitcast when loading the model, so that inference code don't need to deal with missing metadata in safetensor. + // if (self._id == .buffer_id) { + // // Allow to use bitcast outside of compilation. + // var res = self; + // res._shape = res_shape; + // return res; + // } const loc = self.getContext().location(@src(), "bitCast({s})", .{@tagName(dt)}); const op = dialect.stablehlo.bitcast_convert( @@ -2132,12 +2133,11 @@ pub const Tensor = struct { /// Reshapes the input Tensor with the given shape. pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { - const output_shape = self._shape.reshape(output_shape_); - if (self._id == .buffer_id) { - var res = self; - res._shape = output_shape; - return res; - } + var output_shape = self._shape.reshape(output_shape_); + output_shape._dtype = self.dtype(); + // TODO: it would be nice to allow reshape when loading the model, + // so that inference code don't need to deal with missing metadata in safetensor. + const tensor_type = mlirx.tensorType(self.getContext().mlirCtx(), output_shape); const loc = self.getContext().location(@src(), "reshape({f})", .{output_shape}); const reshape_value = dialect.stablehlo.reshape(self.getContext().mlirCtx(), self.value(), tensor_type, loc); From faa87aa0d3686757d83dbba4dbbb2e310533d930 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 2 Sep 2025 14:33:45 +0200 Subject: [PATCH 63/67] refactor moe code --- examples/mixtral/mixtral.zig | 21 ++++-- examples/mixtral/test.zig | 54 +++++++++++----- zml/aio/torch.zig | 2 +- zml/nn.zig | 122 +++++++++++++++++++++++------------ 4 files changed, 137 insertions(+), 62 deletions(-) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/mixtral.zig index 0baf81f7..40c3fdd3 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/mixtral.zig @@ -281,12 +281,25 @@ const RmsNorm = struct { } }; +pub const Router = struct { + weight: zml.Tensor, + bias: zml.Tensor, + const expert_per_token: u32 = 4; + + pub fn forward(self: Router, input: zml.Tensor) zml.Tensor { + const linear: zml.nn.Linear = .{ .weight = self.weight, .bias = self.bias }; + const gating = linear.forward(input).convert(.f32).softmax(-1); + + return zml.nn.moe.hardGating(gating, 4); + } +}; + const MoE = struct { - router: zml.nn.Linear, experts: Mlp, + router: Router, pub const OnDisk = struct { - router: zml.nn.Linear, + router: Router, experts: struct { down_proj_bias: zml.Tensor, down_proj_blocks: zml.Tensor, @@ -324,7 +337,7 @@ const MoE = struct { pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { log.warn("compiling moe with {f}", .{input}); - const gating = self.router.forward(input).convert(.f32).softmax(.expert); + const gating = self.router.forward(input); return zml.nn.mixtureOfExperts(Mlp, self.experts, input, gating, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 1.5 }); } @@ -384,7 +397,7 @@ const MoE = struct { } }; -const Mlp = struct { +pub const Mlp = struct { gate_up_proj: zml.nn.BlockScaledLinear, // {.out = intermediate_size * 2, .d = hidden_size / block_size, .d_block = block_size } down_proj: zml.nn.BlockScaledLinear, // {.out = hidden_size * 2, .d = intermediate_size / block_size, .d_block = block_size } diff --git a/examples/mixtral/test.zig b/examples/mixtral/test.zig index dbe32816..5aceaec9 100644 --- a/examples/mixtral/test.zig +++ b/examples/mixtral/test.zig @@ -103,32 +103,22 @@ pub fn asyncMain() !void { var activation_store = try zml.aio.detectFormatAndOpen(allocator, cli.activations); defer activation_store.deinit(); for (activation_store.buffers.keys(), activation_store.buffers.values()) |k, *v| { - if (std.mem.endsWith(u8, k, ".mlp.in.0") or std.mem.endsWith(u8, k, ".mlp.out.0")) { - v.* = v.squeeze(0).withTags(.{ .s, .d }); + if (std.mem.indexOf(u8, k, ".layers.") != null and (std.mem.endsWith(u8, k, ".in.0") or std.mem.endsWith(u8, k, ".out.0"))) { + if (v.dim(0) == 1 and v.rank() == 3) { + v.* = v.squeeze(0).withTags(.{ .s, .d }); + } } + log.info("Loaded activations: {s} -> {f}", .{ k, v }); // if (std.mem.endsWith(u8, k, "layers.23.mlp.experts.in.1")) log.info("Routing {s}: {d:32.2}", .{k, v.*}); // if (std.mem.endsWith(u8, k, "layers.23.mlp.experts.in.2")) log.info("Routing weights {s}: {d:32.2}", .{k, v.*}); + if (std.mem.endsWith(u8, k, ".0.mlp.experts.in.2")) log.info("Routing weights {s}: {d:32.2}", .{ k, v.* }); } // Test implementation try testImplementation(platform, mixtral, mixtral_weights, activation_store); } -pub fn dequantize(self: zml.nn.BlockScaledLinear) zml.Tensor { - // Bitcast to our actual type. This allows to load weights in a packed layout. - const blocks_0 = self.blocks.bitCast(self.blocks_dtype); - const blocks = blocks_0.merge(.{ .d_block = .{ .d_block, .bitcast } }); - - const scale = self.scale.bitCast(.f8e8m0); - - const dequantized_weight: zml.Tensor = .mul( - blocks.convert(.bf16), - scale.convert(.bf16).appendAxes(.{.d_block}), - ); - return dequantized_weight.merge(.{ .d = .{ .d, .d_block } }).contiguous(.{ .d, .out }); -} - fn testImplementation( platform: zml.Platform, mixtral: Mixtral, @@ -142,6 +132,7 @@ fn testImplementation( try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.o_proj", mixtral.model.layers[0].self_attn.o_proj, mixtral_weights.model.layers[0].self_attn.o_proj, 2e-2); try zml.testing.testLayer(platform, store, "model.model.layers.0.input_layernorm", mixtral.model.layers[0].input_layernorm, mixtral_weights.model.layers[0].input_layernorm, 1e-3); try zml.testing.testLayer(platform, store, "model.model.layers.0.post_attention_layernorm", mixtral.model.layers[0].post_attention_layernorm, mixtral_weights.model.layers[0].post_attention_layernorm, 1e-3); + try zml.testing.testLayer(platform, store, "model.model.layers.0.mlp.router", mixtral.model.layers[0].mlp.router, mixtral_weights.model.layers[0].mlp.router, 1e-2); const allocator = std.heap.smp_allocator; { @@ -178,5 +169,36 @@ fn testImplementation( log.info("All good for dequantize down_proj", .{}); } + { + const moe: MoeAllToAll = .{ .experts = mixtral.model.layers[0].mlp.experts }; + + try zml.testing.testLayer(platform, store, "model.model.layers.0.mlp.experts", moe, .{ .experts = mixtral_weights.model.layers[0].mlp.experts }, 5e-2); + } + try zml.testing.testLayer(platform, store, "model.model.layers.0.mlp", mixtral.model.layers[0].mlp, mixtral_weights.model.layers[0].mlp, 1e-2); } + +fn dequantize(self: zml.nn.BlockScaledLinear) zml.Tensor { + // Bitcast to our actual type. This allows to load weights in a packed layout. + const blocks_0 = self.blocks.bitCast(self.blocks_dtype); + const blocks = blocks_0.merge(.{ .d_block = .{ .d_block, .bitcast } }); + + const scale = self.scale.bitCast(.f8e8m0); + + const dequantized_weight: zml.Tensor = .mul( + blocks.convert(.bf16), + scale.convert(.bf16).appendAxes(.{.d_block}), + ); + return dequantized_weight.merge(.{ .d = .{ .d, .d_block } }).contiguous(.{ .d, .out }); +} + +const MoeAllToAll = struct { + experts: Mixtral.Mlp, + + /// Fork of Mixtral.Moe where we also feed the gating. + pub fn forward(self: MoeAllToAll, input: zml.Tensor, routing: zml.Tensor, gating: zml.Tensor) zml.Tensor { + _ = routing; + + return zml.nn.mixtureOfExpertsAllToAll(Mixtral.Mlp, self.experts, input, gating.withTags(.{ .s, .expert })); + } +}; diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig index c14d511f..3f1511d7 100644 --- a/zml/aio/torch.zig +++ b/zml/aio/torch.zig @@ -36,7 +36,7 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore const py_values = try eval.evaluate(tmp_alloc, ops, true); // file ownership is transferred to the BufferStore - var res = try zml.aio.BufferStore.initWithFiles(allocator, &.{torch_file.buffer_file}); + var res = try zml.aio.BufferStore.initWithFiles(allocator, &.{mmap_file}); try torch_file.parseModel(py_values, &res); return res; } diff --git a/zml/nn.zig b/zml/nn.zig index 5082b195..7a1f00e9 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -142,8 +142,8 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te stdx.debug.assert(opts.experts_per_token > 0, "mixtureOfExperts expects opts.experts_per_token > 0, got {}", .{opts}); if (num_tokens == 1) { - // Only one tokens: most experts are unused. - // select active experts and compute with that. + // Only one tokens: most experts are unused, experts have at most one token. + // Select active experts and compute with that. const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); const per_token_score = routing.values.div(routing.values.sum(.top_expert)); @@ -177,11 +177,82 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te break :tpe tpe; } else num_tokens; - // if (3 * tokens_per_expert <= 2 * num_tokens) { - if (false) { - // Lot of tokens, each experts chose their tokens and compute on that. - // It means that some tokens may have only one expert assigned. - // Each token will get assigned to at least one expert IIF the input gating is sums up to 1 (typically softmax output). + if (3 * tokens_per_expert <= 2 * num_tokens) { + const routing, const tokens_ids_per_expert = moe.dispatchTokens(gating, .{ + .experts_per_token = opts.experts_per_token, + .tokens_per_expert = tokens_per_expert, + }); + const scores_per_expert = routing.gather(.{ .s = tokens_ids_per_expert }, .{}); + const input_per_expert = input.gather(.{ .s = tokens_ids_per_expert }, .{}); + var output_per_expert = experts.forward(input_per_expert); + output_per_expert = output_per_expert.mul(scores_per_expert.convert(output_per_expert.dtype()).broad(output_per_expert.shape())); + + // Reverse engineer the normal output shape that one expert would have produced for all tokens. + // If this fall short, we could use the "sliced_expert" strategy and call forward ourselves. + const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .top_token = .s }).setDim(.s, num_tokens); + const output = zml.Tensor.scatterSlices( + .zeros(output_shape), + .{ .s = tokens_ids_per_expert }, + output_per_expert, + .{ .update_fn = zml.Tensor.ScatterOpts.increment }, + ); + + log.warn("mixtureOfExperts({s}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), input, gating, tokens_per_expert, num_tokens }); + return output; + } else { + const routing = gating.print().topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); + const per_token_score = routing.values.div(routing.values.sum(.top_expert)).convert(gating.dtype()); + + // hard_gating is eql to gating where only the top-2 experts have a non-zero score. + const hard_gating = zml.Tensor.scatterSlices( + .zeros(gating.shape()), + .{ .expert = routing.indices }, + per_token_score, + .{ .indices_are_unique = true }, + ).withSharding(.{.expert}); + + return mixtureOfExpertsAllToAll(Expert, experts, input, hard_gating.print()); + } +} + +/// Send all tokens to all experts, and apply gating. +pub fn mixtureOfExpertsAllToAll(Expert: type, experts: Expert, input: Tensor, gating: Tensor) Tensor { + log.warn("mixtureOfExperts({s}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), input, gating }); + const num_experts = gating.dim(.expert); + // TODO: `input.insertAxes(0, .{.expert}).repeat1d(.expert, num_experts)` is too verbose for just broadcasting along a new axis` + const output_per_expert = experts.forward(input.insertAxes(0, .{.expert}).repeat1d(.expert, num_experts)); + return output_per_expert.dot(gating.convert(input.dtype()), .expert); +} + +pub const moe = struct { + /// Given `(token, expert) -> scores`, + /// keeps only the top-k expert per token, and normalize the scores accordingly. + /// Non selected experts will have a 0 score. + pub fn hardGating(gating: zml.Tensor, experts_per_token: u32) Tensor { + const routing = gating.topK(.{ .top_expert = .expert }, experts_per_token, .{}); + const per_token_score = routing.values.div(routing.values.sum(.top_expert)).convert(gating.dtype()); + return zml.Tensor.scatterSlices( + .zeros(gating.shape()), + .{ .expert = routing.indices }, + per_token_score, + .{ .indices_are_unique = true }, + ); + } + + /// Lot of tokens, each experts chose their tokens. + /// It means that some tokens may have only one expert assigned. + /// Each token will get assigned to at least one expert IIF the input gating is sums up to 1 (typically softmax output). + /// Returns the actual `(token, expert) -> scores` used. + pub fn dispatchTokens( + gating: zml.Tensor, + opts: struct { + // TODO take ratio + tokens_per_expert: u32, + experts_per_token: u32, + }, + ) [2]Tensor { + const num_experts = gating.dim(.expert); + const token_pref = gating.argsort(.expert, .{ .descending = true }); var expert_rank: zml.Tensor = .scatterSlices( .zeros(gating.shape().withDtype(.i32)), @@ -191,7 +262,7 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te ); // The pow(expert_rank) here means that we strongly favor top 1 over top 2 and top 2 over top 3. // expert_routing: (expert, top_token) -> token - const expert_routing = gating.pow(expert_rank.convert(gating.dtype())).topK(.{ .top_token = .s }, tokens_per_expert, .{}); + const expert_routing = gating.pow(expert_rank.convert(gating.dtype())).topK(.{ .top_token = .s }, opts.tokens_per_expert, .{}); const scores_per_expert = gating.gather(.{ .s = expert_routing.indices }, .{}); // Update the gating coefficient to account for the expert routing. @@ -212,42 +283,11 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te ); // Then normalize so the sum of experts scores for one token sums up to 1. gating_v3 = gating_v3.div(gating_v3.sum(.expert)); - // Recompute scores_per_expert using the updated gating. const tokens_ids_per_expert = expert_routing.indices.transpose(.{ .expert, .top_token }); - const scores_per_expert_v2 = gating_v3.gather(.{ .s = expert_routing.indices }, .{}).transpose(.{ .expert, .top_token }); - const input_per_expert = input.gather(.{ .s = tokens_ids_per_expert }, .{}); - var output_per_expert = experts.forward(input_per_expert); - output_per_expert = output_per_expert.mul(scores_per_expert_v2.convert(output_per_expert.dtype()).broad(output_per_expert.shape())); - // Reverse engineer the normal output shape that one expert would have produced for all tokens. - // If this fall short, we could use the "sliced_expert" strategy and call forward ourselves. - const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .top_token = .s }).setDim(.s, num_tokens); - const output = zml.Tensor.scatterSlices( - .zeros(output_shape), - .{ .s = tokens_ids_per_expert }, - output_per_expert, - .{ .update_fn = zml.Tensor.ScatterOpts.increment }, - ); - - log.warn("mixtureOfExperts({s}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), input, gating, tokens_per_expert, num_tokens }); - return output; - } else { - // We send all tokens to all experts, then discard result from non-topk experts using hard_gating. - const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); - const per_token_score = routing.values.div(routing.values.sum(.top_expert)).convert(gating.dtype()); - - // hard_gating is eql to gating where only the top-2 experts have a non-zero score. - const hard_gating = zml.Tensor.scatterSlices( - .zeros(gating.shape()), - .{ .expert = routing.indices }, - per_token_score, - .{ .indices_are_unique = true }, - ).withSharding(.{.expert}); - log.warn("mixtureOfExperts({s}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), input, gating }); - const output_per_expert = experts.forward(input.insertAxes(0, .{.expert}).repeat1d(.expert, num_experts)); - return output_per_expert.dot(hard_gating.convert(input.dtype()), .expert); + return .{ gating_v3, tokens_ids_per_expert }; } -} +}; test mixtureOfExperts { const asynk = @import("async"); From ac771828b5365a365a6609f72a7ad4dec3cf75c0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 3 Sep 2025 10:00:15 +0200 Subject: [PATCH 64/67] mixtral -> Mixtral --- examples/mixtral/{mixtral.zig => Mixtral.zig} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename examples/mixtral/{mixtral.zig => Mixtral.zig} (99%) diff --git a/examples/mixtral/mixtral.zig b/examples/mixtral/Mixtral.zig similarity index 99% rename from examples/mixtral/mixtral.zig rename to examples/mixtral/Mixtral.zig index 40c3fdd3..6ee60214 100644 --- a/examples/mixtral/mixtral.zig +++ b/examples/mixtral/Mixtral.zig @@ -124,7 +124,7 @@ pub fn forward( ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Mixtral ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); + const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index.print(), kv_cache }); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } @@ -290,7 +290,7 @@ pub const Router = struct { const linear: zml.nn.Linear = .{ .weight = self.weight, .bias = self.bias }; const gating = linear.forward(input).convert(.f32).softmax(-1); - return zml.nn.moe.hardGating(gating, 4); + return zml.nn.moe.hardGating(gating, expert_per_token).convert(input.dtype()); } }; @@ -338,7 +338,7 @@ const MoE = struct { pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { log.warn("compiling moe with {f}", .{input}); const gating = self.router.forward(input); - return zml.nn.mixtureOfExperts(Mlp, self.experts, input, gating, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 1.5 }); + return zml.nn.mixtureOfExperts(Mlp, self.experts, input, gating, .{ .experts_per_token = Router.expert_per_token, .tokens_per_expert_ratio = 1.5 }); } pub fn loadSharded(self: MoE, store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, platform: zml.Platform) !zml.Bufferized(MoE) { From c6fa067583eff6b839ac45f89de058ff1df3b9a0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 3 Sep 2025 10:27:25 +0200 Subject: [PATCH 65/67] fix moe --- zml/nn.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 7a1f00e9..ae5a4236 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -136,7 +136,7 @@ const MoeOpts = struct { /// - experts: .{ .expert, .d_out, .d } expert layer (need to have a .forward method). /// -> output: .{ .s, .d_out } pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Tensor, opts: MoeOpts) Tensor { - log.warn("mixtureOfExperts({s}, {f}, {f}, {f})", .{ @typeName(Expert), experts, input, gating }); + log.warn("mixtureOfExperts({s}, {f}, {f})", .{ @typeName(Expert), input, gating }); const num_tokens = input.dim(.s); const num_experts = gating.dim(.expert); stdx.debug.assert(opts.experts_per_token > 0, "mixtureOfExperts expects opts.experts_per_token > 0, got {}", .{opts}); @@ -182,7 +182,7 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te .experts_per_token = opts.experts_per_token, .tokens_per_expert = tokens_per_expert, }); - const scores_per_expert = routing.gather(.{ .s = tokens_ids_per_expert }, .{}); + const scores_per_expert = routing.transpose(.{ .expert, .s }).gather(.{ .s = tokens_ids_per_expert }, .{}); const input_per_expert = input.gather(.{ .s = tokens_ids_per_expert }, .{}); var output_per_expert = experts.forward(input_per_expert); output_per_expert = output_per_expert.mul(scores_per_expert.convert(output_per_expert.dtype()).broad(output_per_expert.shape())); @@ -200,7 +200,7 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te log.warn("mixtureOfExperts({s}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), input, gating, tokens_per_expert, num_tokens }); return output; } else { - const routing = gating.print().topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); + const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); const per_token_score = routing.values.div(routing.values.sum(.top_expert)).convert(gating.dtype()); // hard_gating is eql to gating where only the top-2 experts have a non-zero score. @@ -211,7 +211,7 @@ pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Te .{ .indices_are_unique = true }, ).withSharding(.{.expert}); - return mixtureOfExpertsAllToAll(Expert, experts, input, hard_gating.print()); + return mixtureOfExpertsAllToAll(Expert, experts, input, hard_gating); } } From e54c28ed812dfcc535e47ef5506bfb55bf304472 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 16:21:51 +0200 Subject: [PATCH 66/67] handle printing strided HostBuffer --- stdx/fmt.zig | 52 ++++++++++++++++++++++++---------------------- zml/hostbuffer.zig | 15 ++++++++----- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 145db768..3251e893 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -10,15 +10,15 @@ fn FmtSlice(T: type) type { pub fn formatNumber(f: @This(), writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { return switch (@typeInfo(T)) { - .comptime_float, .float => try formatFloatSlice(f.slice, n, writer), - .comptime_int, .int => try formatIntSlice(f.slice, n, writer), - .bool => try formatBoolSlice(f.slice, n, writer), + .comptime_float, .float => try formatFloatSlice(f.slice, n, 1, writer), + .comptime_int, .int => try formatIntSlice(f.slice, n, 1, writer), + .bool => try formatBoolSlice(f.slice, n, 1, writer), .@"struct" => if (@hasField(T, "re") and @hasField(T, "im")) { - try formatComplexSlice(f.slice, n, writer); + try formatComplexSlice(f.slice, n, 1, writer); } else if (@hasDecl(T, "toF32")) { - try formatFloatSlice(f.slice, n, writer); + try formatFloatSlice(f.slice, n, 1, writer); } else { - try formatSliceAny(f.slice, n, writer); + try formatSliceAny(f.slice, n, 1, writer); }, else => @compileError("FmtSlice doesn't support type: " ++ @typeName(T)), }; @@ -67,51 +67,53 @@ pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) ! return try writer.alignBufferOptions(s, .{ .alignment = spec.alignment, .fill = spec.fill }); } -pub fn formatSliceCustom(fmt_func: anytype, values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { +pub fn formatSliceCustom(fmt_func: anytype, values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { // use the format "width" for the number of columns instead of individual width. const num_cols: usize = spec.width orelse 12; var my_options = spec; my_options.width = null; - const n: usize = values.len; + // TODO: handle negative strides + const strd: usize = @intCast(stride); + const n: usize = @divTrunc(values.len, strd); _ = try writer.write("{"); if (n <= num_cols) { - for (values, 0..) |v, i| { + for (0..n) |i| { // Force inlining so that the switch and the buffer can be done once. - try @call(.always_inline, fmt_func, .{ v, my_options, writer }); + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); if (i < n - 1) _ = try writer.write(","); } } else { const half = @divFloor(num_cols, 2); - for (values[0..half]) |v| { - try @call(.always_inline, fmt_func, .{ v, my_options, writer }); + for (0..half) |i| { + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); _ = try writer.write(","); } _ = try writer.write(" ..., "); - for (values[n - half ..], 0..) |v, i| { - try @call(.always_inline, fmt_func, .{ v, my_options, writer }); - if (i < half - 1) _ = try writer.write(","); + for (n - half..n) |i| { + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); + if (i < n - 1) _ = try writer.write(","); } } _ = try writer.write("}"); } -pub fn formatSliceAny(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatAny, values, spec, writer); +pub fn formatSliceAny(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatAny, values, spec, stride, writer); } -pub fn formatFloatSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatFloat, values, spec, writer); +pub fn formatFloatSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatFloat, values, spec, stride, writer); } -pub fn formatIntSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatInt, values, spec, writer); +pub fn formatIntSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatInt, values, spec, stride, writer); } -pub fn formatComplexSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatComplex, values, spec, writer); +pub fn formatComplexSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatComplex, values, spec, stride, writer); } -pub fn formatBoolSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatBool, values, spec, writer); +pub fn formatBoolSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatBool, values, spec, stride, writer); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 91406bd6..66b51328 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -361,12 +361,17 @@ pub const HostBuffer = struct { try writer.splatByteAll(' ', indent_level); switch (self.dtype()) { inline else => |dt| { - const values = self.items(dt.toZigType()); + const T = dt.toZigType(); + const n: i64 = self._shape.dim(0); + // TODO: handle negative strides + const byte_stride: i64 = self._strides[0]; + const elem_strides: i64 = @divExact(byte_stride, @sizeOf(T)); + const values: []const T = @ptrCast(@alignCast(self._data[0..@intCast(n * byte_stride)])); switch (comptime dt.class()) { - .float => try stdx.fmt.formatFloatSlice(values, options, writer), - .integer => try stdx.fmt.formatIntSlice(values, options, writer), - .complex => try stdx.fmt.formatComplexSlice(values, options, writer), - .bool => try stdx.fmt.formatBoolSlice(values, options, writer), + .float => try stdx.fmt.formatFloatSlice(values, options, elem_strides, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, elem_strides, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, elem_strides, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, elem_strides, writer), } }, } From 8e7f8799d8f8a0feca5c2ebe84ff3666170eac27 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 16:54:31 +0200 Subject: [PATCH 67/67] fix sinks --- examples/mixtral/Mixtral.zig | 5 +-- examples/mixtral/test.zig | 36 +++++++++++++++- stdx/fmt.zig | 2 +- zml/aio/torch/file.zig | 2 +- zml/nn.zig | 8 ++-- zml/tensor.zig | 80 ++++++++++++++++++++++++++++-------- zml/testing.zig | 2 +- 7 files changed, 105 insertions(+), 30 deletions(-) diff --git a/examples/mixtral/Mixtral.zig b/examples/mixtral/Mixtral.zig index 6ee60214..3826e3de 100644 --- a/examples/mixtral/Mixtral.zig +++ b/examples/mixtral/Mixtral.zig @@ -124,7 +124,7 @@ pub fn forward( ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Mixtral ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index.print(), kv_cache }); + const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } @@ -471,8 +471,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const softmax_bias = q.dot(self.sinks.withTags(.{.hd}), .hd); - std.log.warn("q: {f}, sinks: {f} -> softmax_bias: {f}", .{ q, self.sinks, softmax_bias }); + const softmax_bias = self.sinks.withTags(.{.h}); const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true, .softmax_bias = softmax_bias }); // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); diff --git a/examples/mixtral/test.zig b/examples/mixtral/test.zig index 5aceaec9..32b31761 100644 --- a/examples/mixtral/test.zig +++ b/examples/mixtral/test.zig @@ -109,10 +109,14 @@ pub fn asyncMain() !void { } } - log.info("Loaded activations: {s} -> {f}", .{ k, v }); + // log.info("Loaded activations: {s} -> {f}", .{ k, v }); + if (std.mem.indexOf(u8, k, ".22.self_attn.in.") != null) { + log.info("Self attn input {s}: {d:32.3}", .{ k, v.* }); + } + // if (std.mem.endsWith(u8, k, "layers.23.mlp.experts.in.1")) log.info("Routing {s}: {d:32.2}", .{k, v.*}); // if (std.mem.endsWith(u8, k, "layers.23.mlp.experts.in.2")) log.info("Routing weights {s}: {d:32.2}", .{k, v.*}); - if (std.mem.endsWith(u8, k, ".0.mlp.experts.in.2")) log.info("Routing weights {s}: {d:32.2}", .{ k, v.* }); + // if (std.mem.endsWith(u8, k, ".0.mlp.experts.in.2")) log.info("Routing weights {s}: {d:32.2}", .{ k, v.* }); } // Test implementation @@ -135,6 +139,34 @@ fn testImplementation( try zml.testing.testLayer(platform, store, "model.model.layers.0.mlp.router", mixtral.model.layers[0].mlp.router, mixtral_weights.model.layers[0].mlp.router, 1e-2); const allocator = std.heap.smp_allocator; + { + const input: zml.HostBuffer = store.buffers.get("model.model.layers.22.self_attn.in.0").?; + const kv_shape = zml.Shape.init(.{ .layer = 1, .k = input.dim(.s), .h = 8, .hd = 64 }, .bf16); + const self_attn_mod = try zml.compileModel( + allocator, + Mixtral.SelfAttn.forward, + mixtral.model.layers[22].self_attn, + .{ + store.buffers.get("model.model.layers.22.self_attn.in.0").?.shape(), + zml.Shape.scalar(.u32), + Mixtral.KvCache.initShape(kv_shape), + }, + platform, + ); + defer self_attn_mod.deinit(); + + const self_attn_exe = self_attn_mod.prepare(mixtral_weights.model.layers[22].self_attn); + const out, _ = self_attn_exe.call(.{ + try input.toDevice(platform), + try .scalar(platform, 0, .u32), + try Mixtral.KvCache.initBuffer(kv_shape, platform), + }); + + const expected = store.buffers.get("model.model.layers.22.self_attn.out.0").?; + try zml.testing.expectClose(expected, out, 1e-2); + log.info("All good for self_attn", .{}); + } + { const w = mixtral.model.layers[22].mlp.experts.gate_up_proj; const dequant_mod = try zml.compileFn(allocator, dequantize, .{.{ diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 3251e893..8662bfe2 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -1,6 +1,6 @@ const std = @import("std"); -pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { +pub fn slice(T: type, any_slice: []const T) FmtSlice(T) { return .{ .slice = any_slice }; } diff --git a/zml/aio/torch/file.zig b/zml/aio/torch/file.zig index 0307d51c..fe8811fd 100644 --- a/zml/aio/torch/file.zig +++ b/zml/aio/torch/file.zig @@ -480,7 +480,7 @@ test "Read pickle (zipped)" { // torch.save({ "model": model, "tensor": tensor}, "simple.pt") const file = try asynk.File.open("zml/aio/torch/simple.pt", .{ .mode = .read_only }); const mmap_file = try zml.aio.MemoryMappedFile.init(file); - var store = try zml.aio.BufferStore.init(testing.allocator, &.{mmap_file}); + var store = try zml.aio.BufferStore.initWithFiles(testing.allocator, &.{mmap_file}); defer store.deinit(); { diff --git a/zml/nn.zig b/zml/nn.zig index ae5a4236..28958420 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -705,7 +705,7 @@ test "invFreq Yarn" { var inv_freq: @TypeOf(yarn_freq) = undefined; _invFreq(yarn_conf, &inv_freq); for (yarn_freq, inv_freq, 0..) |expected, actual, i| { - errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, yarn_freq, inv_freq }); + errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, stdx.fmt.slice(f32, &yarn_freq), stdx.fmt.slice(f32, &inv_freq) }); try std.testing.expectApproxEqRel(expected, actual, 1e-5); } try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); @@ -1261,9 +1261,9 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { log.warn("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); attn_weights = attn_weights.convert(.f32); attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { - // The split is needed because we also split q ourselves, - // but the contiguous seems brittle, what we want is a kind of "broadTranspose". - const bias = softmax_bias.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }).contiguous(.{ .h, .q, .hq }); + // The split is needed because we also split q ourselves. + // TODO: consider letting the user do that. + const bias = softmax_bias.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); break :attn attn_weights.convert(.f32).softmaxBiased(.k, bias).convert(q.dtype()); } else attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); diff --git a/zml/tensor.zig b/zml/tensor.zig index caa2eb13..0b4f84fa 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1316,33 +1316,77 @@ pub const Tensor = struct { /// Computes softmax along the given axis. /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) - pub fn softmax(self: Tensor, axis_: anytype) Tensor { - const a = self.axis(axis_); - const max_val = self.max(a); - const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), self.dtype())); + pub fn softmax(x: Tensor, axis_: anytype) Tensor { + const a = x.axis(axis_); + const max_val = x.max(a); - const exp_diff_max = self.sub(max_val).exp(); - const res = exp_diff_max.div(exp_diff_max.sum(a)); + const exp_diff_max = x.sub(max_val).exp(); + const y = exp_diff_max.div(exp_diff_max.sum(a)); // If a row is full -inf return full 0 instead of full nan, // this fix attention when mask hides a full row. - return row_mask.broad(self.shape()).select(res, .scalar(0, self.dtype())); + const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), x.dtype())); + return .select(row_mask.broad(y.shape()), y, .scalar(0, x.dtype())); + } + + test softmax { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _softmax(x: Tensor) Tensor { + return x.softmax(-1); + } + }; + + const logits_d = try zml.Buffer.fromArray( + platform, + [4]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }, + ); + const res = try zml.testing.compileAndCall(platform, Local._softmax, .{logits_d}); + try std.testing.expectEqual( + [4]f32{ 2.0 / 10.0, 1.0 / 10.0, 4.0 / 10.0, 3.0 / 10.0 }, + try res.getValue([4]f32), + ); } /// Computes softmax, but adds a bias to the sum of exponentiel. - /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) - pub fn softmaxBiased(self: Tensor, axis_: anytype, bias: ?Tensor) Tensor { - const a = self.axis(axis_); + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + exp(bias) ) + /// + /// Note: the bias is passed pre-exponential, allowing to use precision tricks. + pub fn softmaxBiased(x: Tensor, axis_: anytype, bias: Tensor) Tensor { + const a = x.axis(axis_); + const b = bias.convert(x.dtype()).broad(x.shape().setDim(a, 1)); + const max_val: Tensor = maximum(x.max(a), b); + const exp_diff_max = x.sub(max_val).exp(); + const bias_diff_max = b.sub(max_val).exp(); + const y = exp_diff_max.div(exp_diff_max.sum(a).add(bias_diff_max)); - if (bias == null) return self.softmax(axis_); - const b = bias.?.convert(self.dtype()).broad(self.shape().setDim(a, 1)); - const max_val: Tensor = maximum(self.max(a), b); - const exp_diff_max = self.sub(max_val).exp(); - const bias_diff_max = b.mul(.exp(.negate(max_val))); - const res = exp_diff_max.div(exp_diff_max.sum(a).add(bias_diff_max)); + const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), x.dtype())); + return .select(row_mask.broad(y.shape()), y, .scalar(0, x.dtype())); + } - // The bias means that denominator won't be 0, we don't need to handle that case. - return res; + test softmaxBiased { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _softmaxBiased(x: Tensor, bias: Tensor) Tensor { + return x.softmaxBiased(-1, bias); + } + }; + + const logits_d = try zml.Buffer.fromArray( + platform, + [4]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }, + ); + const bias_d = try zml.Buffer.fromArray(platform, [1]f32{@log(5.0)}); + + try zml.testing.expectClose( + zml.HostBuffer.fromArray(&[4]f32{ 2.0 / 15.0, 1.0 / 15.0, 4.0 / 15.0, 3.0 / 15.0 }), + try zml.testing.compileAndCall(platform, Local._softmaxBiased, .{ logits_d, bias_d }), + 1e-5, + ); } /// Returns a Tensor containing the log of the sum of exponential over the given axis. diff --git a/zml/testing.zig b/zml/testing.zig index a4b91718..1f5a521a 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -92,7 +92,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { const right_data = right.items(R); for (left_data, right_data, 0..) |l, r, i| { if (!approxEq(f32, zml.floats.floatCast(f32, l), zml.floats.floatCast(f32, r), tolerance)) { - log.err("left.data != right_data.\n < {d:40.3} \n > {d:40.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ stdx.fmt.slice(center(left_data, i)), stdx.fmt.slice(center(right_data, i)), i, left_data[i], right_data[i] }); + log.err("left.data != right_data.\n < {d:40.3} \n > {d:40.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ stdx.fmt.slice(L, center(left_data, i)), stdx.fmt.slice(R, center(right_data, i)), i, left_data[i], right_data[i] }); return error.TestUnexpectedResult; } }