diff --git a/async/async.zig b/async/async.zig index f14608b8..a62aa881 100644 --- a/async/async.zig +++ b/async/async.zig @@ -10,6 +10,8 @@ const coro = @import("coro.zig"); const executor = @import("executor.zig"); const stack = @import("stack.zig"); +pub const Coro = coro.Coro; + test { std.testing.refAllDecls(@This()); std.testing.refAllDecls(coro); @@ -65,6 +67,47 @@ fn FrameExx(comptime func: anytype, comptime argsT: type, comptime returnT: type }; } +/// Create a Coro +/// self and stack pointers must remain stable for the lifetime of +/// the coroutine. +pub fn async2(comptime func: anytype, args: anytype, result_dst: anytype) !*coro.Coro { + const Sig = stdx.meta.FnSignature(func, @TypeOf(args)); + var new_stack = try AsyncThread.current.stack_allocator.create(); + if (@TypeOf(result_dst) != *Sig.ReturnPayloadT) { + stdx.debug.compileError("async2 expects result_dst to be a *{s} got {s}", .{ @typeName(Sig.ReturnPayloadT), @typeName(@TypeOf(result_dst)) }); + } + + // Note: I'm not a big fan of this struct. It would be simpler if res_dst was part of the Coro itself. + const storage = try new_stack.push(struct { res_dst: *anyopaque, args: Sig.ArgsT }); + storage.args = args; + storage.res_dst = result_dst; + const Helpers = Helpers_async2(Sig); + const frame = try Coro.initFromStack(Helpers.wrapfn, new_stack, storage); + frame.switchIn(); + return frame; +} + +pub fn awaitAll(futures: []const *coro.Coro) void { + for (futures) |fut| fut.@"await"(); +} + +fn Helpers_async2(comptime Sig: stdx.meta.Signature) type { + return struct { + pub fn wrapfn() void { + const storage_raw = Coro.current().storage; + const storage: *struct { res_dst: *anyopaque, args: Sig.ArgsT } = @alignCast(@ptrCast(storage_raw)); + + const res_dst: *Sig.ReturnT = @alignCast(@ptrCast(storage.res_dst)); + const res = @call(.auto, Sig.Func.Value, storage.args); + if (Sig.ReturnErrorSet) |_| { + res_dst.* = res catch |err| std.debug.panic("coroutine failed: {} with {}", .{ Sig.Func, err }); + } else { + res_dst.* = res; + } + } + }; +} + pub fn asyncc(comptime func: anytype, args: anytype) !FrameEx(func, @TypeOf(args)) { const Signature = stdx.meta.FnSignature(func, @TypeOf(args)); const new_stack = try AsyncThread.current.stack_allocator.create(); diff --git a/async/coro.zig b/async/coro.zig index be26b95b..e4172ed6 100644 --- a/async/coro.zig +++ b/async/coro.zig @@ -104,7 +104,7 @@ pub fn xsuspendSafe() Error!void { thread_state.switchOut(coro.resumer); } -const Coro = struct { +pub const Coro = struct { /// Coroutine status const Status = enum { Start, @@ -137,7 +137,7 @@ const Coro = struct { // empty } - fn initFromStack(func: *const fn () void, stack_: stack.Stack, storage: ?*anyopaque) !Frame { + pub fn initFromStack(func: *const fn () void, stack_: stack.Stack, storage: ?*anyopaque) !Frame { // try StackOverflow.setMagicNumber(stack.full); var stack__ = stack_; const coro = try stack__.push(Coro); @@ -175,11 +175,27 @@ const Coro = struct { pub fn format(self: Coro, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { _ = fmt; _ = options; - try writer.print("Coro{{.id = {any}, .status = {s}}}", .{ + try writer.print("Coro{{.id = {any}, .status = {s}, .storage={x}, .stack_start={x}, .stack_used={x}}}", .{ self.id, @tagName(self.status), + @intFromPtr(self.storage), + @intFromPtr(self.stack.full), + @intFromPtr(self.stack.used.ptr), }); } + + pub fn @"await"(self: *Coro) void { + while (self.status != .Done) xsuspend(); + std.debug.assert(self.status == .Done); + } + + pub fn switchIn(self: *Coro) void { + thread_state.switchIn(self); + } + + pub fn current() *Coro { + return thread_state.current_coro.?; + } }; const VoidSignature = CoroT.Signature.init((struct { @@ -260,6 +276,10 @@ const CoroT = struct { Sig.Func.Value, storage.args, ); + switch (@typeInfo(Sig.ReturnT)) { + .error_union => _ = storage.retval catch unreachable, + else => {}, + } } }; } diff --git a/examples/custom_call/BUILD b/examples/custom_call/BUILD new file mode 100644 index 00000000..bc605442 --- /dev/null +++ b/examples/custom_call/BUILD @@ -0,0 +1,10 @@ +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "custom_call", + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig new file mode 100644 index 00000000..e8bdefd6 --- /dev/null +++ b/examples/custom_call/main.zig @@ -0,0 +1,153 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +const cuda = zml.context.cuda; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +fn add_op_host_func(data: *anyopaque) callconv(.c) void { + const ctx = @as(*AddOp, @alignCast(@ptrCast(data))); + + const result = std.mem.bytesAsValue(f32, ctx.result.asHostBuffer().mutBytes()); + result.* = ctx.a.asHostBuffer().item(f32, 0) + ctx.b.asHostBuffer().item(f32, 0); +} + +pub const AddOp = struct { + pub var type_id: i64 = undefined; + const Self = @This(); + + allocator: std.mem.Allocator, + platform: zml.Platform, + + a: zml.Buffer, + b: zml.Buffer, + result: zml.Buffer, + + results: [1]zml.Buffer = undefined, + stream: *anyopaque = undefined, + + pub fn init( + allocator: std.mem.Allocator, + platform: zml.Platform, + io_shape: zml.Shape, + ) !AddOp { + const host_buffer = try zml.HostBuffer.empty(allocator, io_shape); + defer host_buffer.deinit(allocator); + + const a = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + const b = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + const result = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + + return .{ + .a = a, + .b = b, + .result = result, + .platform = platform, + .allocator = allocator, + }; + } + + pub fn deinit(self: *Self) void { + self.a.deinit(); + self.b.deinit(); + self.result.deinit(); + } + + pub fn call(self: *Self, a: zml.Buffer, b: zml.Buffer) !void { + if (self.platform.target == .cuda) { + cuda.memcpyToHostAsync(self.a.asHostBuffer().mutBytes(), a.opaqueDeviceMemoryDataPointer(), self.stream); + cuda.memcpyToHostAsync(self.b.asHostBuffer().mutBytes(), b.opaqueDeviceMemoryDataPointer(), self.stream); + + _ = cuda.cuLaunchHostFunc(self.stream, @ptrCast(&add_op_host_func), @ptrCast(self)); + + cuda.memcpyToDeviceAsync(self.results[0].opaqueDeviceMemoryDataPointer(), self.result.asHostBuffer().bytes(), self.stream); + return; + } + + if (self.platform.target == .cpu) { + const result = std.mem.bytesAsValue(f32, self.results[0].asHostBuffer().mutBytes()); + result.* = a.asHostBuffer().item(f32, 0) + b.asHostBuffer().item(f32, 0); + return; + } + + return error.UnsupportedTarget; + } +}; + +const Layer = struct { + pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + const result = zml.custom_call(AddOp, .{ a.print(), b.print() }, &[_]zml.Shape{a.shape()}, .{}); + return result[0]; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const shape = zml.Shape.init(.{}, .f32); + + const buffers: zml.aio.BufferStore.Buffers = .{}; + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); + + const compiled = try compilation.awaitt(); + + var executable = compiled.prepare(model_weights); + defer executable.deinit(); + + executable = try executable.withExecutionContext(); + + try platform.registerFFIType(AddOp); + var add_op: AddOp = try .init(allocator, platform, shape); + defer add_op.deinit(); + try executable.bind(AddOp, &add_op); + + var input_a = [1]f32{1.0}; + var input_buffer_a = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_a), .{}); + defer input_buffer_a.deinit(); + + var input_b = [1]f32{1.0}; + var input_buffer_b = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_b), .{}); + defer input_buffer_b.deinit(); + + var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + defer result.deinit(); + + var cpu_result = try result.toHostAlloc(arena); + + log.warn( + "\nThe result of {d} + {d} = {d}\n", + .{ &input_a, &input_b, cpu_result.items(f32) }, + ); +} diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index ee9c852f..182672d4 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -46,6 +46,7 @@ pub const LlamaLM = struct { self.model.max_seq_len = @intCast(options.max_seq_len); self.model.num_heads = @intCast(config.num_attention_heads); self.model.num_kv_heads = @intCast(config.num_key_value_heads); + self.model.norm.eps = config.rms_norm_eps; self.model.rope_opts = .{ .layout = if (config.hf_rope_impl) .sequential else .interleaved, .freq_base = config.rope_theta, diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index d4fb6ee4..24a2cae7 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -163,29 +163,26 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - pub fn Context(comptime T: type) type { - return struct { - pub fn get(self: *const ExecutionContext, api: *const Api) ApiError!*T { - const type_id: TypeId = .{ .type_id = T.type_id }; - var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = @constCast(self.inner()), - .type_id = @constCast(&type_id.toCStruct()), - }); - const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + pub fn getUserDataRaw(self: *const ExecutionContext, api: *const Api, type_id: i64) ?*anyopaque { + var type_id_struct: TypeId = .{ .type_id = type_id }; + var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ + .ctx = @constCast(self.inner()), + .type_id = &type_id_struct, + }); + const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); - if (result) |ffi_error| { - const err = Error.fromInner(ffi_error); - defer err.destroy(api); - log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); - // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - return error.Unknown; - } + return null; + } + return ret.data; + } - if (ret.data == null) return error.NotFound; - return @ptrCast(@alignCast(ret.data.?)); - } - }; + pub fn getUserData(self: *const ExecutionContext, T: type, api: *const Api) ?*T { + return @alignCast(@ptrCast(self.getUserDataRaw(api, T.type_id))); } pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 58d9ac29..fe37ab81 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -578,9 +578,12 @@ pub const Device = opaque { } pub fn memoryStats(self: *const Device, api: *const Api) ApiError!MemoryStats { - const ret = try api.call(.PJRT_Device_MemoryStats, .{ + const ret = api.call(.PJRT_Device_MemoryStats, .{ .device = self.inner(), - }); + }) catch |err| switch (err) { + error.Unimplemented => return std.mem.zeroes(MemoryStats), + else => return err, + }; return .{ .bytes_in_use = @intCast(ret.bytes_in_use), .peak_bytes_in_use = @intCast(ret.peak_bytes_in_use), @@ -1295,19 +1298,18 @@ pub const NamedValue = extern struct { pub const FFI = extern struct { inner: *const c.PJRT_FFI, + // we clone the struct to force fields to be initialized by the caller. pub const UserData = extern struct { type_id: i64, user_data: *anyopaque, + deleter: ?*fn (*anyopaque) void = null, fn toCStruct(self: UserData) c.PJRT_FFI_UserData { - return .{ - .type_id = self.type_id, - .data = self.user_data, - }; + return @bitCast(self); } }; - pub const RegisterFfiOptions = struct { + pub const RegisterOptions = struct { traits: RegisterHandlerTraits = @enumFromInt(0), }; @@ -1319,7 +1321,7 @@ pub const FFI = extern struct { target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, - options: RegisterFfiOptions, + options: RegisterOptions, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .api_version = 1, @@ -1372,7 +1374,3 @@ pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { command_buffer_compatible = c.PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE, _, }; - -pub const CustomCallRegistry = extern struct { - inner: *const c.PJRT_FFI_Register_Handler, -}; diff --git a/stdx/math.zig b/stdx/math.zig index da0bec79..1e1e1318 100644 --- a/stdx/math.zig +++ b/stdx/math.zig @@ -21,3 +21,7 @@ pub inline fn roundeven(x: anytype) @TypeOf(x) { extern fn @"llvm.roundeven"(@TypeOf(x)) @TypeOf(x); }.@"llvm.roundeven"(x); } + +pub fn cdiv(numerator: anytype, denominator: anytype) @TypeOf(numerator) { + return (numerator + denominator - 1) / denominator; +} diff --git a/zml/buffer.zig b/zml/buffer.zig index 36f61255..7518fe09 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -49,7 +49,7 @@ pub const Buffer = struct { pub const FromOptions = struct { wait: bool = true, - memory: ?pjrt.Memory.Kind = null, + memory: ?Memory = null, }; /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -89,15 +89,20 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); - const memory = for (memories) |m| { - const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; - } else return error.NotFound; - args.memory = memory; - } else { + if (platform.target == .cpu or opts.memory == null) { args.device = devices[i]; + } else { + const memory = opts.memory.?; + const device_memories = try devices[i].addressableMemories(platform.pjrt_api); + // TODO measure the cost of this and consider caching on Zig side inside the platform. + const selected_memory = for (device_memories) |m| { + const kind = m.kind(platform.pjrt_api); + if (kind == memory.toPjrtMemory()) break m; + } else { + log.warn("Platform {s} doesn't have memory {s}", .{ @tagName(platform.target), @tagName(memory) }); + return error.NotFound; + }; + args.memory = selected_memory; } const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); @@ -179,10 +184,9 @@ pub const Buffer = struct { return try from(platform, host_buffer, opts); } - pub fn asPinnedHostBuffer(self: Buffer) HostBuffer { - // TODO restore assert - // const memory = self.getMemory().kind(self._api); - // stdx.debug.assert(memory == .pinned_host, "asPinnedHostBuffer({}) expects a buffer allocated on host memory, got {}. see `toMemory`", .{ self, memory }); + pub fn asHostBuffer(self: Buffer) HostBuffer { + const memory = self.getMemory().kind(self._api); + stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({}) expects a buffer allocated on host memory, got {}. see `copyToMemory`", .{ self, memory }); const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } @@ -299,6 +303,11 @@ pub const Buffer = struct { }; } + pub fn opaqueDeviceMemoryDataPointer(self: Buffer) *anyopaque { + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); + return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + } + /// Fetches the content of the given buffer into a stack variable of the given type. pub fn getValue(self: Buffer, T: type) !T { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); @@ -394,13 +403,31 @@ pub const Buffer = struct { return @reduce(.Or, self._shape._sharding_info); } - pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { + pub const CopyToMemoryOpts = struct { + wait: bool = true, + }; + + pub fn copyToMemory(self: Buffer, platform: Platform, memory: Memory, opts: CopyToMemoryOpts) !Buffer { + const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory.toPjrtMemory()); + if (pjrt_memory == null) { + stdx.debug.panic("Memory destination `{s}` for {}", .{ memory.pjrtName(), self }); + } + var new_shards: Buffer.Shards = .{}; for (self._shards.slice()) |shard| { - const new_shard = try shard.copyToMemory(self._api, memory); + const new_shard = try shard.copyToMemory(self._api, pjrt_memory.?); new_shards.appendAssumeCapacity(new_shard); } + if (opts.wait) { + for (new_shards.constSlice()) |shard| { + const event = shard.getReadyEvent(self._api); + if (event) |e| { + try e.awaitBlocking(self._api); + } + } + } + return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; } diff --git a/zml/context.zig b/zml/context.zig index bde867f9..f13e0cda 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -7,16 +7,19 @@ const runfiles = @import("runfiles"); const runtimes = @import("runtimes"); const stdx = @import("stdx"); -const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Target = @import("platform.zig").Target; -const zml_platform = @import("platform.zig"); -const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); -const PlatformsMap = std.EnumArray(Target, ?Platform); +const zml = struct { + pub const Platform = @import("platform.zig").Platform; + pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; + pub const Shape = @import("shape.zig").Shape; + pub const Target = @import("platform.zig").Target; + pub const custom_call = @import("custom_call.zig"); + pub const platform = @import("platform.zig"); +}; + +const PjrtApiMap = std.EnumArray(zml.Target, ?*const pjrt.Api); +const PlatformsMap = std.EnumArray(zml.Target, ?zml.Platform); const log = std.log.scoped(.@"zml/context"); test { @@ -94,7 +97,7 @@ pub const Context = struct { return .{ .platforms = PlatformsMap.initFill(null) }; } - fn platformToLibrary(comptime target: Target) []const u8 { + fn platformToLibrary(comptime target: zml.Target) []const u8 { const ext = switch (builtin.os.tag) { .windows => ".dll", .macos, .ios, .watchos => ".dylib", @@ -105,7 +108,7 @@ pub const Context = struct { }; } - pub fn pjrtApi(target: Target) *const pjrt.Api { + pub fn pjrtApi(target: zml.Target) *const pjrt.Api { return Context.apis.get(target).?; } @@ -119,12 +122,12 @@ pub const Context = struct { self.* = undefined; } - const prefered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; + const prefered_targets = [_]zml.Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; /// Automatically selects the best Platform loaded in the current Context. /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn autoPlatform(self: *Context, opts: Platform.CreateOptions) Platform { + pub fn autoPlatform(self: *Context, opts: zml.Platform.CreateOptions) zml.Platform { stdx.debug.assert(prefered_targets.len == apis.values.len, "New target need to be inserted inside `zml.Context.preferred_targets`", .{}); return self.platformByPreferences(opts, &prefered_targets); @@ -133,7 +136,7 @@ pub const Context = struct { /// Given a list of preferred targets to select the best Platform /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn platformByPreferences(self: *Context, opts: Platform.CreateOptions, prefered: []const Target) Platform { + pub fn platformByPreferences(self: *Context, opts: zml.Platform.CreateOptions, prefered: []const zml.Target) zml.Platform { // Try prefered targets. for (prefered) |target| { if (apis.get(target) == null) continue; @@ -150,7 +153,7 @@ pub const Context = struct { // CPU should only be use as fallback. if (target == .cpu) continue; if (entry.value.* == null) continue; - if (std.mem.indexOfScalar(Target, prefered, target) != null) continue; + if (std.mem.indexOfScalar(zml.Target, prefered, target) != null) continue; return self.platform(target, opts) catch |err| { log.err("Failed to load platform .{s}: {}", .{ @tagName(target), err }); continue; @@ -164,25 +167,25 @@ pub const Context = struct { }; } - pub fn platform(self: *Context, target: Target, opts: Platform.CreateOptions) !Platform { + pub fn platform(self: *Context, target: zml.Target, opts: zml.Platform.CreateOptions) !zml.Platform { if (self.platforms.get(target)) |p| { return p; } const api = Context.apis.get(target); if (api == null) return error.PlatformNotCompiled; - const p = try Platform.init(target, api.?, opts); + const p = try zml.Platform.init(target, api.?, opts); if (p.getDevices().len == 0) { log.err("No device found for platform {} !", .{target}); return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); + try zml.custom_call.registerInternalCustomCalls(p); + return p; } - pub fn printAvailablePlatforms(self: Context, selected: Platform) void { + pub fn printAvailablePlatforms(self: Context, selected: zml.Platform) void { // List available targets log.info("Available Platforms:", .{}); const selected_prefix = "✅"; @@ -190,7 +193,7 @@ pub const Context = struct { const selected_postfix = "(AUTO-SELECTED)"; const not_selected_postfix = ""; - for (zml_platform.available_targets) |target| { + for (zml.platform.available_targets) |target| { log.info(" {s} {s} {s}", .{ if (target == selected.target) selected_prefix else not_selected_prefix, @tagName(target), @@ -211,12 +214,10 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; }; const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { + pub fn registerZmlCustomCalls(platform: zml.Platform) !void { const ffi = platform.pjrt_api.ffi() orelse { log.warn("Registering custom calls failed: No FFI Extension found in {s} PJRT Plugin.", .{@tagName(platform.target)}); return; @@ -235,12 +236,12 @@ const CustomCall = struct { std.debug.assert(user_ctx_ptr.dtype == .u64); const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); + const input_buffers = stdx.stackSlice(8, zml.HostBuffer, call_frame.args.len); for (input_buffers, 0..) |*b, i| { b.* = hostBufferFromPinnedBuffer(call_frame.args.buffers()[i]); } - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); + const output_buffers = stdx.stackSlice(8, zml.HostBuffer, call_frame.results.len); for (output_buffers, 0..) |*b, i| { b.* = hostBufferFromPinnedBuffer(call_frame.results.buffers()[i]); } @@ -250,9 +251,9 @@ const CustomCall = struct { } }; -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { +fn getShape(buffer_desc: *const pjrt.ffi.Buffer) zml.Shape { // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { + const dt: zml.DataType = switch (buffer_desc.dtype) { .invalid => @panic("invalid ffi"), .pred => .bool, .i8 => .i8, @@ -260,17 +261,17 @@ fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { .i32 => .i32, .i64 => .i64, .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), + inline else => |t| @field(zml.DataType, @tagName(t)), }; - return Shape.init(buffer_desc.dims(), dt); + return zml.Shape.init(buffer_desc.dims(), dt); } /// Create a HostBuffer from a ffi description of a buffer. /// Normally the ffi describe device buffer but we assume they are located in pinned memory, /// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { +fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) zml.HostBuffer { const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( + return zml.HostBuffer.fromBytes( buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()], ); diff --git a/zml/custom_call.zig b/zml/custom_call.zig new file mode 100644 index 00000000..a0da18a4 --- /dev/null +++ b/zml/custom_call.zig @@ -0,0 +1,225 @@ +const std = @import("std"); + +const asynk = @import("async"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const Buffer = @import("buffer.zig").Buffer; +const CompilationContext = @import("module.zig").CompilationContext; +const cuda = @import("context.zig").cuda; +const DataType = @import("dtype.zig").DataType; +const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const mlirx = @import("mlirx.zig"); +const pjrtx = @import("pjrtx.zig"); +const ffi = pjrtx.ffi; +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; +const log = std.log.scoped(.@"zml/custom_call"); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn CustomCallInputsType(CustomOp: type) type { + const ArgsT = stdx.meta.FnArgs(CustomOp.call); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + + const args = @typeInfo(ArgsT).@"struct".fields; + + for (args[1..args.len]) |field| { + if (field.type != Buffer) { + @compileError("Expected all arguments in custom call `call` to be of type zml.Buffer"); + } + } + + return [args.len - 1]Tensor; +} + +pub const CustomCallOptions = struct { + output_operand_aliases: []const i64 = &.{}, + copy_inputs_to_host_pinned: bool = false, + register_ffi_options: pjrt.FFI.RegisterOptions = .{ + .traits = @enumFromInt(0), + }, +}; + +pub fn custom_call( + comptime CustomOp: type, + inputs: CustomCallInputsType(CustomOp), + res_shapes_: []const Shape, + comptime opts: CustomCallOptions, +) []Tensor { + stdx.debug.assert(@hasDecl(CustomOp, "call"), "CustomOp must have a call method", .{}); + const op_name = @typeName(CustomOp); + + const ctx = CompilationContext.current(); + ctx._num_custom_calls += 1; + const allocator = ctx.allocator(); + const mlir_ctx = ctx.mlirCtx(); + const platform = ctx._platform; + const pjrt_api = platform.pjrt_api; + const ffi_ = pjrt_api.ffi(); + + if (ffi_ == null) { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); + } + + const ffi_func = proxy(CustomOp, opts); + const target_name = "callback_" ++ op_name; + + ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.register_ffi_options) catch unreachable; + log.info("{s} / Registered custom call with target name \"{s}\" with proxy func {*}", .{ op_name, target_name, &ffi_func }); + + const custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; + for (inputs, 0..) |t, i| { + custom_call_inputs[i] = t.value(); + } + + const res_types = allocator.alloc(mlir.Type, 8) catch unreachable; + for (res_shapes_, 0..) |sh, i| { + res_types[i] = mlirx.tensorType(mlir_ctx, sh); + } + + const frontend_attributes: mlir.Attribute = .dict(mlir_ctx, &.{}); + + const op = dialect.stablehlo.custom_call( + mlir_ctx, + custom_call_inputs[0..inputs.len], + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, + .has_side_effect = true, + .output_operand_aliases = opts.output_operand_aliases, + }, + res_types[0..res_shapes_.len], + mlir_ctx.location(@src()), + ); + + var custom_call_outputs = allocator.alloc(Tensor, 8) catch unreachable; + + for (custom_call_outputs[0..res_shapes_.len], res_shapes_, 0..) |*t, sh, i| { + t.* = Tensor._result(sh, op.result(i)); + } + + return custom_call_outputs[0..res_shapes_.len]; +} + +fn proxy(comptime CustomOp: type, opts: CustomCallOptions) ffi.Handler { + stdx.debug.assertComptime(@hasField(CustomOp, "platform"), "CustomCall context struct needs to contain a 'platform: zml.Platform' field in struct {s}", .{@typeName(CustomOp)}); + + return struct { + const Self = @This(); + + pub fn proxy(call_frame: *ffi.CallFrame) callconv(.C) ?*ffi.Error { + if (call_frame.registeringHook()) return null; + const execution_context = call_frame.ctx; + const user_ctx: *CustomOp = execution_context.getUserData(CustomOp, call_frame.api) orelse + stdx.debug.panic("An exe was called but its execution context {s} was never bound", .{@typeName(CustomOp)}); + const platform: Platform = user_ctx.platform; + log.debug("Callback called with ffi api: {*} but pjrt_api: {*}", .{ call_frame.api, platform.pjrt_api }); + + if (@hasField(CustomOp, "stream") and platform.target != .cpu) { + const stream = call_frame.api.stream(execution_context); + user_ctx.stream = stream; + } else { + log.info("No stream field provided in container {s} for custom call", .{@typeName(CustomOp)}); + } + + var callback_args: std.meta.ArgsTuple(@TypeOf(CustomOp.call)) = undefined; + callback_args[0] = user_ctx; + + inline for (1..callback_args.len) |i| { + const ffi_buffer = call_frame.args.buffers()[i - 1]; + const ffi_buffer_shape = getShape(ffi_buffer); + var zml_buffer: Buffer = undefined; + + if (platform.target == .cpu) { + zml_buffer = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + zml_buffer = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch unreachable; + } + callback_args[i] = zml_buffer; + } + + for (0..call_frame.results.len) |i| { + const ffi_buffer = call_frame.results.buffers()[i]; + const ffi_buffer_shape = getShape(ffi_buffer); + + if (platform.target == .cpu) { + user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + } + + @call(.auto, CustomOp.call, callback_args) catch |err| { + stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(CustomOp), err }); + }; + + return null; + } + }.proxy; +} + +pub fn getShape(ffi_buffer: *const ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .pred => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} + +/// Internal custom calls. +/// These are not meant to be used by users, but rather by the library itself. +pub const custom_call_internal_types = [_]type{ + Print, +}; + +pub fn registerInternalCustomCalls( + platform: Platform, +) !void { + inline for (custom_call_internal_types) |custom_call_type| { + try platform.registerFFIType(custom_call_type); + log.info("Registered internal custom call {s} with type_id {d}", .{ @typeName(custom_call_type), custom_call_type.type_id }); + } +} + +pub const Print = struct { + pub var type_id: i64 = undefined; + const Self = @This(); + + // TODO: is this a good idea to use the platform to + platform: Platform, + + results: [1]Buffer = undefined, + + pub fn init(platform: Platform) Print { + return .{ + .platform = platform, + }; + } + + pub fn call(_: *Self, input: Buffer) !void { + std.log.defaultLog(.info, .zml, "Device buffer: {}: {}", .{ input.shape(), input.asHostBuffer().pretty() }); + } +}; diff --git a/zml/exe.zig b/zml/exe.zig index 0923f17c..7761c1c2 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -8,6 +8,7 @@ const Bufferized = @import("tensor.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); +const custom_call = @import("custom_call.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; @@ -154,7 +155,7 @@ pub const BaseExe = struct { exe: *pjrt.LoadedExecutable, /// The execution context for this executable. - context: ?*pjrt.ExecuteContext = null, + exe_context: ?*pjrt.ExecuteContext = null, /// Pre-allocated slice of buffers to use as inputs when the module is called. input_per_device: []const [*]*pjrt.Buffer, @@ -181,7 +182,7 @@ pub const BaseExe = struct { parent_allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, - args: struct { input_shapes: []const Shape, result_shapes: []const Shape, n_devices: u8 }, + args: struct { input_shapes: []const Shape, result_shapes: []const Shape, n_devices: u8, need_execution_context: bool }, ) !BaseExe { var arena = std.heap.ArenaAllocator.init(parent_allocator); errdefer arena.deinit(); @@ -205,7 +206,8 @@ pub const BaseExe = struct { const all_shapes = try allocator.alloc(Shape, n_in + n_out); @memcpy(all_shapes[0..n_in], args.input_shapes); @memcpy(all_shapes[n_in..], args.result_shapes); - return .{ + + var base_exe: BaseExe = .{ .platform = platform, .exe = exe, .ready_buffer_count = 0, @@ -215,12 +217,20 @@ pub const BaseExe = struct { .output_per_device = output_per_device, .input_shapes = all_shapes[0..n_in], .result_shapes = all_shapes[n_in..], + .exe_context = if (args.need_execution_context) try platform.pjrt_api.createExecuteContext() else null, ._arena = arena, }; + // Those may not needed, but also it's annoying if they don't get registered automatically. + if (args.need_execution_context) { + inline for (custom_call.custom_call_internal_types) |T| { + try base_exe.bind(T, .init(platform)); + } + } + return base_exe; } pub fn deinit(self: BaseExe) void { - if (self.context) |ctx| { + if (self.exe_context) |ctx| { ctx.deinit(self.platform.pjrt_api); } self._arena.deinit(); @@ -244,16 +254,16 @@ pub const BaseExe = struct { // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, - .context = self.context, + .context = self.exe_context, }) catch |err| { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.deinit(self.platform.pjrt_api); + // } + // } } pub fn _unsafeAssignResults(self: BaseExe, T: type, result: *T) void { @@ -285,6 +295,24 @@ pub const BaseExe = struct { stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); } + pub fn bind(self: *BaseExe, comptime T: type, value: T) !void { + const storage = try self._arena.allocator().create(T); + storage.* = value; + try self.bindPtr(T, storage); + } + + pub fn bindPtr(self: BaseExe, comptime T: type, user_data: *T) !void { + stdx.debug.assert(self.exe_context != null, "Exe doesn't have an execution exe_context", .{}); + const pjrt_api = self.platform.pjrt_api; + + if (pjrt_api.ffi()) |ffi| { + try ffi.addUserData(pjrt_api, self.exe_context.?, .{ .type_id = T.type_id, .user_data = user_data }); + log.debug("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), @intFromPtr(user_data), T.type_id, self.exe_context.? }); + } else { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(self.platform.target)}); + } + } + pub fn serialize(self: BaseExe, writer: anytype) !void { var executable = try self.exe.getExecutable(self.platform.pjrt_api); var serialize_result = try executable.serialize(self.platform.pjrt_api); @@ -314,7 +342,7 @@ pub const BaseExe = struct { pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ - .n_in = self.input_buffer_count, + .input_shapes = self.input_shapes, .result_shapes = self.result_shapes, .n_devices = self.num_devices, }); @@ -348,6 +376,14 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + pub fn bind(self: *Self, comptime T: type, value: T) !void { + try self.inner.bind(T, value); + } + + pub fn bindPtr(self: Self, comptime T: type, value: *T) !void { + try self.inner.bindPtr(T, value); + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 8f6fcb1c..8a44f9af 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -176,10 +176,20 @@ pub const HostBuffer = struct { return ptr[0..self._shape.count()]; } + pub fn item(self: HostBuffer, comptime T: type, index: usize) T { + stdx.debug.assert(index < self._shape.count(), "index {} out of bounds for shape {}", .{ index, self._shape }); + return self.items(T)[index]; + } + pub fn mutItems(self: HostBuffer, comptime T: type) []T { return @constCast(self.items(T)); } + pub fn mutitem(self: HostBuffer, comptime T: type, index: usize) T { + stdx.debug.assert(index < self._shape.count(), "index {} out of bounds for shape {}", .{ index, self._shape }); + return self.mutItems(T)[index]; + } + pub fn bytes(self: HostBuffer) []const u8 { stdx.debug.assert(self.isContiguous(), "{} isn't contiguous, can't interpret as []const u8", .{self}); return self._data[0..self._shape.byteSize()]; diff --git a/zml/module.zig b/zml/module.zig index f031e815..8eb64f77 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -49,6 +49,7 @@ pub const CompilationContext = struct { _mlir_canonicalizer: mlir.PassManager, _module: mlir.Module, + _num_custom_calls: u32 = 0, _blocks: std.BoundedArray(TaggedBlock, 64) = .{}, _fn_cache: FnCache = .{}, @@ -233,6 +234,7 @@ pub const CompilationContext = struct { .input_shapes = f.args_shapes, .result_shapes = f.res_shapes, .n_devices = sharding.num_replicas * sharding.num_partitions, + .need_execution_context = self._num_custom_calls > 0, }, ); } @@ -899,6 +901,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m // Use lld from libllvm instead of invoking the ld.lld binary. // This saves us from having to sandbox it. setFlag(&options, "xla_gpu_use_inprocess_lld", true); + setFlag(&options, "xla_gpu_autotune_level", 0); }, else => {}, } @@ -1157,9 +1160,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1175,7 +1178,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/nn.zig b/zml/nn.zig index ffdb7a9c..adc06237 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -175,17 +175,23 @@ pub const RopeOpts = struct { /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { - const content = try std.json.Value.jsonParse(allocator, source, options); - if (content == .null) return .default; + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } - if (content != .object) return error.InvalidEnumTag; + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { + if (source == .null) return .default; - const obj = content.object; + if (source != .object) return error.InvalidEnumTag; + + const obj = source.object; const impl = obj.get("rope_type") orelse return error.MissingField; if (impl != .string) return error.InvalidEnumTag; if (std.mem.eql(u8, impl.string, "llama3")) { - // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, undefined, content, .{ .ignore_unknown_fields = true }) }; + return .{ .llama3 = try std.json.innerParseFromValue(Llama3, allocator, source, options) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; diff --git a/zml/ops.zig b/zml/ops.zig index e0af64e8..aaba299e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -764,33 +764,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - return customCall( - "zmlHostBufferCallback", - inputs, - output_shapes, - .{ - .callback = @intFromPtr(callback), - .user_context = @intFromPtr(blkctx), - }, - .{ - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = opts.output_operand_aliases, - }, - ); -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 1795efc3..eabfaa10 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -215,6 +215,13 @@ pub const Event = opaque { return self.inner().getEventError(api); } + pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { + if (self.isReady(api)) { + return; + } + try self.inner().await_(api); + } + pub fn await_(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); @@ -272,14 +279,14 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, .context = args.context, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { diff --git a/zml/platform.zig b/zml/platform.zig index a7bc87c9..ea197b4f 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -53,6 +53,7 @@ pub const Platform = struct { pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; pub fn sharding(self: Platform) Sharding { + // TODO: store me inside the platform itself to avoid expensive ffi calls // replicas run the same function but with different inputs, // while partitions contribute to one evaluation over a shared input. // Inside an inference process, we generally don't want replicas, diff --git a/zml/tensor.zig b/zml/tensor.zig index 74bf1779..ee0da3a4 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -13,6 +13,8 @@ const Memory = @import("buffer.zig").Buffer.Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); +const custom_call_ = @import("custom_call.zig"); +const custom_call = custom_call_.custom_call; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -3825,22 +3827,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - // TODO: find a way of doing print that doesn't involve a H2D copy. - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.log.defaultLog(.info, .zml, "Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); - // This is true because of the operand aliases. - // Since the result is already pointing to the input we don't need to modify the buffer. - std.debug.assert(host_buffer._data == outputs[0]._data); + return custom_call(custom_call_.Print, .{input}, &[_]Shape{input.shape()}, .{ .output_operand_aliases = &.{0}, .copy_inputs_to_host_pinned = true })[0]; } }; diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 06bcd6fa..597d3ee6 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -1,9 +1,10 @@ const builtin = @import("builtin"); + const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { .macos => MacOsTracer, - .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + .linux => FakeTracer, else => FakeTracer, }; @@ -60,6 +61,10 @@ const MacOsTracer = struct { }; } + pub fn deinit(self: *const MacOsTracer) void { + _ = self; + } + pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { const interval_id = c.os_signpost_id_generate(self.logger); c.zml_os_signpost_event(self.logger, interval_id, message); @@ -83,6 +88,10 @@ const FakeTracer = struct { return .{}; } + pub fn deinit(self: *const FakeTracer) void { + _ = self; + } + pub fn event(self: *const FakeTracer, message: [:0]const u8) void { _ = self; _ = message; diff --git a/zml/zml.zig b/zml/zml.zig index 45abfccd..546e986c 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -34,6 +34,7 @@ pub const torch = @import("torch.zig"); pub const tokenizer = @import("zml/tokenizer"); pub const call = ops.call; +pub const custom_call = @import("custom_call.zig").custom_call; pub const compile = exe.compile; pub const compileWithPrefix = exe.compileWithPrefix; pub const compileFn = exe.compileFn;