From 971ea706be4aabd24be40f5e90c294bf151da886 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 9 Apr 2025 18:02:21 +0000 Subject: [PATCH 1/2] poc --- examples/mnist/mnist.zig | 21 +++++++++- pjrt/pjrt.zig | 16 ++++++-- zml/context.zig | 83 ++++++++++++++++++++++++++++++++++++++++ zml/ops.zig | 52 +++++++++++++++++++++++++ zml/pjrtx.zig | 2 +- zml/tensor.zig | 14 +++++++ 6 files changed, 183 insertions(+), 5 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c49300a9..c41e221e 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -20,7 +20,7 @@ const Mnist = struct { bias: zml.Tensor, pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.matmul(input).add(self.bias).relu(); + return self.weight.matmul(input.print2()).add(self.bias).relu(); } }; @@ -108,6 +108,25 @@ pub fn asyncMain() !void { var result: zml.Buffer = mnist.call(.{input}); defer result.deinit(); + log.info( + \\✅ RECOGNIZED DIGIT: + \\ +-------------+ + \\{s} + \\ +-------------+ + \\ + , .{digits[try result.getValue(u8)]}); + } + { + const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); + var sample: [28 * 28]u8 align(16) = undefined; + _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); + var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); + defer input.deinit(); + + printDigit(sample); + var result: zml.Buffer = mnist.call(.{input}); + defer result.deinit(); + log.info( \\✅ RECOGNIZED DIGIT: \\ +-------------+ diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9dbd5aac..53fe2b18 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -502,6 +502,16 @@ pub const Device = opaque { }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) []*const Memory { + const ret = api.call(.PJRT_Device_AddressableMemories, .{ + .device = self.inner(), + }) catch unreachable; + if (ret.memories) |memories| { + return @constCast(@ptrCast(memories[0..ret.num_memories])); + } + return &.{}; + } }; pub const DeviceDescription = opaque { @@ -843,7 +853,7 @@ pub const Buffer = opaque { return @ptrCast(ret.event); } - pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!?*Buffer { + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), .dst_memory = @ptrCast(@constCast(dst_memory)), @@ -925,8 +935,8 @@ pub const Memory = opaque { pub fn kind(self: *const Memory, api: *const Api) Kind { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable[0..ret.kind_size]; - return std.meta.stringToEnum(Kind, kind_) orelse unreachable; + const kind_ = ret.kind orelse unreachable; + return std.meta.stringToEnum(Kind, kind_[0..ret.kind_size]) orelse unreachable; } pub fn kindId(self: *const Memory, api: *const Api) u32 { diff --git a/zml/context.zig b/zml/context.zig index 03de62a3..fa898039 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -1,6 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); +const asynk = @import("async"); const c = @import("c"); const mlir = @import("mlir"); const runfiles = @import("runfiles"); @@ -8,6 +9,7 @@ const runtimes = @import("runtimes"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; +const bufferTypeFromDtype = @import("buffer.zig").bufferTypeFromDtype; const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); @@ -214,6 +216,7 @@ pub const Context = struct { } pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; + pub const HostCall = fn (?*anyopaque, []const HostBuffer) void; }; const CustomCall = struct { @@ -222,6 +225,7 @@ const CustomCall = struct { if (registry) |reg| { try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback); + try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCall", @tagName(platform.target), &hostBufferCall); } else { stdx.debug.panic("Registering custom calls failed", .{}); } @@ -251,6 +255,35 @@ const CustomCall = struct { callback(user_ctx, input_buffers, output_buffers); return null; } + + fn hostBufferCall(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; + std.debug.assert(callback_attr.dtype == .u64); + const callback: *const Context.HostCall = @ptrFromInt(callback_attr.get(usize)); + + const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; + std.debug.assert(user_ctx_ptr.dtype == .u64); + const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); + + const pjrt_api_ptr = call_frame.attrs.getByName(.scalar, "pjrt_api") orelse unreachable; + std.debug.assert(pjrt_api_ptr.dtype == .u64); + const pjrt_api: *const pjrt.Api = @ptrFromInt(pjrt_api_ptr.get(usize)); + + const pjrt_client_ptr = call_frame.attrs.getByName(.scalar, "pjrt_client") orelse unreachable; + std.debug.assert(pjrt_client_ptr.dtype == .u64); + const pjrt_client: *pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); + + const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); + for (input_buffers, 0..) |*b, i| { + b.* = copyBuffer(pjrt_api, pjrt_client, call_frame.args.get(i)); + } + + callback(user_ctx, input_buffers); + // asynk.callBlocking(callback, .{ user_ctx, input_buffers }); + return null; + } }; fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { @@ -278,3 +311,53 @@ fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { buffer_desc.data[0..buffer_shape.byteSize()], ); } + +fn copyBuffer(pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { + const buffer_shape = getShape(buffer_desc); + + const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { + var res: [Shape.MAX_RANK]i64 = undefined; + for (0..Shape.MAX_RANK) |i| { + res[i] = @intCast(Shape.MAX_RANK - i - 1); + } + break :blk res; + }; + + const device = pjrt_client.getAddressableDevices(pjrt_api)[0]; + const memories = pjrt_client.addressableMemories(pjrt_api); + + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(pjrt_api) == pjrt.Memory.Kind.pinned_host) { + selected_mem = mem; + } + } + + const pjrt_buffer = pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ + .data = buffer_desc.data, + .element_type = bufferTypeFromDtype(buffer_shape.dtype()), + .dims = buffer_shape.dims(), + // TODO: exposes sharding in the API. + .device = device, + .layout = .{ + .tiled = .{ + .minor_to_major = minor_to_major[Shape.MAX_RANK - buffer_shape.rank() ..], + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .stream = null, + }) catch @panic("failed to createViewOfDeviceBuffer"); + + const copied_buffer = pjrt_buffer.copyToMemory(pjrt_api, selected_mem) catch unreachable; + + const opaqueDataPointer = copied_buffer.getOpaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable; + const sizeInbytes = copied_buffer.getOnDeviceSizeInBytes(pjrt_api) catch unreachable; + const data = @as([*]const u8, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + const end: usize = @intCast(sizeInbytes); + + return HostBuffer.fromBytes( + buffer_shape, + data[0..end], + ); +} diff --git a/zml/ops.zig b/zml/ops.zig index 1765888a..c7fe2244 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -821,6 +821,58 @@ pub fn addHostCallback( return res; } +pub const HostCallOpt = struct { + has_side_effect: bool = true, +}; + +pub fn addHostCall( + callback: *const Context.HostCall, + blkctx: ?*anyopaque, + inputs: []const Tensor, + opts: HostCallOpt, +) []Tensor { + const ctx = CompilationContext.current(); + + const mlir_ctx = ctx.mlirCtx(); + const backend_config = mlir.Attribute.dict(mlir_ctx, &.{ + .{ "callback", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(callback))) }, + .{ "user_context", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(blkctx))) }, + .{ "pjrt_api", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_api))) }, + .{ "pjrt_client", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_client))) }, + }); + + const values = stdx.stackSlice(8, mlir.Value, inputs.len); + for (inputs, values) |i, *v| { + v.* = ctx.getValue(i); + } + const res_types = stdx.stackSlice(8, mlir.Type, inputs.len); + for (res_types, inputs) |*r, i| { + r.* = mlir.ext.RankedTensorType.fromShape(mlir_ctx, i.shape()).as(mlir.Type); + } + + const loc = ctx.mlirCtx().location(@src()); + const op = dialect.stablehlo.custom_call( + ctx.mlirCtx(), + values, + .{ + .call_target_name = "zmlHostBufferCall", + .api_version = .typed_ffi, + .backend_config = backend_config, + .has_side_effect = opts.has_side_effect, + .output_operand_aliases = &.{}, + }, + res_types, + loc, + ); + + const res = ctx.allocator().alloc(Tensor, inputs.len) catch @panic("OOM"); + for (res, inputs, 0..) |*r, input, i| { + r.* = Tensor._result(input.shape(), op.result(i)); + } + + return res; +} + pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index bd094ead..dbfa82a0 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -182,7 +182,7 @@ pub const Buffer = opaque { } pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(self.inner().copyToMemory(api, memory_)); + return @ptrCast(try self.inner().copyToMemory(api, memory_)); } pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { diff --git a/zml/tensor.zig b/zml/tensor.zig index b248ea20..040dc59b 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -3800,6 +3800,20 @@ pub const Tensor = struct { std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); std.debug.assert(host_buffer.data.ptr == outputs[0].data.ptr); } + + pub fn print2(input: Tensor) Tensor { + return ops.addHostCall( + &printCallback2, + null, + &.{input}, + .{ .has_side_effect = true }, + )[0]; + } + + fn printCallback2(_: ?*anyopaque, inputs: []const HostBuffer) void { + const host_buffer = inputs[0]; + std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); + } }; fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { From 3301e85016ba1098c6612881d8db373103526258 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 11 Apr 2025 13:38:48 +0000 Subject: [PATCH 2/2] working wip --- examples/mnist/mnist.zig | 67 ++++++++++++------ pjrt/ffi.zig | 52 ++++++++++++-- pjrt/pjrt.zig | 90 +++++++++++++++++++----- zml/context.zig | 148 +++++++++++++++++++++++++++++++++++++-- zml/exe.zig | 12 ++-- zml/pjrtx.zig | 3 + zml/tensor.zig | 18 +++-- zml/zml.zig | 1 + 8 files changed, 328 insertions(+), 63 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c41e221e..ed423e2f 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -10,6 +10,15 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; +pub fn Pouet(comptime T: type) type { + _ = T; // autofix + return struct { + const Self = @This(); + + pub fn pouet() callconv(.C) void {} + }; +} + /// Model definition const Mnist = struct { fc1: Layer, @@ -20,7 +29,7 @@ const Mnist = struct { bias: zml.Tensor, pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.matmul(input.print2()).add(self.bias).relu(); + return self.weight.matmul(input.print()).add(self.bias).relu(); } }; @@ -28,6 +37,11 @@ const Mnist = struct { pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); var x = input.flattenAll().convert(.f32); + // const t = zml.custom_call(struct { + // pub fn call(self: *const Mnist, input: zml.Buffer) zml.Buffer { + // return pure zig + // } + // }.call, .{self}, .{input}, .{}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { x = zml.call(layer, .forward, .{x}); @@ -49,8 +63,12 @@ pub fn asyncMain() !void { // log.info("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); + const compilation_options = zml.CompilationOptions{ + .xla_dump_to = "/tmp/zml/hugo", + }; + // // Auto-select platform - const platform = context.autoPlatform(.{}); + const platform = context.autoPlatform(.{}).withCompilationOptions(compilation_options); context.printAvailablePlatforms(platform); // Parse program args @@ -98,6 +116,12 @@ pub fn asyncMain() !void { // inference - can be looped { + var execution_context = try context.getExecutionCtx(platform); + const attached_value_id = try execution_context.attach("toto", platform.compilation_options); + // std.debug.print("Attached value ptr: {*}\n", .{platform.compilation_options}); + std.debug.print("Attached value id: {s}\n", .{attached_value_id}); + defer execution_context.deinit(); + const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); @@ -105,7 +129,7 @@ pub fn asyncMain() !void { defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); + var result: zml.Buffer = mnist.call(.{input}, &execution_context); defer result.deinit(); log.info( @@ -116,25 +140,28 @@ pub fn asyncMain() !void { \\ , .{digits[try result.getValue(u8)]}); } - { - const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); - var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); - defer input.deinit(); + // { + // const execution_context = try zml.ExecuteContext.init(platform); + // defer execution_context.deinit(); - printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); - defer result.deinit(); + // const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); + // var sample: [28 * 28]u8 align(16) = undefined; + // _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); + // var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); + // defer input.deinit(); - log.info( - \\✅ RECOGNIZED DIGIT: - \\ +-------------+ - \\{s} - \\ +-------------+ - \\ - , .{digits[try result.getValue(u8)]}); - } + // printDigit(sample); + // var result: zml.Buffer = mnist.call(.{input}, execution_context); + // defer result.deinit(); + + // log.info( + // \\✅ RECOGNIZED DIGIT: + // \\ +-------------+ + // \\{s} + // \\ +-------------+ + // \\ + // , .{digits[try result.getValue(u8)]}); + // } } fn printDigit(digit: [28 * 28]u8) void { diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 39e7c248..357b7d5c 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -191,10 +191,12 @@ pub const ExecutionContext = opaque { // } // } - pub fn get(self: *ExecutionContext, api: *const Api, type_id: *TypeId) ApiError!*anyopaque { + pub fn get(self: *const ExecutionContext, api: *const Api, type_name: []const u8) ApiError!*anyopaque { + const type_id = getTypeId(type_name); + std.debug.print("get {s} with type_id: {d} from {any}\n", .{ type_name, type_id.type_id, @constCast(self.inner()) }); var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = self.inner(), - .type_id = @ptrCast(@alignCast(type_id)), + .ctx = @constCast(self.inner()), + .type_id = @constCast(&type_id.toCStruct()), }); const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); @@ -210,9 +212,49 @@ pub const ExecutionContext = opaque { return ret.data.?; } + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { + var ret = pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ + .ctx = @constCast(self.inner()), + .task = @ptrCast(@alignCast(task)), + .data = @ptrCast(@alignCast(data)), + }); + + const result = api.inner().XLA_FFI_ThreadPool_Schedule.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + std.debug.print("error: {any} \n", .{err}); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } + + fn getTypeId(type_name: []const u8) TypeId { + const id: i64 = @bitCast(std.hash.Fnv1a_64.hash(type_name)); + + return .{ + .type_id = id, + }; + } + // TODO getDeviceOrdinal() }; +const TypeId = extern struct { + type_id: i64, + + pub fn toCStruct(self: TypeId) c.XLA_FFI_TypeId { + return c.XLA_FFI_TypeId{ + .type_id = self.type_id, + }; + } +}; + +const Task = fn (*anyopaque) void; + const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -222,10 +264,6 @@ const ByteSpan = extern struct { } }; -pub const TypeId = extern struct { - type_id: i64, -}; - pub const DataType = enum(c.XLA_FFI_DataType) { invalid = c.XLA_FFI_DataType_INVALID, pred = c.XLA_FFI_DataType_PRED, diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 53fe2b18..bdb423af 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,7 +31,7 @@ fn pjrtStructSize(comptime T: type) usize { return @field(c, typedef_name ++ "_STRUCT_SIZE"); } -inline fn pjrtStruct(v: anytype) @TypeOf(v) { +pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { var ret = v; ret.struct_size = pjrtStructSize(@TypeOf(v)); return ret; @@ -111,6 +111,7 @@ pub const Api = struct { inline fn call(self: *const Api, comptime method: Funcs, arg: CallFnArgType(method)) ApiError!@TypeOf(arg) { var ret = pjrtStruct(arg); const fn_ptr = @field(&self.inner, @tagName(method)).?; + std.debug.print("Calling {s}\n", .{@tagName(method)}); const result = fn_ptr(&ret); if (@TypeOf(result) == void) { return ret; @@ -160,9 +161,14 @@ pub const Api = struct { return state.str; } - pub fn customCallRegistry(api: *const Api) ?CustomCallRegistry { + pub fn createExecuteContext(api: *const Api) ApiError!*ExecuteContext { + const ret = try api.call(.PJRT_ExecuteContext_Create, .{}); + return @ptrCast(ret.context.?); + } + + pub fn ffi(api: *const Api) ?FFI { if (api.lookupExtension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { - return .{ .inner = ext.register_handler.? }; + return .{ .inner = ext }; } return null; } @@ -567,6 +573,14 @@ pub const SerializeResult = struct { } }; +pub const ExecuteContext = opaque { + pub fn deinit(self: *ExecuteContext, api: *const Api) void { + _ = api.call(.PJRT_ExecuteContext_Destroy, .{ + .context = @ptrCast(self), + }) catch {}; + } +}; + pub const Executable = opaque { const inner = InnerMixin(c.PJRT_Executable).inner; @@ -633,6 +647,7 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: *ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { var options = pjrtStruct(c.PJRT_ExecuteOptions{ @@ -643,6 +658,7 @@ pub const LoadedExecutable = opaque { .launch_id = 0, .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices.ptr), .num_non_donatable_input_indices = args.non_donatable_input_indices.len, + .context = @ptrCast(args.context), }); _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), @@ -1047,21 +1063,6 @@ pub const AsyncHostToDeviceTransferManager = opaque { } }; -pub const ExecutionContext = opaque { - const inner = InnerMixin(c.PJRT_ExecutionContext).inner; - - pub fn init(api: *const Api) ApiError!*ExecutionContext { - const ret = try api.call(.PJRT_ExecutionContext_Create, .{}); - return @ptrCast(ret.context.?); - } - - pub fn deinit(self: *ExecutionContext, api: *const Api) void { - _ = api.call(.PJRT_ExecutionContext_Destroy, .{ - .context = self.inner(), - }) catch unreachable; - } -}; - pub const NamedValue = extern struct { comptime { std.debug.assert(@sizeOf(NamedValue) == @sizeOf(c.PJRT_NamedValue)); @@ -1167,6 +1168,59 @@ pub const NamedValue = extern struct { } }; +pub const FFI = extern struct { + inner: *const c.PJRT_FFI, + + pub const UserData = extern struct { + type_id: i64, + user_data: *anyopaque, + + fn toCStruct(self: UserData) c.PJRT_FFI_UserData { + return .{ + .type_id = self.type_id, + .data = self.user_data, + }; + } + }; + + pub fn customCallRegistry(self: *const FFI) ?CustomCallRegistry { + return .{ .inner = self.inner.register_handler.? }; + } + + pub fn registerTypeId(self: *const FFI, api: *const Api, type_name: []const u8) ApiError!i64 { + var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ + .type_name = type_name.ptr, + .type_name_size = type_name.len, + .type_id = getTypeId(type_name), + }); + const result = self.inner.type_id_register.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + return pjrt_error.getCode(api).toApiError(); + } + + return ret.type_id; + } + + pub fn addUserData(self: *const FFI, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { + var ret = pjrtStruct(c.PJRT_FFI_UserData_Add_Args{ + .context = @ptrCast(context), + .user_data = user_data.toCStruct(), + }); + const result = self.inner.user_data_add.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + log.err("[addUserData] {s}", .{pjrt_error.getMessage(api)}); + return pjrt_error.getCode(api).toApiError(); + } + } + + pub fn getTypeId(type_name: []const u8) i64 { + // Use FNV-1a 64-bit hash algorithm + return @bitCast(std.hash.Fnv1a_64.hash(type_name)); + } +}; + // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub const CustomCallRegistry = extern struct { diff --git a/zml/context.zig b/zml/context.zig index fa898039..cf5df2fc 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -9,11 +9,13 @@ const runtimes = @import("runtimes"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; +const Tensor = @import("tensor.zig").Tensor; const bufferTypeFromDtype = @import("buffer.zig").bufferTypeFromDtype; const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const CompilationOptions = @import("platform.zig").CompilationOptions; const Shape = @import("shape.zig").Shape; const Target = @import("platform.zig").Target; const zml_platform = @import("platform.zig"); @@ -39,6 +41,7 @@ pub const Context = struct { inline for (comptime std.enums.values(runtimes.Platform)) |t| { if (runtimes.load(t)) |api| { Context.apis.set(t, api); + CustomCall.registerZmlCustomCalls(api, t) catch unreachable; } else |_| {} } } @@ -179,12 +182,16 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); return p; } + pub fn getExecutionCtx(self: *Context, platform_: Platform) !ExecuteContext { + _ = self; // autofix + + return try ExecuteContext.init(platform_); + } + pub fn printAvailablePlatforms(self: Context, selected: Platform) void { // List available targets log.info("Available Platforms:", .{}); @@ -219,13 +226,60 @@ pub const Context = struct { pub const HostCall = fn (?*anyopaque, []const HostBuffer) void; }; +pub const ExecuteContext = struct { + pub const ValueId = []const u8; + + execute_ctx: *pjrt.ExecuteContext, + _pjrt_api: *const pjrt.Api, + + pub fn init(platform: Platform) !ExecuteContext { + const pjrt_api = platform.pjrt_api; + const execute_ctx = try pjrt_api.createExecuteContext(); + + const ffi = pjrt_api.ffi().?; + + const type_id = try ffi.registerTypeId(pjrt_api, "platform"); + + try ffi.addUserData(pjrt_api, execute_ctx, .{ .type_id = type_id, .user_data = @constCast(&platform.target) }); + + return .{ + .execute_ctx = execute_ctx, + ._pjrt_api = pjrt_api, + }; + } + + pub inline fn attach(self: *ExecuteContext, name: []const u8, value: anytype) !ValueId { + const ffi = self._pjrt_api.ffi().?; + const type_name = name; + const type_id = try ffi.registerTypeId(self._pjrt_api, type_name); + std.debug.print("Registered type id: {d}\n", .{type_id}); + + const user_data = @constCast(&value); + std.debug.print("User data: {*} {any}\n", .{ user_data, user_data }); + try ffi.addUserData(self._pjrt_api, self.execute_ctx, .{ .type_id = type_id, .user_data = user_data }); + return type_name; + } + + pub fn deinit(self: *ExecuteContext) void { + self.execute_ctx.deinit(self._pjrt_api); + } +}; + const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { - const registry = platform.pjrt_api.customCallRegistry(); + pub fn registerZmlCustomCalls(pjrt_api: *const pjrt.Api, target: Target) !void { + const ffi = pjrt_api.ffi(); + + if (ffi == null) { + std.debug.print("No FFI available for platform: {s}\n", .{@tagName(target)}); + return error.NoFfiAvailable; + } + + const registry = ffi.?.customCallRegistry(); if (registry) |reg| { - try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback); - try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCall", @tagName(platform.target), &hostBufferCall); + std.debug.print("Registering custom calls for platform: {s}\n", .{@tagName(target)}); + try reg.registerFfi(pjrt_api, "zmlHostBufferCallback", @tagName(target), &hostBufferCallback); + try reg.registerFfi(pjrt_api, "zmlHostBufferCall", @tagName(target), &hostBufferCall); } else { stdx.debug.panic("Registering custom calls failed", .{}); } @@ -234,6 +288,14 @@ const CustomCall = struct { fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; + const execution_context = call_frame.ctx.?; + const toto_pq = execution_context.get(call_frame.api.?, "toto") catch unreachable; + std.debug.print("platform_pq: {any}\n", .{toto_pq}); + + const toto: *CompilationOptions = @ptrCast(@alignCast(toto_pq)); + // execution_context.scheduleTask(call_frame.api.?, my_task, lastone) catch unreachable; + std.debug.print("toto : {s}\n", .{toto.xla_dump_to.?}); + const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; std.debug.assert(callback_attr.dtype == .u64); const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); @@ -276,16 +338,38 @@ const CustomCall = struct { const pjrt_client: *pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); + var lastone: *pjrt.Buffer = undefined; for (input_buffers, 0..) |*b, i| { b.* = copyBuffer(pjrt_api, pjrt_client, call_frame.args.get(i)); + lastone = copyBuffer2(pjrt_api, pjrt_client, call_frame.args.get(i)); } + const execution_context = call_frame.ctx.?; + const platform_pq = execution_context.get(call_frame.api.?, "toto") catch unreachable; + std.debug.print("platform_pq: {any}\n", .{platform_pq}); + + const platform = @as(*CompilationOptions, @ptrCast(@alignCast(platform_pq))); + // execution_context.scheduleTask(call_frame.api.?, my_task, lastone) catch unreachable; + std.debug.print("platform: {any}\n", .{platform.xla_dump_to}); callback(user_ctx, input_buffers); - // asynk.callBlocking(callback, .{ user_ctx, input_buffers }); return null; } }; +fn my_task(data: *anyopaque) void { + for (0..10) |i| { + _ = i; // autofix + std.debug.print("Hello from my_task: {any}\n", .{data}); + } +} + +pub fn printCallback2(pjrt_api: *const pjrt.Api, buf: *pjrt.Buffer) void { + for (0..10) |i| { + _ = i; // autofix + std.debug.print("Device buffer: {any}: {any}", .{ buf.getDimensions(pjrt_api), buf.getElementType(pjrt_api) }); + } +} + fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { // log.warn("received buffer {}", .{buffer_desc}); const dt: DataType = switch (buffer_desc.dtype) { @@ -333,6 +417,8 @@ fn copyBuffer(pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, buffer_desc: } } + std.debug.print("selected memory: {s}\n", .{selected_mem}); + const pjrt_buffer = pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ .data = buffer_desc.data, .element_type = bufferTypeFromDtype(buffer_shape.dtype()), @@ -361,3 +447,51 @@ fn copyBuffer(pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, buffer_desc: data[0..end], ); } + +fn copyBuffer2(pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, buffer_desc: *const pjrt.ffi.Buffer) *pjrt.Buffer { + const buffer_shape = getShape(buffer_desc); + + const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { + var res: [Shape.MAX_RANK]i64 = undefined; + for (0..Shape.MAX_RANK) |i| { + res[i] = @intCast(Shape.MAX_RANK - i - 1); + } + break :blk res; + }; + + const device = pjrt_client.getAddressableDevices(pjrt_api)[0]; + const memories = pjrt_client.addressableMemories(pjrt_api); + + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(pjrt_api) == pjrt.Memory.Kind.pinned_host) { + selected_mem = mem; + } + } + + std.debug.print("selected memory: {s}\n", .{selected_mem}); + + const pjrt_buffer = pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ + .data = buffer_desc.data, + .element_type = bufferTypeFromDtype(buffer_shape.dtype()), + .dims = buffer_shape.dims(), + // TODO: exposes sharding in the API. + .device = device, + .layout = .{ + .tiled = .{ + .minor_to_major = minor_to_major[Shape.MAX_RANK - buffer_shape.rank() ..], + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .stream = null, + }) catch @panic("failed to createViewOfDeviceBuffer"); + + // if (@hasDecl(c, "ZML_RUNTIME_CPU")) { + // return pjrt_buffer; + // } + + const copied_buffer = pjrt_buffer.copyToMemory(pjrt_api, selected_mem) catch unreachable; + + return copied_buffer; +} diff --git a/zml/exe.zig b/zml/exe.zig index 946410c6..ded63d35 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -8,6 +8,7 @@ const pjrt = @import("pjrtx.zig"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("tensor.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; +const ExecuteContext = @import("context.zig").ExecuteContext; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; @@ -191,12 +192,12 @@ pub const BaseExe = struct { self._arena.deinit(); } - pub fn call(self: BaseExe) void { + pub fn call(self: BaseExe, execute_context: *ExecuteContext) void { stdx.debug.assert(self.input_buffer_count == self.ready_buffer_count, "BaseExe isn't ready to be called, expected {} buffer inputs got {}", .{ self.input_buffer_count, self.ready_buffer_count }); - return self._unsafeCall(); + return self._unsafeCall(execute_context); } - pub fn _unsafeCall(self: BaseExe) void { + pub fn _unsafeCall(self: BaseExe, execute_context: *ExecuteContext) void { var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; const sharding = self.platform.sharding(); @@ -209,6 +210,7 @@ pub const BaseExe = struct { // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, + .context = execute_context.execute_ctx, }) catch unreachable; for (events[0..sharding.num_partitions]) |e| { @@ -279,10 +281,10 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return self.inner.platform; } - pub fn call(self: Self, args: Bufferized(ArgsT)) Bufferized(ReturnT) { + pub fn call(self: Self, args: Bufferized(ArgsT), execute_context: *ExecuteContext) Bufferized(ReturnT) { const total_ready = fillBuffers(&args, self.inner.input_per_device, self.inner.ready_buffer_count); std.debug.assert(total_ready == self.inner.input_buffer_count); - self.inner._unsafeCall(); + self.inner._unsafeCall(execute_context); var result: Bufferized(ReturnT) = undefined; assignRawBuffers(&result, self.inner.platform, self.inner.output_per_device, self.inner.result_shapes); return result; diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index dbfa82a0..b9d5ee5f 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -8,6 +8,7 @@ pub const ffi = pjrt.ffi; pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; +pub const ExecuteContext = pjrt.ExecuteContext; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; @@ -262,6 +263,7 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: *ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { @@ -271,6 +273,7 @@ pub const LoadedExecutable = opaque { .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, + .context = args.context, } }); } diff --git a/zml/tensor.zig b/zml/tensor.zig index 040dc59b..d2a9b95d 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -3796,9 +3796,12 @@ pub const Tensor = struct { } fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); - std.debug.assert(host_buffer.data.ptr == outputs[0].data.ptr); + for (0..10) |i| { + _ = i; // autofix + const host_buffer = inputs[0]; + std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); + std.debug.assert(host_buffer.data.ptr == outputs[0].data.ptr); + } } pub fn print2(input: Tensor) Tensor { @@ -3810,9 +3813,12 @@ pub const Tensor = struct { )[0]; } - fn printCallback2(_: ?*anyopaque, inputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); + pub fn printCallback2(_: ?*anyopaque, inputs: []const HostBuffer) void { + for (0..10) |i| { + _ = i; // autofix + const host_buffer = inputs[0]; + std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); + } } }; diff --git a/zml/zml.zig b/zml/zml.zig index 18282b93..98766a62 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -7,6 +7,7 @@ pub const Buffer = @import("buffer.zig").Buffer; pub const Bufferized = @import("tensor.zig").Bufferized; pub const CompilationOptions = @import("platform.zig").CompilationOptions; pub const Context = @import("context.zig").Context; +pub const ExecuteContext = @import("context.zig").ExecuteContext; pub const Data = @import("dtype.zig").Data; pub const DataType = @import("dtype.zig").DataType; pub const HostBuffer = @import("hostbuffer.zig").HostBuffer;