From 971ea706be4aabd24be40f5e90c294bf151da886 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 9 Apr 2025 18:02:21 +0000 Subject: [PATCH 1/5] poc --- examples/mnist/mnist.zig | 21 +++++++++- pjrt/pjrt.zig | 16 ++++++-- zml/context.zig | 83 ++++++++++++++++++++++++++++++++++++++++ zml/ops.zig | 52 +++++++++++++++++++++++++ zml/pjrtx.zig | 2 +- zml/tensor.zig | 14 +++++++ 6 files changed, 183 insertions(+), 5 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c49300a9..c41e221e 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -20,7 +20,7 @@ const Mnist = struct { bias: zml.Tensor, pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.matmul(input).add(self.bias).relu(); + return self.weight.matmul(input.print2()).add(self.bias).relu(); } }; @@ -108,6 +108,25 @@ pub fn asyncMain() !void { var result: zml.Buffer = mnist.call(.{input}); defer result.deinit(); + log.info( + \\✅ RECOGNIZED DIGIT: + \\ +-------------+ + \\{s} + \\ +-------------+ + \\ + , .{digits[try result.getValue(u8)]}); + } + { + const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); + var sample: [28 * 28]u8 align(16) = undefined; + _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); + var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); + defer input.deinit(); + + printDigit(sample); + var result: zml.Buffer = mnist.call(.{input}); + defer result.deinit(); + log.info( \\✅ RECOGNIZED DIGIT: \\ +-------------+ diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9dbd5aac..53fe2b18 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -502,6 +502,16 @@ pub const Device = opaque { }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) []*const Memory { + const ret = api.call(.PJRT_Device_AddressableMemories, .{ + .device = self.inner(), + }) catch unreachable; + if (ret.memories) |memories| { + return @constCast(@ptrCast(memories[0..ret.num_memories])); + } + return &.{}; + } }; pub const DeviceDescription = opaque { @@ -843,7 +853,7 @@ pub const Buffer = opaque { return @ptrCast(ret.event); } - pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!?*Buffer { + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), .dst_memory = @ptrCast(@constCast(dst_memory)), @@ -925,8 +935,8 @@ pub const Memory = opaque { pub fn kind(self: *const Memory, api: *const Api) Kind { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable[0..ret.kind_size]; - return std.meta.stringToEnum(Kind, kind_) orelse unreachable; + const kind_ = ret.kind orelse unreachable; + return std.meta.stringToEnum(Kind, kind_[0..ret.kind_size]) orelse unreachable; } pub fn kindId(self: *const Memory, api: *const Api) u32 { diff --git a/zml/context.zig b/zml/context.zig index 03de62a3..fa898039 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -1,6 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); +const asynk = @import("async"); const c = @import("c"); const mlir = @import("mlir"); const runfiles = @import("runfiles"); @@ -8,6 +9,7 @@ const runtimes = @import("runtimes"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; +const bufferTypeFromDtype = @import("buffer.zig").bufferTypeFromDtype; const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); @@ -214,6 +216,7 @@ pub const Context = struct { } pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; + pub const HostCall = fn (?*anyopaque, []const HostBuffer) void; }; const CustomCall = struct { @@ -222,6 +225,7 @@ const CustomCall = struct { if (registry) |reg| { try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback); + try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCall", @tagName(platform.target), &hostBufferCall); } else { stdx.debug.panic("Registering custom calls failed", .{}); } @@ -251,6 +255,35 @@ const CustomCall = struct { callback(user_ctx, input_buffers, output_buffers); return null; } + + fn hostBufferCall(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; + std.debug.assert(callback_attr.dtype == .u64); + const callback: *const Context.HostCall = @ptrFromInt(callback_attr.get(usize)); + + const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; + std.debug.assert(user_ctx_ptr.dtype == .u64); + const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); + + const pjrt_api_ptr = call_frame.attrs.getByName(.scalar, "pjrt_api") orelse unreachable; + std.debug.assert(pjrt_api_ptr.dtype == .u64); + const pjrt_api: *const pjrt.Api = @ptrFromInt(pjrt_api_ptr.get(usize)); + + const pjrt_client_ptr = call_frame.attrs.getByName(.scalar, "pjrt_client") orelse unreachable; + std.debug.assert(pjrt_client_ptr.dtype == .u64); + const pjrt_client: *pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); + + const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); + for (input_buffers, 0..) |*b, i| { + b.* = copyBuffer(pjrt_api, pjrt_client, call_frame.args.get(i)); + } + + callback(user_ctx, input_buffers); + // asynk.callBlocking(callback, .{ user_ctx, input_buffers }); + return null; + } }; fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { @@ -278,3 +311,53 @@ fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { buffer_desc.data[0..buffer_shape.byteSize()], ); } + +fn copyBuffer(pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { + const buffer_shape = getShape(buffer_desc); + + const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { + var res: [Shape.MAX_RANK]i64 = undefined; + for (0..Shape.MAX_RANK) |i| { + res[i] = @intCast(Shape.MAX_RANK - i - 1); + } + break :blk res; + }; + + const device = pjrt_client.getAddressableDevices(pjrt_api)[0]; + const memories = pjrt_client.addressableMemories(pjrt_api); + + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(pjrt_api) == pjrt.Memory.Kind.pinned_host) { + selected_mem = mem; + } + } + + const pjrt_buffer = pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ + .data = buffer_desc.data, + .element_type = bufferTypeFromDtype(buffer_shape.dtype()), + .dims = buffer_shape.dims(), + // TODO: exposes sharding in the API. + .device = device, + .layout = .{ + .tiled = .{ + .minor_to_major = minor_to_major[Shape.MAX_RANK - buffer_shape.rank() ..], + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .stream = null, + }) catch @panic("failed to createViewOfDeviceBuffer"); + + const copied_buffer = pjrt_buffer.copyToMemory(pjrt_api, selected_mem) catch unreachable; + + const opaqueDataPointer = copied_buffer.getOpaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable; + const sizeInbytes = copied_buffer.getOnDeviceSizeInBytes(pjrt_api) catch unreachable; + const data = @as([*]const u8, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + const end: usize = @intCast(sizeInbytes); + + return HostBuffer.fromBytes( + buffer_shape, + data[0..end], + ); +} diff --git a/zml/ops.zig b/zml/ops.zig index 1765888a..c7fe2244 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -821,6 +821,58 @@ pub fn addHostCallback( return res; } +pub const HostCallOpt = struct { + has_side_effect: bool = true, +}; + +pub fn addHostCall( + callback: *const Context.HostCall, + blkctx: ?*anyopaque, + inputs: []const Tensor, + opts: HostCallOpt, +) []Tensor { + const ctx = CompilationContext.current(); + + const mlir_ctx = ctx.mlirCtx(); + const backend_config = mlir.Attribute.dict(mlir_ctx, &.{ + .{ "callback", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(callback))) }, + .{ "user_context", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(blkctx))) }, + .{ "pjrt_api", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_api))) }, + .{ "pjrt_client", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_client))) }, + }); + + const values = stdx.stackSlice(8, mlir.Value, inputs.len); + for (inputs, values) |i, *v| { + v.* = ctx.getValue(i); + } + const res_types = stdx.stackSlice(8, mlir.Type, inputs.len); + for (res_types, inputs) |*r, i| { + r.* = mlir.ext.RankedTensorType.fromShape(mlir_ctx, i.shape()).as(mlir.Type); + } + + const loc = ctx.mlirCtx().location(@src()); + const op = dialect.stablehlo.custom_call( + ctx.mlirCtx(), + values, + .{ + .call_target_name = "zmlHostBufferCall", + .api_version = .typed_ffi, + .backend_config = backend_config, + .has_side_effect = opts.has_side_effect, + .output_operand_aliases = &.{}, + }, + res_types, + loc, + ); + + const res = ctx.allocator().alloc(Tensor, inputs.len) catch @panic("OOM"); + for (res, inputs, 0..) |*r, input, i| { + r.* = Tensor._result(input.shape(), op.result(i)); + } + + return res; +} + pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index bd094ead..dbfa82a0 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -182,7 +182,7 @@ pub const Buffer = opaque { } pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(self.inner().copyToMemory(api, memory_)); + return @ptrCast(try self.inner().copyToMemory(api, memory_)); } pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { diff --git a/zml/tensor.zig b/zml/tensor.zig index b248ea20..040dc59b 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -3800,6 +3800,20 @@ pub const Tensor = struct { std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); std.debug.assert(host_buffer.data.ptr == outputs[0].data.ptr); } + + pub fn print2(input: Tensor) Tensor { + return ops.addHostCall( + &printCallback2, + null, + &.{input}, + .{ .has_side_effect = true }, + )[0]; + } + + fn printCallback2(_: ?*anyopaque, inputs: []const HostBuffer) void { + const host_buffer = inputs[0]; + std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); + } }; fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { From 5a80a6438cb6190f17689a852597c0cd7962a2a5 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 11 Apr 2025 13:38:48 +0000 Subject: [PATCH 2/5] working wip working version without unnecessary file remove so --- examples/mnist/mnist.zig | 57 +++--- pjrt/ffi.zig | 82 +++++---- pjrt/pjrt.zig | 89 ++++++++-- runtimes/cuda/BUILD.bazel | 6 + runtimes/cuda/libpjrt_cuda.BUILD.bazel | 2 +- stdx/meta.zig | 8 + zml/buffer.zig | 22 +++ zml/context.zig | 151 +--------------- zml/exe.zig | 32 ++++ zml/ffi.zig | 235 +++++++++++++++++++++++++ zml/module.zig | 12 +- zml/ops.zig | 104 ----------- zml/pjrtx.zig | 3 + zml/tensor.zig | 35 +--- zml/test_buffer.zig | 73 ++++++++ zml/zml.zig | 3 + 16 files changed, 548 insertions(+), 366 deletions(-) create mode 100644 zml/ffi.zig create mode 100644 zml/test_buffer.zig diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c41e221e..d76c2796 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -2,7 +2,7 @@ const std = @import("std"); const asynk = @import("async"); const zml = @import("zml"); - +const pjrt = zml.pjrt; const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ @@ -10,6 +10,29 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; +pub const PrintOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn beforeCustomCall(input: zml.Tensor, another_input: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + log.info("Compiling PrintOp for target: {s}", .{@tagName(input.getContext().target())}); + return .{ input, another_input.add(another_input) }; + } + + pub fn call(self: *Self, input: zml.Buffer, another_input: zml.Buffer) !struct { zml.Buffer, zml.Buffer } { + var output = input; + log.info("Input 0: {any}", .{output.getMemory().kind(self._platform.pjrt_api)}); + output = try output.copyToMemory(self.getPlatform(), .host_pinned); + log.info("After copy 0: {any}", .{output.getMemory().kind(self._platform.pjrt_api)}); + return .{ try output.copyToMemory(self.getPlatform(), .device), another_input }; + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + /// Model definition const Mnist = struct { fc1: Layer, @@ -20,19 +43,20 @@ const Mnist = struct { bias: zml.Tensor, pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.matmul(input.print2()).add(self.bias).relu(); + return self.weight.matmul(input).add(self.bias).relu(); } }; /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); + log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); var x = input.flattenAll().convert(.f32); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { x = zml.call(layer, .forward, .{x}); } - return x.argMax(0).indices.convert(.u8); + const r = x.argMax(0).indices.convert(.u8).print(PrintOp, x); + return r[0]; } }; @@ -49,7 +73,7 @@ pub fn asyncMain() !void { // log.info("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); - // // Auto-select platform + // Auto-select platform const platform = context.autoPlatform(.{}); context.printAvailablePlatforms(platform); @@ -85,7 +109,7 @@ pub fn asyncMain() !void { const compiled_mnist = try compilation.awaitt(); log.info("✅ Compiled model in {d}ms", .{start_time.read() / std.time.ns_per_ms}); - const mnist = compiled_mnist.prepare(model_weights); + const mnist = compiled_mnist.prepare(model_weights).withExecutionContext(); defer mnist.deinit(); log.info("✅ Weights transferred in {d}ms", .{start_time.read() / std.time.ns_per_ms}); @@ -98,25 +122,10 @@ pub fn asyncMain() !void { // inference - can be looped { - const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); - var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); - defer input.deinit(); + const print_op_ctx: PrintOp = .{ ._platform = &platform }; + try mnist.attach(&print_op_ctx); + std.debug.print("Attached print op to model: {}\n", .{print_op_ctx}); - printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); - defer result.deinit(); - - log.info( - \\✅ RECOGNIZED DIGIT: - \\ +-------------+ - \\{s} - \\ +-------------+ - \\ - , .{digits[try result.getValue(u8)]}); - } - { const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 39e7c248..484d230c 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -165,54 +165,74 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - // pub fn attach(self: *ExecutionContext, api: *const Api, value: anytype) ApiError!void { - // // register type id ==> typeid - // const typename_ = "zml." ++ @typeName(@TypeOf(value)); + pub fn Context(comptime T: type) type { + return struct { + pub fn get(self: *const ExecutionContext, api: *const Api) ApiError!*T { + const type_id = getTypeId(@typeName(T)); + var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ + .ctx = @constCast(self.inner()), + .type_id = @constCast(&type_id.toCStruct()), + }); + const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); - // var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Register_Args{ - // .ctx = self.inner(), - // .handler = @ptrCast(@alignCast(handler)), - // }); - // const result = api.inner().XLA_FFI_ExecutionContext_Register.?(&ret); + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); - // var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Register_Args{ - // .ctx = self.inner(), - // .handler = @ptrCast(@alignCast(handler)), - // }); - // const result = api.inner().XLA_FFI_ExecutionContext_Register.?(&ret); + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } - // if (result) |ffi_error| { - // const err = Error.fromInner(ffi_error); - // defer err.destroy(api); - // log.err("[ExecutionContext.register] {s}", .{err.getMessage(api)}); + if (ret.data == null) return error.NotFound; + return @ptrCast(@alignCast(ret.data.?)); + } + }; + } - // // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - // return error.Unknown; - // } - // } - - pub fn get(self: *ExecutionContext, api: *const Api, type_id: *TypeId) ApiError!*anyopaque { - var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = self.inner(), - .type_id = @ptrCast(@alignCast(type_id)), + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { + var ret = pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ + .ctx = @constCast(self.inner()), + .task = @ptrCast(@alignCast(task)), + .data = @ptrCast(@alignCast(data)), }); - const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + + const result = api.inner().XLA_FFI_ThreadPool_Schedule.?(&ret); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); defer err.destroy(api); + std.debug.print("error: {any} \n", .{err}); log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); // TODO(Corentin): Retrieve error code from Error when implemented in XLA. return error.Unknown; } + } - return ret.data.?; + fn getTypeId(type_name: []const u8) TypeId { + const id: i64 = @bitCast(std.hash.Fnv1a_64.hash(type_name)); + + return .{ + .type_id = id, + }; } // TODO getDeviceOrdinal() }; +const TypeId = extern struct { + type_id: i64, + + pub fn toCStruct(self: TypeId) c.XLA_FFI_TypeId { + return c.XLA_FFI_TypeId{ + .type_id = self.type_id, + }; + } +}; + +const Task = fn (*anyopaque) void; + const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -222,10 +242,6 @@ const ByteSpan = extern struct { } }; -pub const TypeId = extern struct { - type_id: i64, -}; - pub const DataType = enum(c.XLA_FFI_DataType) { invalid = c.XLA_FFI_DataType_INVALID, pred = c.XLA_FFI_DataType_PRED, @@ -438,7 +454,7 @@ pub const Error = opaque { pub const inner = TransmuteMixin(Error, c.XLA_FFI_Error).to; pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; - pub fn create(api: *const Api, error_code: ErrorCode, message: [:0]const u8) *Error { + pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret = pjrtStruct(c.XLA_FFI_Error_Create_Args{ .message = message.ptr, .errc = @intFromEnum(error_code), diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 53fe2b18..d15103af 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,7 +31,7 @@ fn pjrtStructSize(comptime T: type) usize { return @field(c, typedef_name ++ "_STRUCT_SIZE"); } -inline fn pjrtStruct(v: anytype) @TypeOf(v) { +pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { var ret = v; ret.struct_size = pjrtStructSize(@TypeOf(v)); return ret; @@ -160,9 +160,14 @@ pub const Api = struct { return state.str; } - pub fn customCallRegistry(api: *const Api) ?CustomCallRegistry { + pub fn createExecuteContext(api: *const Api) ApiError!*ExecuteContext { + const ret = try api.call(.PJRT_ExecuteContext_Create, .{}); + return @ptrCast(ret.context.?); + } + + pub fn ffi(api: *const Api) ?FFI { if (api.lookupExtension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { - return .{ .inner = ext.register_handler.? }; + return .{ .inner = ext }; } return null; } @@ -567,6 +572,14 @@ pub const SerializeResult = struct { } }; +pub const ExecuteContext = opaque { + pub fn deinit(self: *ExecuteContext, api: *const Api) void { + _ = api.call(.PJRT_ExecuteContext_Destroy, .{ + .context = @ptrCast(self), + }) catch {}; + } +}; + pub const Executable = opaque { const inner = InnerMixin(c.PJRT_Executable).inner; @@ -633,6 +646,7 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { var options = pjrtStruct(c.PJRT_ExecuteOptions{ @@ -643,6 +657,7 @@ pub const LoadedExecutable = opaque { .launch_id = 0, .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices.ptr), .num_non_donatable_input_indices = args.non_donatable_input_indices.len, + .context = @ptrCast(args.context), }); _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), @@ -1047,21 +1062,6 @@ pub const AsyncHostToDeviceTransferManager = opaque { } }; -pub const ExecutionContext = opaque { - const inner = InnerMixin(c.PJRT_ExecutionContext).inner; - - pub fn init(api: *const Api) ApiError!*ExecutionContext { - const ret = try api.call(.PJRT_ExecutionContext_Create, .{}); - return @ptrCast(ret.context.?); - } - - pub fn deinit(self: *ExecutionContext, api: *const Api) void { - _ = api.call(.PJRT_ExecutionContext_Destroy, .{ - .context = self.inner(), - }) catch unreachable; - } -}; - pub const NamedValue = extern struct { comptime { std.debug.assert(@sizeOf(NamedValue) == @sizeOf(c.PJRT_NamedValue)); @@ -1167,6 +1167,59 @@ pub const NamedValue = extern struct { } }; +pub const FFI = extern struct { + inner: *const c.PJRT_FFI, + + pub const UserData = extern struct { + type_id: i64, + user_data: *anyopaque, + + fn toCStruct(self: UserData) c.PJRT_FFI_UserData { + return .{ + .type_id = self.type_id, + .data = self.user_data, + }; + } + }; + + pub fn customCallRegistry(self: *const FFI) ?CustomCallRegistry { + return .{ .inner = self.inner.register_handler.? }; + } + + pub fn registerTypeId(self: *const FFI, api: *const Api, type_name: []const u8) ApiError!i64 { + var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ + .type_name = type_name.ptr, + .type_name_size = type_name.len, + .type_id = getTypeId(type_name), + }); + const result = self.inner.type_id_register.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + return pjrt_error.getCode(api).toApiError(); + } + + return ret.type_id; + } + + pub fn addUserData(self: *const FFI, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { + var ret = pjrtStruct(c.PJRT_FFI_UserData_Add_Args{ + .context = @ptrCast(context), + .user_data = user_data.toCStruct(), + }); + const result = self.inner.user_data_add.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + log.err("[addUserData] {s}", .{pjrt_error.getMessage(api)}); + return pjrt_error.getCode(api).toApiError(); + } + } + + pub fn getTypeId(type_name: []const u8) i64 { + // Use FNV-1a 64-bit hash algorithm + return @bitCast(std.hash.Fnv1a_64.hash(type_name)); + } +}; + // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub const CustomCallRegistry = extern struct { diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index a7f8bf94..ed50ebf5 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -1,5 +1,11 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") +exports_files( + ["libpjrt_cuda.so"], + visibility = ["//visibility:public"], +) + cc_library( name = "zmlxcuda_lib", srcs = ["zmlxcuda.c"], diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index 8881a016..fbf100a7 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -25,7 +25,7 @@ copy_to_directory( cc_import( name = "libpjrt_cuda", data = [":sandbox"], - shared_library = "libpjrt_cuda.so", + shared_library = "@zml//runtimes/cuda:libpjrt_cuda.so", add_needed = ["libzmlxcuda.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", diff --git a/stdx/meta.zig b/stdx/meta.zig index 7aa90531..40754158 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -164,6 +164,14 @@ pub fn FnResult(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } +pub fn FnReturnNoError(comptime func: anytype) type { + const ReturnT = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(ReturnT)) { + .error_union => |info| info.payload, + else => ReturnT, + }; +} + pub fn Head(Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { diff --git a/zml/buffer.zig b/zml/buffer.zig index e0829d39..d1d380f4 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -357,6 +357,28 @@ pub const Buffer = struct { return shard.memory(self._api); } + pub fn copyToMemory(self: Buffer, platform: Platform, kind: Memory) !Buffer { + const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == kind.toPjrtMemory()) { + selected_mem = mem; + } + } + + var shards: Shards = .{}; + for (self._shards.constSlice()) |shard| { + const pjrt_buffer = try shard.copyToMemory(self._api, selected_mem); + shards.appendAssumeCapacity(pjrt_buffer); + } + + return .{ + ._api = platform.pjrt_api, + ._shape = self._shape, + ._shards = shards, + }; + } + fn hasShardedAxis(self: Buffer) bool { if (self._shards.len == 1) return false; return @reduce(.Or, self._shape._sharding_info); diff --git a/zml/context.zig b/zml/context.zig index fa898039..a7af4d9d 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -9,11 +9,12 @@ const runtimes = @import("runtimes"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; -const bufferTypeFromDtype = @import("buffer.zig").bufferTypeFromDtype; +const Tensor = @import("tensor.zig").Tensor; const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const CompilationOptions = @import("platform.zig").CompilationOptions; const Shape = @import("shape.zig").Shape; const Target = @import("platform.zig").Target; const zml_platform = @import("platform.zig"); @@ -179,8 +180,6 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); return p; } @@ -214,150 +213,4 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; - pub const HostCall = fn (?*anyopaque, []const HostBuffer) void; }; - -const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { - const registry = platform.pjrt_api.customCallRegistry(); - - if (registry) |reg| { - try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback); - try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCall", @tagName(platform.target), &hostBufferCall); - } else { - stdx.debug.panic("Registering custom calls failed", .{}); - } - } - - fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; - std.debug.assert(callback_attr.dtype == .u64); - const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); - - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; - std.debug.assert(user_ctx_ptr.dtype == .u64); - const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.args.get(i)); - } - - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.results.get(i)); - } - - callback(user_ctx, input_buffers, output_buffers); - return null; - } - - fn hostBufferCall(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; - std.debug.assert(callback_attr.dtype == .u64); - const callback: *const Context.HostCall = @ptrFromInt(callback_attr.get(usize)); - - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; - std.debug.assert(user_ctx_ptr.dtype == .u64); - const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - - const pjrt_api_ptr = call_frame.attrs.getByName(.scalar, "pjrt_api") orelse unreachable; - std.debug.assert(pjrt_api_ptr.dtype == .u64); - const pjrt_api: *const pjrt.Api = @ptrFromInt(pjrt_api_ptr.get(usize)); - - const pjrt_client_ptr = call_frame.attrs.getByName(.scalar, "pjrt_client") orelse unreachable; - std.debug.assert(pjrt_client_ptr.dtype == .u64); - const pjrt_client: *pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); - - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = copyBuffer(pjrt_api, pjrt_client, call_frame.args.get(i)); - } - - callback(user_ctx, input_buffers); - // asynk.callBlocking(callback, .{ user_ctx, input_buffers }); - return null; - } -}; - -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { - // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { - .invalid => @panic("invalid ffi"), - .pred => .bool, - .s8 => .i8, - .s16 => .i16, - .s32 => .i32, - .s64 => .i64, - .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), - }; - return Shape.init(buffer_desc.dims(), dt); -} - -/// Create a HostBuffer from a ffi description of a buffer. -/// Normally the ffi describe device buffer but we assume they are located in pinned memory, -/// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { - const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( - buffer_shape, - buffer_desc.data[0..buffer_shape.byteSize()], - ); -} - -fn copyBuffer(pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { - const buffer_shape = getShape(buffer_desc); - - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var res: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - res[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk res; - }; - - const device = pjrt_client.getAddressableDevices(pjrt_api)[0]; - const memories = pjrt_client.addressableMemories(pjrt_api); - - var selected_mem: *const pjrt.Memory = undefined; - for (memories) |mem| { - if (mem.kind(pjrt_api) == pjrt.Memory.Kind.pinned_host) { - selected_mem = mem; - } - } - - const pjrt_buffer = pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ - .data = buffer_desc.data, - .element_type = bufferTypeFromDtype(buffer_shape.dtype()), - .dims = buffer_shape.dims(), - // TODO: exposes sharding in the API. - .device = device, - .layout = .{ - .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - buffer_shape.rank() ..], - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - .stream = null, - }) catch @panic("failed to createViewOfDeviceBuffer"); - - const copied_buffer = pjrt_buffer.copyToMemory(pjrt_api, selected_mem) catch unreachable; - - const opaqueDataPointer = copied_buffer.getOpaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable; - const sizeInbytes = copied_buffer.getOnDeviceSizeInBytes(pjrt_api) catch unreachable; - const data = @as([*]const u8, @ptrFromInt(@intFromPtr(opaqueDataPointer))); - const end: usize = @intCast(sizeInbytes); - - return HostBuffer.fromBytes( - buffer_shape, - data[0..end], - ); -} diff --git a/zml/exe.zig b/zml/exe.zig index 946410c6..44b35623 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -155,6 +155,8 @@ pub const BaseExe = struct { /// Allocator backing memory _arena: std.heap.ArenaAllocator, + _context: ?*pjrt.ExecuteContext = null, + pub fn init(parent_allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, args: struct { n_in: u32, result_shapes: []const Shape, n_devices: u8 }) !BaseExe { var arena = std.heap.ArenaAllocator.init(parent_allocator); errdefer arena.deinit(); @@ -188,6 +190,9 @@ pub const BaseExe = struct { } pub fn deinit(self: BaseExe) void { + if (self._context) |ctx| { + ctx.deinit(self.platform.pjrt_api); + } self._arena.deinit(); } @@ -209,6 +214,7 @@ pub const BaseExe = struct { // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, + .context = if (self._context) |ctx| ctx else null, }) catch unreachable; for (events[0..sharding.num_partitions]) |e| { @@ -271,6 +277,32 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + pub fn withExecutionContext(self: Self) Self { + stdx.debug.assert(self.inner._context == null, "Exe already has an execution context", .{}); + var new: Self = .{ .inner = self.inner }; + const pjrt_execute_context = self.inner.platform.pjrt_api.createExecuteContext() catch unreachable; + new.inner._context = pjrt_execute_context; + log.info("[FFI] Created {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); + return new; + } + + pub fn attach(self: Self, value: anytype) !void { + stdx.debug.assert(self.inner._context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = self.inner.platform.pjrt_api; + const ffi = pjrt_api.ffi().?; + const ValueT = @TypeOf(value); + const type_name = switch (@typeInfo(ValueT)) { + .pointer => @typeName(std.meta.Child(ValueT)), + else => @typeName(ValueT), // todo error + }; + + const type_id = try ffi.registerTypeId(pjrt_api, type_name); + + const user_data: *anyopaque = @ptrCast(@constCast(value)); + log.info("[FFI] Attach {any}@{x} of type {d} on {any}\n", .{ value, user_data, type_id, self.inner._context.? }); + try ffi.addUserData(pjrt_api, self.inner._context.?, .{ .type_id = type_id, .user_data = user_data }); + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } diff --git a/zml/ffi.zig b/zml/ffi.zig new file mode 100644 index 00000000..88746331 --- /dev/null +++ b/zml/ffi.zig @@ -0,0 +1,235 @@ +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; +const dtype = @import("dtype.zig"); +const pjrt = @import("pjrtx.zig"); +const buffer = @import("buffer.zig"); +const mlir = @import("mlir.zig"); +const module = @import("module.zig"); +const ops = @import("ops.zig"); +const tensor = @import("tensor.zig"); +const shape = @import("shape.zig"); +const Buffer = buffer.Buffer; +const CompilationContext = module.CompilationContext; +const DataType = dtype.DataType; +const Tensor = tensor.Tensor; +const Shape = shape.Shape; + +const log = std.log.scoped(.@"zml/custom_call"); + +fn nbInputTensors(comptime args: type) u32 { + if (@typeInfo(args) != .@"struct") { + @compileError("Expected struct type"); + } + return @typeInfo(args).@"struct".fields.len - 1; +} + +fn nbOutputTensors(comptime args: type) u32 { + if (@typeInfo(args) != .@"struct") { + @compileError("Expected struct type"); + } + return @typeInfo(args).@"struct".fields.len; +} + +fn isCustomCallContainer(custom_op: type) bool { + return @hasDecl(custom_op, "call"); +} + +fn hasBeforeCustomCallHook(custom_op: type) bool { + return @hasDecl(custom_op, "beforeCustomCall"); +} + +pub fn CustomCallInputType(custom_op: type) type { + return [nbInputTensors(stdx.meta.FnArgs(custom_op.call))]Tensor; +} + +pub fn CustomCallOutputType(custom_op: type) type { + return [nbOutputTensors(stdx.meta.FnReturnNoError(custom_op.call))]Tensor; +} + +pub fn custom_call( + comptime custom_op: type, + inputs: CustomCallInputType(custom_op), +) CustomCallOutputType(custom_op) { + stdx.debug.assert(isCustomCallContainer(custom_op), "custom_op must have a call method", .{}); + + const ctx = inputs[0].getContext(); + const mlir_ctx = ctx.mlirCtx(); + const platform = ctx._platform; + const pjrt_api = platform.pjrt_api; + const ffi = pjrt_api.ffi().?; + const registry = ffi.customCallRegistry().?; + const ffi_func = proxy(custom_op); + const target_name = "callback_" ++ @typeName(custom_op); + registry.registerFfi(pjrt_api, target_name, @tagName(ctx._platform.target), &ffi_func) catch unreachable; + log.info("Registered custom call {s} for {s} with proxy func {*}", .{ target_name, @typeName(custom_op), &ffi_func }); + + const custom_call_inputs = stdx.stackSlice(8, mlir.Value, inputs.len); + var res_shapes: []Shape = undefined; + var res_types: []mlir.Type = undefined; + + if (hasBeforeCustomCallHook(custom_op)) { + const before_custom_call = custom_op.beforeCustomCall; + if (@typeInfo(@TypeOf(before_custom_call)) != .@"fn") { + stdx.debug.panic("beforeCustomCall must be a function"); + } + + var args: std.meta.ArgsTuple(@TypeOf(before_custom_call)) = undefined; + inline for (0..args.len) |i| { + args[i] = inputs[i]; + } + + const before_custom_call_outputs = ctx.callFunc(@typeName(custom_op), before_custom_call, args) catch |err| { + std.debug.panic("Error in {any} beforeCustomCall func: {any}\n", .{ @typeName(custom_op), err }); + }; + std.debug.print("beforeCustomCall outputs: {any}\n", .{before_custom_call_outputs}); + res_shapes = stdx.stackSlice(8, Shape, before_custom_call_outputs.len); + res_types = stdx.stackSlice(8, mlir.Type, before_custom_call_outputs.len); + inline for (before_custom_call_outputs, 0..) |o, i| { + res_shapes[i] = o.shape(); + res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o.shape()).as(mlir.Type); + } + + inline for (before_custom_call_outputs, 0..) |output, i| { + custom_call_inputs[i] = output.value(); + } + } else { + log.warn("No beforeCustomCall function found for {s}, expecting return type to be args type", .{@typeName(custom_op)}); + for (custom_call_inputs, inputs) |*input, tensor_| { + input.* = tensor_.value(); + } + + res_shapes = stdx.stackSlice(8, Shape, inputs.len); + res_types = stdx.stackSlice(8, mlir.Type, inputs.len); + inline for (inputs, 0..) |input, i| { + res_shapes[i] = input.shape(); + res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, input.shape()).as(mlir.Type); + } + } + + const op = dialect.stablehlo.custom_call( + mlir_ctx, + custom_call_inputs, + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), + .has_side_effect = true, + .output_operand_aliases = &.{}, + }, + res_types, + mlir_ctx.location(@src()), + ); + + var custom_call_outputs: CustomCallOutputType(custom_op) = undefined; + for (&custom_call_outputs, res_shapes, 0..) |*r, shape_, i| { + r.* = Tensor._result(shape_, op.result(i)); + } + + return custom_call_outputs; +} + +fn view_buf(pjrt_api: *const pjrt.Api, pjrt_client: *const pjrt.Client, buffer_desc: *const pjrt.ffi.Buffer) Buffer { + const buffer_shape = getShape(buffer_desc); + const device = pjrt_client.getAddressableDevices(pjrt_api)[0]; + + const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { + var res: [Shape.MAX_RANK]i64 = undefined; + for (0..Shape.MAX_RANK) |i| { + res[i] = @intCast(Shape.MAX_RANK - i - 1); + } + break :blk res; + }; + + const pjrt_buffer = pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ + .data = buffer_desc.data, + .element_type = buffer.bufferTypeFromDtype(buffer_shape.dtype()), + .dims = buffer_shape.dims(), + .device = device, + .layout = .{ + .tiled = .{ + .minor_to_major = minor_to_major[Shape.MAX_RANK - buffer_shape.rank() ..], + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .stream = null, + }) catch @panic("failed to createViewOfDeviceBuffer"); + + var shards: Buffer.Shards = .{}; + shards.appendAssumeCapacity(pjrt_buffer); + + return .{ + ._api = pjrt_api, + ._shape = buffer_shape, + ._shards = shards, + }; +} + +fn proxy(comptime custom_op: type) pjrt.ffi.Handler { + return struct { + const Self = @This(); + + pub fn proxy(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + const execution_context = call_frame.ctx.?; + const user_ctx: *custom_op = pjrt.ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api.?) catch unreachable; + const platform = user_ctx._platform; + const pjrt_api = platform.pjrt_api; + const pjrt_client = platform.pjrt_client; + + const input_buffers = stdx.stackSlice(8, Buffer, call_frame.args.len); + for (input_buffers, 0..) |*b, i| { + b.* = view_buf(pjrt_api, pjrt_client, call_frame.args.get(i)); + } + + const output_buffers = stdx.stackSlice(8, Buffer, call_frame.results.len); + for (output_buffers, 0..) |*b, i| { + b.* = view_buf(pjrt_api, pjrt_client, call_frame.results.get(i)); + } + + var args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; + args[0] = user_ctx; + inline for (1..args.len) |i| { + args[i] = input_buffers[i - 1]; + } + + const outputs = @call(.auto, custom_op.call, args) catch |err| { + std.debug.print("Error on custom call {any}: {any}\n", .{ @typeName(custom_op), err }); + const ffi_error = pjrt.ffi.Error.create(call_frame.api.?, pjrt.ffi.ErrorCode.unknown, @errorName(err)); + return ffi_error; + }; + + if (@typeInfo(@TypeOf(outputs)) == .array) { + for (outputs, 0..) |output, i| { + std.debug.print("output {any}: {any}\n", .{ i, output }); + } + } else { + std.debug.print("output: {any}\n", .{outputs}); + } + + return null; + } + }.proxy; +} + +fn getShape(ffi_buffer: *const pjrt.ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .pred => .bool, + .s8 => .i8, + .s16 => .i16, + .s32 => .i32, + .s64 => .i64, + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} diff --git a/zml/module.zig b/zml/module.zig index 011e6fe6..e7112a36 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -220,8 +220,8 @@ pub const CompilationContext = struct { break :blk loaded_executable; }; - log.debug("******** ZML generated MLIR ********", .{}); - log.debug("{}", .{module.op().mlirFormatter(.{})}); + log.warn("******** ZML generated MLIR ********", .{}); + log.warn("{}", .{module.op().mlirFormatter(.{})}); if (timer) |*t| { const time_ms = @divFloor(t.lap(), std.time.ns_per_ms); @@ -1154,9 +1154,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1172,7 +1172,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/ops.zig b/zml/ops.zig index c7fe2244..d11885c1 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -769,110 +769,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - const ctx = CompilationContext.current(); - - const mlir_ctx = ctx.mlirCtx(); - const backend_config = mlir.Attribute.dict(mlir_ctx, &.{ - .{ "callback", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(callback))) }, - .{ "user_context", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(blkctx))) }, - }); - - const values = stdx.stackSlice(8, mlir.Value, inputs.len); - for (inputs, values) |i, *v| { - v.* = ctx.getValue(i.toMemory(.host_pinned)); - } - const res_types = stdx.stackSlice(8, mlir.Type, output_shapes.len); - for (res_types, output_shapes) |*r, o| { - r.* = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o).as(mlir.Type); - } - - const loc = ctx.mlirCtx().location(@src()); - const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), - values, - .{ - .call_target_name = "zmlHostBufferCallback", - .api_version = .typed_ffi, - .backend_config = backend_config, - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = opts.output_operand_aliases, - }, - res_types, - loc, - ); - - const res = ctx.allocator().alloc(Tensor, output_shapes.len) catch @panic("OOM"); - for (res, output_shapes, 0..) |*r, o, i| { - r.* = Tensor._result(o, op.result(i)).toMemory(.device); - } - - return res; -} - -pub const HostCallOpt = struct { - has_side_effect: bool = true, -}; - -pub fn addHostCall( - callback: *const Context.HostCall, - blkctx: ?*anyopaque, - inputs: []const Tensor, - opts: HostCallOpt, -) []Tensor { - const ctx = CompilationContext.current(); - - const mlir_ctx = ctx.mlirCtx(); - const backend_config = mlir.Attribute.dict(mlir_ctx, &.{ - .{ "callback", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(callback))) }, - .{ "user_context", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(blkctx))) }, - .{ "pjrt_api", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_api))) }, - .{ "pjrt_client", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_client))) }, - }); - - const values = stdx.stackSlice(8, mlir.Value, inputs.len); - for (inputs, values) |i, *v| { - v.* = ctx.getValue(i); - } - const res_types = stdx.stackSlice(8, mlir.Type, inputs.len); - for (res_types, inputs) |*r, i| { - r.* = mlir.ext.RankedTensorType.fromShape(mlir_ctx, i.shape()).as(mlir.Type); - } - - const loc = ctx.mlirCtx().location(@src()); - const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), - values, - .{ - .call_target_name = "zmlHostBufferCall", - .api_version = .typed_ffi, - .backend_config = backend_config, - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = &.{}, - }, - res_types, - loc, - ); - - const res = ctx.allocator().alloc(Tensor, inputs.len) catch @panic("OOM"); - for (res, inputs, 0..) |*r, input, i| { - r.* = Tensor._result(input.shape(), op.result(i)); - } - - return res; -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index dbfa82a0..c03606d5 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -8,6 +8,7 @@ pub const ffi = pjrt.ffi; pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; +pub const ExecuteContext = pjrt.ExecuteContext; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; @@ -262,6 +263,7 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { @@ -271,6 +273,7 @@ pub const LoadedExecutable = opaque { .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, + .context = args.context, } }); } diff --git a/zml/tensor.zig b/zml/tensor.zig index 040dc59b..dbf21afd 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -8,8 +8,10 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; +const custom_call = @import("ffi.zig").custom_call; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const Memory = @import("buffer.zig").Buffer.Memory; +const ffi = @import("ffi.zig"); const meta = @import("meta.zig"); const mlir = @import("mlir.zig"); const Location = mlir.Location; @@ -3782,37 +3784,8 @@ pub const Tensor = struct { }.binaryOpHelper; } - /// Insert code that will print the content of the given buffer at runtime. - /// Only for debug purpose, it inserts device to host synchronization - /// so it will slow down the program execution. - pub fn print(input: Tensor) Tensor { - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); - std.debug.assert(host_buffer.data.ptr == outputs[0].data.ptr); - } - - pub fn print2(input: Tensor) Tensor { - return ops.addHostCall( - &printCallback2, - null, - &.{input}, - .{ .has_side_effect = true }, - )[0]; - } - - fn printCallback2(_: ?*anyopaque, inputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); + pub fn print(self: Tensor, print_op: type, another_tensor: Tensor) ffi.CustomCallOutputType(print_op) { + return custom_call(print_op, .{ self, another_tensor }); } }; diff --git a/zml/test_buffer.zig b/zml/test_buffer.zig new file mode 100644 index 00000000..2028e3ff --- /dev/null +++ b/zml/test_buffer.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const testing = std.testing; +const zml = @import("zml.zig"); +const Buffer = zml.Buffer; +const Shape = zml.Shape; +const DataType = zml.DataType; + +test "Buffer.fromSlice basic usage" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + const data = [_]f32{ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 }; + var buf = try Buffer.fromSlice(platform, .{ 2, 3 }, &data); + defer buf.deinit(); + + try testing.expectEqual(Shape.init(.{ 2, 3 }, .f32), buf.shape()); + + const result = try buf.getValue([6]f32); + try testing.expectEqualSlices(f32, &data, &result); +} + +test "Buffer.constant creation" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + // Test scalar constant + { + var buf = try Buffer.constant(platform, Shape.init(.{}, .f32), 42.0); + defer buf.deinit(); + + const val = try buf.getValue(f32); + try testing.expectEqual(@as(f32, 42.0), val); + } + + // Test multi-dimensional constant + { + var buf = try Buffer.constant(platform, Shape.init(.{ 2, 3 }, .i32), 7); + defer buf.deinit(); + + const result = try buf.getValue([6]i32); + try testing.expectEqualSlices(i32, &[_]i32{7} ** 6, &result); + } +} + +test "Buffer.fromArray" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + const arr = [2][3]u8{ + [_]u8{ 1, 2, 3 }, + [_]u8{ 4, 5, 6 }, + }; + + var buf = try Buffer.fromArray(platform, arr); + defer buf.deinit(); + + try testing.expectEqual(Shape.init(.{ 2, 3 }, .u8), buf.shape()); + + const result = try buf.getValue([6]u8); + try testing.expectEqualSlices(u8, &[_]u8{ 1, 2, 3, 4, 5, 6 }, &result); +} + +test "Buffer dtype and shape operations" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + var buf = try Buffer.constant(platform, Shape.init(.{ 2, 3, 4 }, .f32), 1.0); + defer buf.deinit(); + + try testing.expectEqual(DataType.f32, buf.dtype()); + try testing.expectEqual(@as(u4, 3), buf.rank()); + try testing.expectEqualSlices(i64, &[_]i64{ 2, 3, 4 }, buf.dims()); +} diff --git a/zml/zml.zig b/zml/zml.zig index 18282b93..5b6353f9 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -5,6 +5,7 @@ pub const Buffer = @import("buffer.zig").Buffer; pub const Bufferized = @import("tensor.zig").Bufferized; +pub const CompilationContext = @import("module.zig").CompilationContext; pub const CompilationOptions = @import("platform.zig").CompilationOptions; pub const Context = @import("context.zig").Context; pub const Data = @import("dtype.zig").Data; @@ -18,6 +19,7 @@ pub const Tensor = @import("tensor.zig").Tensor; // Namespaces pub const context = @import("context.zig"); +pub const ffi = @import("ffi.zig"); pub const exe = @import("exe.zig"); pub const floats = @import("floats.zig"); pub const helpers = @import("helpers.zig"); @@ -34,6 +36,7 @@ pub const torch = @import("torch.zig"); pub const tokenizer = @import("zml/tokenizer"); pub const call = ops.call; +pub const custom_call = ffi.custom_call; pub const compile = exe.compile; pub const compileFn = exe.compileFn; pub const compileModel = exe.compileModel; From 28ea603c00593e274af6bcf54c7d195735e44788 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 16 Apr 2025 10:57:32 +0000 Subject: [PATCH 3/5] non working on gpu --- examples/custom_call/BUILD | 10 +++ examples/custom_call/main.zig | 158 ++++++++++++++++++++++++++++++++++ examples/mnist/mnist.zig | 3 + zml/context.zig | 3 - zml/ffi.zig | 138 ++++++++++++++++++----------- zml/zml.zig | 1 - 6 files changed, 261 insertions(+), 52 deletions(-) create mode 100644 examples/custom_call/BUILD create mode 100644 examples/custom_call/main.zig diff --git a/examples/custom_call/BUILD b/examples/custom_call/BUILD new file mode 100644 index 00000000..bc605442 --- /dev/null +++ b/examples/custom_call/BUILD @@ -0,0 +1,10 @@ +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "custom_call", + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig new file mode 100644 index 00000000..fbdc0836 --- /dev/null +++ b/examples/custom_call/main.zig @@ -0,0 +1,158 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +pub const AddOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn beforeCustomCall(a: zml.Tensor, b: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + return .{ a, b }; + } + + pub fn call(self: *Self, a: zml.HostBuffer, _: zml.HostBuffer) !zml.HostBuffer { + _ = self; // autofix + log.info("mem a: {any}", .{a.items(f32)}); + return a; + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + +pub const LogResultOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn beforeCustomCall(result: zml.Tensor) zml.Tensor { + return result; + } + + pub fn call(self: *Self, result: zml.HostBuffer) !zml.HostBuffer { + _ = self; // autofix + log.info("LogResultOp mem r: {any}", .{result.items(f32)[0]}); + return result; + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + +pub const LogValuesVoidOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn call(self: *Self, result: zml.HostBuffer, a: zml.HostBuffer, b: zml.HostBuffer) !void { + _ = self; // autofix + log.info("LogValuesVoidOp mem result: {any}", .{result.items(f32)}); + log.info("LogValuesVoidOp mem a: {any}", .{a.items(f32)}); + log.info("LogValuesVoidOp mem b: {any}", .{b.items(f32)}); + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + +/// Model definition +const Layer = struct { + pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + // const result = zml.custom_call(AddOp, .{ a.toMemory(.host_pinned), b.toMemory(.host_pinned) }); + const result = a.add(b); + // zml.custom_call(LogValuesVoidOp, .{ result, a, b }); + return zml.custom_call(LogResultOp, .{result}); + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Arena allocator for BufferStore etc. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const shape = zml.Shape.init(.{1}, .f32); + + // We manually produce a BufferStore. You would not normally do that. + // A BufferStore is usually created by loading model data from a file. + const buffers: zml.aio.BufferStore.Buffers = .{}; + + // the actual BufferStore + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + // A clone of our model, consisting of shapes. We only need shapes for compiling. + // We use the BufferStore to infer the shapes. + const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + + // Start compiling. This uses the inferred shapes from the BufferStore. + // The shape of the input tensor, we have to pass in manually. + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + + // Produce a bufferized weights struct from the fake BufferStore. + // This is like the inferred shapes, but with actual values. + // We will need to send those to the computation device later. + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); // for good practice + + // Wait for compilation to finish + const compiled = try compilation.awaitt(); + + // pass the model weights to the compiled module to create an executable module + var executable = compiled.prepare(model_weights).withExecutionContext(); + defer executable.deinit(); + + const add_op_ctx: AddOp = .{ ._platform = &platform }; + try executable.attach(&add_op_ctx); + const log_result_op_ctx: LogResultOp = .{ ._platform = &platform }; + try executable.attach(&log_result_op_ctx); + // const log_values_op_ctx: LogValuesVoidOp = .{ ._platform = &platform }; + // try executable.attach(&log_values_op_ctx); + + // prepare input buffers + var input_a = [1]f32{1.0}; + var input_buffer_a = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_a)); + defer input_buffer_a.deinit(); + var input_b = [1]f32{1.0}; + var input_buffer_b = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_b)); + defer input_buffer_b.deinit(); + + // call our executable module + var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + defer result.deinit(); + + // fetch the result to CPU memory + const cpu_result = try result.toHostAlloc(arena); + std.debug.print( + "\nThe result of {d} + {d} = {d}\n", + .{ &input_a, &input_b, cpu_result.items(f32) }, + ); +} diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index d76c2796..f5b99b17 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -7,6 +7,9 @@ const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ .log_level = .info, + .log_scope_levels = &[_]std.log.ScopeLevel{ + .{ .scope = .@"zml/module", .level = .debug }, + }, .logFn = asynk.logFn(std.log.defaultLog), }; diff --git a/zml/context.zig b/zml/context.zig index a7af4d9d..4cb9a7ee 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -1,7 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); -const asynk = @import("async"); const c = @import("c"); const mlir = @import("mlir"); const runfiles = @import("runfiles"); @@ -9,12 +8,10 @@ const runtimes = @import("runtimes"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; -const Tensor = @import("tensor.zig").Tensor; const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; -const CompilationOptions = @import("platform.zig").CompilationOptions; const Shape = @import("shape.zig").Shape; const Target = @import("platform.zig").Target; const zml_platform = @import("platform.zig"); diff --git a/zml/ffi.zig b/zml/ffi.zig index 88746331..70373fd1 100644 --- a/zml/ffi.zig +++ b/zml/ffi.zig @@ -3,6 +3,7 @@ const assert = std.debug.assert; const testing = std.testing; const builtin = @import("builtin"); +const asynk = @import("asynk"); const stdx = @import("stdx"); const dialect = struct { @@ -11,12 +12,14 @@ const dialect = struct { const dtype = @import("dtype.zig"); const pjrt = @import("pjrtx.zig"); const buffer = @import("buffer.zig"); +const hostbuffer = @import("hostbuffer.zig"); const mlir = @import("mlir.zig"); const module = @import("module.zig"); const ops = @import("ops.zig"); const tensor = @import("tensor.zig"); const shape = @import("shape.zig"); const Buffer = buffer.Buffer; +const HostBuffer = hostbuffer.HostBuffer; const CompilationContext = module.CompilationContext; const DataType = dtype.DataType; const Tensor = tensor.Tensor; @@ -24,41 +27,40 @@ const Shape = shape.Shape; const log = std.log.scoped(.@"zml/custom_call"); -fn nbInputTensors(comptime args: type) u32 { - if (@typeInfo(args) != .@"struct") { - @compileError("Expected struct type"); - } - return @typeInfo(args).@"struct".fields.len - 1; -} +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; -fn nbOutputTensors(comptime args: type) u32 { - if (@typeInfo(args) != .@"struct") { - @compileError("Expected struct type"); - } - return @typeInfo(args).@"struct".fields.len; -} - -fn isCustomCallContainer(custom_op: type) bool { - return @hasDecl(custom_op, "call"); -} - -fn hasBeforeCustomCallHook(custom_op: type) bool { - return @hasDecl(custom_op, "beforeCustomCall"); -} +const CallbackT = HostBuffer; pub fn CustomCallInputType(custom_op: type) type { - return [nbInputTensors(stdx.meta.FnArgs(custom_op.call))]Tensor; + const ArgsT = stdx.meta.FnArgs(custom_op.call); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + return [@typeInfo(ArgsT).@"struct".fields.len - 1]Tensor; } pub fn CustomCallOutputType(custom_op: type) type { - return [nbOutputTensors(stdx.meta.FnReturnNoError(custom_op.call))]Tensor; + const ReturnT = stdx.meta.FnReturnNoError(custom_op.call); + + if (@typeInfo(ReturnT) != .@"struct") { + @compileError("Expected struct type"); + } + + if (ReturnT == CallbackT) { + return Tensor; + } + + return [@typeInfo(ReturnT).@"struct".fields.len]Tensor; } pub fn custom_call( comptime custom_op: type, inputs: CustomCallInputType(custom_op), ) CustomCallOutputType(custom_op) { - stdx.debug.assert(isCustomCallContainer(custom_op), "custom_op must have a call method", .{}); + stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); const ctx = inputs[0].getContext(); const mlir_ctx = ctx.mlirCtx(); @@ -75,7 +77,7 @@ pub fn custom_call( var res_shapes: []Shape = undefined; var res_types: []mlir.Type = undefined; - if (hasBeforeCustomCallHook(custom_op)) { + if (@hasDecl(custom_op, "beforeCustomCall")) { const before_custom_call = custom_op.beforeCustomCall; if (@typeInfo(@TypeOf(before_custom_call)) != .@"fn") { stdx.debug.panic("beforeCustomCall must be a function"); @@ -89,15 +91,27 @@ pub fn custom_call( const before_custom_call_outputs = ctx.callFunc(@typeName(custom_op), before_custom_call, args) catch |err| { std.debug.panic("Error in {any} beforeCustomCall func: {any}\n", .{ @typeName(custom_op), err }); }; - std.debug.print("beforeCustomCall outputs: {any}\n", .{before_custom_call_outputs}); - res_shapes = stdx.stackSlice(8, Shape, before_custom_call_outputs.len); - res_types = stdx.stackSlice(8, mlir.Type, before_custom_call_outputs.len); - inline for (before_custom_call_outputs, 0..) |o, i| { + var before_custom_call_outputs_array: []Tensor = undefined; + + if (@TypeOf(before_custom_call_outputs) == Tensor) { + before_custom_call_outputs_array = stdx.stackSlice(8, Tensor, 1); + before_custom_call_outputs_array[0] = before_custom_call_outputs; + } else { + before_custom_call_outputs_array = stdx.stackSlice(8, Tensor, @typeInfo(@TypeOf(before_custom_call_outputs)).@"struct".fields.len); + inline for (@typeInfo(@TypeOf(before_custom_call_outputs)).@"struct".fields, 0..) |field, i| { + before_custom_call_outputs_array[i] = @field(before_custom_call_outputs, field.name); + } + } + + res_shapes = stdx.stackSlice(8, Shape, before_custom_call_outputs_array.len); + res_types = stdx.stackSlice(8, mlir.Type, before_custom_call_outputs_array.len); + + for (before_custom_call_outputs_array, 0..) |o, i| { res_shapes[i] = o.shape(); res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o.shape()).as(mlir.Type); } - inline for (before_custom_call_outputs, 0..) |output, i| { + for (before_custom_call_outputs_array, 0..) |output, i| { custom_call_inputs[i] = output.value(); } } else { @@ -128,15 +142,24 @@ pub fn custom_call( mlir_ctx.location(@src()), ); + // if (CustomCallOutputType(custom_op) == void) { + // return; + // } + + if (CustomCallOutputType(custom_op) == Tensor) { + return Tensor._result(res_shapes[0], op.result(0)); + } + var custom_call_outputs: CustomCallOutputType(custom_op) = undefined; + for (&custom_call_outputs, res_shapes, 0..) |*r, shape_, i| { - r.* = Tensor._result(shape_, op.result(i)); + r.* = Tensor._result(shape_, op.result(i)).toMemory(.device); } return custom_call_outputs; } -fn view_buf(pjrt_api: *const pjrt.Api, pjrt_client: *const pjrt.Client, buffer_desc: *const pjrt.ffi.Buffer) Buffer { +fn view_buf(pjrt_api: *const pjrt.Api, pjrt_client: *const pjrt.Client, buffer_desc: *const pjrt.ffi.Buffer) CallbackT { const buffer_shape = getShape(buffer_desc); const device = pjrt_client.getAddressableDevices(pjrt_api)[0]; @@ -183,22 +206,26 @@ fn proxy(comptime custom_op: type) pjrt.ffi.Handler { const user_ctx: *custom_op = pjrt.ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api.?) catch unreachable; const platform = user_ctx._platform; const pjrt_api = platform.pjrt_api; + _ = pjrt_api; // autofix const pjrt_client = platform.pjrt_client; - - const input_buffers = stdx.stackSlice(8, Buffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = view_buf(pjrt_api, pjrt_client, call_frame.args.get(i)); - } - - const output_buffers = stdx.stackSlice(8, Buffer, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - b.* = view_buf(pjrt_api, pjrt_client, call_frame.results.get(i)); - } + _ = pjrt_client; // autofix var args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; args[0] = user_ctx; inline for (1..args.len) |i| { - args[i] = input_buffers[i - 1]; + const buffer_desc = call_frame.args.get(i - 1); + const buffer_shape = getShape(buffer_desc); + const b = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); + + const data_ptr: usize = @intFromPtr(b.data.ptr); + const page_off = data_ptr % std.heap.page_size_min; + const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); + std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE | std.posix.PROT.READ) catch |e| { + log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); + }; + + args[i] = b; + std.debug.print("input buffer {any}: {any}\n", .{ i, b.data }); } const outputs = @call(.auto, custom_op.call, args) catch |err| { @@ -207,12 +234,27 @@ fn proxy(comptime custom_op: type) pjrt.ffi.Handler { return ffi_error; }; - if (@typeInfo(@TypeOf(outputs)) == .array) { - for (outputs, 0..) |output, i| { - std.debug.print("output {any}: {any}\n", .{ i, output }); - } - } else { - std.debug.print("output: {any}\n", .{outputs}); + std.debug.print("output: {any}\n", .{outputs}); + + // const output_buffers = stdx.stackSlice(8, CallbackT, call_frame.results.len); + // for (output_buffers, 0..) |*b, i| { + // b.* = view_buf2(pjrt_api, pjrt_client, call_frame.results.get(i)); + // } + + const output_buffers = stdx.stackSlice(8, CallbackT, call_frame.results.len); + for (output_buffers, 0..) |*b, i| { + const buffer_desc = call_frame.results.get(i); + const buffer_shape = getShape(buffer_desc); + b.* = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); + + const data_ptr: usize = @intFromPtr(b.data.ptr); + const page_off = data_ptr % std.heap.page_size_min; + const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); + std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE | std.posix.PROT.READ) catch |e| { + log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); + }; + + std.debug.print("output buffer {any}: {any}\n", .{ i, b.data }); } return null; diff --git a/zml/zml.zig b/zml/zml.zig index 5b6353f9..38e600a9 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -5,7 +5,6 @@ pub const Buffer = @import("buffer.zig").Buffer; pub const Bufferized = @import("tensor.zig").Bufferized; -pub const CompilationContext = @import("module.zig").CompilationContext; pub const CompilationOptions = @import("platform.zig").CompilationOptions; pub const Context = @import("context.zig").Context; pub const Data = @import("dtype.zig").Data; From 932a7cdb25d0a59caf17859c2f8fc1857bcff088 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 17 Apr 2025 08:54:47 +0000 Subject: [PATCH 4/5] wip --- MODULE.bazel | 2 +- MODULE.bazel.lock | 168 ++++++++++++++++- examples/MODULE.bazel | 2 +- examples/MODULE.bazel.lock | 170 +++++++++++++++++- examples/custom_call/main.zig | 29 ++- pjrt/ffi.zig | 20 ++- pjrt/pjrt.zig | 8 +- runtimes/cpu/cpu.bzl | 12 +- runtimes/cuda/cuda.bzl | 4 +- runtimes/cuda/libpjrt_cuda.BUILD.bazel | 2 +- runtimes/rocm/rocm.bzl | 4 +- .../xla/20250409.0-7167d71/MODULE.bazel | 37 ++++ .../20250409.0-7167d71/overlay/MODULE.bazel | 1 + .../xla/20250409.0-7167d71/overlay/tsl.bzl | 19 ++ .../20250409.0-7167d71/overlay/workspace.bzl | 60 +++++++ .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 +++++ ...ler-registration-API-to-the-FFI-PjRt.patch | 132 ++++++++++++++ .../xla/20250409.0-7167d71/source.json | 15 ++ third_party/modules/xla/metadata.json | 3 +- zml/buffer.zig | 36 ++-- zml/exe.zig | 2 +- zml/ffi.zig | 123 +++++++++---- zml/module.zig | 6 +- zml/tensor.zig | 6 +- 24 files changed, 796 insertions(+), 106 deletions(-) create mode 100644 third_party/modules/xla/20250409.0-7167d71/MODULE.bazel create mode 120000 third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel create mode 100644 third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl create mode 100644 third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl create mode 100644 third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch create mode 100644 third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch create mode 100644 third_party/modules/xla/20250409.0-7167d71/source.json diff --git a/MODULE.bazel b/MODULE.bazel index 1bcf2717..0311cca8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,7 +24,7 @@ bazel_dep(name = "rules_zig", version = "20250314.0-b9739c6") bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a") bazel_dep(name = "toolchains_protoc", version = "0.3.7") bazel_dep(name = "with_cfg.bzl", version = "0.9.1") -bazel_dep(name = "xla", version = "20250317.1-71c67e2") +bazel_dep(name = "xla", version = "20250409.0-7167d71") bazel_dep(name = "zig-protobuf", version = "20250318.0-930153e") bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index f6792256..22e6de13 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -83,7 +83,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.17.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.20.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", @@ -150,7 +150,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.53.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", @@ -269,8 +269,8 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/MODULE.bazel": "39ee3d15edbf09f536a52c42b4297fe1f08a239e28964555f578f301159033f3", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/source.json": "57af375068c7292d88062b2a3c2cae546f6e262f653f03388b5d73207f7482e3", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/MODULE.bazel": "d9049cabd0becdf8d26377927dda874e2febc803401269814776f232fde91b7c", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/source.json": "7870121042c518ca6109fa16fb070b4dd6762f42aefdd9ddcc259194ca6627cd", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -350,8 +350,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.3.0/source.json": "c55ed591aa5009401ddf80ded9762ac32c358d2517ee7820be981e2de9756cf3", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.53.0/MODULE.bazel": "795c21aaad2574f8cb8163a03808ef4a4a33c0b0a81c0749ca489563561b83c1", "https://bcr.bazel.build/modules/rules_uv/0.53.0/source.json": "bd665e54519407989760844261e02b184ea4f141e65e399f75cd6a3ebf02b649", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", @@ -380,7 +380,7 @@ "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", - "usagesDigest": "wD+r6kULzwkrPbJnhFpRl2hIsJZIwSSaeEUj3z6v0i4=", + "usagesDigest": "ai1Q9Iw9PMxCbfOYikwNfCbFYAEzeR5Pirndq3g0LFo=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2029,6 +2029,160 @@ ] ] } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", + "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ] + ] + } } } } diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index 2fcb0573..6a60ea53 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -4,7 +4,7 @@ bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "rules_zig", version = "20250314.0-b9739c6") bazel_dep(name = "platforms", version = "0.0.11") bazel_dep(name = "zml", version = "0.1.0") -bazel_dep(name = "aspect_bazel_lib", version = "2.11.0") +bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") bazel_dep(name = "rules_oci", version = "2.0.0") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 4c09f13c..060ab953 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -15,7 +15,6 @@ "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", @@ -84,7 +83,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.17.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.20.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", @@ -152,7 +151,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.53.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", @@ -187,7 +186,6 @@ "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", "https://bcr.bazel.build/modules/apple_support/1.17.1/source.json": "6b2b8c74d14e8d485528a938e44bdb72a5ba17632b9e14ef6e68a5ee96c8347f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "ae12288421b12faeb09bc4cfb3c05945ce66df5c55fca8a2f2f66ae9f21c7acb", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.11.0/MODULE.bazel": "cb1ba9f9999ed0bc08600c221f532c1ddd8d217686b32ba7d45b0713b5131452", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", @@ -273,8 +271,8 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/MODULE.bazel": "39ee3d15edbf09f536a52c42b4297fe1f08a239e28964555f578f301159033f3", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/source.json": "57af375068c7292d88062b2a3c2cae546f6e262f653f03388b5d73207f7482e3", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/MODULE.bazel": "d9049cabd0becdf8d26377927dda874e2febc803401269814776f232fde91b7c", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/source.json": "7870121042c518ca6109fa16fb070b4dd6762f42aefdd9ddcc259194ca6627cd", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -356,8 +354,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.3.0/source.json": "c55ed591aa5009401ddf80ded9762ac32c358d2517ee7820be981e2de9756cf3", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.53.0/MODULE.bazel": "795c21aaad2574f8cb8163a03808ef4a4a33c0b0a81c0749ca489563561b83c1", "https://bcr.bazel.build/modules/rules_uv/0.53.0/source.json": "bd665e54519407989760844261e02b184ea4f141e65e399f75cd6a3ebf02b649", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", @@ -387,7 +385,7 @@ "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", - "usagesDigest": "wD+r6kULzwkrPbJnhFpRl2hIsJZIwSSaeEUj3z6v0i4=", + "usagesDigest": "ai1Q9Iw9PMxCbfOYikwNfCbFYAEzeR5Pirndq3g0LFo=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2508,6 +2506,160 @@ ] ] } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", + "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ] + ] + } } } } diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig index fbdc0836..cc6d2216 100644 --- a/examples/custom_call/main.zig +++ b/examples/custom_call/main.zig @@ -18,7 +18,7 @@ pub const AddOp = struct { return .{ a, b }; } - pub fn call(self: *Self, a: zml.HostBuffer, _: zml.HostBuffer) !zml.HostBuffer { + pub fn call(self: *Self, a: zml.Buffer, _: zml.Buffer) !zml.Buffer { _ = self; // autofix log.info("mem a: {any}", .{a.items(f32)}); return a; @@ -34,13 +34,12 @@ pub const LogResultOp = struct { _platform: *const zml.Platform, - pub fn beforeCustomCall(result: zml.Tensor) zml.Tensor { - return result; - } + // pub fn beforeCustomCall(result: zml.Tensor) zml.Tensor { + // return result; + // } - pub fn call(self: *Self, result: zml.HostBuffer) !zml.HostBuffer { - _ = self; // autofix - log.info("LogResultOp mem r: {any}", .{result.items(f32)[0]}); + pub fn call(self: *Self, result: zml.Buffer) !zml.Buffer { + log.info("LogResultOp result: {*}", .{try result._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._platform.pjrt_api)}); return result; } @@ -70,9 +69,19 @@ pub const LogValuesVoidOp = struct { const Layer = struct { pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { // const result = zml.custom_call(AddOp, .{ a.toMemory(.host_pinned), b.toMemory(.host_pinned) }); - const result = a.add(b); + // const a_ = a.toMemory(.host_pinned); + // const b_ = b.toMemory(.host_pinned); + const a_ = a.toMemory(.host_pinned); + const b_ = b.toMemory(.host_pinned); + const c = a_.clamp(b_, a_); + const d = c.toMemory(.device); + const on_device = a.add(d).toMemory(.host_pinned); + // var result = zml.custom_call(LogResultOp, .{on_device}); + // const result = on_device.toMemory(.device).add(b); // zml.custom_call(LogValuesVoidOp, .{ result, a, b }); - return zml.custom_call(LogResultOp, .{result}); + // var result = a.add(b).toMemory(.host_pinned); + var result = zml.custom_call(LogResultOp, .{on_device}); + return result.add(zml.Tensor.scalar(5, .f32)); } }; @@ -147,6 +156,8 @@ pub fn asyncMain() !void { // call our executable module var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + const result_mem = result.getMemory(); + std.debug.print("<<< Result memory: {any}\n", .{result_mem.kind(platform.pjrt_api)}); defer result.deinit(); // fetch the result to CPU memory diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 484d230c..a9c39cef 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -190,6 +190,24 @@ pub const ExecutionContext = opaque { }; } + pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { + var ret = pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ + .ctx = @constCast(self.inner()), + }); + const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.getDeviceOrdinal] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + return ret.device_ordinal; + } + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { var ret = pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ .ctx = @constCast(self.inner()), @@ -217,8 +235,6 @@ pub const ExecutionContext = opaque { .type_id = id, }; } - - // TODO getDeviceOrdinal() }; const TypeId = extern struct { diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index d15103af..d2cb698d 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -414,7 +414,7 @@ pub const Client = opaque { dims: []const i64, element_type: BufferType, layout: MemoryLayout, - device: *const Device, + memory: *const Memory, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.C) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.C) void {} }.call, @@ -431,7 +431,7 @@ pub const Client = opaque { .num_dims = args.dims.len, .element_type = @intFromEnum(args.element_type), .layout = @ptrCast(@constCast(&layout)), - .device = @ptrCast(@constCast(args.device)), + .memory = @ptrCast(@constCast(args.memory)), .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, .stream = if (args.stream) |stream| stream else 0, @@ -852,10 +852,10 @@ pub const Buffer = opaque { } pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { - const ret = try api.call(.PJRT_Buffer_OpaqueDeviceMemoryDataPointer, .{ + const ret = try api.call(.PJRT_Buffer_UnsafePointer, .{ .buffer = self.inner(), }); - return ret.device_memory_ptr.?; + return @ptrFromInt(ret.buffer_pointer); } pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64) ApiError!?*Event { diff --git a/runtimes/cpu/cpu.bzl b/runtimes/cpu/cpu.bzl index d3613802..9e0b0374 100644 --- a/runtimes/cpu/cpu.bzl +++ b/runtimes/cpu/cpu.bzl @@ -25,22 +25,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "1cda1325095c12bd0019838d28ee92d811ac478d22ed3c08020d5a0cd2d9f34a", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "a799573bc238bfbfa18c415002280902e4547f7e12fb09a9a7e73e8b35b58b9b", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "35af82d9e5c70d16ac15f4c18024a2dd5ed2faefc89940eafe3d5350d2cbd9e7", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "31f6fcc1a5341ea7cd6eaf5c441189e5f6399a37dfafb4a45699edacf44c742c", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "da4deaf850d715997614768b2fc0283595ee8181133ab3243d65635e3439de69", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "91a18c0fb2c82f28953b5134833415638422925099ee6498987d583f178588b5", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index 82d1e448..2c116ef0 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -202,8 +202,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "64029cd3d68118b166198e3246877ed706ed35eb732b1770a9bf530b5b0a8ab4", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "9e2d1741de8531274c4c2ffe77f3a82003591827294e2a2773cc09f33cc2137e", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index fbf100a7..8881a016 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -25,7 +25,7 @@ copy_to_directory( cc_import( name = "libpjrt_cuda", data = [":sandbox"], - shared_library = "@zml//runtimes/cuda:libpjrt_cuda.so", + shared_library = "libpjrt_cuda.so", add_needed = ["libzmlxcuda.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index c190a9ee..9d41c106 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -127,8 +127,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "13a2ced965a44a0e8ad0d752c8ac5aa99107b17b41bd850967d40b82e102ec50", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "4bdb7690f6e7ac0faa1c29299663269781c2b5d33fbe28c1ff4bc33ce9ec01ff", ) return mctx.extension_metadata( diff --git a/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel b/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel new file mode 100644 index 00000000..50cafb9e --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel @@ -0,0 +1,37 @@ +module( + name = "xla", + version = "20250409.0-7167d71", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.11") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.20.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl", "python_version_repo") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "llvm-raw", + "stablehlo", +) + +llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") +llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel b/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel new file mode 120000 index 00000000..3ef3b743 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel @@ -0,0 +1 @@ +/mnt/hugo/zml/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl new file mode 100644 index 00000000..192e06f8 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl @@ -0,0 +1,60 @@ +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//third_party/triton:workspace.bzl", triton = "repo") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + pybind11_bazel() + triton() + llvm("llvm-raw") + stablehlo() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "//third_party/grpc:generate_cc_env_fix.patch", + "//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", + "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..f367b12b --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,132 @@ +From 490985facbe2fe6462b2006a99514b1c7e8a0775 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Wed, 5 Feb 2025 19:25:03 +0100 +Subject: [PATCH 1/7] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 ++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 +++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 33 ++++++++++++++++++++++++++- + 3 files changed, 53 insertions(+), 1 deletion(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index 8e8a4baa53..551c35b461 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,8 +69,13 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index d9f6d163d8..3dbd4df089 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -69,12 +69,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + size_t struct_size; + PJRT_Extension_Type type; + PJRT_Extension_Base* next; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 8898088a23..746861f697 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,16 +13,21 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" + #include "absl/strings/string_view.h" ++#include "absl/strings/str_format.h" ++#include "xla/ffi/api/c_api.h" ++#include "xla/ffi/api/ffi.h" + #include "xla/ffi/execution_context.h" + #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -68,6 +73,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, +@@ -75,6 +105,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + /*next=*/next, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.39.5 (Apple Git-154) diff --git a/third_party/modules/xla/20250409.0-7167d71/source.json b/third_party/modules/xla/20250409.0-7167d71/source.json new file mode 100644 index 00000000..36797b83 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "xla-7167d711e572e1da17d6c4a87ab1f3661c8f2ac0", + "url": "https://github.com/openxla/xla/archive/7167d711e572e1da17d6c4a87ab1f3661c8f2ac0.tar.gz", + "integrity": "sha256-QxwcMAFT4lQLHp+mHh3nhWo9qwbTf9+kiOMWmNuXoVI=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" + } +} diff --git a/third_party/modules/xla/metadata.json b/third_party/modules/xla/metadata.json index 3916998f..01c87137 100644 --- a/third_party/modules/xla/metadata.json +++ b/third_party/modules/xla/metadata.json @@ -18,7 +18,8 @@ "20250122.0-cc075be", "20250123.0-cc075be", "20250204-0-6789523", - "20250317.0-71c67e2" + "20250317.0-71c67e2", + "20250409.0-7167d71" ], "yanked_versions": {} } diff --git a/zml/buffer.zig b/zml/buffer.zig index d1d380f4..184eb808 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -44,23 +44,6 @@ pub const Buffer = struct { } }; - pub const Shard = struct { - api: *const pjrt.Api, - buffer: *pjrt.Buffer, - ready_event: ?*pjrt.Event = null, - ready: bool = false, - - pub fn awaitt(self: *Shard) !void { - if (self.ready) { - return; - } - if (self.ready_event orelse self.buffer.getReadyEvent(self.api)) |ev| { - try ev.awaitt(self.api); - } - self.ready = true; - } - }; - _shape: Shape, _api: *const pjrt.Api, _shards: Shards, @@ -279,6 +262,22 @@ pub const Buffer = struct { return res; } + pub fn getValueFromDataInMemory(self: Buffer, T: type) !T { + stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); + const data = try self.dataInMemory(); + const value = std.mem.bytesAsValue(T, @constCast(data)); + return value.*; + } + + pub fn dataInMemory(self: Buffer) ![]const u8 { + const shard_buffer = self._shards.get(0); + const opaqueDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); + const sizeInbytes = try shard_buffer.getOnDeviceSizeInBytes(self._api); + const data = @as([*]const u8, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + const end: usize = @intCast(sizeInbytes); + return data[0..end]; + } + /// Copies the content of the Buffer back to host, in the given buffer, /// and return a new `HostBuffer` object with the same shape. /// The returned `HostBuffer` doesn't own the memory. @@ -358,7 +357,8 @@ pub const Buffer = struct { } pub fn copyToMemory(self: Buffer, platform: Platform, kind: Memory) !Buffer { - const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device + const device = try self._shards.get(0).getDevice(self._api); + const memories = device.addressableMemories(self._api); var selected_mem: *const pjrt.Memory = undefined; for (memories) |mem| { if (mem.kind(platform.pjrt_api) == kind.toPjrtMemory()) { diff --git a/zml/exe.zig b/zml/exe.zig index 44b35623..7e71588f 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -299,7 +299,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { const type_id = try ffi.registerTypeId(pjrt_api, type_name); const user_data: *anyopaque = @ptrCast(@constCast(value)); - log.info("[FFI] Attach {any}@{x} of type {d} on {any}\n", .{ value, user_data, type_id, self.inner._context.? }); + log.info("[FFI] Attach {any}@{x} of type {d} on {any}", .{ value, user_data, type_id, self.inner._context.? }); try ffi.addUserData(pjrt_api, self.inner._context.?, .{ .type_id = type_id, .user_data = user_data }); } diff --git a/zml/ffi.zig b/zml/ffi.zig index 70373fd1..06ba1418 100644 --- a/zml/ffi.zig +++ b/zml/ffi.zig @@ -16,14 +16,16 @@ const hostbuffer = @import("hostbuffer.zig"); const mlir = @import("mlir.zig"); const module = @import("module.zig"); const ops = @import("ops.zig"); +const platform = @import("platform.zig"); const tensor = @import("tensor.zig"); const shape = @import("shape.zig"); const Buffer = buffer.Buffer; const HostBuffer = hostbuffer.HostBuffer; const CompilationContext = module.CompilationContext; const DataType = dtype.DataType; -const Tensor = tensor.Tensor; +const Platform = platform.Platform; const Shape = shape.Shape; +const Tensor = tensor.Tensor; const log = std.log.scoped(.@"zml/custom_call"); @@ -32,7 +34,7 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; -const CallbackT = HostBuffer; +const CallbackT = Buffer; pub fn CustomCallInputType(custom_op: type) type { const ArgsT = stdx.meta.FnArgs(custom_op.call); @@ -64,8 +66,8 @@ pub fn custom_call( const ctx = inputs[0].getContext(); const mlir_ctx = ctx.mlirCtx(); - const platform = ctx._platform; - const pjrt_api = platform.pjrt_api; + const platform_ = ctx._platform; + const pjrt_api = platform_.pjrt_api; const ffi = pjrt_api.ffi().?; const registry = ffi.customCallRegistry().?; const ffi_func = proxy(custom_op); @@ -128,6 +130,10 @@ pub fn custom_call( } } + const frontend_attributes = mlir.Attribute.dict(mlir_ctx, &.{ + .{ "_xla_buffer_placement", .string(mlir_ctx, @tagName(Buffer.Memory.host_pinned.toPjrtMemory())) }, + }); + const op = dialect.stablehlo.custom_call( mlir_ctx, custom_call_inputs, @@ -135,6 +141,7 @@ pub fn custom_call( .call_target_name = target_name, .api_version = .typed_ffi, .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, .has_side_effect = true, .output_operand_aliases = &.{}, }, @@ -147,21 +154,37 @@ pub fn custom_call( // } if (CustomCallOutputType(custom_op) == Tensor) { - return Tensor._result(res_shapes[0], op.result(0)); + var ret = Tensor._result(res_shapes[0], op.result(0)); + ret._output_memory_kind = .host_pinned; + return ret; } var custom_call_outputs: CustomCallOutputType(custom_op) = undefined; for (&custom_call_outputs, res_shapes, 0..) |*r, shape_, i| { - r.* = Tensor._result(shape_, op.result(i)).toMemory(.device); + r.* = Tensor._result(shape_, op.result(i)); } return custom_call_outputs; } -fn view_buf(pjrt_api: *const pjrt.Api, pjrt_client: *const pjrt.Client, buffer_desc: *const pjrt.ffi.Buffer) CallbackT { +fn view_buf( + platform_: *const Platform, + device: *const pjrt.Device, + buffer_desc: *const pjrt.ffi.Buffer, +) Buffer { + const pjrt_api = platform_.pjrt_api; + const pjrt_client = platform_.pjrt_client; const buffer_shape = getShape(buffer_desc); - const device = pjrt_client.getAddressableDevices(pjrt_api)[0]; + + const kind: Buffer.Memory = .device; + const memories = device.addressableMemories(pjrt_api); + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform_.pjrt_api) == kind.toPjrtMemory()) { + selected_mem = mem; + } + } const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { var res: [Shape.MAX_RANK]i64 = undefined; @@ -171,11 +194,12 @@ fn view_buf(pjrt_api: *const pjrt.Api, pjrt_client: *const pjrt.Client, buffer_d break :blk res; }; + std.debug.print("view_buf: {any} {any} {any}\n", .{ buffer_desc, buffer_shape, selected_mem.kind(pjrt_api) }); const pjrt_buffer = pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ .data = buffer_desc.data, .element_type = buffer.bufferTypeFromDtype(buffer_shape.dtype()), .dims = buffer_shape.dims(), - .device = device, + .memory = selected_mem, .layout = .{ .tiled = .{ .minor_to_major = minor_to_major[Shape.MAX_RANK - buffer_shape.rank() ..], @@ -203,29 +227,56 @@ fn proxy(comptime custom_op: type) pjrt.ffi.Handler { pub fn proxy(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; const execution_context = call_frame.ctx.?; + const device_id = execution_context.getDeviceOrdinal(call_frame.api.?) catch unreachable; const user_ctx: *custom_op = pjrt.ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api.?) catch unreachable; - const platform = user_ctx._platform; - const pjrt_api = platform.pjrt_api; - _ = pjrt_api; // autofix - const pjrt_client = platform.pjrt_client; - _ = pjrt_client; // autofix + + const platform_: *const Platform = user_ctx._platform; + const pjrt_api = platform_.pjrt_api; + const pjrt_client = platform_.pjrt_client; + + const device = pjrt_client.getAddressableDevices(pjrt_api)[@as(u32, @intCast(device_id))]; + log.info("Proxy call_frame {*} on device {} (ordinal: {d})", .{ call_frame, device, device_id }); var args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; args[0] = user_ctx; inline for (1..args.len) |i| { const buffer_desc = call_frame.args.get(i - 1); + + log.info("FFI Buffer {} ({d}/{d})", .{ buffer_desc, i, call_frame.args.len }); + // log.info("Pouet >>>>>>>>>> {d}", .{buffer_desc.data[0..1]}); const buffer_shape = getShape(buffer_desc); - const b = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); + _ = buffer_shape; // autofix - const data_ptr: usize = @intFromPtr(b.data.ptr); - const page_off = data_ptr % std.heap.page_size_min; - const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); - std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE | std.posix.PROT.READ) catch |e| { - log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); - }; + // const host_buffer = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); + // log.info("Host buffer {any}", .{host_buffer.items(f32)}); + const zml_buf = view_buf(platform_, device, buffer_desc); + log.info(">> zml_buf {any}", .{zml_buf.getMemory().kind(pjrt_api)}); + // std.debug.print("zml_buf YAYA {any}: {any}\n", .{ i, zml_buf.getValueFromDataInMemory(f32) catch unreachable }); + // const zml_buf_mem = zml_buf.copyToMemory(platform_.*, .host_pinned) catch unreachable; + // const event = zml_buf_mem._shards.get(0).getReadyEvent(pjrt_api); + // _ = event; // autofix + // std.debug.print(" >> is ready : {any}\n", .{event.?.isReady(pjrt_api)}); + // std.time.sleep(std.time.ns_per_s); + // event.?.await_(pjrt_api) catch unreachable; + // std.debug.print(" >> is ready2 : {any}\n", .{event.?.isReady(pjrt_api)}); + // const dd = zml_buf_mem.getValueFromDataInMemory(f32) catch unreachable; + // std.debug.print("zml_buf YOYO {any}: {any}\n", .{ i, dd }); + // log.info("zml_buf mem {any}", .{zml_buf_mem.getMemory().kind(pjrt_api)}); - args[i] = b; - std.debug.print("input buffer {any}: {any}\n", .{ i, b.data }); + // defer zml_buf.deinit(); + + // const buffer_shape = getShape(buffer_desc); + // const b = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); + + // const data_ptr: usize = @intFromPtr(b.data.ptr); + // const page_off = data_ptr % std.heap.page_size_min; + // const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); + // std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE | std.posix.PROT.READ) catch |e| { + // log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); + // }; + + args[i] = zml_buf; + // std.debug.print("input buffer {any}: {any}\n", .{ i, b.data }); } const outputs = @call(.auto, custom_op.call, args) catch |err| { @@ -241,21 +292,21 @@ fn proxy(comptime custom_op: type) pjrt.ffi.Handler { // b.* = view_buf2(pjrt_api, pjrt_client, call_frame.results.get(i)); // } - const output_buffers = stdx.stackSlice(8, CallbackT, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - const buffer_desc = call_frame.results.get(i); - const buffer_shape = getShape(buffer_desc); - b.* = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); + // const output_buffers = stdx.stackSlice(8, CallbackT, call_frame.results.len); + // for (output_buffers, 0..) |*b, i| { + // const buffer_desc = call_frame.results.get(i); + // const buffer_shape = getShape(buffer_desc); + // b.* = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); - const data_ptr: usize = @intFromPtr(b.data.ptr); - const page_off = data_ptr % std.heap.page_size_min; - const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); - std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE | std.posix.PROT.READ) catch |e| { - log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); - }; + // const data_ptr: usize = @intFromPtr(b.data.ptr); + // const page_off = data_ptr % std.heap.page_size_min; + // const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); + // std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE | std.posix.PROT.READ) catch |e| { + // log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); + // }; - std.debug.print("output buffer {any}: {any}\n", .{ i, b.data }); - } + // std.debug.print("output buffer {any}: {any}\n", .{ i, b.data }); + // } return null; } diff --git a/zml/module.zig b/zml/module.zig index e7112a36..1fad6d37 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -196,6 +196,9 @@ pub const CompilationContext = struct { pjrt_location = try std.fs.path.joinZ(arena, &.{ module_dir.?, "module.pjrt" }); } + log.info("******** ZML generated MLIR ********", .{}); + log.info("{}", .{module.op().mlirFormatter(.{})}); + const loaded_executable: *pjrt.LoadedExecutable = blk: { if (pjrt_location) |pjrt_loc| { if (loadPjrtExecutable(arena, self._platform, pjrt_loc)) |exe| { @@ -220,9 +223,6 @@ pub const CompilationContext = struct { break :blk loaded_executable; }; - log.warn("******** ZML generated MLIR ********", .{}); - log.warn("{}", .{module.op().mlirFormatter(.{})}); - if (timer) |*t| { const time_ms = @divFloor(t.lap(), std.time.ns_per_ms); if (time_ms > 1000) log.info("Compilation took {d:.3}s", .{stdx.math.divFloat(f32, time_ms, 1000)}); diff --git a/zml/tensor.zig b/zml/tensor.zig index dbf21afd..72a30324 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -208,8 +208,6 @@ pub const Tensor = struct { const ctx = self.getContext(); const mlir_ctx = ctx.mlirCtx(); if (ctx.target() == .cpu) return self; - var res = self; - res._output_memory_kind = kind; const memory_kind = @tagName(kind.toPjrtMemory()); @@ -225,7 +223,9 @@ pub const Tensor = struct { .api_version = .original, }, &.{self.value().getType()}, mlir_ctx.location(@src())); - return _result(res._shape, op.result(0)); + var res = _result(self._shape, op.result(0)); + res._output_memory_kind = kind; + return res; }, .buffer_id => { var res = self; From 802c8249eeb1c72facca27c7a627180387ead4b4 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 22 Apr 2025 15:27:48 +0000 Subject: [PATCH 5/5] backup h200 --- async/async.zig | 44 +++++++ examples/custom_call/main.zig | 129 +++++++++++++------- examples/mnist/mnist.zig | 26 +++- pjrt/ffi.zig | 37 ++++-- pjrt/pjrt.zig | 35 ++++-- runtimes/cuda/cuda.zig | 3 + stdx/meta.zig | 2 +- zml/buffer.zig | 26 ++-- zml/context.zig | 63 ++++++++++ zml/exe.zig | 6 +- zml/ffi.zig | 224 +++++++++++++--------------------- zml/ops.zig | 4 +- zml/pjrtx.zig | 7 ++ zml/platform.zig | 1 + 14 files changed, 381 insertions(+), 226 deletions(-) diff --git a/async/async.zig b/async/async.zig index 5cee9947..23aab5af 100644 --- a/async/async.zig +++ b/async/async.zig @@ -219,6 +219,50 @@ pub const AsyncThread = struct { return try aio.run(&executor_, mainFunc, .{}, mainStack); } + + pub fn mainWithArgs(allocator: std.mem.Allocator, comptime mainFunc: anytype, args: anytype) !stdx.meta.FnResultNoError(mainFunc) { + if (xev.dynamic) try xev.detect(); + var thread_pool = XevThreadPool.init(.{}); + defer { + thread_pool.shutdown(); + thread_pool.deinit(); + } + + var loop = try xev.Loop.init(.{ + .thread_pool = &thread_pool, + }); + defer loop.deinit(); + + var executor_ = aio.Executor.init(&loop); + + var async_notifier = try xev.Async.init(); + defer async_notifier.deinit(); + + var waiters_queue: threading.WaiterQueue = undefined; + waiters_queue.init(); + + var stack_allocator = stack.StackAllocator.init(allocator); + defer stack_allocator.deinit(); + + var asyncThread: AsyncThread = .{ + .executor = &executor_, + .stack_allocator = &stack_allocator, + .loop = &loop, + .thread_pool = &thread_pool, + .async_notifier = &async_notifier, + .waiters_queue = &waiters_queue, + }; + AsyncThread.current = &asyncThread; + + var c2: xev.Completion = undefined; + async_notifier.wait(AsyncThread.current.loop, &c2, AsyncThread, AsyncThread.current, &AsyncThread.wakerCallback); + + // allocate the main coroutine stack, on the current thread's stack! + var mainStackData: stack.Stack.Data = undefined; + const mainStack = stack.Stack.init(&mainStackData); + + return try aio.run(&executor_, mainFunc, args, mainStack); + } }; pub fn getStdIn() !File { diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig index cc6d2216..099c6e4e 100644 --- a/examples/custom_call/main.zig +++ b/examples/custom_call/main.zig @@ -3,7 +3,7 @@ const zml = @import("zml"); const asynk = @import("async"); pub const std_options: std.Options = .{ - .log_level = .info, + .log_level = .warn, .logFn = asynk.logFn(std.log.defaultLog), }; @@ -14,14 +14,23 @@ pub const AddOp = struct { _platform: *const zml.Platform, - pub fn beforeCustomCall(a: zml.Tensor, b: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - return .{ a, b }; + pub fn beforeCustomCall(a: zml.Tensor, b: zml.Tensor) struct { zml.Tensor, zml.Tensor, zml.Tensor } { + return .{ a, b, zml.Tensor.scalar(0, .f32) }; } - pub fn call(self: *Self, a: zml.Buffer, _: zml.Buffer) !zml.Buffer { + pub fn call(self: *Self, a: zml.Buffer, b: zml.Buffer, result: zml.Buffer) !struct { zml.Buffer, zml.Buffer, zml.Buffer } { _ = self; // autofix - log.info("mem a: {any}", .{a.items(f32)}); - return a; + // const a_ = try a.getValue(f32); + // const b_ = try b.getValue(f32); + + // log.warn("{s}@{*} a value: {d}", .{ @typeName(Self), self, a_ }); + // log.warn("{s}@{*} b value: {d}", .{ @typeName(Self), self, b_ }); + + // const add_result = a_ + b_; + // try result.setValue(f32, add_result); + // log.warn("{s}@{*} result value: {d}", .{ @typeName(Self), self, try result.getValue(f32) }); + + return .{ a, b, result }; } fn getPlatform(self: *Self) zml.Platform { @@ -34,13 +43,14 @@ pub const LogResultOp = struct { _platform: *const zml.Platform, - // pub fn beforeCustomCall(result: zml.Tensor) zml.Tensor { - // return result; - // } + pub fn beforeCustomCall(value: zml.Tensor) zml.Tensor { + return value; + } - pub fn call(self: *Self, result: zml.Buffer) !zml.Buffer { - log.info("LogResultOp result: {*}", .{try result._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._platform.pjrt_api)}); - return result; + pub fn call(self: *Self, value: zml.Buffer) !zml.Buffer { + _ = self; // autofix + // log.warn("{s}@{*} value: {d}", .{ @typeName(Self), self, try value.getValue(f32) }); + return value; } fn getPlatform(self: *Self) zml.Platform { @@ -67,21 +77,13 @@ pub const LogValuesVoidOp = struct { /// Model definition const Layer = struct { + // a = 40x128 bf16 ou f32 pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { - // const result = zml.custom_call(AddOp, .{ a.toMemory(.host_pinned), b.toMemory(.host_pinned) }); - // const a_ = a.toMemory(.host_pinned); - // const b_ = b.toMemory(.host_pinned); - const a_ = a.toMemory(.host_pinned); - const b_ = b.toMemory(.host_pinned); - const c = a_.clamp(b_, a_); - const d = c.toMemory(.device); - const on_device = a.add(d).toMemory(.host_pinned); - // var result = zml.custom_call(LogResultOp, .{on_device}); - // const result = on_device.toMemory(.device).add(b); - // zml.custom_call(LogValuesVoidOp, .{ result, a, b }); - // var result = a.add(b).toMemory(.host_pinned); - var result = zml.custom_call(LogResultOp, .{on_device}); - return result.add(zml.Tensor.scalar(5, .f32)); + var result = a.add(b); + // const results = zml.custom_call(AddOp, .{ a, b }); + // result = results[0].mul(results[1]); + result = zml.custom_call(LogResultOp, .{result}); + return result.add(a); } }; @@ -106,7 +108,7 @@ pub fn asyncMain() !void { const platform = context.autoPlatform(.{}); context.printAvailablePlatforms(platform); - const shape = zml.Shape.init(.{1}, .f32); + const shape = zml.Shape.init(.{ 40, 128 }, .bf16); // We manually produce a BufferStore. You would not normally do that. // A BufferStore is usually created by loading model data from a file. @@ -146,24 +148,61 @@ pub fn asyncMain() !void { // const log_values_op_ctx: LogValuesVoidOp = .{ ._platform = &platform }; // try executable.attach(&log_values_op_ctx); - // prepare input buffers - var input_a = [1]f32{1.0}; - var input_buffer_a = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_a)); - defer input_buffer_a.deinit(); - var input_b = [1]f32{1.0}; - var input_buffer_b = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_b)); - defer input_buffer_b.deinit(); + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); - // call our executable module - var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); - const result_mem = result.getMemory(); - std.debug.print("<<< Result memory: {any}\n", .{result_mem.kind(platform.pjrt_api)}); - defer result.deinit(); + for (0..2) |i| { + log.warn("Iteration {d}", .{i}); - // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); - std.debug.print( - "\nThe result of {d} + {d} = {d}\n", - .{ &input_a, &input_b, cpu_result.items(f32) }, - ); + // prepare input buffers + var input_buffer_a = try createRandomBuffer(allocator, platform, shape, random); + defer input_buffer_a.deinit(); + var input_buffer_b = try createRandomBuffer(allocator, platform, shape, random); + defer input_buffer_b.deinit(); + + var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + defer result.deinit(); + + // fetch the result to CPU memory + // const cpu_result = try result.toHostAlloc(arena); + // log.warn( + // "\nThe result of {d} + {d} = {d}\n", + // .{ &input_a, &input_b, cpu_result.items(f32) }, + // ); + + std.time.sleep(1 * std.time.ns_per_s); + } +} + +fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const data = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = random.float(f64); + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return zml.Buffer.from(platform, host_buffer); } diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index f5b99b17..816dba39 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -58,8 +58,30 @@ const Mnist = struct { for (layers) |layer| { x = zml.call(layer, .forward, .{x}); } - const r = x.argMax(0).indices.convert(.u8).print(PrintOp, x); - return r[0]; + return relu(x).argMax(0).indices.convert(.u8); + } + + pub fn relu(input: zml.Tensor) zml.Tensor { + return zml.ops.addHostCallback( + &reluCallback, + null, + &.{input}, + &.{input.shape()}, + .{}, + )[0]; + } + + fn reluCallback(_: ?*anyopaque, inputs: []const zml.HostBuffer, outputs: []const zml.HostBuffer) void { + const in = inputs[0].items(f32); + const out = outputs[0].mutItems(f32); + std.log.defaultLog(.info, .zml, "in: {*}[0..{}] = {e:.3}", .{ in.ptr, in.len, in }); + + var i: usize = 0; + for (in) |x| { + out[i] = @max(x, 0); + i += 1; + } + std.log.defaultLog(.info, .zml, "out: {*}[0..{}] = {e:.3}", .{ out.ptr, out.len, out }); } }; diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index a9c39cef..13b3df1a 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -91,22 +91,20 @@ fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; - pub fn getStream(self: *const Api, context: ?*ExecutionContext) ApiError!*anyopaque { + pub fn stream(self: *const Api, context: ?*ExecutionContext) *Stream { var ret = pjrtStruct(c.XLA_FFI_Stream_Get_Args{ .ctx = if (context) |ctx| ctx.inner() else null, }); const result = self.inner().XLA_FFI_Stream_Get.?(&ret); - if (result) |ffi_error| { const err = Error.fromInner(ffi_error); defer err.destroy(self); log.err("[Api.getStream] {s}", .{err.getMessage(self)}); - // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - return error.Unknown; + @panic("failed to get stream"); } - return ret.stream.?; + return @ptrCast(ret.stream.?); } pub fn allocateDeviceMemory(self: *const Api, context: ?*ExecutionContext, size: usize, alignment: usize) ApiError!*anyopaque { @@ -249,6 +247,8 @@ const TypeId = extern struct { const Task = fn (*anyopaque) void; +pub const Stream = opaque {}; + const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -285,11 +285,19 @@ pub const DataType = enum(c.XLA_FFI_DataType) { f8e4m3fnuz = c.XLA_FFI_DataType_F8E4M3FNUZ, }; +pub const DevicePtr = enum(usize) { + _, + + pub fn asPtr(ptr: DevicePtr) [*]u8 { + return @ptrFromInt(@intFromEnum(ptr)); + } +}; + pub const Buffer = extern struct { struct_size: usize, extension_start: ?*c.XLA_FFI_Extension_Base, dtype: DataType, - data: [*]u8, + data: DevicePtr, rank: u64, _dims: [*]const i64, @@ -306,7 +314,7 @@ pub const Buffer = extern struct { _ = fmt; _ = options; - try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data) }); + try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data.asPtr()) }); } }; @@ -321,9 +329,8 @@ pub const Args = extern struct { buffer = c.XLA_FFI_ArgType_BUFFER, }; - pub fn get(self: Args, i: usize) *const Buffer { - std.debug.assert(self.types[0..self.len][i] == .buffer); - return self.ptr[0..self.len][i]; + pub fn buffers(self: Args) []*const Buffer { + return self.ptr[0..self.len]; } }; @@ -338,9 +345,8 @@ pub const Rets = extern struct { buffer = c.XLA_FFI_RetType_BUFFER, }; - pub fn get(self: Rets, i: usize) *const Buffer { - std.debug.assert(self.types[0..self.len][i] == .buffer); - return self.ptr[0..self.len][i]; + pub fn buffers(self: Rets) []*const Buffer { + return self.ptr[0..self.len]; } }; @@ -380,6 +386,11 @@ pub const Attrs = extern struct { data: [*]const u8, }; + pub fn slice(self: Array, T: type) []const T { + const ptr: [*]const T = @alignCast(@ptrCast(self.data)); + return ptr[0..self.len]; + } + pub fn getByIndex(self: Attrs, comptime attr_type: AttrType, index: usize) ?*const @FieldType(Attr, @tagName(attr_type)) { const attr = self.ptr[0..self.len][index]; const actual_type = self.types[index]; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index d2cb698d..4a7b7339 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -414,7 +414,8 @@ pub const Client = opaque { dims: []const i64, element_type: BufferType, layout: MemoryLayout, - memory: *const Memory, + device: *const Device, + memory_kind: ?Memory.Kind, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.C) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.C) void {} }.call, @@ -424,6 +425,16 @@ pub const Client = opaque { pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { const layout = args.layout.toCStruct(); + var memory: ?*const Memory = null; + + if (args.memory_kind) |kind| { + for (args.device.addressableMemories(api)) |m| { + if (m.kind(api) == kind) { + memory = m; + } + } + } + const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), .device_buffer_ptr = @ptrCast(@constCast(args.data)), @@ -431,7 +442,8 @@ pub const Client = opaque { .num_dims = args.dims.len, .element_type = @intFromEnum(args.element_type), .layout = @ptrCast(@constCast(&layout)), - .memory = @ptrCast(@constCast(args.memory)), + .device = @ptrCast(@constCast(args.device)), + .memory = if (memory) |m| @ptrCast(@constCast(m)) else null, .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, .stream = if (args.stream) |stream| stream else 0, @@ -449,20 +461,21 @@ pub const Client = opaque { return &.{}; } - pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!*Buffer { - const ret = try api.call(.PJRT_Client_DMA_Map, .{ + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + const args: Api.CallFnArgType(.PJRT_Client_DmaMap) = .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), .size = @intCast(data.len), - }); - return @ptrCast(ret.buffer.?); + }; + std.debug.print("DMA map {any} \n", .{args}); + _ = try api.call(.PJRT_Client_DmaMap, args); } - pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) void { - _ = api.call(.PJRT_Client_DMA_Unmap, .{ + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + _ = try api.call(.PJRT_Client_DmaUnmap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), - }) catch unreachable; + }); } pub const CreateBuffersForAsyncHostToDeviceArgs = struct { @@ -852,10 +865,10 @@ pub const Buffer = opaque { } pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { - const ret = try api.call(.PJRT_Buffer_UnsafePointer, .{ + const ret = try api.call(.PJRT_Buffer_OpaqueDeviceMemoryDataPointer, .{ .buffer = self.inner(), }); - return @ptrFromInt(ret.buffer_pointer); + return ret.device_memory_ptr.?; } pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64) ApiError!?*Event { diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index bc749693..37a81480 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -62,6 +62,9 @@ pub fn load() !*const pjrt.Api { log.warn("Detected {s} in LD_LIBRARY_PATH. This can lead to undefined behaviors and crashes", .{nvidiaLibsPath}); } + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB", "8", 1); + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_PREALLOCATE", "true", 1); + // CUDA path has to be set _before_ loading the PJRT plugin. // See https://github.com/openxla/xla/issues/21428 try setupXlaGpuCudaDirFlag(); diff --git a/stdx/meta.zig b/stdx/meta.zig index 40754158..ee8d998d 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -164,7 +164,7 @@ pub fn FnResult(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } -pub fn FnReturnNoError(comptime func: anytype) type { +pub fn FnResultNoError(comptime func: anytype) type { const ReturnT = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); return switch (@typeInfo(ReturnT)) { .error_union => |info| info.payload, diff --git a/zml/buffer.zig b/zml/buffer.zig index 184eb808..c038636a 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -4,6 +4,7 @@ const testing = std.testing; const asynk = @import("async"); const stdx = @import("stdx"); +const ops = @import("ops.zig"); const Context = @import("context.zig").Context; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -91,7 +92,7 @@ pub const Buffer = struct { .dims = buf.shape().dims(), .byte_strides = byte_strides, .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, + .host_buffer_semantics = .MutableZeroCopy, }, }); @@ -217,23 +218,17 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const anyopaque, device_data: *anyopaque) Buffer { - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var res: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - res[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk res; - }; - const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, .element_type = bufferTypeFromDtype(shape_.dtype()), .dims = shape_.dims(), // TODO: exposes sharding in the API. .device = platform.getDevices()[0], + // TODO: expose memory in the API. + .memory_kind = null, .layout = .{ .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], + .minor_to_major = ops.minorToMajor(shape_.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, @@ -262,6 +257,16 @@ pub const Buffer = struct { return res; } + pub fn setValue(self: Buffer, T: type, value: anytype) !void { + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + const shard_buffer = self._shards.get(0); + const opaqueDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); + const sizeInBytes = try shard_buffer.getOnDeviceSizeInBytes(self._api); + stdx.debug.assert(sizeInBytes == @sizeOf(T), "Buffer size {d} does not match type size {d}", .{ sizeInBytes, @sizeOf(T) }); + const data = @as(*T, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + data.* = value; + } + pub fn getValueFromDataInMemory(self: Buffer, T: type) !T { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); const data = try self.dataInMemory(); @@ -277,7 +282,6 @@ pub const Buffer = struct { const end: usize = @intCast(sizeInbytes); return data[0..end]; } - /// Copies the content of the Buffer back to host, in the given buffer, /// and return a new `HostBuffer` object with the same shape. /// The returned `HostBuffer` doesn't own the memory. diff --git a/zml/context.zig b/zml/context.zig index 4cb9a7ee..65c6f381 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -37,6 +37,7 @@ pub const Context = struct { inline for (comptime std.enums.values(runtimes.Platform)) |t| { if (runtimes.load(t)) |api| { Context.apis.set(t, api); + if (t == .cuda) cuda.init(); } else |_| {} } } @@ -211,3 +212,65 @@ pub const Context = struct { } } }; + +pub const cuda = struct { + var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); + var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); + pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); + + pub const MemcpyKind = enum(c_int) { + host_to_host = 0, + host_to_device = 1, + device_to_host = 2, + device_to_device = 3, + inferred = 4, + }; + + const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + const MemcpyBlocking = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind) callconv(.C) c_int; + const StreamSynchronize = *const fn (stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + + pub fn init() void { + var cudart = std.DynLib.open("libcudart.so.12") catch { + log.err("cudart not found, callback will segfault", .{}); + return; + }; + defer cudart.close(); + + _memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { + @panic("cudaMemcpyAsync not found"); + }; + _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { + @panic("cudaMemcpy not found"); + }; + streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { + @panic("cudaStreamSynchronize not found"); + }; + } + + pub fn memcpyToHostBlocking(dst: []u8, src: pjrt.ffi.DevicePtr) void { + const err = _memcpyBlocking(dst.ptr, src.asPtr(), dst.len, .device_to_host); + check(err); + } + + pub fn memcpyToDeviceBlocking(dst: pjrt.ffi.DevicePtr, src: []const u8) void { + const err = _memcpyBlocking(dst.asPtr(), src.ptr, src.len, .host_to_device); + check(err); + } + + pub fn memcpyToDeviceAsync(dst: pjrt.ffi.DevicePtr, src: []const u8, stream: ?*pjrt.ffi.Stream) void { + const err = _memcpyAsync(dst.asPtr(), src.ptr, src.len, .host_to_device, stream); + check(err); + } + + pub fn memcpyToHostAsync(dst: []u8, src: pjrt.ffi.DevicePtr, stream: ?*pjrt.ffi.Stream) void { + const err = _memcpyAsync(dst.ptr, src.asPtr(), dst.len, .device_to_host, stream); + check(err); + } + + pub fn check(err: c_int) void { + if (err == 0) return; + log.err("CUDA ERROR {d}", .{err}); + @panic("CUDA error"); + } +}; diff --git a/zml/exe.zig b/zml/exe.zig index 7e71588f..1d6f263f 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -12,7 +12,7 @@ const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; -const log = std.log.scoped(.zml); +const log = std.log.scoped(.@"zml/exe"); test { std.testing.refAllDecls(@This()); @@ -282,7 +282,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { var new: Self = .{ .inner = self.inner }; const pjrt_execute_context = self.inner.platform.pjrt_api.createExecuteContext() catch unreachable; new.inner._context = pjrt_execute_context; - log.info("[FFI] Created {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); + log.info("Created context execution {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); return new; } @@ -299,7 +299,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { const type_id = try ffi.registerTypeId(pjrt_api, type_name); const user_data: *anyopaque = @ptrCast(@constCast(value)); - log.info("[FFI] Attach {any}@{x} of type {d} on {any}", .{ value, user_data, type_id, self.inner._context.? }); + log.info("Attached {s}@{x} with type id {d} on {any}", .{ type_name, user_data, type_id, self.inner._context.? }); try ffi.addUserData(pjrt_api, self.inner._context.?, .{ .type_id = type_id, .user_data = user_data }); } diff --git a/zml/ffi.zig b/zml/ffi.zig index 06ba1418..5fff04cf 100644 --- a/zml/ffi.zig +++ b/zml/ffi.zig @@ -1,11 +1,11 @@ const std = @import("std"); -const assert = std.debug.assert; const testing = std.testing; const builtin = @import("builtin"); -const asynk = @import("asynk"); +const asynk = @import("async"); const stdx = @import("stdx"); +const context = @import("context.zig"); const dialect = struct { const stablehlo = @import("mlir/dialects").stablehlo; }; @@ -19,6 +19,7 @@ const ops = @import("ops.zig"); const platform = @import("platform.zig"); const tensor = @import("tensor.zig"); const shape = @import("shape.zig"); +const cuda = context.cuda; const Buffer = buffer.Buffer; const HostBuffer = hostbuffer.HostBuffer; const CompilationContext = module.CompilationContext; @@ -34,35 +35,49 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; -const CallbackT = Buffer; +// todo: check deallocs pub fn CustomCallInputType(custom_op: type) type { - const ArgsT = stdx.meta.FnArgs(custom_op.call); + const ArgsT = stdx.meta.FnArgs(custom_op.beforeCustomCall); if (@typeInfo(ArgsT) != .@"struct") { @compileError("Expected struct type"); } - return [@typeInfo(ArgsT).@"struct".fields.len - 1]Tensor; + // todo check typeof first input then others args + return [@typeInfo(ArgsT).@"struct".fields.len]Tensor; } pub fn CustomCallOutputType(custom_op: type) type { - const ReturnT = stdx.meta.FnReturnNoError(custom_op.call); + const ReturnT = stdx.meta.FnResultNoError(custom_op.call); if (@typeInfo(ReturnT) != .@"struct") { @compileError("Expected struct type"); } - if (ReturnT == CallbackT) { + if (ReturnT == Buffer) { return Tensor; } - return [@typeInfo(ReturnT).@"struct".fields.len]Tensor; } +pub fn lenOutputBeforeCustomCall(custom_op: type) usize { + const ReturnT = stdx.meta.FnResultNoError(custom_op.beforeCustomCall); + + if (@typeInfo(ReturnT) != .@"struct") { + @compileError("Expected struct type"); + } + + if (ReturnT == Tensor) { + return 1; + } + return @typeInfo(ReturnT).@"struct".fields.len; +} + pub fn custom_call( comptime custom_op: type, inputs: CustomCallInputType(custom_op), ) CustomCallOutputType(custom_op) { stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); + const op_name = @typeName(custom_op); const ctx = inputs[0].getContext(); const mlir_ctx = ctx.mlirCtx(); @@ -71,11 +86,12 @@ pub fn custom_call( const ffi = pjrt_api.ffi().?; const registry = ffi.customCallRegistry().?; const ffi_func = proxy(custom_op); - const target_name = "callback_" ++ @typeName(custom_op); - registry.registerFfi(pjrt_api, target_name, @tagName(ctx._platform.target), &ffi_func) catch unreachable; - log.info("Registered custom call {s} for {s} with proxy func {*}", .{ target_name, @typeName(custom_op), &ffi_func }); + const target_name = "callback_" ++ op_name; - const custom_call_inputs = stdx.stackSlice(8, mlir.Value, inputs.len); + registry.registerFfi(pjrt_api, target_name, @tagName(platform_.target), &ffi_func) catch unreachable; + log.info("{s} / Registered custom call with target_name \"{s}\" with proxy func {*}", .{ op_name, target_name, &ffi_func }); + + var custom_call_inputs: []mlir.Value = undefined; var res_shapes: []Shape = undefined; var res_types: []mlir.Type = undefined; @@ -91,7 +107,7 @@ pub fn custom_call( } const before_custom_call_outputs = ctx.callFunc(@typeName(custom_op), before_custom_call, args) catch |err| { - std.debug.panic("Error in {any} beforeCustomCall func: {any}\n", .{ @typeName(custom_op), err }); + stdx.debug.panic("Error in {any} beforeCustomCall func: {any}\n", .{ @typeName(custom_op), err }); }; var before_custom_call_outputs_array: []Tensor = undefined; @@ -105,19 +121,20 @@ pub fn custom_call( } } + custom_call_inputs = stdx.stackSlice(8, mlir.Value, lenOutputBeforeCustomCall(custom_op)); res_shapes = stdx.stackSlice(8, Shape, before_custom_call_outputs_array.len); res_types = stdx.stackSlice(8, mlir.Type, before_custom_call_outputs_array.len); for (before_custom_call_outputs_array, 0..) |o, i| { res_shapes[i] = o.shape(); + custom_call_inputs[i] = o.value(); res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o.shape()).as(mlir.Type); } - - for (before_custom_call_outputs_array, 0..) |output, i| { - custom_call_inputs[i] = output.value(); - } } else { - log.warn("No beforeCustomCall function found for {s}, expecting return type to be args type", .{@typeName(custom_op)}); + // todo : check if args len != outputs len + log.warn("{s} / No beforeCustomCall function found, we will expect that the return type of call func will be like args type", .{op_name}); + custom_call_inputs = stdx.stackSlice(8, mlir.Value, inputs.len); + for (custom_call_inputs, inputs) |*input, tensor_| { input.* = tensor_.value(); } @@ -131,6 +148,7 @@ pub fn custom_call( } const frontend_attributes = mlir.Attribute.dict(mlir_ctx, &.{ + .{ "_xla_compute_type", .string(mlir_ctx, "host") }, .{ "_xla_buffer_placement", .string(mlir_ctx, @tagName(Buffer.Memory.host_pinned.toPjrtMemory())) }, }); @@ -149,14 +167,8 @@ pub fn custom_call( mlir_ctx.location(@src()), ); - // if (CustomCallOutputType(custom_op) == void) { - // return; - // } - if (CustomCallOutputType(custom_op) == Tensor) { - var ret = Tensor._result(res_shapes[0], op.result(0)); - ret._output_memory_kind = .host_pinned; - return ret; + return Tensor._result(res_shapes[0], op.result(0)); } var custom_call_outputs: CustomCallOutputType(custom_op) = undefined; @@ -168,59 +180,10 @@ pub fn custom_call( return custom_call_outputs; } -fn view_buf( - platform_: *const Platform, - device: *const pjrt.Device, - buffer_desc: *const pjrt.ffi.Buffer, -) Buffer { - const pjrt_api = platform_.pjrt_api; - const pjrt_client = platform_.pjrt_client; - const buffer_shape = getShape(buffer_desc); - - const kind: Buffer.Memory = .device; - const memories = device.addressableMemories(pjrt_api); - var selected_mem: *const pjrt.Memory = undefined; - for (memories) |mem| { - if (mem.kind(platform_.pjrt_api) == kind.toPjrtMemory()) { - selected_mem = mem; - } - } - - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var res: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - res[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk res; - }; - - std.debug.print("view_buf: {any} {any} {any}\n", .{ buffer_desc, buffer_shape, selected_mem.kind(pjrt_api) }); - const pjrt_buffer = pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ - .data = buffer_desc.data, - .element_type = buffer.bufferTypeFromDtype(buffer_shape.dtype()), - .dims = buffer_shape.dims(), - .memory = selected_mem, - .layout = .{ - .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - buffer_shape.rank() ..], - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - .stream = null, - }) catch @panic("failed to createViewOfDeviceBuffer"); - - var shards: Buffer.Shards = .{}; - shards.appendAssumeCapacity(pjrt_buffer); - - return .{ - ._api = pjrt_api, - ._shape = buffer_shape, - ._shards = shards, - }; -} - fn proxy(comptime custom_op: type) pjrt.ffi.Handler { + const op_name = @typeName(custom_op); + _ = op_name; // autofix + return struct { const Self = @This(); @@ -235,77 +198,62 @@ fn proxy(comptime custom_op: type) pjrt.ffi.Handler { const pjrt_client = platform_.pjrt_client; const device = pjrt_client.getAddressableDevices(pjrt_api)[@as(u32, @intCast(device_id))]; - log.info("Proxy call_frame {*} on device {} (ordinal: {d})", .{ call_frame, device, device_id }); + _ = device; // autofix + const ffi_buffer = call_frame.args.buffers()[0]; + // const buffer_on_device = Buffer.asViewOfDeviceBuffer(platform_.*, getShape(ffi_buffer), null, ffi_buffer.data.asPtr()); + // const buffer_pinned = buffer_on_device.copyToMemory(platform_.*, .host_pinned) catch unreachable; + // _ = buffer_pinned; // autofix - var args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; - args[0] = user_ctx; - inline for (1..args.len) |i| { - const buffer_desc = call_frame.args.get(i - 1); + const ctx = call_frame.ctx; + const stream = call_frame.api.?.stream(@constCast(ctx)); + _ = stream; // autofix + const host = HostBuffer.empty(std.heap.smp_allocator, getShape(ffi_buffer)) catch unreachable; + cuda.memcpyToHostAsync(@constCast(host.data), ffi_buffer.data, null); + _ = cuda.streamSynchronize(null); + // log.info("{s} / Proxy call_frame {*} on device {} (ordinal: {d})", .{ op_name, call_frame, device, device_id }); - log.info("FFI Buffer {} ({d}/{d})", .{ buffer_desc, i, call_frame.args.len }); - // log.info("Pouet >>>>>>>>>> {d}", .{buffer_desc.data[0..1]}); - const buffer_shape = getShape(buffer_desc); - _ = buffer_shape; // autofix + // var callback_args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; + // callback_args[0] = user_ctx; - // const host_buffer = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); - // log.info("Host buffer {any}", .{host_buffer.items(f32)}); - const zml_buf = view_buf(platform_, device, buffer_desc); - log.info(">> zml_buf {any}", .{zml_buf.getMemory().kind(pjrt_api)}); - // std.debug.print("zml_buf YAYA {any}: {any}\n", .{ i, zml_buf.getValueFromDataInMemory(f32) catch unreachable }); - // const zml_buf_mem = zml_buf.copyToMemory(platform_.*, .host_pinned) catch unreachable; - // const event = zml_buf_mem._shards.get(0).getReadyEvent(pjrt_api); - // _ = event; // autofix - // std.debug.print(" >> is ready : {any}\n", .{event.?.isReady(pjrt_api)}); - // std.time.sleep(std.time.ns_per_s); - // event.?.await_(pjrt_api) catch unreachable; - // std.debug.print(" >> is ready2 : {any}\n", .{event.?.isReady(pjrt_api)}); - // const dd = zml_buf_mem.getValueFromDataInMemory(f32) catch unreachable; - // std.debug.print("zml_buf YOYO {any}: {any}\n", .{ i, dd }); - // log.info("zml_buf mem {any}", .{zml_buf_mem.getMemory().kind(pjrt_api)}); + // inline for (1..callback_args.len) |i| { + // const ffi_buffer = call_frame.args.buffers()[i - 1]; + // log.info("{s} / FFI Buffer arg: {} ({d}/{d})", .{ op_name, ffi_buffer, i, call_frame.args.len }); - // defer zml_buf.deinit(); + // const buffer_on_device = Buffer.asViewOfDeviceBuffer(platform_.*, getShape(ffi_buffer), null, ffi_buffer.data.asPtr()); + // const buffer_device_pjrt_buffer = buffer_on_device._shards.get(0).getOpaqueDeviceMemoryDataPointer(pjrt_api); + // log.info("{s} / ZML Buffer on device: {} {any} ({d}/{d})", .{ op_name, buffer_on_device, buffer_device_pjrt_buffer, i, call_frame.args.len }); - // const buffer_shape = getShape(buffer_desc); - // const b = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); + // const buffer_pinned = buffer_on_device.copyToMemory(platform_.*, .host_pinned) catch unreachable; + // const buffer_pinned_pjrt_buffer = buffer_pinned._shards.get(0).getOpaqueDeviceMemoryDataPointer(pjrt_api); + // log.info("{s} / ZML Buffer pinned: {} {any} ({d}/{d})", .{ op_name, buffer_pinned, buffer_pinned_pjrt_buffer, i, call_frame.args.len }); - // const data_ptr: usize = @intFromPtr(b.data.ptr); - // const page_off = data_ptr % std.heap.page_size_min; - // const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); - // std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE | std.posix.PROT.READ) catch |e| { - // log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); - // }; - - args[i] = zml_buf; - // std.debug.print("input buffer {any}: {any}\n", .{ i, b.data }); - } - - const outputs = @call(.auto, custom_op.call, args) catch |err| { - std.debug.print("Error on custom call {any}: {any}\n", .{ @typeName(custom_op), err }); - const ffi_error = pjrt.ffi.Error.create(call_frame.api.?, pjrt.ffi.ErrorCode.unknown, @errorName(err)); - return ffi_error; - }; - - std.debug.print("output: {any}\n", .{outputs}); - - // const output_buffers = stdx.stackSlice(8, CallbackT, call_frame.results.len); - // for (output_buffers, 0..) |*b, i| { - // b.* = view_buf2(pjrt_api, pjrt_client, call_frame.results.get(i)); + // callback_args[i] = buffer_pinned; // } - // const output_buffers = stdx.stackSlice(8, CallbackT, call_frame.results.len); - // for (output_buffers, 0..) |*b, i| { - // const buffer_desc = call_frame.results.get(i); - // const buffer_shape = getShape(buffer_desc); - // b.* = HostBuffer.fromBytes(buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()]); + // const zml_results = @call(.auto, custom_op.call, callback_args) catch |err| { + // stdx.debug.panic("Error while calling {any} call func: {any}\n", .{ @typeName(custom_op), err }); + // }; - // const data_ptr: usize = @intFromPtr(b.data.ptr); - // const page_off = data_ptr % std.heap.page_size_min; - // const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); - // std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE | std.posix.PROT.READ) catch |e| { - // log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); - // }; + // var results_array: []Buffer = undefined; + // if (@TypeOf(zml_results) == Buffer) { + // results_array = stdx.stackSlice(8, Buffer, 1); + // results_array[0] = zml_results; + // } else { + // results_array = stdx.stackSlice(8, Buffer, @typeInfo(@TypeOf(zml_results)).@"struct".fields.len); + // inline for (@typeInfo(@TypeOf(zml_results)).@"struct".fields, 0..) |field, i| { + // results_array[i] = @field(zml_results, field.name); + // } + // } - // std.debug.print("output buffer {any}: {any}\n", .{ i, b.data }); + // for (results_array, 0..) |zml_buffer, i| { + // const ffi_buffer = call_frame.results.buffers()[i]; + // _ = ffi_buffer; // autofix + // log.info("{s} / FFI Buffer result ptr: {} ({d}/{d})", .{ op_name, call_frame.results.ptr[i], i + 1, call_frame.results.len }); + + // const zml_buffer_pjrt_buffer = zml_buffer._shards.get(0).getOpaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable; + // log.info("{s} / ZML Buffer result: {} {any} ({d}/{d})", .{ op_name, zml_buffer, zml_buffer_pjrt_buffer, i + 1, call_frame.results.len }); + + // // cuda.memcpyToDeviceBlocking(ffi_buffer.data, zml_buffer.dataInMemory() catch unreachable); // } return null; diff --git a/zml/ops.zig b/zml/ops.zig index d11885c1..3f70997b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1201,13 +1201,13 @@ inline fn toI64(values: anytype) []i64 { } const _MINOR_TO_MAJOR = blk: { - var ret: [Shape.MAX_RANK]usize = undefined; + var ret: [Shape.MAX_RANK]i64 = undefined; for (0..Shape.MAX_RANK) |i| { ret[i] = @intCast(Shape.MAX_RANK - i - 1); } break :blk ret; }; -fn minorToMajor(rank: u8) []const usize { +pub fn minorToMajor(rank: u8) []const i64 { return _MINOR_TO_MAJOR[_MINOR_TO_MAJOR.len - rank ..]; } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index c03606d5..287cc3b2 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -67,6 +67,13 @@ pub const Client = opaque { return @ptrCast(buffer); } + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaMap(api, data); + } + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaUnmap(api, data); + } + pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { return @ptrCast(try asynk.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); } diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..ba6f3ac3 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -125,6 +125,7 @@ const _CreateOptions = struct { }; pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.fromBool("should_stage_host_to_device_transfers", false)); switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform"));