diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6c2a9df0..2517fa19 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -128,9 +128,11 @@ pub fn generateText( tracer.frameEnd(frame_id, try std.fmt.bufPrintZ(tracer_buffer, "Generated token {}/{}", .{ i + 1, output_tokens_len })); - // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + // extract the generated token from the buffer, async + var fut_tok_buf = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + _ = try fut_tok_buf.awaitt(); const generated_token = generated_token_buffer[0]; + // de-tokenize generated token into a string const chunk = try tokenizer_decoder.next(@intCast(generated_token)) orelse unreachable; num_tokens_generated = i; @@ -166,7 +168,7 @@ const params = clap.parseParamsComptime( \\--seed random seed (optional) \\--seq-len sequence length \\--create-options platform creation options JSON, defaults to {} - \\--no-llama3 skip prompt template + \\--no-llama3 skip prompt template \\--sharding default: true: sharding on or off ); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9cfa5fb5..651a1a55 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -732,7 +732,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, diff --git a/zml/aio.zig b/zml/aio.zig index 55155c48..07f7c2a9 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -561,8 +561,7 @@ pub fn loadModelBuffersWithPrefix( return res; } -/// Takes a bufferized version of a `model`, ie a mirror struct of the `model`, and deinit all the -/// Buffer found. +/// deprecated, use zml.buffers.deinitAll() pub fn unloadBuffers(model: anytype) void { zml.meta.visit((struct { fn cb(_: void, buffer: *zml.Buffer) void { @@ -590,6 +589,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; obj.* = try zml.Buffer.from(platform, buf_with_metadata); + _ = try obj.awaitt(); } else { return error.BufferNotFound; }; diff --git a/zml/buffer.zig b/zml/buffer.zig index a9fba67a..ce752c2c 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -27,12 +27,30 @@ const log = std.log.scoped(.zml); /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { + pub const Shard = struct { + // TODO: we should use SoA instead of AoS because of the "ready" bool. + // the Buffer struct is already huge. + buffer: *pjrt.Buffer, + ready_event: ?*pjrt.Event = null, + ready: bool = false, + + pub fn awaitt(self: *Shard, api: *const pjrt.Api) pjrt.ApiError!void { + if (self.ready) { + return; + } + if (self.ready_event orelse self.buffer.getReadyEvent(api)) |ev| { + try ev.awaitt(api); + } + self.ready = true; + } + }; + _shape: Shape, _api: *const pjrt.Api, _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; - pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const Shards = std.BoundedArray(Shard, MAX_NUM_SHARDS); /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { @@ -56,49 +74,35 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - .{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + }); + res._shards.appendAssumeCapacity(.{ + .buffer = pjrt_buffer, + .ready_event = event, }); - - frames.appendAssumeCapacity(frame); } - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); - res._shards.appendAssumeCapacity(pjrt_buffer); - } return res; } - /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { - stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); - stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); - var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); - return .{ - ._api = platform.pjrt_api, - ._shape = shape_, - ._shards = shards, - }; + pub fn awaitt(self: *Buffer) pjrt.ApiError!*Buffer { + for (self._shards.slice()) |*shard| { + try shard.awaitt(self._api); + } + return self; } /// Copies the given Zig slice to the accelerator memory and @@ -108,6 +112,11 @@ pub const Buffer = struct { return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); } + pub fn fromSlice2(platform: Platform, dimz: anytype, s: anytype) !Buffer { + const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); + return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); + } + /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { @@ -115,6 +124,10 @@ pub const Buffer = struct { return try from(platform, host_buffer); } + pub fn fromBytes(platform: Platform, shape_: Shape, data: []const u8) !Buffer { + return from(platform, HostBuffer.fromBytes(shape_, data)); + } + /// Creates a Buffer with a single element. pub fn scalar(platform: Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); @@ -168,7 +181,9 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, .data = &bytes }; - return try from(platform, host_buffer); + var buf = try from(platform, host_buffer); + _ = try buf.awaitt(); + return buf; } test constant { @@ -193,6 +208,7 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. + /// The returned buffer will be considered as ready to read. pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const anyopaque, device_data: *anyopaque) !Buffer { const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { var res: [Shape.MAX_RANK]i64 = undefined; @@ -220,7 +236,12 @@ pub const Buffer = struct { }); var shards: Shards = .{}; - shards.appendAssumeCapacity(pjrt_buffer); + shards.appendAssumeCapacity(.{ + .buffer = pjrt_buffer, + // We don't have event here so we assume it's ready + .ready_event = null, + .ready = true, + }); return .{ ._api = platform.pjrt_api, ._shape = shape_, @@ -233,9 +254,9 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.awaitt(self._api); } return res; } @@ -245,22 +266,23 @@ pub const Buffer = struct { /// The returned `HostBuffer` doesn't own the memory. pub fn toHost(self: Buffer, output: []u8) !HostBuffer { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, output); + var hb = HostBuffer.fromBytes(self.shape(), output); + hb._event = maybe_event; + hb._api = self._api; + hb._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return hb; } /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); + var output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); - if (maybe_event) |event| { - try event.await_(self._api); - } + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, @constCast(output.data)); + output._event = maybe_event; + output._api = self._api; + output._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; return output; } @@ -268,8 +290,8 @@ pub const Buffer = struct { /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *const Buffer) void { - for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._api); + for (self._shards.constSlice()) |shard| { + shard.buffer.deinit(self._api); } } @@ -386,3 +408,27 @@ test bufferTypeFromDtype { try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); } } + +/// await all buffers in the given struct +pub fn awaitAll(buffers: anytype) pjrt.ApiError!void { + var err: ?pjrt.ApiError = null; + meta.visit((struct { + fn cb(err_ptr: *?pjrt.ApiError, buffer: *Buffer) void { + _ = buffer.awaitt() catch |e| { + err_ptr.* = e; + return; + }; + } + }).cb, &err, buffers); + + return err orelse {}; +} + +/// deinit all buffers in the given struct +pub fn deinitAll(buffers: anytype) void { + meta.visit((struct { + fn cb(_: void, buffer: *Buffer) void { + buffer.deinit(); + } + }).cb, {}, buffers); +} diff --git a/zml/exe.zig b/zml/exe.zig index b34d5e9b..db87ddf7 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -203,18 +203,13 @@ pub const BaseExe = struct { .arguments = self.input_per_device, .num_args = self.input_buffer_count, .results = self.output_per_device, + // Create a local inste .events = events[0..sharding.num_partitions], // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, }) catch unreachable; - - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } } pub fn serialize(self: BaseExe, writer: anytype) !void { @@ -235,13 +230,27 @@ pub const BaseExe = struct { self.ready_buffer_count += n; } - pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { - var shards: Buffer.Shards = .{}; - for (self.output_per_device) |dev_out| { - shards.appendAssumeCapacity(dev_out[i]); - } + pub fn numOutputs(self: BaseExe) u32 { + return @intCast(self.result_shapes.len); + } - return Buffer.fromPjrtBuffers(self.platform, self.result_shapes[i], shards.constSlice()); + /// Takes pjrt.Buffer from each device and make it into a Buffer. + /// Set the event of the buffer to the event of the call. + pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { + std.debug.assert(i < self.numOutputs()); + var shards: Buffer.Shards = .{}; + for (0..self.num_devices) |dev| { + shards.appendAssumeCapacity(.{ + .buffer = self.output_per_device[dev][i], + .ready_event = null, + .ready = false, + }); + } + return .{ + ._shape = self.result_shapes[i], + ._api = self.platform.pjrt_api, + ._shards = shards, + }; } }; @@ -283,7 +292,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { std.debug.assert(total_ready == self.inner.input_buffer_count); self.inner._unsafeCall(); var result: Bufferized(ReturnT) = undefined; - assignRawBuffers(&result, self.inner.platform, self.inner.output_per_device, self.inner.result_shapes); + assignRawBuffers(&result, self.inner); return result; } }; @@ -316,7 +325,7 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { const model_sharding = ctx.buffers.len; stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {}-sharded tensor into a {}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); for (buffer._shards.constSlice(), 0..) |shard, d| { - ctx.buffers[d][ctx.index] = shard; + ctx.buffers[d][ctx.index] = shard.buffer; } ctx.index += 1; } @@ -325,31 +334,20 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { } /// Visit the given struct and override tensors by creating a new one using the provided PJRT buffers. -fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Buffer, buffer_shapes: []Shape) void { +fn assignRawBuffers(v: anytype, exe: BaseExe) void { const LocalContext = struct { - index: u32, - platform: Platform, - buffers: []const [*]*pjrt.Buffer, - buffer_shapes: []Shape, - }; - var local_ctx: LocalContext = .{ - .index = 0, - .platform = platform, - .buffers = buffers, - .buffer_shapes = buffer_shapes, - }; - meta.visit((struct { - fn cb(ctx: *LocalContext, buffer: *Buffer) void { + index: u32 = 0, + exe: *const BaseExe, + + fn cb(ctx: *@This(), buffer: *Buffer) void { const i = ctx.index; ctx.index += 1; - if (i >= ctx.buffer_shapes.len) return; + if (i >= ctx.exe.result_shapes.len) return; - var shards: Buffer.Shards = .{}; - for (ctx.buffers) |buff| { - shards.appendAssumeCapacity(buff[i]); - } - buffer.* = Buffer.fromPjrtBuffers(ctx.platform, ctx.buffer_shapes[i], shards.constSlice()); + buffer.* = ctx.exe.getOutputBuffer(i); } - }).cb, &local_ctx, v); - stdx.debug.internalAssert(local_ctx.index == buffer_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ buffers.len, @typeName(@TypeOf(v)), local_ctx.index }); + }; + var local_ctx: LocalContext = .{ .exe = &exe }; + meta.visit(LocalContext.cb, &local_ctx, v); + stdx.debug.internalAssert(local_ctx.index == exe.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ exe.result_shapes.len, @typeName(@TypeOf(v)), local_ctx.index }); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index cd7e3a60..da96cff9 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -1,6 +1,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); + const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -22,6 +24,9 @@ pub const HostBuffer = struct { managed: u5, unmanaged, } = .unmanaged, + _ready: bool = true, + _api: ?*const pjrt.Api = null, + _event: ?*pjrt.Event = null, /// Allocates a HostBuffer with the given shape. /// The memory is left undefined. @@ -255,6 +260,21 @@ pub const HostBuffer = struct { }; } + pub fn awaitt(self: *HostBuffer) !*HostBuffer { + if (self._ready) { + return self; + } + if (self._event) |ev| { + if (self._api) |api| { + try ev.awaitt(api); + } else { + stdx.debug.panic("Got an event but no API. This should not happen.", .{}); + } + } + self._ready = true; + return self; + } + pub fn format( self: HostBuffer, comptime fmt: []const u8, diff --git a/zml/module.zig b/zml/module.zig index 0c1bc19f..a003896d 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -905,8 +905,8 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables setFlag(&options, "xla_gpu_enable_triton_gemm", false); setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); - setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); - setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); + // setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); + // setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); diff --git a/zml/nn.zig b/zml/nn.zig index 6f756e99..9ba5a5c1 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -934,7 +934,7 @@ pub const PartialSoftmax = struct { /// Returns intermediary results to allow aggregating later. pub fn partialSoftmax(self: Tensor, axis: anytype) PartialSoftmax { const a = self.axis(axis); - const max_val = self.max(a); + const max_val = self.maximum(Tensor.constant(self.shape(), DataType.constant(self.dtype(), 0))).max(a); const out = self.sub(max_val.broad(self.shape())).exp(); return .{ .values = out, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 7408e074..ae39af8f 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -17,7 +17,6 @@ const log = std.log.scoped(.zml); pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; -pub const Buffer = pjrt.Buffer; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; @@ -63,13 +62,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return buffer; + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -130,6 +125,62 @@ pub const Client = opaque { } }; +pub const Buffer = opaque { + pub const inner = InnerMixin(pjrt.Buffer).inner; + + pub fn deinit(self: *Buffer, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { + return try self.inner().getDevice(api); + } + + pub fn delete(self: *Buffer, api: *const Api) void { + self.inner().delete(api); + } + + pub fn isDeleted(self: *const Buffer, api: *const Api) bool { + return self.inner().isDeleted(api); + } + + pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { + return self.inner().isOnCpu(api); + } + + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { + return @ptrCast(try self.inner().toHostBuffer(api, dst)); + } + + pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { + return self.inner().getElementType(api); + } + + pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { + return self.inner().getDimensions(api); + } + + pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { + return try self.inner().getUnpaddedDimensions(api); + } + + pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { + return try self.inner().getOnDeviceSizeInBytes(api); + } + + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { + return @ptrCast(self.inner().copyToDevice(api, device)); + } + + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { + return @ptrCast(self.inner().getReadyEvent(api)); + } + + pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { + return try self.inner().getOpaqueDeviceMemoryDataPointer(api); + } +}; + pub const Event = opaque { pub const inner = InnerMixin(pjrt.Event).inner; @@ -145,7 +196,8 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) !void { + pub const await_ = awaitt; + pub fn awaitt(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); if (self.isReady(api)) { @@ -201,13 +253,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { diff --git a/zml/tensor.zig b/zml/tensor.zig index 23ab003b..f2e37497 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2419,6 +2419,10 @@ pub const Tensor = struct { /// of the operator which is backend specific. update_fn: *const fn (Tensor, Tensor) Tensor = increment, + pub fn maximum(old_value: Tensor, new_value: Tensor) Tensor { + return old_value.maximum(new_value.convert(old_value.dtype())); + } + pub fn increment(old_value: Tensor, new_value: Tensor) Tensor { return old_value.add(new_value.convert(old_value.dtype())); } diff --git a/zml/testing.zig b/zml/testing.zig index e686b603..4ae4140c 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -41,11 +41,13 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { .{ try left_.toHostAlloc(allocator), true } else .{ left_, false }; + _ = try left.awaitt(); var right: zml.HostBuffer, const should_free_right = if (@TypeOf(right_) == zml.Buffer) .{ try right_.toHostAlloc(allocator), true } else .{ right_, false }; + _ = try right.awaitt(); defer { if (should_free_left) left.deinit(allocator); diff --git a/zml/zml.zig b/zml/zml.zig index 088ad687..8c075bcf 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -18,6 +18,7 @@ pub const Tensor = @import("tensor.zig").Tensor; // Namespaces pub const context = @import("context.zig"); +pub const buffer = @import("buffer.zig"); pub const exe = @import("exe.zig"); pub const floats = @import("floats.zig"); pub const helpers = @import("helpers.zig");