From 39ef2d22fb07c3820e1874159109e4f1e76b5ec5 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 9 Apr 2025 15:28:16 +0200 Subject: [PATCH 01/24] add llama3 style of Rope --- examples/llama/llama.zig | 12 +-- zml/nn.zig | 166 ++++++++++++++++++++++++++++++--------- 2 files changed, 137 insertions(+), 41 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 96630359..2eba908d 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -1,14 +1,14 @@ const std = @import("std"); +const testing = std.testing; + const stdx = @import("stdx"); const zml = @import("zml"); - -const testing = std.testing; const Buffer = zml.Buffer; const Tensor = zml.Tensor; const ShapeOf = zml.ShapeOf; - const gguf = zml.io.gguf; const expectClose = zml.testing.expectClose; + const log = std.log.scoped(.llama); /// Llama architecture, using huggingface transformers naming. @@ -27,6 +27,7 @@ pub const LlamaLM = struct { max_position_embeddings: usize, rms_norm_eps: f32, hf_rope_impl: bool = true, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, }; pub const Options = struct { @@ -48,8 +49,9 @@ pub const LlamaLM = struct { self.model.num_heads = @intCast(config.num_attention_heads); self.model.num_kv_heads = @intCast(config.num_key_value_heads); self.model.rope_opts = .{ - .impl = if (config.hf_rope_impl) .sequential else .interleaved, + .layout = if (config.hf_rope_impl) .sequential else .interleaved, .freq_base = config.rope_theta, + .scaling = config.rope_scaling, }; for (self.model.layers) |*layer| { layer.self_attn.num_heads = self.model.num_heads; @@ -131,7 +133,7 @@ pub const Llama = struct { num_heads: i64 = 32, num_kv_heads: i64 = 32, rope_opts: zml.nn.RopeOpts = .{ - .impl = .interleaved, + .layout = .interleaved, .freq_base = 10_000, }, diff --git a/zml/nn.zig b/zml/nn.zig index c3120d26..62cb926d 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1,21 +1,20 @@ //! Common layer definition and functions for Neural Networks (NN) const std = @import("std"); -const stdx = @import("stdx"); - -const cuda = @import("nn/cuda.zig"); -const helpers = @import("helpers.zig"); -const meta = @import("meta.zig"); -const ops = @import("ops.zig"); -const zml = @import("zml.zig"); - -const DataType = @import("dtype.zig").DataType; -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - const assert = std.debug.assert; -const log = std.log.scoped(.@"zml/tensor"); const testing = std.testing; +const stdx = @import("stdx"); + +const DataType = @import("dtype.zig").DataType; +const helpers = @import("helpers.zig"); +const meta = @import("meta.zig"); +const cuda = @import("nn/cuda.zig"); +const ops = @import("ops.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; +const zml = @import("zml.zig"); + +const log = std.log.scoped(.@"zml/tensor"); test { _ = cuda; std.testing.refAllDecls(@This()); @@ -142,13 +141,46 @@ test normalizeL2 { } pub const RopeOpts = struct { - /// There are two implementations corresponding to how to split `x` in real/imag parts. + layout: Layout = .sequential, + freq_base: f32 = 10_000, + scaling: Scaling = .default, + + /// There are two layouts corresponding to how to split `x` in real/imag parts. /// * Interleaved means that the real/imag of each scalar is contiguous. /// * Sequential means that you first read all real values then all imag values. - pub const Implementation = enum { interleaved, sequential }; + /// Typically HF models use sequential, while GGUF use interleaved. + pub const Layout = enum { interleaved, sequential }; - impl: Implementation, - freq_base: f32 = 10_000, + /// There are several ways to init the scaling aka "inv_freq" + pub const Scaling = union(enum) { + default: void, + custom: []const f32, + llama3: Llama3, + + pub const Llama3 = struct { + factor: f32, + high_freq_factor: f32, + low_freq_factor: f32, + original_max_position_embeddings: u32, + }; + + /// Read a Rope scaling config from HF config.json format. + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { + const content = try std.json.Value.jsonParse(allocator, source, options); + if (content != .object) return error.InvalidEnumTag; + + const obj = content.object; + const impl = obj.get("rope_type") orelse return error.MissingField; + if (impl != .string) return error.InvalidEnumTag; + if (std.mem.eql(u8, impl.string, "llama3")) { + // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, undefined, content, .{ .ignore_unknown_fields = true }) }; + } else { + log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); + return .{ .default = {} }; + } + } + }; }; /// Rotary position embedding modify queries and keys tensor before compute Q * K in self attention. @@ -170,10 +202,10 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={})", .{x}); break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); }; - const x_real, const x_imag = splitRealImg(x, opts.impl); + const x_real, const x_imag = splitRealImg(x, opts.layout); // compute sin and cos in f32 before downcasting to x type. - const inv_freq = invFreq(x.dim(.hd), opts.freq_base, .f32).withTags(.{.hd}); + const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); const cos = inv_freq_pos.cos().convert(x.dtype()).broad(x_real.shape()); const sin = inv_freq_pos.sin().convert(x.dtype()).broad(x_real.shape()); @@ -183,13 +215,13 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); // flatten last dimensions - return mergeRealImg(y_real, y_imag, opts.impl); + return mergeRealImg(y_real, y_imag, opts.layout); } -pub fn splitRealImg(x: Tensor, impl: RopeOpts.Implementation) [2]Tensor { +pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { const n = x.dim(-1); - return switch (impl) { + return switch (layout) { .sequential => .{ x.slice1d(-1, .{ .end = @divExact(n, 2) }), x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), @@ -201,8 +233,8 @@ pub fn splitRealImg(x: Tensor, impl: RopeOpts.Implementation) [2]Tensor { }; } -pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, impl: RopeOpts.Implementation) Tensor { - return switch (impl) { +pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Tensor { + return switch (layout) { .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), .interleaved => Tensor.concatenate(&.{ x_real.appendAxes(.{.interleaved_real_img}), @@ -212,21 +244,83 @@ pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, impl: RopeOpts.Implementatio } /// {exp( - n * ln(10_000) / N ) | n in [0..N] } -pub fn invFreq(N: i64, theta: f32, dtype: DataType) Tensor { - const freq = -@log(theta) / @as(f32, @floatFromInt(N)); - const range = Tensor.arange(.{ .start = 0, .end = N, .step = 2 }, dtype).scale(freq); - return range.exp(); +pub fn invFreq(N: i64, opts: RopeOpts) Tensor { + const allocator = zml.module.CompilationContext.current().allocator(); + const N_half: usize = @intCast(@divExact(N, 2)); + const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); + _invFreq(opts, inv_freq); + return zml.Tensor.constantTensor(.fromSlice(.{@divExact(N, 2)}, inv_freq)); +} + +fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { + const N = inv_freq.len; + // Default frequencies + for (0.., inv_freq) |n, *f| { + f.* = @exp(-@log(opts.freq_base) * stdx.math.divFloat(f32, n, N)); + } + + switch (opts.scaling) { + .default => {}, + .custom => { + stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {}, got {}", .{ N, opts.scaling.custom.len }); + @memcpy(inv_freq, opts.scaling.custom); + }, + .llama3 => |s| { + // https://arxiv.org/pdf/2309.16039 + // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. + // In Llama3 they used a higher base freq and also downscaled low frequencies. + std.debug.assert(s.low_freq_factor < s.high_freq_factor); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.high_freq_factor * (2 * std.math.pi) / M; + const f_low = s.low_freq_factor * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Linear interpolation for middle freq + const lerp: f64 = (inv_freq[n] - f_low) / (f_high - f_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, + } +} + +test invFreq { + // Llama 3.2-1B config + const llama_conf: RopeOpts = .{ + .freq_base = 500_000, + .scaling = .{ .llama3 = .{ + .factor = 32, + .high_freq_factor = 4, + .low_freq_factor = 1, + .original_max_position_embeddings = 8192, + } }, + }; + const llama_freq = [_]f32{ 1.000000000000e+00, 6.636012792587e-01, 4.403666257858e-01, 2.922278344631e-01, 1.939227581024e-01, 1.286873817444e-01, 8.539710193872e-02, 5.666961893439e-02, 3.760603070259e-02, 2.495540864766e-02, 1.656044088304e-02, 1.098952908069e-02, 7.292665075511e-03, 4.839421249926e-03, 3.211446106434e-03, 1.290548010729e-03, 4.295567050576e-04, 9.708286233945e-05, 1.946163865796e-05, 1.291476746701e-05, 8.570255886298e-06, 5.687232260243e-06, 3.774054448513e-06, 2.504467147446e-06, 1.661967417022e-06, 1.102883629756e-06, 7.318749339902e-07, 4.856731266045e-07, 3.222932889457e-07, 2.138742303259e-07, 1.419272024350e-07, 9.418306490261e-08 }; + + var inv_freq: @TypeOf(llama_freq) = undefined; + _invFreq(llama_conf, &inv_freq); + for (llama_freq, inv_freq, 0..) |expected, actual, i| { + errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, llama_freq, inv_freq }); + try std.testing.expectApproxEqRel(expected, actual, 1e-5); + } } test "real/img" { const platform = zml.testing.env(); const Fns = struct { - fn testSplitMergeIsId(impl: RopeOpts.Implementation) Tensor { + fn testSplitMergeIsId(layout: RopeOpts.Layout) Tensor { const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, impl); - const y = mergeRealImg(real, imag, impl); - const real2, const imag2 = splitRealImg(y, impl); + const real, const imag = splitRealImg(x, layout); + const y = mergeRealImg(real, imag, layout); + const real2, const imag2 = splitRealImg(y, layout); return real.cmp(.EQ, real2).flatten(0).convert(.i32).sum(-1).add( imag.cmp(.EQ, imag2).flatten(0).convert(.i32).sum(-1), ); @@ -312,13 +406,13 @@ test rope { { // Convert input to the requested format const real, const imag = splitRealImg(input, .sequential); - input = mergeRealImg(real, imag, opts.impl); + input = mergeRealImg(real, imag, opts.layout); } var res = rope(input, null, opts).squeeze(0); { // Convert back to sequential - const real, const imag = splitRealImg(res, opts.impl); + const real, const imag = splitRealImg(res, opts.layout); res = mergeRealImg(real, imag, .sequential); } return res; @@ -328,8 +422,8 @@ test rope { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. const x = try zml.Buffer.fromSlice(platform, .{ .b = 1, .s = 5, .hd = 4 }, &[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5); - const res1 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .impl = .interleaved } }); - const res2 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .impl = .sequential } }); + const res1 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }); + const res2 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }); try zml.testing.expectClose(res1, res2, 1e-4); } From a059e6db8f7f223a7f45792e62424732b86caf70 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 9 Apr 2025 15:37:46 +0200 Subject: [PATCH 02/24] promote rmsNorm to zml.nn --- examples/llama/llama.zig | 9 +-------- zml/nn.zig | 9 +++++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 2eba908d..8314a307 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -6,8 +6,6 @@ const zml = @import("zml"); const Buffer = zml.Buffer; const Tensor = zml.Tensor; const ShapeOf = zml.ShapeOf; -const gguf = zml.io.gguf; -const expectClose = zml.testing.expectClose; const log = std.log.scoped(.llama); @@ -223,12 +221,7 @@ const RmsNorm = struct { /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: Tensor) Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); - // upcast to improve precision - const xf32 = x.convert(.f32); - const mean = xf32.mul(xf32).mean(.d); - const rsqrt = Tensor.rsqrt(mean.addConstant(self.eps)).convert(x.dtype()); - const normalized = x.mul(rsqrt.broad(x.shape())); - + const normalized = zml.nn.rmsNorm(x, .d, self.eps); return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); } }; diff --git a/zml/nn.zig b/zml/nn.zig index 62cb926d..104d793f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -107,6 +107,15 @@ pub const LayerNorm = struct { } }; +pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x.axis(axis); + // upcast to improve precision + const xf32 = x.convert(.f32); + const mean = xf32.mul(xf32).mean(ax); + const rsqrt = Tensor.rsqrt(mean.addConstant(eps)).convert(x.dtype()); + return x.mul(rsqrt.broad(x.shape())); +} + /// Center and scale by the variance. /// normalize(x, eps) = (x - mean(x)) / sqrt(var(x) + eps) /// Work on the last axis. From f8b2cd75922d1a28f808000d85dda6fe8a216476 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 10 Apr 2025 11:28:52 +0200 Subject: [PATCH 03/24] buffered tensor print --- zml/tensor.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index 79c2f2c9..8d68361b 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -3796,8 +3796,11 @@ pub const Tensor = struct { } fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { + // TODO: find a way of doing print that doesn't involve a H2D copy. const host_buffer = inputs[0]; - std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); + std.log.defaultLog(.info, .zml, "Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); + // This is true because of the operand aliases. + // Since the result is already pointing to the input we don't need to modify the buffer. std.debug.assert(host_buffer.data.ptr == outputs[0].data.ptr); } }; From ef44232a1b86b38af2973b9e47dfc928a19f88e6 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 10 Apr 2025 11:29:34 +0200 Subject: [PATCH 04/24] HostBuffer.mutItems --- zml/hostbuffer.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 1885a001..13f160c5 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -169,6 +169,10 @@ pub const HostBuffer = struct { return ptr[0..self._shape.count()]; } + pub fn mutItems(self: HostBuffer, comptime T: type) []T { + return @constCast(self.items(T)); + } + pub fn shape(self: HostBuffer) Shape { return self._shape; } From 8fb6f983265bed3411e4e1aa7bad2c606eb1728b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 10 Apr 2025 15:00:26 +0200 Subject: [PATCH 05/24] rename ffi types --- pjrt/ffi.zig | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 39e7c248..8ae77478 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -229,10 +229,10 @@ pub const TypeId = extern struct { pub const DataType = enum(c.XLA_FFI_DataType) { invalid = c.XLA_FFI_DataType_INVALID, pred = c.XLA_FFI_DataType_PRED, - s8 = c.XLA_FFI_DataType_S8, - s16 = c.XLA_FFI_DataType_S16, - s32 = c.XLA_FFI_DataType_S32, - s64 = c.XLA_FFI_DataType_S64, + i8 = c.XLA_FFI_DataType_S8, + i16 = c.XLA_FFI_DataType_S16, + i32 = c.XLA_FFI_DataType_S32, + i64 = c.XLA_FFI_DataType_S64, u8 = c.XLA_FFI_DataType_U8, u16 = c.XLA_FFI_DataType_U16, u32 = c.XLA_FFI_DataType_U32, @@ -346,6 +346,11 @@ pub const Attrs = extern struct { dtype: DataType, len: usize, data: [*]const u8, + + pub fn slice(self: Array, T: type) []const T { + const ptr: [*]const T = @alignCast(@ptrCast(self.data)); + return ptr[0..self.len]; + } }; pub fn getByIndex(self: Attrs, comptime attr_type: AttrType, index: usize) ?*const @FieldType(Attr, @tagName(attr_type)) { From ab2d3db6884fb5d11e3ccfaa609bf7d6a921f6ee Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 10 Apr 2025 15:00:44 +0200 Subject: [PATCH 06/24] pass tags to the callback --- zml/context.zig | 21 +++++++++++++++------ zml/ops.zig | 31 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/zml/context.zig b/zml/context.zig index 03de62a3..529ff9ef 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -230,22 +230,35 @@ const CustomCall = struct { fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; + const err = "malformed zmlHostBufferCallback custom call"; + const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse @panic(err); std.debug.assert(callback_attr.dtype == .u64); const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; + const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse @panic(err); std.debug.assert(user_ctx_ptr.dtype == .u64); const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); + const tags_arr = call_frame.attrs.getByName(.array, "__tags") orelse @panic(err); + std.debug.assert(tags_arr.dtype == .i64); + var tags = tags_arr.slice([*:0]const u8); + const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); for (input_buffers, 0..) |*b, i| { b.* = hostBufferFromPinnedBuffer(call_frame.args.get(i)); + for (b._shape._tags.slice()) |*t| { + t.* = tags[0]; + tags = tags[1..]; + } } const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); for (output_buffers, 0..) |*b, i| { b.* = hostBufferFromPinnedBuffer(call_frame.results.get(i)); + for (b._shape._tags.slice()) |*t| { + t.* = tags[0]; + tags = tags[1..]; + } } callback(user_ctx, input_buffers, output_buffers); @@ -258,10 +271,6 @@ fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { const dt: DataType = switch (buffer_desc.dtype) { .invalid => @panic("invalid ffi"), .pred => .bool, - .s8 => .i8, - .s16 => .i16, - .s32 => .i32, - .s64 => .i64, .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), inline else => |t| @field(DataType, @tagName(t)), }; diff --git a/zml/ops.zig b/zml/ops.zig index 1765888a..01eda102 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -782,11 +782,42 @@ pub fn addHostCallback( opts: HostCallbackOpt, ) []Tensor { const ctx = CompilationContext.current(); + var tags = std.ArrayListUnmanaged(i64).initCapacity(ctx.allocator(), (inputs.len + output_shapes.len) * Shape.MAX_RANK) catch @panic("OOM"); + defer tags.deinit(ctx.allocator()); + + var readable_tags = std.ArrayList(u8).initCapacity(ctx.allocator(), (inputs.len + output_shapes.len) * 32) catch @panic("OOM"); + defer readable_tags.deinit(); + + readable_tags.appendSlice("(") catch {}; + for (inputs) |input| { + readable_tags.appendSlice(".{") catch {}; + for (input.shape().tags()) |t| { + const ptr: u64 = @intFromPtr(t); + tags.appendAssumeCapacity(@bitCast(ptr)); + readable_tags.appendSlice(std.mem.span(t)) catch {}; + readable_tags.append(',') catch {}; + } + readable_tags.appendSlice("},") catch {}; + } + readable_tags.appendSlice(") -> (") catch {}; + for (output_shapes) |output| { + readable_tags.appendSlice(".{") catch {}; + for (output.tags()) |t| { + const ptr: u64 = @intFromPtr(t); + tags.appendAssumeCapacity(@bitCast(ptr)); + readable_tags.appendSlice(std.mem.span(t)) catch {}; + readable_tags.append(',') catch {}; + } + readable_tags.appendSlice("},") catch {}; + } + readable_tags.appendSlice(")") catch {}; const mlir_ctx = ctx.mlirCtx(); const backend_config = mlir.Attribute.dict(mlir_ctx, &.{ .{ "callback", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(callback))) }, .{ "user_context", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(blkctx))) }, + .{ "tags", .string(mlir_ctx, readable_tags.items) }, + .{ "__tags", .dense(mlir_ctx, .i64, tags.items) }, }); const values = stdx.stackSlice(8, mlir.Value, inputs.len); From 31919e590c3572338150d1bb75664fba3e99d9d0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 14 Apr 2025 10:40:55 +0200 Subject: [PATCH 07/24] HostBuffer.choose --- zml/hostbuffer.zig | 34 ++++++++++++++++++++++++++++++++-- zml/tensor.zig | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 13f160c5..8e87c6a2 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -257,6 +257,31 @@ pub const HostBuffer = struct { return self.slice1d(ax, .{ .start = start, .end = start + 1 }).squeeze(ax); } + pub fn choose(self: HostBuffer, offsets: anytype) HostBuffer { + const off, const tags = Shape.parseDimensions(offsets); + + var sh = self._shape; + var _strides: std.BoundedArray(i64, Shape.MAX_RANK) = if (self._strides) |s| + .{ .buffer = s, .len = self.rank() } + else + self._shape.computeStrides(); + + var offset: i64 = 0; + + for (off.constSlice(), tags.constSlice()) |o, t| { + const ax = sh.axis(t); + offset += o * _strides.orderedRemove(ax); + sh = sh.remove(ax); + } + + return .{ + ._shape = sh, + .data = self.data[@intCast(offset)..], + ._strides = _strides.buffer, + ._memory = .unmanaged, + }; + } + pub fn squeeze(self: HostBuffer, axis_: anytype) HostBuffer { const ax = self._shape.axis(axis_); stdx.debug.assert(self.dim(ax) == 1, "squeeze expects a 1-d axis got {} in {}", .{ ax, self }); @@ -309,15 +334,20 @@ pub const HostBuffer = struct { try writer.writeByteNTimes(' ', indent_level); return switch (self.dtype()) { inline else => |dt| { + const fmt = comptime switch (dt.class()) { + .integer => "{d}", + .float => "{e:.3}", + .bool, .complex => "{any}", + }; const values = self.items(dt.toZigType()); // Write first rows const num_cols: u32 = 12; const n: u64 = @intCast(self.dim(0)); if (n <= num_cols) { - try writer.print("{any},\n", .{values[0..n]}); + try writer.print(fmt ++ ",\n", .{values[0..n]}); } else { const half = @divExact(num_cols, 2); - try writer.print("{any}, ..., {any},\n", .{ values[0..half], values[n - half ..] }); + try writer.print(fmt ++ ", ..., " ++ fmt ++ ",\n", .{ values[0..half], values[n - half ..] }); } }, }; diff --git a/zml/tensor.zig b/zml/tensor.zig index 8d68361b..ef45c36a 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1526,8 +1526,15 @@ pub const Tensor = struct { pub const Slice = struct { start: i64 = 0, - end: ?i64 = null, - step: i64 = 1, + end: i64 = to_the_end, + step: i32 = 1, + singleton: bool = false, + + pub fn single(offset: i64) Slice { + return .{ .start = offset, .end = offset + 1, .singleton = true }; + } + + const to_the_end = std.math.maxInt(i64); }; /// Slices the input Tensor over the given axis using the given parameters. @@ -1549,13 +1556,13 @@ pub const Tensor = struct { const args: Slice = .{ .start = self.wrapIndex(a, s.start), - .end = if (s.end) |end| self.wrapIndex(a, end) else self.dim(a), + .end = if (s.end == Slice.to_the_end) self.dim(a) else self.wrapIndex(a, s.end), .step = s.step, }; start_indices[a] = args.start; - limit_indices[a] = args.end.?; + limit_indices[a] = args.end; strides[a] = args.step; - res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end.? - args.start, args.step) catch unreachable); + res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); } const mlir_ctx = self.getContext().mlirCtx(); @@ -1571,7 +1578,11 @@ pub const Tensor = struct { loc, ); - return _result(res_shape, slice_op.result(0)); + var res = _result(res_shape, slice_op.result(0)); + for (slices, 0..) |s, a| { + if (s.singleton) res_shape = res_shape.remove(a); + } + return res.reshape(res_shape); } test slice { @@ -1606,8 +1617,17 @@ pub const Tensor = struct { } pub fn choose1d(self: Tensor, axis_: anytype, i: i64) Tensor { - // TODO: this use case could be handled directly by slice if we added a .single field - return self.slice1d(axis_, .{ .start = i, .end = i + 1 }).squeeze(axis_); + return self.slice1d(axis_, .single(i)); + } + + pub fn choose(self: Tensor, offsets: anytype) Tensor { + const off, const tags = Shape.parseDimensions(offsets); + var slices = [_]Slice{.{}} ** MAX_RANK; + for (off, tags) |o, t| { + const ax = self.axis(t); + slices[ax] = .single(o); + } + return self.slice(slices[0..self.rank()]); } /// Concatenates the input Tensors along the given axis. @@ -1913,7 +1933,7 @@ pub const Tensor = struct { const result_type = mlir.ext.RankedTensorType.fromShape(ctx, val.shape()); const loc = ctx.location(@src()); const elem_type = mlir.ext.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {}", .{val.shape()}); - const constant_op = dialect.stablehlo.constant(ctx, result_type, elem_type, val.data, loc); + const constant_op = dialect.stablehlo.constant(ctx, result_type, elem_type, val.data[0..val.shape().byteSize()], loc); return _result(val.shape(), constant_op.result(0)); } From 7cead132a6b71c47cf011282427ff5aec424fc71 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 14 Apr 2025 15:11:49 +0200 Subject: [PATCH 08/24] mprotect output hostbuffers --- zml/context.zig | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/zml/context.zig b/zml/context.zig index 529ff9ef..5b882559 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -259,6 +259,17 @@ const CustomCall = struct { t.* = tags[0]; tags = tags[1..]; } + // Note: this seems to be a bug in XLA. + // The pinned memory is created in one thread, and the callback is called on another thread. + // It seems the memory protection information is not correctly propagated through all threads. + // So we need to explicitly enable writing here. + // TODO: fix this, we don't want to do syscalls on every callback call. + const data_ptr: usize = @intFromPtr(b.data.ptr); + const page_off = data_ptr % std.heap.page_size_min; + const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); + std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE) catch |e| { + log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); + }; } callback(user_ctx, input_buffers, output_buffers); From aa6cca9ea1ddf6da8c3700cb027525d2a04f3a1c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 15 Apr 2025 09:45:11 +0200 Subject: [PATCH 09/24] async.Queue fixup! async.Queue --- async/async.zig | 69 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/async/async.zig b/async/async.zig index 5cee9947..73e1a785 100644 --- a/async/async.zig +++ b/async/async.zig @@ -1,14 +1,15 @@ const std = @import("std"); + const stdx = @import("stdx"); const xev = @import("xev").Dynamic; +const XevThreadPool = @import("xev").ThreadPool; + +const aio = @import("asyncio.zig"); +const channel_mod = @import("channel.zig"); const coro = @import("coro.zig"); const executor = @import("executor.zig"); -const channel_mod = @import("channel.zig"); -const aio = @import("asyncio.zig"); const stack = @import("stack.zig"); -const XevThreadPool = @import("xev").ThreadPool; - pub const Condition = struct { inner: executor.Condition, @@ -67,6 +68,64 @@ pub fn asyncc(comptime func: anytype, args: anytype) !FrameEx(func, @TypeOf(args }; } +pub fn Queue(comptime func: anytype) type { + const Args = stdx.meta.FnArgs(func); + const Ret = stdx.meta.FnResult(func); + + stdx.debug.assertComptime(@sizeOf(Args) <= @sizeOf(stack.Stack.Data), "Queue expects arguments to fit inside {} bytes, got {}", .{ @sizeOf(stack.Stack.Data), @sizeOf(Args) }); + stdx.debug.assertComptime(Ret == void or @typeInfo(Ret) == .error_union, "Queue expects a function returning void or an error, got {}", .{func}); + const Task = struct { + const T = @This(); + + _task: XevThreadPool.Task = .{ .callback = &T.run }, + active: bool = false, + args: Args, + res: ?Ret = null, + + pub fn run(task_: *XevThreadPool.Task) void { + const task: *T = @alignCast(@fieldParentPtr("_task", task_)); + task.res = @call(.auto, func, task.args); + @atomicStore(bool, &task.active, false, .monotonic); + } + }; + + return struct { + data: []Task = &.{}, + next: usize = 0, + + const Q = @This(); + + pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) !Q { + const data = try allocator.alloc(Task, capacity); + for (data) |*d| d.active = false; + return .{ .data = data }; + } + + pub fn push(q: *Q, args: stdx.meta.FnArgs(func)) void { + const next = q.next; + const task: *Task = &q.data[next]; + while (@atomicLoad(bool, &task.active, .acquire)) {} + + task.* = .{ .active = true, .args = args }; + AsyncThread.current.thread_pool.schedule(XevThreadPool.Batch.from(&task._task)); + q.next = (next + 1) % q.data.len; + } + + /// Busy wait. + pub fn waitSync(q: *Q) Ret { + for (q.data) |*task| { + while (@atomicLoad(bool, &task.active, .acquire)) {} + if (Ret != void) if (task.res) |res| try res; + } + } + + pub fn deinit(q: *Q, allocator: std.mem.Allocator) void { + allocator.free(q.data); + q.data = &.{}; + } + }; +} + pub fn callBlocking(comptime func: anytype, args: anytype) stdx.meta.FnSignature(func, @TypeOf(args)).ReturnT { const Signature = stdx.meta.FnSignature(func, @TypeOf(args)); @@ -156,7 +215,7 @@ pub const threading = struct { }; pub const AsyncThread = struct { - threadlocal var current: *AsyncThread = undefined; + var current: *AsyncThread = undefined; executor: *aio.Executor, stack_allocator: *stack.StackAllocator, From 7c953416c81dfa3b0bb8f3b1d58507973127eba3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 15 Apr 2025 11:45:30 +0200 Subject: [PATCH 10/24] zml: accelerate HostBuffer.choose --- zml/aio.zig | 17 ++++++------ zml/buffer.zig | 2 +- zml/hostbuffer.zig | 68 ++++++++++++++++++++++++---------------------- zml/shape.zig | 19 ++++++++----- 4 files changed, 57 insertions(+), 49 deletions(-) diff --git a/zml/aio.zig b/zml/aio.zig index 98299908..7d005f8b 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -1,11 +1,9 @@ -const asynk = @import("async"); -const builtin = @import("builtin"); -const c = @import("c"); const std = @import("std"); -const stdx = @import("stdx"); +const builtin = @import("builtin"); -const zml = @import("zml.zig"); -const posix = @import("posix.zig"); +const asynk = @import("async"); +const c = @import("c"); +const stdx = @import("stdx"); pub const gguf = @import("aio/gguf.zig"); pub const nemo = @import("aio/nemo.zig"); @@ -13,10 +11,11 @@ pub const safetensors = @import("aio/safetensors.zig"); pub const tinyllama = @import("aio/tinyllama.zig"); pub const torch = @import("aio/torch.zig"); pub const yaml = @import("aio/yaml.zig"); +const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const posix = @import("posix.zig"); +const zml = @import("zml.zig"); pub const log = std.log.scoped(.@"zml/aio"); -const HostBuffer = @import("hostbuffer.zig").HostBuffer; - test { std.testing.refAllDecls(@This()); std.testing.refAllDecls(gguf); @@ -450,7 +449,7 @@ test populateModel { // Create a fake HostBuffer, we use the given integer to identify the created buffer. fn _newHostBuffer(n: u32) zml.HostBuffer { - return .{ ._shape = zml.Shape.init(.{n}, .f16), .data = undefined }; + return .{ ._shape = zml.Shape.init(.{n}, .f16), ._strides = undefined, .data = undefined }; } }; diff --git a/zml/buffer.zig b/zml/buffer.zig index e0829d39..6ceba327 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -88,7 +88,7 @@ pub const Buffer = struct { } else 0; const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); - const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); + const byte_strides = host_buffer.strides(); var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 8e87c6a2..84553c07 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -17,7 +17,7 @@ test { /// If the memory is `.unmanaged` it doesn't need to be freed (eg memory mapped, or tracked elsewhere). pub const HostBuffer = struct { _shape: Shape, - _strides: ?[Shape.MAX_RANK]i64 = null, + _strides: [Shape.MAX_RANK]i64, data: []const u8, _memory: union(enum) { managed: std.mem.Alignment, @@ -30,6 +30,7 @@ pub const HostBuffer = struct { pub fn empty(allocator: std.mem.Allocator, sh: Shape) !HostBuffer { return .{ ._shape = sh, + ._strides = sh.computeStrides().buffer, .data = try allocator.alignedAlloc(u8, 64, sh.byteSize()), ._memory = .{ .managed = .@"64" }, }; @@ -42,6 +43,7 @@ pub const HostBuffer = struct { stdx.debug.assert(shape_.byteSize() == data_.len, "shape {} and data {} don't match", .{ shape_.byteSize(), data_.len }); return .{ ._shape = shape_, + ._strides = shape_.computeStrides().buffer, .data = data_, ._memory = .unmanaged, }; @@ -64,6 +66,7 @@ pub const HostBuffer = struct { std.debug.assert(shape_.count() == s.len); return .{ ._shape = shape_, + ._strides = shape_.computeStrides().buffer, .data = @alignCast(std.mem.sliceAsBytes(s)), ._memory = .unmanaged, }; @@ -93,6 +96,7 @@ pub const HostBuffer = struct { const sh = parseArrayInfo(T); return .{ ._shape = sh, + ._strides = sh.computeStrides().buffer, .data = @alignCast(std.mem.sliceAsBytes(arr_ptr)), // Array are typically stack allocated and don't need to be freed. ._memory = .unmanaged, @@ -181,9 +185,9 @@ pub const HostBuffer = struct { return self._shape.dtype(); } - pub fn strides(self: *const HostBuffer) ?[]const i64 { + pub fn strides(self: *const HostBuffer) []const i64 { // Pass strides per pointer otherwise we return a pointer to this stack frame. - return if (self._strides) |*strd| strd[0..self.rank()] else null; + return self._strides[0..self._shape.rank()]; } // TODO: rename .data into ._data and make it a [*]u8 @@ -208,7 +212,7 @@ pub const HostBuffer = struct { } pub fn isContiguous(self: HostBuffer) bool { - const _strides = self._strides orelse return true; + const _strides = self._strides; const cont_strides = self._shape.computeStrides(); for (self._shape.dims(), _strides[0..self.rank()], cont_strides.constSlice()) |d, stride, cont_stride| { if (d != 1 and stride != cont_stride) return false; @@ -220,6 +224,7 @@ pub const HostBuffer = struct { stdx.debug.assert(self.isContiguous(), "reshape expects a contiguous tensor, got: {}", .{self}); var res = self; res._shape = self._shape.reshape(shape_); + res._strides = res._shape.computeStrides().buffer; return res; } @@ -239,15 +244,11 @@ pub const HostBuffer = struct { stdx.debug.assert(end >= 1 and end <= d, "slice1d({}, {}) expects the slice end to be between 1 and {} got: {}", .{ self, ax, d, s }); stdx.debug.assert(start < end, "slice1d({}, {}) expects the slice start ({}) to be smaller than the end ({}), got: {}", .{ self, ax, start, end, s }); - // If strides weren't set it means original buffer is contiguous. - // But it won't be anymore after slicing. The strides don't change though. - const _strides = self._strides orelse self._shape.computeStrides().buffer; - const offset: usize = @intCast(start * _strides[ax]); + const offset: usize = @intCast(start * self._strides[ax]); return .{ ._shape = self.shape().set(ax, end - start), .data = self.data[offset..], - // When axis is 0, we stay contiguous. - ._strides = if (ax == 0) self._strides else _strides, + ._strides = self._strides, ._memory = .unmanaged, }; } @@ -257,43 +258,43 @@ pub const HostBuffer = struct { return self.slice1d(ax, .{ .start = start, .end = start + 1 }).squeeze(ax); } - pub fn choose(self: HostBuffer, offsets: anytype) HostBuffer { + pub fn chooseItems(self: HostBuffer, T: type, offsets: anytype) []const T { const off, const tags = Shape.parseDimensions(offsets); + // TODO: this is a bit too restrictive, choosing slice could extract a contiguous + // slice out of a non-contiguous tensor. + std.debug.assert(self.isContiguous()); + var sh = self._shape; - var _strides: std.BoundedArray(i64, Shape.MAX_RANK) = if (self._strides) |s| - .{ .buffer = s, .len = self.rank() } - else - self._shape.computeStrides(); - var offset: i64 = 0; - + var last_sliced_ax: i8 = -1; for (off.constSlice(), tags.constSlice()) |o, t| { const ax = sh.axis(t); - offset += o * _strides.orderedRemove(ax); - sh = sh.remove(ax); + offset += o * self._strides[ax]; + sh._dims.buffer[ax] = 1; + last_sliced_ax = @max(ax, last_sliced_ax); } - return .{ - ._shape = sh, - .data = self.data[@intCast(offset)..], - ._strides = _strides.buffer, - ._memory = .unmanaged, - }; + if (last_sliced_ax > 0) { + for (0..@intCast(last_sliced_ax)) |ax| + std.debug.assert(sh._dims.buffer[ax] == 1); + } + + const ptr: [*]const T = @alignCast(@ptrCast(self.data.ptr[@bitCast(offset)..])); + return ptr[0..sh.count()]; } pub fn squeeze(self: HostBuffer, axis_: anytype) HostBuffer { const ax = self._shape.axis(axis_); stdx.debug.assert(self.dim(ax) == 1, "squeeze expects a 1-d axis got {} in {}", .{ ax, self }); - var _strides: ?[Shape.MAX_RANK]i64 = self._strides; - if (self._strides) |strydes| { - std.mem.copyForwards(i64, _strides.?[0 .. Shape.MAX_RANK - 1], strydes[1..]); - } + var strd: std.BoundedArray(i64, Shape.MAX_RANK) = .{ .buffer = self._strides, .len = self.rank() }; + _ = strd.orderedRemove(ax); + return .{ ._shape = self.shape().drop(ax), .data = self.data, - ._strides = _strides, + ._strides = strd.buffer, ._memory = self._memory, }; } @@ -304,9 +305,12 @@ pub const HostBuffer = struct { options: std.fmt.FormatOptions, writer: anytype, ) !void { - _ = fmt; _ = options; - try writer.print("HostBuffer(.{_})", .{self._shape}); + if (std.mem.eql(u8, fmt, "v")) { + try writer.print("HostBuffer(.{_})@0x{x}", .{ self._shape, @intFromPtr(self.data.ptr) }); + } else { + try writer.print("HostBuffer(.{_})", .{self._shape}); + } } /// Formatter for a HostBuffer that also print the values not just the shape. diff --git a/zml/shape.zig b/zml/shape.zig index 8da8519b..db70a9f2 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -749,15 +749,20 @@ pub const Shape = struct { } pub fn computeStrides(self: Shape) std.BoundedArray(i64, MAX_RANK) { - const base_stride = self.dtype().sizeOf(); const rk = self.rank(); - var strides: std.BoundedArray(i64, MAX_RANK) = .{ .len = @intCast(self.rank()) }; + var strides: std.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; if (rk == 0) return strides; - strides.buffer[rk - 1] = base_stride; - for (1..rk) |i| { - const j = @as(usize, rk) - 1 - i; - strides.buffer[j] = self._dims.get(j + 1) * strides.buffer[j + 1]; - } + + const V = @Vector(MAX_RANK, i64); + const rank_mask = std.simd.iota(u8, MAX_RANK) < @as(@Vector(MAX_RANK, u8), @splat(rk)); + // For each axis compute the product of all following dimensions + // and the element size in bytes. + var d: V = @bitCast(self._dims.buffer); + d = @select(i64, rank_mask, d, @as(V, @splat(1))); + d = std.simd.shiftElementsLeft(d, 1, self.dtype().sizeOf()); + d = std.simd.prefixScan(.Mul, -1, d); + + strides.buffer = @bitCast(d); return strides; } From 870cbcf3262a433d7c6403f174279530804a84f3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 17 Apr 2025 13:46:54 +0200 Subject: [PATCH 11/24] stdx.fmt: format slices this is an improved version from what was implemented in "prettyPrinter" but I had to dig a bit into the fmt internal to achieve it. Signed-off-by: Guillaume Wenzek --- stdx/BUILD.bazel | 1 + stdx/fmt.zig | 155 +++++++++++++++++++++++++++++++++++++++++++++ stdx/stdx.zig | 1 + zml/hostbuffer.zig | 62 +++++++++++------- 4 files changed, 195 insertions(+), 24 deletions(-) create mode 100644 stdx/fmt.zig diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index b1701259..f2fb4ea4 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -5,6 +5,7 @@ zig_library( srcs = [ "debug.zig", "flags.zig", + "fmt.zig", "io.zig", "json.zig", "math.zig", diff --git a/stdx/fmt.zig b/stdx/fmt.zig new file mode 100644 index 00000000..3934de1d --- /dev/null +++ b/stdx/fmt.zig @@ -0,0 +1,155 @@ +const std = @import("std"); + +pub const Fmt = union(enum) { + int: IntFmt, + float: FloatFmt, + generic: void, + + pub fn parse(T: type, comptime fmt_: []const u8) Fmt { + return switch (@typeInfo(T)) { + .float, .comptime_float => .{ .float = FloatFmt.parseComptime(fmt_) }, + .int, .comptime_int => .{ .int = IntFmt.parseComptime(fmt_) }, + else => .{ .generic = {} }, + }; + } +}; + +pub const FullFormatOptions = struct { + fmt: Fmt, + options: std.fmt.FormatOptions, +}; + +pub const IntFmt = struct { + base: u8, + case: std.fmt.Case = .lower, + + pub fn parseComptime(comptime fmt_: []const u8) IntFmt { + return parse(fmt_) catch @panic("invalid fmt for int: " ++ fmt_); + } + + pub fn parse(fmt_: []const u8) error{InvalidArgument}!IntFmt { + return if (fmt_.len == 0 or std.mem.eql(u8, fmt_, "d")) + .{ .base = 10, .case = .lower } + else if (std.mem.eql(u8, fmt_, "x")) + .{ .base = 16, .case = .lower } + else if (std.mem.eql(u8, fmt_, "X")) + .{ .base = 16, .case = .upper } + else if (std.mem.eql(u8, fmt_, "o")) + .{ .base = 8, .case = .upper } + else + // TODO: unicode/ascii + error.InvalidArgument; + } +}; + +pub const FloatFmt = enum(u8) { + scientific = @intFromEnum(std.fmt.format_float.Format.scientific), + decimal = @intFromEnum(std.fmt.format_float.Format.decimal), + hex, + + pub fn parseComptime(comptime fmt_: []const u8) FloatFmt { + return parse(fmt_) catch @panic("invalid fmt for float: " ++ fmt_); + } + + pub fn parse(fmt_: []const u8) error{InvalidArgument}!FloatFmt { + return if (fmt_.len == 0 or std.mem.eql(u8, fmt_, "e")) + .scientific + else if (std.mem.eql(u8, fmt_, "d")) + .decimal + else if (std.mem.eql(u8, fmt_, "x")) + .hex + else + error.InvalidArgument; + } +}; + +pub fn formatValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_float, .float => try formatFloatValue(value, full, writer), + .comptime_int, .int => try formatIntValue(value, full, writer), + else => try formatAnyValue(value, full, writer), + }; +} + +pub fn formatFloatValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { + const formatFloat = std.fmt.format_float.formatFloat; + var buf: [std.fmt.format_float.bufferSize(.decimal, f64)]u8 = undefined; + + const x = switch (@typeInfo(@TypeOf(value))) { + .@"struct" => value.toF32(), + .float => value, + else => @compileError("formatFloatValue expects a float, got: " ++ @typeName(@TypeOf(value))), + }; + const s_or_err = switch (full.fmt.float) { + .scientific => formatFloat(&buf, x, .{ .mode = .scientific, .precision = full.options.precision }), + .decimal => formatFloat(&buf, x, .{ .mode = .decimal, .precision = full.options.precision }), + .hex => hex: { + var buf_stream = std.io.fixedBufferStream(&buf); + std.fmt.formatFloatHexadecimal(x, full.options, buf_stream.writer()) catch unreachable; + break :hex buf_stream.getWritten(); + }, + }; + + const s = s_or_err catch "(float)"; + return std.fmt.formatBuf(s, full.options, writer); +} + +pub fn formatIntValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { + switch (@typeInfo(@TypeOf(value))) { + .int => {}, + else => @compileError("formatIntValue expects an int, got: " ++ @typeName(@TypeOf(value))), + } + return std.fmt.formatInt(value, full.fmt.int.base, full.fmt.int.case, full.options, writer); +} + +pub fn formatAnyValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { + var buf: [48]u8 = undefined; + const s = std.fmt.bufPrint(&buf, "{any}", .{value}) catch blk: { + buf[45..].* = "...".*; + break :blk buf[0..]; + }; + return std.fmt.formatBuf(s, full.options, writer); +} + +pub fn formatSliceCustom(fmt_func: anytype, values: anytype, full: FullFormatOptions, writer: anytype) !void { + + // Write first rows + const num_cols: usize = full.options.width orelse 12; + const n: usize = values.len; + _ = try writer.write("{"); + if (n <= num_cols) { + for (values, 0..) |v, i| { + // Force inlining so that the switch and the buffer can be done once. + try @call(.always_inline, fmt_func, .{ v, full, writer }); + if (i < n - 1) _ = try writer.write(","); + } + } else { + const half = @divFloor(num_cols, 2); + for (values[0..half]) |v| { + try @call(.always_inline, fmt_func, .{ v, full, writer }); + _ = try writer.write(","); + } + _ = try writer.write(" ..., "); + for (values[n - half ..], 0..) |v, i| { + try @call(.always_inline, fmt_func, .{ v, full, writer }); + if (i < half - 1) _ = try writer.write(","); + } + } + _ = try writer.write("}"); +} + +pub fn formatAny(values: anytype, full: FullFormatOptions, writer: anytype) !void { + return try formatSliceCustom(formatAnyValue, values, full, writer); +} + +pub fn formatFloatSlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { + return try formatSliceCustom(formatFloatValue, values, full, writer); +} + +pub fn formatIntSlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { + return try formatSliceCustom(formatIntValue, values, full, writer); +} + +pub fn formatAnySlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { + return try formatSliceCustom(formatAnyValue, values, full, writer); +} diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 96707c80..226c1225 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -1,5 +1,6 @@ pub const debug = @import("debug.zig"); pub const flags = @import("flags.zig"); +pub const fmt = @import("fmt.zig"); pub const io = @import("io.zig"); pub const json = @import("json.zig"); pub const math = @import("math.zig"); diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 84553c07..6f30a75d 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; +const floats = @import("floats.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -323,39 +324,52 @@ pub const HostBuffer = struct { x: HostBuffer, pub fn format(self: PrettyPrinter, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { - _ = fmt; - _ = options; - try prettyPrint(self.x, writer); + const fmt_: stdx.fmt.Fmt = switch (self.x.dtype().class()) { + .integer => .parse(i32, fmt), + .float => .parse(f32, fmt), + else => .parse(void, fmt), + }; + try prettyPrint(self.x, writer, .{ .fmt = fmt_, .options = options }); } }; - pub fn prettyPrint(self: HostBuffer, writer: anytype) !void { - return self.prettyPrintIndented(4, 0, writer); + pub fn prettyPrint(self: HostBuffer, writer: anytype, options: stdx.fmt.FullFormatOptions) !void { + return self.prettyPrintIndented(writer, 4, 0, options); } - fn prettyPrintIndented(self: HostBuffer, num_rows: u8, indent_level: u8, writer: anytype) !void { - if (self.rank() == 1) { - try writer.writeByteNTimes(' ', indent_level); + fn prettyPrintIndented(self: HostBuffer, writer: anytype, num_rows: u8, indent_level: u8, options: stdx.fmt.FullFormatOptions) !void { + if (self.rank() == 0) { + // Special case input tensor is a scalar return switch (self.dtype()) { inline else => |dt| { - const fmt = comptime switch (dt.class()) { - .integer => "{d}", - .float => "{e:.3}", - .bool, .complex => "{any}", + const val: dt.toZigType() = self.items(dt.toZigType())[0]; + return switch (comptime dt.class()) { + // Since we have custom floats, we need to explicitly convert to float32 ourselves. + .float => stdx.fmt.formatFloatValue(floats.floatCast(f32, val), options, writer), + .integer => stdx.fmt.formatIntValue(val, options, writer), + .bool, .complex => stdx.fmt.formatAnyValue(val, options, writer), }; - const values = self.items(dt.toZigType()); - // Write first rows - const num_cols: u32 = 12; - const n: u64 = @intCast(self.dim(0)); - if (n <= num_cols) { - try writer.print(fmt ++ ",\n", .{values[0..n]}); - } else { - const half = @divExact(num_cols, 2); - try writer.print(fmt ++ ", ..., " ++ fmt ++ ",\n", .{ values[0..half], values[n - half ..] }); - } }, }; } + if (self.rank() == 1) { + // Print a contiguous slice of items from the buffer in one line. + // The number of items printed is controlled by the user through format syntax. + try writer.writeByteNTimes(' ', indent_level); + switch (self.dtype()) { + inline else => |dt| { + const values = self.items(dt.toZigType()); + switch (comptime dt.class()) { + .float => try stdx.fmt.formatFloatSlice(values, options, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, writer), + .bool, .complex => try stdx.fmt.formatAnySlice(values, options, writer), + } + }, + } + _ = try writer.write("\n"); + return; + } + // TODO: consider removing the \n if dim is 1 for this axis. try writer.writeByteNTimes(' ', indent_level); _ = try writer.write("{\n"); defer { @@ -368,7 +382,7 @@ pub const HostBuffer = struct { for (0..@min(num_rows, n)) |d| { const di: i64 = @intCast(d); const sliced_self = self.slice1d(0, .{ .start = di, .end = di + 1 }).squeeze(0); - try sliced_self.prettyPrintIndented(num_rows, indent_level + 2, writer); + try sliced_self.prettyPrintIndented(writer, num_rows, indent_level + 2, options); } if (n < num_rows) return; @@ -381,7 +395,7 @@ pub const HostBuffer = struct { for (@max(n - num_rows, num_rows)..n) |d| { const di: i64 = @intCast(d); const sliced_self = self.slice1d(0, .{ .start = di, .end = di + 1 }).squeeze(0); - try sliced_self.prettyPrintIndented(num_rows, indent_level + 2, writer); + try sliced_self.prettyPrintIndented(writer, num_rows, indent_level + 2, options); } } }; From 74434b9947739c9673d3d63a4d2821fe0074f6d5 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 17 Apr 2025 16:38:20 +0200 Subject: [PATCH 12/24] explicit cudaCopy for hostBufferCallback --- pjrt/ffi.zig | 13 ++--- pjrt/pjrt.zig | 6 +-- zml/buffer.zig | 4 +- zml/context.zig | 138 +++++++++++++++++++++++++++++++++++++----------- zml/module.zig | 6 ++- zml/ops.zig | 4 +- zml/tensor.zig | 20 +++++++ 7 files changed, 147 insertions(+), 44 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 8ae77478..c653aff3 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -88,10 +88,12 @@ fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { }; } +pub const Stream = opaque {}; + pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; - pub fn getStream(self: *const Api, context: ?*ExecutionContext) ApiError!*anyopaque { + pub fn stream(self: *const Api, context: ?*ExecutionContext) *Stream { var ret = pjrtStruct(c.XLA_FFI_Stream_Get_Args{ .ctx = if (context) |ctx| ctx.inner() else null, }); @@ -102,11 +104,10 @@ pub const Api = opaque { defer err.destroy(self); log.err("[Api.getStream] {s}", .{err.getMessage(self)}); - // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - return error.Unknown; + @panic("failed to get stream"); } - return ret.stream.?; + return @ptrCast(ret.stream.?); } pub fn allocateDeviceMemory(self: *const Api, context: ?*ExecutionContext, size: usize, alignment: usize) ApiError!*anyopaque { @@ -375,8 +376,8 @@ pub const Attrs = extern struct { pub const CallFrame = extern struct { struct_size: usize, extension_start: ?*ExtensionBase, - api: ?*const Api, - ctx: ?*const ExecutionContext, + api: *const Api, + ctx: ?*ExecutionContext, stage: ExecutionStage, args: Args, results: Rets, diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 2b0e65b6..c5e2118b 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,7 +31,7 @@ fn pjrtStructSize(comptime T: type) usize { return @field(c, typedef_name ++ "_STRUCT_SIZE"); } -inline fn pjrtStruct(v: anytype) @TypeOf(v) { +pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { var ret = v; ret.struct_size = pjrtStructSize(@TypeOf(v)); return ret; @@ -414,7 +414,7 @@ pub const Client = opaque { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.C) void {} }.call, on_delete_callback_arg: ?*anyopaque = null, - stream: ?isize = null, + stream: ?*const ffi.Stream = null, }; pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { @@ -429,7 +429,7 @@ pub const Client = opaque { .device = @ptrCast(@constCast(args.device)), .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, - .stream = if (args.stream) |stream| stream else 0, + .stream = @bitCast(@intFromPtr(args.stream)), }); return @ptrCast(ret.buffer.?); } diff --git a/zml/buffer.zig b/zml/buffer.zig index 6ceba327..89f572dd 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -233,7 +233,7 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. - pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const anyopaque, device_data: *anyopaque) Buffer { + pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const pjrt.ffi.Stream, device_data: *anyopaque) Buffer { const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { var res: [Shape.MAX_RANK]i64 = undefined; for (0..Shape.MAX_RANK) |i| { @@ -255,7 +255,7 @@ pub const Buffer = struct { .tile_dims_sizes = &.{}, }, }, - .stream = @bitCast(@as(usize, @intFromPtr(stream))), + .stream = stream, }) catch @panic("failed to createViewOfDeviceBuffer"); var shards: Shards = .{}; diff --git a/zml/context.zig b/zml/context.zig index 5b882559..092d3d89 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -24,6 +24,8 @@ test { std.testing.refAllDecls(Context); } +var _active_platform: Platform = undefined; + /// Every program using ZML must start with a `zml.Context.init(.{});` /// The ZML context contains global state to interact with the different /// devices available on your system. @@ -38,6 +40,7 @@ pub const Context = struct { if (runtimes.load(t)) |api| { Context.apis.set(t, api); } else |_| {} + if (t == .cuda) cuda.init(); } } }.call); @@ -128,7 +131,10 @@ pub const Context = struct { pub fn autoPlatform(self: *Context, opts: Platform.CreateOptions) Platform { stdx.debug.assert(prefered_targets.len == apis.values.len, "New target need to be inserted inside `zml.Context.preferred_targets`", .{}); - return self.platformByPreferences(opts, &prefered_targets); + const p = self.platformByPreferences(opts, &prefered_targets); + // TODO properly propagate the platform to the callbacks + _active_platform = p; + return p; } /// Given a list of preferred targets to select the best Platform @@ -230,6 +236,7 @@ const CustomCall = struct { fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; + // TODO correctly pass the callback w/o embedding ptr in MLIR const err = "malformed zmlHostBufferCallback custom call"; const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse @panic(err); std.debug.assert(callback_attr.dtype == .u64); @@ -241,42 +248,90 @@ const CustomCall = struct { const tags_arr = call_frame.attrs.getByName(.array, "__tags") orelse @panic(err); std.debug.assert(tags_arr.dtype == .i64); - var tags = tags_arr.slice([*:0]const u8); + const tags = tags_arr.slice([*:0]const u8); const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.args.get(i)); - for (b._shape._tags.slice()) |*t| { - t.* = tags[0]; - tags = tags[1..]; - } - } - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.results.get(i)); - for (b._shape._tags.slice()) |*t| { - t.* = tags[0]; - tags = tags[1..]; + + // TODO correctly pass the platform w/o global + const platform = _active_platform; + if (platform.target == .cpu) { + for (input_buffers, 0..) |*b, i| { + b.* = hostViewOfDeviceBuffer(call_frame.args.get(i)); } - // Note: this seems to be a bug in XLA. - // The pinned memory is created in one thread, and the callback is called on another thread. - // It seems the memory protection information is not correctly propagated through all threads. - // So we need to explicitly enable writing here. - // TODO: fix this, we don't want to do syscalls on every callback call. - const data_ptr: usize = @intFromPtr(b.data.ptr); - const page_off = data_ptr % std.heap.page_size_min; - const page_start: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(data_ptr - page_off); - std.posix.mprotect(page_start[0 .. b.data.len + page_off], std.posix.PROT.WRITE) catch |e| { - log.err("Failed to protect memory of buffer {}: {}", .{ b, e }); - }; + for (output_buffers, 0..) |*b, i| { + b.* = hostViewOfDeviceBuffer(call_frame.results.get(i)); + } + applyTags(tags, input_buffers, output_buffers); + + callback(user_ctx, input_buffers, output_buffers); + return null; } + if (platform.target != .cuda) @panic("hostBufferCallback are only supported on cpu and cuda for now"); + const stream = call_frame.*.api.stream(call_frame.ctx); + for (input_buffers, 0..) |*b, i| { + const d = call_frame.args.get(i); + b.* = HostBuffer.empty(std.heap.smp_allocator, getShape(d)) catch @panic("OOM"); + const memcpy_err = cuda.memcpyAsync(@constCast(b.data.ptr), d.data, b.shape().byteSize(), .device_to_host, stream); + std.debug.assert(memcpy_err == 0); + } + for (output_buffers, 0..) |*b, i| { + const d = call_frame.results.get(i); + b.* = HostBuffer.empty(std.heap.smp_allocator, getShape(d)) catch @panic("OOM"); + } + applyTags(tags, input_buffers, output_buffers); + + const d2h_err = cuda.streamSynchronize(stream); + std.debug.assert(d2h_err == 0); callback(user_ctx, input_buffers, output_buffers); + + for (output_buffers, 0..) |*b, i| { + const d = call_frame.args.get(i); + b.deinit(std.heap.smp_allocator); + const memcpy_err = cuda.memcpyAsync(d.data, b.data.ptr, b.shape().byteSize(), .host_to_device, stream); + std.debug.assert(memcpy_err == 0); + } + for (input_buffers) |*b| { + b.deinit(std.heap.smp_allocator); + } + const h2d_err = cuda.streamSynchronize(stream); + std.debug.assert(h2d_err == 0); + return null; } }; +const cuda = struct { + var memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); + var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); + + pub const MemcpyKind = enum(c_int) { + host_to_host = 0, + host_to_device = 1, + device_to_host = 2, + device_to_device = 3, + }; + + const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + const StreamSynchronize = *const fn (stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + + pub fn init() void { + var cudart = std.DynLib.open("libcudart.so.12") catch { + log.err("cudart not found, callback will segfault", .{}); + return; + }; + defer cudart.close(); + + memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { + @panic("cudaMemcpyAsync not found"); + }; + streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { + @panic("cudaStreamSynchronize not found"); + }; + } +}; + fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { // log.warn("received buffer {}", .{buffer_desc}); const dt: DataType = switch (buffer_desc.dtype) { @@ -288,13 +343,36 @@ fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { return Shape.init(buffer_desc.dims(), dt); } -/// Create a HostBuffer from a ffi description of a buffer. -/// Normally the ffi describe device buffer but we assume they are located in pinned memory, -/// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { +/// Create a Device from a ffi description of a buffer. +/// Normally the ffi describe device buffer but when platform == cpu it's the same addresspace +fn hostViewOfDeviceBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { const buffer_shape = getShape(buffer_desc); return HostBuffer.fromBytes( buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()], ); } + +fn deviceBuffer(platform: Platform, buffer_desc: *const pjrt.ffi.Buffer, stream: *const pjrt.ffi.Stream) Buffer { + const buffer_shape = getShape(buffer_desc); + return Buffer.asViewOfDeviceBuffer(platform, buffer_shape, stream, buffer_desc.data); +} + +fn applyTags(flat_tags: []const [*:0]const u8, in: []HostBuffer, out: []HostBuffer) void { + var tags = flat_tags; + for (in) |*b| { + for (b._shape._tags.slice()) |*t| { + t.* = tags[0]; + tags = tags[1..]; + } + } + + for (out) |*b| { + for (b._shape._tags.slice()) |*t| { + t.* = tags[0]; + tags = tags[1..]; + } + } + + std.debug.assert(tags.len == 0); +} diff --git a/zml/module.zig b/zml/module.zig index 011e6fe6..773cb274 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -540,9 +540,13 @@ pub const CompilationContext = struct { } } + pub fn numPartitions(self: CompilationContext) u8 { + return self._platform.sharding().num_partitions; + } + pub fn getShardingAttr(self: CompilationContext, shape: Shape) mlir.Attribute { const ctx = self.mlirCtx(); - const num_partitions = self._platform.sharding().num_partitions; + const num_partitions = self.numPartitions(); var sharding_str: std.BoundedArray(u8, 128) = .{}; writeShardingRepresentation(shape, num_partitions, sharding_str.writer()) catch unreachable; return mlir.Attribute.string(ctx, sharding_str.constSlice()); diff --git a/zml/ops.zig b/zml/ops.zig index 01eda102..cf651a53 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -822,7 +822,7 @@ pub fn addHostCallback( const values = stdx.stackSlice(8, mlir.Value, inputs.len); for (inputs, values) |i, *v| { - v.* = ctx.getValue(i.toMemory(.host_pinned)); + v.* = ctx.getValue(i); } const res_types = stdx.stackSlice(8, mlir.Type, output_shapes.len); for (res_types, output_shapes) |*r, o| { @@ -846,7 +846,7 @@ pub fn addHostCallback( const res = ctx.allocator().alloc(Tensor, output_shapes.len) catch @panic("OOM"); for (res, output_shapes, 0..) |*r, o, i| { - r.* = Tensor._result(o, op.result(i)).toMemory(.device); + r.* = Tensor._result(o, op.result(i)); } return res; diff --git a/zml/tensor.zig b/zml/tensor.zig index ef45c36a..b8898d53 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -176,6 +176,7 @@ pub const Tensor = struct { var res = self; res._shape = self._shape.withSharding(axes_); + if (ctx.numPartitions() <= 1) return self; const op = dialect.stablehlo.custom_call( mlir_ctx, &.{self.value()}, @@ -1535,6 +1536,25 @@ pub const Tensor = struct { } const to_the_end = std.math.maxInt(i64); + + pub fn format( + self: Slice, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + if (self.singleton) { + try writer.print("[{}]", .{self.start}); + } else if (self.end == to_the_end and self.step == 1) { + try writer.print("[{}..]", .{self.start}); + } else if (self.step == 1) { + try writer.print("[{}..{}]", .{ self.start, self.end }); + } else { + try writer.print("[{}..{}:{}]", .{ self.start, self.end, self.step }); + } + } }; /// Slices the input Tensor over the given axis using the given parameters. From 5cc95c2a5e4903abc3bec60d6fd9a29d4de90497 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 18 Apr 2025 10:59:11 +0200 Subject: [PATCH 13/24] make it work using cuda memcpy blocking --- pjrt/ffi.zig | 20 ++++++---- zml/context.zig | 98 ++++++++++++++++++++++++++++++++----------------- zml/ops.zig | 17 ++++++--- zml/tensor.zig | 2 +- 4 files changed, 91 insertions(+), 46 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index c653aff3..48c924ad 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -254,11 +254,19 @@ pub const DataType = enum(c.XLA_FFI_DataType) { f8e4m3fnuz = c.XLA_FFI_DataType_F8E4M3FNUZ, }; +pub const DevicePtr = enum(usize) { + _, + + pub fn asPtr(ptr: DevicePtr) [*]u8 { + return @ptrFromInt(@intFromEnum(ptr)); + } +}; + pub const Buffer = extern struct { struct_size: usize, extension_start: ?*c.XLA_FFI_Extension_Base, dtype: DataType, - data: [*]u8, + data: DevicePtr, rank: u64, _dims: [*]const i64, @@ -290,9 +298,8 @@ pub const Args = extern struct { buffer = c.XLA_FFI_ArgType_BUFFER, }; - pub fn get(self: Args, i: usize) *const Buffer { - std.debug.assert(self.types[0..self.len][i] == .buffer); - return self.ptr[0..self.len][i]; + pub fn buffers(self: Args) []*const Buffer { + return self.ptr[0..self.len]; } }; @@ -307,9 +314,8 @@ pub const Rets = extern struct { buffer = c.XLA_FFI_RetType_BUFFER, }; - pub fn get(self: Rets, i: usize) *const Buffer { - std.debug.assert(self.types[0..self.len][i] == .buffer); - return self.ptr[0..self.len][i]; + pub fn buffers(self: Rets) []*const Buffer { + return self.ptr[0..self.len]; } }; diff --git a/zml/context.zig b/zml/context.zig index 092d3d89..38087e50 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -250,60 +250,66 @@ const CustomCall = struct { std.debug.assert(tags_arr.dtype == .i64); const tags = tags_arr.slice([*:0]const u8); - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); + const host_inputs = stdx.stackSlice(8, HostBuffer, call_frame.args.len); + const host_outputs = stdx.stackSlice(8, HostBuffer, call_frame.results.len); // TODO correctly pass the platform w/o global const platform = _active_platform; if (platform.target == .cpu) { - for (input_buffers, 0..) |*b, i| { - b.* = hostViewOfDeviceBuffer(call_frame.args.get(i)); + for (host_inputs, call_frame.args.buffers()) |*host, device| { + host.* = hostViewOfDeviceBuffer(device); } - for (output_buffers, 0..) |*b, i| { - b.* = hostViewOfDeviceBuffer(call_frame.results.get(i)); + for (host_outputs, call_frame.results.buffers()) |*host, device| { + host.* = hostViewOfDeviceBuffer(device); } - applyTags(tags, input_buffers, output_buffers); + applyTags(tags, host_inputs, host_outputs); - callback(user_ctx, input_buffers, output_buffers); + callback(user_ctx, host_inputs, host_outputs); return null; } if (platform.target != .cuda) @panic("hostBufferCallback are only supported on cpu and cuda for now"); const stream = call_frame.*.api.stream(call_frame.ctx); - for (input_buffers, 0..) |*b, i| { - const d = call_frame.args.get(i); - b.* = HostBuffer.empty(std.heap.smp_allocator, getShape(d)) catch @panic("OOM"); - const memcpy_err = cuda.memcpyAsync(@constCast(b.data.ptr), d.data, b.shape().byteSize(), .device_to_host, stream); - std.debug.assert(memcpy_err == 0); - } - for (output_buffers, 0..) |*b, i| { - const d = call_frame.results.get(i); - b.* = HostBuffer.empty(std.heap.smp_allocator, getShape(d)) catch @panic("OOM"); - } - applyTags(tags, input_buffers, output_buffers); - const d2h_err = cuda.streamSynchronize(stream); std.debug.assert(d2h_err == 0); - callback(user_ctx, input_buffers, output_buffers); + for (host_inputs, call_frame.args.buffers()) |*host, device| { + host.* = HostBuffer.empty(std.heap.smp_allocator, getShape(device)) catch @panic("OOM"); + // log.warn("Readind device memory from {*} to {*}", .{ device.data.asPtr(), b.data }); + cuda.memcpyToHostBlocking(@constCast(host.data), device.data); + // log.info("input {} {}: {}", .{ i, b.shape(), b.pretty() }); + } - for (output_buffers, 0..) |*b, i| { - const d = call_frame.args.get(i); - b.deinit(std.heap.smp_allocator); - const memcpy_err = cuda.memcpyAsync(d.data, b.data.ptr, b.shape().byteSize(), .host_to_device, stream); - std.debug.assert(memcpy_err == 0); + init_host_outputs: for (host_outputs, call_frame.results.buffers()) |*host, device| { + // TODO: in case of input/output aliasing we don't need to allocate here. + // This information can be inferred from the data pointers of the inputs. + for (host_inputs, call_frame.args.buffers()) |host_input, device_input| { + if (device_input.data == device.data) { + host.* = host_input; + log.warn("Aliased host buffer for output: {*}", .{host.data}); + continue :init_host_outputs; + } + } + host.* = HostBuffer.empty(std.heap.smp_allocator, getShape(device)) catch @panic("OOM"); + // log.warn("Allocated host buffer for output: {*}", .{host.data}); } - for (input_buffers) |*b| { - b.deinit(std.heap.smp_allocator); + applyTags(tags, host_inputs, host_outputs); + + callback(user_ctx, host_inputs, host_outputs); + + for (host_outputs, call_frame.results.buffers()) |host, device| { + // log.warn("Writing from {*} to device memory {*}", .{ device.data, host.data.asPtr() }); + cuda.memcpyToDeviceBlocking(device.data, host.data); } - const h2d_err = cuda.streamSynchronize(stream); - std.debug.assert(h2d_err == 0); + for (host_inputs) |*b| b.deinit(std.heap.smp_allocator); + for (host_outputs) |*b| b.deinit(std.heap.smp_allocator); return null; } }; const cuda = struct { - var memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); + var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); + var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); pub const MemcpyKind = enum(c_int) { @@ -311,9 +317,11 @@ const cuda = struct { host_to_device = 1, device_to_host = 2, device_to_device = 3, + inferred = 4, }; const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + const MemcpyBlocking = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind) callconv(.C) c_int; const StreamSynchronize = *const fn (stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; pub fn init() void { @@ -323,13 +331,37 @@ const cuda = struct { }; defer cudart.close(); - memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { + _memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { @panic("cudaMemcpyAsync not found"); }; + _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { + @panic("cudaMemcpy not found"); + }; streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { @panic("cudaStreamSynchronize not found"); }; } + + pub fn memcpyToHostBlocking(dst: []u8, src: pjrt.ffi.DevicePtr) void { + const err = _memcpyBlocking(dst.ptr, src.asPtr(), dst.len, .device_to_host); + check(err); + } + + pub fn memcpyToDeviceBlocking(dst: pjrt.ffi.DevicePtr, src: []const u8) void { + const err = _memcpyBlocking(dst.asPtr(), src.ptr, src.len, .host_to_device); + check(err); + } + + pub fn memcpyToDeviceAsync(dst: pjrt.ffi.DevicePtr, src: []const u8, stream: ?*pjrt.ffi.Stream) void { + const err = _memcpyAsync(dst.asPtr(), src.ptr, src.len, .host_to_device, stream); + check(err); + } + + pub fn check(err: c_int) void { + if (err == 0) return; + log.err("CUDA ERROR {d}", .{err}); + @panic("CUDA error"); + } }; fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { @@ -349,7 +381,7 @@ fn hostViewOfDeviceBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { const buffer_shape = getShape(buffer_desc); return HostBuffer.fromBytes( buffer_shape, - buffer_desc.data[0..buffer_shape.byteSize()], + buffer_desc.data.asPtr()[0..buffer_shape.byteSize()], ); } diff --git a/zml/ops.zig b/zml/ops.zig index cf651a53..14580d9d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -821,12 +821,17 @@ pub fn addHostCallback( }); const values = stdx.stackSlice(8, mlir.Value, inputs.len); - for (inputs, values) |i, *v| { - v.* = ctx.getValue(i); + const operands_layouts = stdx.stackSlice(8, []const usize, inputs.len); + for (inputs, 0..) |input, i| { + values[i] = ctx.getValue(input); + operands_layouts[i] = minorToMajor(input.rank()); } + const res_types = stdx.stackSlice(8, mlir.Type, output_shapes.len); - for (res_types, output_shapes) |*r, o| { - r.* = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o).as(mlir.Type); + const results_layouts = stdx.stackSlice(8, []const usize, output_shapes.len); + for (output_shapes, 0..) |o, i| { + res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o).as(mlir.Type); + results_layouts[i] = minorToMajor(o.rank()); } const loc = ctx.mlirCtx().location(@src()); @@ -838,6 +843,8 @@ pub fn addHostCallback( .api_version = .typed_ffi, .backend_config = backend_config, .has_side_effect = opts.has_side_effect, + .operand_layouts = operands_layouts, + .result_layouts = results_layouts, .output_operand_aliases = opts.output_operand_aliases, }, res_types, @@ -846,7 +853,7 @@ pub fn addHostCallback( const res = ctx.allocator().alloc(Tensor, output_shapes.len) catch @panic("OOM"); for (res, output_shapes, 0..) |*r, o, i| { - r.* = Tensor._result(o, op.result(i)); + r.* = Tensor._result(o, op.result(i)); //.toMemory(.device); } return res; diff --git a/zml/tensor.zig b/zml/tensor.zig index b8898d53..5e48ba21 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -3826,6 +3826,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { + // TODO: find a way of doing print that doesn't involve a H2D copy. return ops.addHostCallback( &printCallback, null, @@ -3836,7 +3837,6 @@ pub const Tensor = struct { } fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - // TODO: find a way of doing print that doesn't involve a H2D copy. const host_buffer = inputs[0]; std.log.defaultLog(.info, .zml, "Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); // This is true because of the operand aliases. From 4e49d6a1b7716284c8a1d5e5e079f96583bde654 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 18 Apr 2025 14:23:29 +0200 Subject: [PATCH 14/24] SIMD copyTags --- zml/context.zig | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/zml/context.zig b/zml/context.zig index 38087e50..cc303d9f 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -262,7 +262,7 @@ const CustomCall = struct { for (host_outputs, call_frame.results.buffers()) |*host, device| { host.* = hostViewOfDeviceBuffer(device); } - applyTags(tags, host_inputs, host_outputs); + copyTags(tags, host_inputs, host_outputs); callback(user_ctx, host_inputs, host_outputs); return null; @@ -274,9 +274,9 @@ const CustomCall = struct { std.debug.assert(d2h_err == 0); for (host_inputs, call_frame.args.buffers()) |*host, device| { host.* = HostBuffer.empty(std.heap.smp_allocator, getShape(device)) catch @panic("OOM"); - // log.warn("Readind device memory from {*} to {*}", .{ device.data.asPtr(), b.data }); + // log.warn("Readind device memory from {*} to {*}", .{ device.data.asPtr(), host.data }); cuda.memcpyToHostBlocking(@constCast(host.data), device.data); - // log.info("input {} {}: {}", .{ i, b.shape(), b.pretty() }); + // log.info("input {} {}: {}", .{ i, host.shape(), host.pretty() }); } init_host_outputs: for (host_outputs, call_frame.results.buffers()) |*host, device| { @@ -285,14 +285,15 @@ const CustomCall = struct { for (host_inputs, call_frame.args.buffers()) |host_input, device_input| { if (device_input.data == device.data) { host.* = host_input; - log.warn("Aliased host buffer for output: {*}", .{host.data}); + std.debug.assert(host_input.shape().eql(getShape(device))); + // log.warn("Aliased host buffer for output: {*}", .{host.data}); continue :init_host_outputs; } } host.* = HostBuffer.empty(std.heap.smp_allocator, getShape(device)) catch @panic("OOM"); // log.warn("Allocated host buffer for output: {*}", .{host.data}); } - applyTags(tags, host_inputs, host_outputs); + copyTags(tags, host_inputs, host_outputs); callback(user_ctx, host_inputs, host_outputs); @@ -390,21 +391,19 @@ fn deviceBuffer(platform: Platform, buffer_desc: *const pjrt.ffi.Buffer, stream: return Buffer.asViewOfDeviceBuffer(platform, buffer_shape, stream, buffer_desc.data); } -fn applyTags(flat_tags: []const [*:0]const u8, in: []HostBuffer, out: []HostBuffer) void { - var tags = flat_tags; +fn copyTags(flat_tags: []const [*:0]const u8, in: []HostBuffer, out: []HostBuffer) void { + var tags = flat_tags.ptr; + var num_read: usize = 0; for (in) |*b| { - for (b._shape._tags.slice()) |*t| { - t.* = tags[0]; - tags = tags[1..]; - } + const r = b.rank(); + b._shape._tags.buffer = tags[0..8].*; + tags = tags[r..]; + num_read += r; } - for (out) |*b| { - for (b._shape._tags.slice()) |*t| { - t.* = tags[0]; - tags = tags[1..]; - } + const r = b.rank(); + b._shape._tags.buffer = tags[0..8].*; + tags = tags[r..]; + num_read += r; } - - std.debug.assert(tags.len == 0); } From 975f24d8c089b93b87920b3660248d60154f0c46 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Apr 2025 11:44:31 +0200 Subject: [PATCH 15/24] memcpyAsync --- zml/context.zig | 33 +++++++++++++++++++------ zml/zml.zig | 66 ++++++++++++++++++++++++------------------------- 2 files changed, 57 insertions(+), 42 deletions(-) diff --git a/zml/context.zig b/zml/context.zig index cc303d9f..cbf4e102 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -270,13 +270,14 @@ const CustomCall = struct { if (platform.target != .cuda) @panic("hostBufferCallback are only supported on cpu and cuda for now"); const stream = call_frame.*.api.stream(call_frame.ctx); - const d2h_err = cuda.streamSynchronize(stream); - std.debug.assert(d2h_err == 0); for (host_inputs, call_frame.args.buffers()) |*host, device| { + // Note: we are using the smp allocator here. + // Typically a given custom call will be called a lot of times with the same shape + // So having a per custom call arena would be great. + // We do need to be careful for the case where two callbacks are called at the same time though. host.* = HostBuffer.empty(std.heap.smp_allocator, getShape(device)) catch @panic("OOM"); // log.warn("Readind device memory from {*} to {*}", .{ device.data.asPtr(), host.data }); - cuda.memcpyToHostBlocking(@constCast(host.data), device.data); - // log.info("input {} {}: {}", .{ i, host.shape(), host.pretty() }); + cuda.memcpyToHostAsync(@constCast(host.data), device.data, stream); } init_host_outputs: for (host_outputs, call_frame.results.buffers()) |*host, device| { @@ -295,13 +296,19 @@ const CustomCall = struct { } copyTags(tags, host_inputs, host_outputs); + cuda.streamSynchronize(stream); + + // TODO we should return an pjrt.ffi.Future to allow the pjrt thread to schedule more work. callback(user_ctx, host_inputs, host_outputs); for (host_outputs, call_frame.results.buffers()) |host, device| { // log.warn("Writing from {*} to device memory {*}", .{ device.data, host.data.asPtr() }); - cuda.memcpyToDeviceBlocking(device.data, host.data); + cuda.memcpyToDeviceAsync(device.data, host.data, stream); } + for (host_inputs) |*b| b.deinit(std.heap.smp_allocator); + // TODO we could return here and schedule the sync/free in another task. + cuda.streamSynchronize(stream); for (host_outputs) |*b| b.deinit(std.heap.smp_allocator); return null; @@ -311,7 +318,7 @@ const CustomCall = struct { const cuda = struct { var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); - var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); + var _streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); pub const MemcpyKind = enum(c_int) { host_to_host = 0, @@ -338,7 +345,7 @@ const cuda = struct { _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { @panic("cudaMemcpy not found"); }; - streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { + _streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { @panic("cudaStreamSynchronize not found"); }; } @@ -348,16 +355,26 @@ const cuda = struct { check(err); } + pub fn memcpyToHostAsync(dst: []u8, src: pjrt.ffi.DevicePtr, stream: *pjrt.ffi.Stream) void { + const err = _memcpyAsync(dst.ptr, src.asPtr(), dst.len, .device_to_host, stream); + check(err); + } + pub fn memcpyToDeviceBlocking(dst: pjrt.ffi.DevicePtr, src: []const u8) void { const err = _memcpyBlocking(dst.asPtr(), src.ptr, src.len, .host_to_device); check(err); } - pub fn memcpyToDeviceAsync(dst: pjrt.ffi.DevicePtr, src: []const u8, stream: ?*pjrt.ffi.Stream) void { + pub fn memcpyToDeviceAsync(dst: pjrt.ffi.DevicePtr, src: []const u8, stream: *pjrt.ffi.Stream) void { const err = _memcpyAsync(dst.asPtr(), src.ptr, src.len, .host_to_device, stream); check(err); } + pub fn streamSynchronize(stream: *pjrt.ffi.Stream) void { + const err = _streamSynchronize(stream); + check(err); + } + pub fn check(err: c_int) void { if (err == 0) return; log.err("CUDA ERROR {d}", .{err}); diff --git a/zml/zml.zig b/zml/zml.zig index 18282b93..26d821db 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -3,56 +3,54 @@ //! compiling it for various accelerators and targets, and executing it. //! -pub const Buffer = @import("buffer.zig").Buffer; -pub const Bufferized = @import("tensor.zig").Bufferized; -pub const CompilationOptions = @import("platform.zig").CompilationOptions; -pub const Context = @import("context.zig").Context; -pub const Data = @import("dtype.zig").Data; -pub const DataType = @import("dtype.zig").DataType; -pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; -pub const Platform = @import("platform.zig").Platform; -pub const Shape = @import("shape.zig").Shape; -pub const ShapeOf = @import("tensor.zig").ShapeOf; -pub const Target = @import("platform.zig").Target; -pub const Tensor = @import("tensor.zig").Tensor; +const std = @import("std"); -// Namespaces -pub const context = @import("context.zig"); -pub const exe = @import("exe.zig"); -pub const floats = @import("floats.zig"); -pub const helpers = @import("helpers.zig"); -pub const nn = @import("nn.zig"); -pub const module = @import("module.zig"); -pub const meta = @import("meta.zig"); -pub const platform = @import("platform.zig"); -pub const mlir = @import("mlir.zig"); -pub const pjrt = @import("pjrtx.zig"); -pub const testing = @import("testing.zig"); -pub const torch = @import("torch.zig"); - -// pub const tokenizer = @import("tokenizer.zig"); pub const tokenizer = @import("zml/tokenizer"); -pub const call = ops.call; +pub const aio = @import("aio.zig"); +pub const sentencepiece = @import("aio/sentencepiece.zig"); +pub const Buffer = @import("buffer.zig").Buffer; +pub const Bufferized = @import("tensor.zig").Bufferized; +pub const bufferTypeFromDtype = @import("buffer.zig").bufferTypeFromDtype; +pub const CompilationOptions = @import("platform.zig").CompilationOptions; +pub const Context = @import("context.zig").Context; +pub const context = @import("context.zig"); +pub const Data = @import("dtype.zig").Data; +pub const DataType = @import("dtype.zig").DataType; +pub const exe = @import("exe.zig"); pub const compile = exe.compile; pub const compileFn = exe.compileFn; pub const compileModel = exe.compileModel; pub const FnExe = exe.FnExe; pub const ModuleExe = exe.ModuleExe; pub const ModuleSignature = exe.ModuleSignature; - +pub const floats = @import("floats.zig"); +pub const helpers = @import("helpers.zig"); +pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; +pub const meta = @import("meta.zig"); +pub const mlir = @import("mlir.zig"); +pub const module = @import("module.zig"); +pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); +pub const call = ops.call; +pub const pjrt = @import("pjrtx.zig"); +pub const Platform = @import("platform.zig").Platform; +pub const platform = @import("platform.zig"); +pub const Shape = @import("shape.zig").Shape; +pub const ShapeOf = @import("tensor.zig").ShapeOf; +pub const Target = @import("platform.zig").Target; +pub const Tensor = @import("tensor.zig").Tensor; +pub const testing = @import("testing.zig"); +pub const torch = @import("torch.zig"); + +// Namespaces +// pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { pub const Tracer = @import("tools/tracer.zig").Tracer; }; -pub const aio = @import("aio.zig"); -pub const sentencepiece = @import("aio/sentencepiece.zig"); - pub const log = std.log.scoped(.zml); -const std = @import("std"); - test { // NOTE : testing entrypoint. // Don't forget to import your module if you want to declare tests declarations that will be run by //zml:test From c7247412839e9833a3e0bb6adce5f2b9533cbfbe Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Apr 2025 12:08:11 +0200 Subject: [PATCH 16/24] Buffer.fromHostWithPadding --- zml/buffer.zig | 14 +++++++++----- zml/zml.zig | 1 - 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 89f572dd..696570a3 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -70,16 +70,20 @@ pub const Buffer = struct { /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { + return fromHostWithPadding(platform, host_buffer, host_buffer.shape()); + } + + pub fn fromHostWithPadding(platform: Platform, host_buffer: HostBuffer, padded_shape: Shape) !Buffer { var res: Buffer = .{ ._api = platform.pjrt_api, - ._shape = host_buffer.shape(), + ._shape = padded_shape, ._shards = .{}, }; // We shard only on the first axis so that the chunks are still contiguous. // TODO: support more advanced sharding specs stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - const sharding_ax: ?u3 = std.simd.firstTrue(host_buffer.shape()._sharding_info); + const sharding_ax: ?u3 = std.simd.firstTrue(padded_shape._sharding_info); const n_partitions = platform.sharding().num_partitions; const chunk_size = if (sharding_ax) |ax| cs: { // This kind of sharding error should be detected earlier on. @@ -87,7 +91,7 @@ pub const Buffer = struct { break :cs @divExact(host_buffer.dim(ax), n_partitions); } else 0; - const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); + const buffer_type = bufferTypeFromDtype(padded_shape.dtype()); const byte_strides = host_buffer.strides(); var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; @@ -105,7 +109,7 @@ pub const Buffer = struct { pjrt.Client.BufferFromHostBufferArgs{ .data = buf.data, .buffer_type = buffer_type, - .dims = buf.shape().dims(), + .dims = padded_shape.dims(), .byte_strides = byte_strides, .device = devices[i], .host_buffer_semantics = .ImmutableOnlyDuringCall, @@ -363,7 +367,7 @@ pub const Buffer = struct { } }; -pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { +fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { return switch (dt) { .bool => .PRED, .f8e4m3b11fnuz => .F8E4M3B11FNUZ, diff --git a/zml/zml.zig b/zml/zml.zig index 26d821db..18482b2a 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -11,7 +11,6 @@ pub const aio = @import("aio.zig"); pub const sentencepiece = @import("aio/sentencepiece.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const Bufferized = @import("tensor.zig").Bufferized; -pub const bufferTypeFromDtype = @import("buffer.zig").bufferTypeFromDtype; pub const CompilationOptions = @import("platform.zig").CompilationOptions; pub const Context = @import("context.zig").Context; pub const context = @import("context.zig"); From 2484028fdd73518100d8d3bd428a549fddca99a9 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 23 Apr 2025 08:26:30 +0200 Subject: [PATCH 17/24] Revert "Buffer.fromHostWithPadding" This reverts commit 0e7bd71bdca516fa797ffd287db528827b4cdf0a. --- zml/buffer.zig | 14 +++++--------- zml/zml.zig | 1 + 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 696570a3..89f572dd 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -70,20 +70,16 @@ pub const Buffer = struct { /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { - return fromHostWithPadding(platform, host_buffer, host_buffer.shape()); - } - - pub fn fromHostWithPadding(platform: Platform, host_buffer: HostBuffer, padded_shape: Shape) !Buffer { var res: Buffer = .{ ._api = platform.pjrt_api, - ._shape = padded_shape, + ._shape = host_buffer.shape(), ._shards = .{}, }; // We shard only on the first axis so that the chunks are still contiguous. // TODO: support more advanced sharding specs stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - const sharding_ax: ?u3 = std.simd.firstTrue(padded_shape._sharding_info); + const sharding_ax: ?u3 = std.simd.firstTrue(host_buffer.shape()._sharding_info); const n_partitions = platform.sharding().num_partitions; const chunk_size = if (sharding_ax) |ax| cs: { // This kind of sharding error should be detected earlier on. @@ -91,7 +87,7 @@ pub const Buffer = struct { break :cs @divExact(host_buffer.dim(ax), n_partitions); } else 0; - const buffer_type = bufferTypeFromDtype(padded_shape.dtype()); + const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides(); var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; @@ -109,7 +105,7 @@ pub const Buffer = struct { pjrt.Client.BufferFromHostBufferArgs{ .data = buf.data, .buffer_type = buffer_type, - .dims = padded_shape.dims(), + .dims = buf.shape().dims(), .byte_strides = byte_strides, .device = devices[i], .host_buffer_semantics = .ImmutableOnlyDuringCall, @@ -367,7 +363,7 @@ pub const Buffer = struct { } }; -fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { +pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { return switch (dt) { .bool => .PRED, .f8e4m3b11fnuz => .F8E4M3B11FNUZ, diff --git a/zml/zml.zig b/zml/zml.zig index 18482b2a..26d821db 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -11,6 +11,7 @@ pub const aio = @import("aio.zig"); pub const sentencepiece = @import("aio/sentencepiece.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const Bufferized = @import("tensor.zig").Bufferized; +pub const bufferTypeFromDtype = @import("buffer.zig").bufferTypeFromDtype; pub const CompilationOptions = @import("platform.zig").CompilationOptions; pub const Context = @import("context.zig").Context; pub const context = @import("context.zig"); From 74f7977632ed122ce986135f679b286965337b0b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 23 Apr 2025 08:42:22 +0200 Subject: [PATCH 18/24] more precise f32 -> bf16 conversion --- zml/floats.zig | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/zml/floats.zig b/zml/floats.zig index 5fa7cde4..74a8b993 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -305,11 +305,23 @@ pub const BFloat16 = packed struct(u16) { pub fn isInf(self: BFloat16) bool { return allBitsOne(self.exponent) and self.mantissa == 0; } + + pub fn toF32(self: BFloat16) f32 { + // Pad the BF16 with zeros 0 + return @bitCast([2]u16{ 0, @bitCast(self) }); + } + + pub fn fromF32(float32: f32) BFloat16 { + var int: u32 = @bitCast(float32); + // Round up if needed. + int += 0x8000; + const parts: [2]u16 = @bitCast(int); + return @bitCast(parts[1]); + } + const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; pub const format = Helpers.format; }; @@ -317,7 +329,7 @@ test BFloat16 { // From https://en.wikipedia.org/wiki/Bfloat16_floating-point_format#Examples try std.testing.expectEqual(BFloat16.fromF32(0), BFloat16{ .sign = 0, .exponent = 0, .mantissa = 0 }); try std.testing.expectEqual(BFloat16.fromF32(-2), BFloat16{ .sign = 1, .exponent = 127 + 1, .mantissa = 0 }); - try std.testing.expectEqual(BFloat16.fromF32(3.02344107628), BFloat16{ .sign = 0, .exponent = 127 + 1, .mantissa = 65 }); + try std.testing.expectEqual(BFloat16.fromF32(3.02344107628), BFloat16{ .sign = 0, .exponent = 127 + 1, .mantissa = 66 }); try std.testing.expectEqual(BFloat16.fromF32(1.0 / 128.0), BFloat16{ .sign = 0, .exponent = 127 - 7, .mantissa = 0 }); try std.testing.expectEqual(std.mem.toBytes(BFloat16.inf.neg()), [_]u8{ 0x80, 0xff }); try std.testing.expectEqual(BFloat16.inf, BFloat16.fromF32(std.math.inf(f32))); From b459fa0c0db3d57590624a9a6b8f110c62a43707 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 23 Apr 2025 08:42:54 +0200 Subject: [PATCH 19/24] better error messages for shape parsing --- zml/shape.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/shape.zig b/zml/shape.zig index db70a9f2..45f81bae 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -58,10 +58,10 @@ pub const Shape = struct { const fv = @field(v, field.name); if (comptime stdx.meta.isInteger(field.type)) { dims_.appendAssumeCapacity(@intCast(fv)); - } else if (comptime isAutoDim(fv)) { + } else if (@TypeOf(fv) == EnumLiteral and comptime isAutoDim(fv)) { dims_.appendAssumeCapacity(-1); } else { - stdx.debug.compileError("Field {s} should be an integer or an auto dimension", .{field.name}); + stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); } if (comptime stdx.meta.isTuple(T)) { tags_.appendAssumeCapacity(TagUnknown); From 6d2484a377e45eb9ad3072c840fe9e5e98102eba Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 23 Apr 2025 10:56:06 +0200 Subject: [PATCH 20/24] chooseItems should assert type --- zml/hostbuffer.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 6f30a75d..17b550b9 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -164,9 +164,8 @@ pub const HostBuffer = struct { /// WARNING: It's only valid if the buffer is contiguous. /// Strided buffers can't use this method. pub fn items(self: HostBuffer, comptime T: type) []const T { - if (DataType.fromZigType(T) != self.dtype()) { - std.debug.panic("Can't reinterpret {} as {s}", .{ self, @typeName(T) }); - } + // TODO we should allow interpreting the output as @Vector(8, f32) when the tensor is f32. + stdx.debug.assert(DataType.fromZigType(T) == self.dtype(), "Can't reinterpret {} as {s}", .{ self, @typeName(T) }); if (!self.isContiguous()) { std.debug.panic("{} isn't contiguous", .{self}); } @@ -265,6 +264,7 @@ pub const HostBuffer = struct { // TODO: this is a bit too restrictive, choosing slice could extract a contiguous // slice out of a non-contiguous tensor. std.debug.assert(self.isContiguous()); + stdx.debug.assert(DataType.fromZigType(T) == self.dtype(), "Can't reinterpret {} as {s}", .{ self, @typeName(T) }); var sh = self._shape; var offset: i64 = 0; From 3173af4e3142368da15f69500644ca712dd162e6 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 24 Apr 2025 13:57:56 +0200 Subject: [PATCH 21/24] Buffer.asPinnedHostBuffer --- pjrt/pjrt.zig | 2 +- zml/buffer.zig | 25 ++++++++----------------- zml/context.zig | 2 +- zml/exe.zig | 14 +++++++++++--- zml/hostbuffer.zig | 1 + 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index c5e2118b..9d20b3af 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -653,7 +653,7 @@ pub const LoadedExecutable = opaque { }); } - pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { + pub fn getExecutable(self: *const LoadedExecutable, api: *const Api) ApiError!*Executable { const ret = try api.call(.PJRT_LoadedExecutable_GetExecutable, .{ .loaded_executable = self.inner(), }); diff --git a/zml/buffer.zig b/zml/buffer.zig index 89f572dd..26b10c59 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -44,23 +44,6 @@ pub const Buffer = struct { } }; - pub const Shard = struct { - api: *const pjrt.Api, - buffer: *pjrt.Buffer, - ready_event: ?*pjrt.Event = null, - ready: bool = false, - - pub fn awaitt(self: *Shard) !void { - if (self.ready) { - return; - } - if (self.ready_event orelse self.buffer.getReadyEvent(self.api)) |ev| { - try ev.awaitt(self.api); - } - self.ready = true; - } - }; - _shape: Shape, _api: *const pjrt.Api, _shards: Shards, @@ -155,6 +138,14 @@ pub const Buffer = struct { return try from(platform, host_buffer); } + pub fn asPinnedHostBuffer(self: Buffer) HostBuffer { + // TODO restore assert + // const memory = self.getMemory().kind(self._api); + // stdx.debug.assert(memory == .pinned_host, "asPinnedHostBuffer({}) expects a buffer allocated on host memory, got {}. see `toMemory`", .{ self, memory }); + const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); + return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); + } + /// Creates a Buffer with a single element. pub fn scalar(platform: Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); diff --git a/zml/context.zig b/zml/context.zig index cbf4e102..7f39d404 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -39,8 +39,8 @@ pub const Context = struct { inline for (comptime std.enums.values(runtimes.Platform)) |t| { if (runtimes.load(t)) |api| { Context.apis.set(t, api); + if (t == .cuda) cuda.init(); } else |_| {} - if (t == .cuda) cuda.init(); } } }.call); diff --git a/zml/exe.zig b/zml/exe.zig index 946410c6..84d7a218 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -1,13 +1,13 @@ const std = @import("std"); + const stdx = @import("stdx"); const aio = @import("aio.zig"); -const meta = @import("meta.zig"); -const pjrt = @import("pjrtx.zig"); - const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("tensor.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; +const meta = @import("meta.zig"); +const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; @@ -244,6 +244,14 @@ pub const BaseExe = struct { return Buffer.fromPjrtBuffers(self.platform, self.result_shapes[i], shards.constSlice()); } + + pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { + return .init(parent_allocator, self.platform, self.exe, .{ + .n_in = self.input_buffer_count, + .result_shapes = self.result_shapes, + .n_devices = self.num_devices, + }); + } }; /// Represents a ZML function, compiled into a PJRT executable. diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 17b550b9..d21e9fc4 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -277,6 +277,7 @@ pub const HostBuffer = struct { } if (last_sliced_ax > 0) { + // TODO better error message for (0..@intCast(last_sliced_ax)) |ax| std.debug.assert(sh._dims.buffer[ax] == 1); } From 0e8ccfca1f5920b0d779a96c460955770d23e2e1 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 30 Apr 2025 10:59:26 +0000 Subject: [PATCH 22/24] updates --- examples/.vscode/settings.json | 2 +- pjrt/pjrt.zig | 6 +++--- zml/buffer.zig | 23 +++++++++++++++++++++++ zml/exe.zig | 16 +++++++++++++--- zml/pjrtx.zig | 2 +- zml/platform.zig | 3 +++ 6 files changed, 44 insertions(+), 8 deletions(-) diff --git a/examples/.vscode/settings.json b/examples/.vscode/settings.json index 3a9f5009..82050af6 100644 --- a/examples/.vscode/settings.json +++ b/examples/.vscode/settings.json @@ -23,7 +23,7 @@ "C_Cpp.clang_format_style": "Microsoft", "zig.formattingProvider": "zls", "zig.path": "x", - "zig.zls.path": "/${workspaceFolder}/tools/zls.sh", + "zig.zls.path": "${workspaceFolder}/tools/zls.sh", "[zig]": { "editor.suggest.insertMode": "replace", "editor.stickyScroll.defaultModel": "foldingProviderModel", diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9d20b3af..65c9b2f8 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -850,7 +850,7 @@ pub const Buffer = opaque { return @ptrCast(ret.event); } - pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!?*Buffer { + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), .dst_memory = @ptrCast(@constCast(dst_memory)), @@ -932,8 +932,8 @@ pub const Memory = opaque { pub fn kind(self: *const Memory, api: *const Api) Kind { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable[0..ret.kind_size]; - return std.meta.stringToEnum(Kind, kind_) orelse unreachable; + const kind_ = ret.kind orelse unreachable; + return std.meta.stringToEnum(Kind, kind_[0..ret.kind_size]) orelse unreachable; } pub fn kindId(self: *const Memory, api: *const Api) u32 { diff --git a/zml/buffer.zig b/zml/buffer.zig index 26b10c59..43276470 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -270,6 +270,29 @@ pub const Buffer = struct { return res; } + pub fn copyToMemory(self: Buffer, platform: Platform, kind: Memory) !Buffer { + const device = try self._shards.get(0).getDevice(self._api); + const memories = try device.addressableMemories(self._api); + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == kind.toPjrtMemory()) { + selected_mem = mem; + } + } + + var shards: Shards = .{}; + for (self._shards.constSlice()) |shard| { + const pjrt_buffer = try shard.copyToMemory(self._api, selected_mem); + shards.appendAssumeCapacity(pjrt_buffer); + } + + return .{ + ._api = platform.pjrt_api, + ._shape = self._shape, + ._shards = shards, + }; + } + /// Copies the content of the Buffer back to host, in the given buffer, /// and return a new `HostBuffer` object with the same shape. /// The returned `HostBuffer` doesn't own the memory. diff --git a/zml/exe.zig b/zml/exe.zig index 84d7a218..5528ce4f 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -200,6 +200,7 @@ pub const BaseExe = struct { var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; const sharding = self.platform.sharding(); + const trace = self.platform.tracer.frameStart("_unsafeCall execute"); self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.input_buffer_count, @@ -210,12 +211,15 @@ pub const BaseExe = struct { // TODO: expose it ? .non_donatable_input_indices = &.{}, }) catch unreachable; + self.platform.tracer.frameEnd(trace, "_unsafeCall execute"); + const trace_events_await = self.platform.tracer.frameStart("_unsafeCall await events"); for (events[0..sharding.num_partitions]) |e| { if (e) |ev| { ev.await_(self.platform.pjrt_api) catch unreachable; } } + self.platform.tracer.frameEnd(trace_events_await, "_unsafeCall await events"); } pub fn serialize(self: BaseExe, writer: anytype) !void { @@ -232,7 +236,7 @@ pub const BaseExe = struct { // } pub fn prepare(self: *BaseExe, x: anytype) void { - const n = fillBuffers(&x, self.input_per_device, self.ready_buffer_count); + const n = fillBuffers(&x, self.platform, self.input_per_device, self.ready_buffer_count); self.ready_buffer_count += n; } @@ -288,7 +292,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { } pub fn call(self: Self, args: Bufferized(ArgsT)) Bufferized(ReturnT) { - const total_ready = fillBuffers(&args, self.inner.input_per_device, self.inner.ready_buffer_count); + const total_ready = fillBuffers(&args, self.inner.platform, self.inner.input_per_device, self.inner.ready_buffer_count); std.debug.assert(total_ready == self.inner.input_buffer_count); self.inner._unsafeCall(); var result: Bufferized(ReturnT) = undefined; @@ -310,7 +314,10 @@ fn splitBuffer(T: type, buffer: []T, lengths: anytype) [lengths.len][]T { } /// Visit the given struct and fill the `buffers` slice with the buffer associated with encountered Tensor. -fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { +fn fillBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { + const trace = platform.tracer.frameStart("fillBuffers_" ++ @typeName(@TypeOf(v))); + defer platform.tracer.frameEnd(trace, "fillBuffers_" ++ @typeName(@TypeOf(v))); + const LocalContext = struct { index: u32, buffers: []const [*]*pjrt.Buffer, @@ -335,6 +342,9 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { /// Visit the given struct and override tensors by creating a new one using the provided PJRT buffers. fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Buffer, buffer_shapes: []Shape) void { + const trace = platform.tracer.frameStart("assignRawBuffers_" ++ @typeName(@TypeOf(v))); + defer platform.tracer.frameEnd(trace, "assignRawBuffers_" ++ @typeName(@TypeOf(v))); + const LocalContext = struct { index: u32, platform: Platform, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index bd094ead..dbfa82a0 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -182,7 +182,7 @@ pub const Buffer = opaque { } pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(self.inner().copyToMemory(api, memory_)); + return @ptrCast(try self.inner().copyToMemory(api, memory_)); } pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..adf8d639 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -7,6 +7,7 @@ const stdx = @import("stdx"); const meta = @import("meta.zig"); const module = @import("module.zig"); const pjrt = @import("pjrtx.zig"); +const Tracer = @import("tools/tracer.zig").Tracer; const log = std.log.scoped(.zml); @@ -27,6 +28,7 @@ pub const Platform = struct { pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, compilation_options: CompilationOptions = .{}, + tracer: Tracer, pub const MAX_NUM_DEVICES: u8 = 32; pub const CreateOptions = _CreateOptions; @@ -43,6 +45,7 @@ pub const Platform = struct { .pjrt_api = api, .pjrt_client = pjrt_client, .compilation_options = .{}, + .tracer = Tracer.init("ai.zml.platform"), }; } From 4c6851b545e9a1efc156ab5d149553bf646ae84d Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 7 Jan 2025 22:55:02 +0100 Subject: [PATCH 23/24] zml: make buffers and execute asynchronous This change makes buffer and execution completely asynchronous. This enable easy pipelining of multiple operations. It doesn't map well to `async` because the PJRT API is already asynchronous, so might as well use it as is. --- examples/mnist/mnist.zig | 2 +- pjrt/pjrt.zig | 2 +- zml/buffer.zig | 56 +++++++++++++++++++++------------------- zml/hostbuffer.zig | 19 ++++++++++++++ zml/pjrtx.zig | 21 +++++---------- 5 files changed, 57 insertions(+), 43 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c49300a9..96221ff1 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -105,7 +105,7 @@ pub fn asyncMain() !void { defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); + var result: zml.Buffer = mnist.call(.{try input.awaitt()}); defer result.deinit(); log.info( diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 65c9b2f8..6528eb89 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -818,7 +818,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, diff --git a/zml/buffer.zig b/zml/buffer.zig index 43276470..b1919e94 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -73,38 +73,39 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - pjrt.Client.BufferFromHostBufferArgs{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - - frames.appendAssumeCapacity(frame); - } - - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } res._shards.appendAssumeCapacity(pjrt_buffer); } + return res; } + pub fn awaitt(self: Buffer) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.awaitt(self._api); + } + } + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -265,7 +266,7 @@ pub const Buffer = struct { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.awaitt(self._api); } return res; } @@ -299,10 +300,11 @@ pub const Buffer = struct { pub fn toHost(self: Buffer, output: []u8) !HostBuffer { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); + var hb = HostBuffer.fromBytes(self.shape(), output); + hb._event = maybe_event; + hb._api = self._api; + hb._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return hb; } /// Copies the content of the Buffer to the host. @@ -311,9 +313,9 @@ pub const Buffer = struct { const output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); - if (maybe_event) |event| { - try event.await_(self._api); - } + output._event = maybe_event; + output._api = self._api; + output._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; return output; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index d21e9fc4..a8577932 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -2,6 +2,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); + const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -24,6 +26,9 @@ pub const HostBuffer = struct { managed: std.mem.Alignment, unmanaged, } = .unmanaged, + _ready: bool = true, + _api: ?*const pjrt.Api = null, + _event: ?*pjrt.Event = null, /// Allocates a HostBuffer with the given shape. /// The memory is left undefined. @@ -301,6 +306,20 @@ pub const HostBuffer = struct { }; } + pub fn awaitt(self: *HostBuffer) !*HostBuffer { + if (self._ready) { + return self; + } + if (self._event) |ev| { + if (self._api) |api| { + try ev.awaitt(api); + } else { + stdx.debug.panic("Got an event but no API. This should not happen.", .{}); + } + } + return self; + } + pub fn format( self: HostBuffer, comptime fmt: []const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index dbfa82a0..91126c72 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -57,13 +57,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return @ptrCast(buffer); + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -153,10 +149,6 @@ pub const Buffer = opaque { return self.inner().isOnCpu(api); } - pub fn memory(self: *const Buffer, api: *const Api) *const Memory { - return self.inner().memory(api); - } - pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { return @ptrCast(try self.inner().toHostBuffer(api, dst)); } @@ -209,7 +201,8 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) ApiError!void { + pub const await_ = awaitt; + pub fn awaitt(self: *Event, api: *const Api) !void { defer self.deinit(api); if (self.isReady(api)) { @@ -265,13 +258,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 80063808eca8d808666f819c44d9d874dd94b328 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 26 May 2025 12:33:31 +0000 Subject: [PATCH 24/24] squash --- MODULE.bazel | 2 +- MODULE.bazel.lock | 168 +++++++++++- async/async.zig | 44 +++ examples/MODULE.bazel.lock | 168 +++++++++++- examples/custom_call/BUILD | 24 ++ examples/custom_call/main.zig | 228 ++++++++++++++++ examples/custom_call/playground.zig | 251 ++++++++++++++++++ examples/llama/main.zig | 9 +- examples/mnist/mnist.zig | 63 ++++- mlir/dialects/stablehlo.zig | 10 + pjrt/ffi.zig | 114 +++++--- pjrt/pjrt.zig | 130 +++++++-- runtimes/cpu/cpu.bzl | 12 +- runtimes/cuda/BUILD.bazel | 6 + runtimes/cuda/cuda.bzl | 4 +- runtimes/cuda/libpjrt_cuda.BUILD.bazel | 3 +- runtimes/rocm/rocm.bzl | 4 +- stdx/meta.zig | 8 + ...ler-registration-API-to-the-FFI-PjRt.patch | 9 +- .../xla/20250409.0-7167d71/MODULE.bazel | 37 +++ .../20250409.0-7167d71/overlay/MODULE.bazel | 1 + .../xla/20250409.0-7167d71/overlay/llvm.bzl | 13 + .../xla/20250409.0-7167d71/overlay/tsl.bzl | 19 ++ .../20250409.0-7167d71/overlay/workspace.bzl | 60 +++++ .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 +++ ...ler-registration-API-to-the-FFI-PjRt.patch | 137 ++++++++++ .../xla/20250409.0-7167d71/source.json | 16 ++ third_party/modules/xla/metadata.json | 3 +- zml/buffer.zig | 118 +++++--- zml/context.zig | 162 +---------- zml/exe.zig | 52 +++- zml/ffi.zig | 244 +++++++++++++++++ zml/hostbuffer.zig | 9 + zml/module.zig | 37 ++- zml/ops.zig | 94 +------ zml/pjrtx.zig | 16 +- zml/platform.zig | 1 + zml/tensor.zig | 57 ++-- zml/test_buffer.zig | 73 +++++ zml/zml.zig | 2 + 40 files changed, 2035 insertions(+), 414 deletions(-) create mode 100644 examples/custom_call/BUILD create mode 100644 examples/custom_call/main.zig create mode 100644 examples/custom_call/playground.zig create mode 100644 third_party/modules/xla/20250409.0-7167d71/MODULE.bazel create mode 120000 third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel create mode 100644 third_party/modules/xla/20250409.0-7167d71/overlay/llvm.bzl create mode 100644 third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl create mode 100644 third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl create mode 100644 third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch create mode 100644 third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch create mode 100644 third_party/modules/xla/20250409.0-7167d71/source.json create mode 100644 zml/ffi.zig create mode 100644 zml/test_buffer.zig diff --git a/MODULE.bazel b/MODULE.bazel index b392ef40..711d470a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,7 +24,7 @@ bazel_dep(name = "rules_zig", version = "20250314.0-b9739c6") bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a") bazel_dep(name = "toolchains_protoc", version = "0.3.7") bazel_dep(name = "with_cfg.bzl", version = "0.9.1") -bazel_dep(name = "xla", version = "20250317.2-71c67e2") +bazel_dep(name = "xla", version = "20250409.0-7167d71") bazel_dep(name = "zig-protobuf", version = "20250318.0-930153e") bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ee52d7cf..e534d743 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -83,7 +83,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.17.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.20.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", @@ -150,7 +150,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", @@ -269,8 +269,8 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/MODULE.bazel": "39ee3d15edbf09f536a52c42b4297fe1f08a239e28964555f578f301159033f3", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/source.json": "57af375068c7292d88062b2a3c2cae546f6e262f653f03388b5d73207f7482e3", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/MODULE.bazel": "d9049cabd0becdf8d26377927dda874e2febc803401269814776f232fde91b7c", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/source.json": "7870121042c518ca6109fa16fb070b4dd6762f42aefdd9ddcc259194ca6627cd", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -350,8 +350,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.3.0/source.json": "c55ed591aa5009401ddf80ded9762ac32c358d2517ee7820be981e2de9756cf3", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", "https://bcr.bazel.build/modules/rules_uv/0.65.0/source.json": "315237fb5f44ada2d537aabb2f89c6ba2296569ed63fc7340ca66d65f5730081", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", @@ -380,7 +380,7 @@ "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", - "usagesDigest": "wD+r6kULzwkrPbJnhFpRl2hIsJZIwSSaeEUj3z6v0i4=", + "usagesDigest": "ai1Q9Iw9PMxCbfOYikwNfCbFYAEzeR5Pirndq3g0LFo=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2066,6 +2066,160 @@ ] ] } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", + "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ] + ] + } } } } diff --git a/async/async.zig b/async/async.zig index 73e1a785..019e12de 100644 --- a/async/async.zig +++ b/async/async.zig @@ -278,6 +278,50 @@ pub const AsyncThread = struct { return try aio.run(&executor_, mainFunc, .{}, mainStack); } + + pub fn mainWithArgs(allocator: std.mem.Allocator, comptime mainFunc: anytype, args: anytype) !stdx.meta.FnResultNoError(mainFunc) { + if (xev.dynamic) try xev.detect(); + var thread_pool = XevThreadPool.init(.{}); + defer { + thread_pool.shutdown(); + thread_pool.deinit(); + } + + var loop = try xev.Loop.init(.{ + .thread_pool = &thread_pool, + }); + defer loop.deinit(); + + var executor_ = aio.Executor.init(&loop); + + var async_notifier = try xev.Async.init(); + defer async_notifier.deinit(); + + var waiters_queue: threading.WaiterQueue = undefined; + waiters_queue.init(); + + var stack_allocator = stack.StackAllocator.init(allocator); + defer stack_allocator.deinit(); + + var asyncThread: AsyncThread = .{ + .executor = &executor_, + .stack_allocator = &stack_allocator, + .loop = &loop, + .thread_pool = &thread_pool, + .async_notifier = &async_notifier, + .waiters_queue = &waiters_queue, + }; + AsyncThread.current = &asyncThread; + + var c2: xev.Completion = undefined; + async_notifier.wait(AsyncThread.current.loop, &c2, AsyncThread, AsyncThread.current, &AsyncThread.wakerCallback); + + // allocate the main coroutine stack, on the current thread's stack! + var mainStackData: stack.Stack.Data = undefined; + const mainStack = stack.Stack.init(&mainStackData); + + return try aio.run(&executor_, mainFunc, args, mainStack); + } }; pub fn getStdIn() !File { diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 1008eb86..612efec4 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -83,7 +83,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.17.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.20.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", @@ -151,7 +151,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", @@ -271,8 +271,8 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/MODULE.bazel": "39ee3d15edbf09f536a52c42b4297fe1f08a239e28964555f578f301159033f3", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/source.json": "57af375068c7292d88062b2a3c2cae546f6e262f653f03388b5d73207f7482e3", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/MODULE.bazel": "d9049cabd0becdf8d26377927dda874e2febc803401269814776f232fde91b7c", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/source.json": "7870121042c518ca6109fa16fb070b4dd6762f42aefdd9ddcc259194ca6627cd", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -354,8 +354,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.3.0/source.json": "c55ed591aa5009401ddf80ded9762ac32c358d2517ee7820be981e2de9756cf3", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", "https://bcr.bazel.build/modules/rules_uv/0.65.0/source.json": "315237fb5f44ada2d537aabb2f89c6ba2296569ed63fc7340ca66d65f5730081", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", @@ -385,7 +385,7 @@ "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", - "usagesDigest": "wD+r6kULzwkrPbJnhFpRl2hIsJZIwSSaeEUj3z6v0i4=", + "usagesDigest": "ai1Q9Iw9PMxCbfOYikwNfCbFYAEzeR5Pirndq3g0LFo=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2291,6 +2291,160 @@ ] ] } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", + "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ] + ] + } } } } diff --git a/examples/custom_call/BUILD b/examples/custom_call/BUILD new file mode 100644 index 00000000..f579a1b0 --- /dev/null +++ b/examples/custom_call/BUILD @@ -0,0 +1,24 @@ +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "custom_call", + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) + +zig_cc_binary( + name = "playground", + copts = ["-mcpu=native"], + extra_srcs = glob([ + "*.ptx", + "*.cubin", + ]), + main = "playground.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig new file mode 100644 index 00000000..0aa99264 --- /dev/null +++ b/examples/custom_call/main.zig @@ -0,0 +1,228 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +const cuda = zml.context.cuda; +const ffi = zml.ffi; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +pub const AddOp = struct { + const Self = @This(); + + platform: zml.Platform, + + buffers: []*const ffi.FFIBuffer = undefined, + stream: *ffi.FFIStream = undefined, + + pub fn call(self: *Self, a_: *const ffi.FFIBuffer, b_: *const ffi.FFIBuffer) !void { + const a_device = zml.Buffer.asViewOfDeviceBuffer(self.platform, ffi.getShape(a_), null, a_.data.asPtr()); + const a = a_device.copyToMemory(self.platform, .host_pinned) catch unreachable; + + const b_device = zml.Buffer.asViewOfDeviceBuffer(self.platform, ffi.getShape(b_), null, b_.data.asPtr()); + const b = b_device.copyToMemory(self.platform, .host_pinned) catch unreachable; + + _ = a.awaitt() catch unreachable; + _ = b.awaitt() catch unreachable; + + const a_value = try a.getValue(f32); + const b_value = try b.getValue(f32); + + const result: f32 = a_value + b_value; + const result_hb = zml.HostBuffer.fromBytes(a_device.shape(), std.mem.asBytes(&result)); + + cuda.memcpyToDeviceAsync(self.buffers[0].data, result_hb.data, self.stream); + } +}; + +pub const LogResultOp = struct { + const Self = @This(); + + platform: zml.Platform, + + buffers: []*const ffi.FFIBuffer = undefined, + stream: *ffi.FFIStream = undefined, + + pub fn call(self: *Self, value: *const ffi.FFIBuffer) !void { + const value_device = zml.Buffer.asViewOfDeviceBuffer(self.platform, ffi.getShape(value), null, value.data.asPtr()); + const value_pinned = value_device.copyToMemory(self.platform, .host_pinned) catch unreachable; + + log.warn("{s}@{*} value: {d}", .{ @typeName(Self), self, try value_pinned.getValue(f32) }); + + const data = try value_pinned.dataInMemory(); + cuda.memcpyToDeviceAsync(self.buffers[0].data, data, self.stream); + } +}; + +pub const LogValuesVoidOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn call(self: *Self, result: zml.HostBuffer, a: zml.HostBuffer, b: zml.HostBuffer) !void { + _ = self; // autofix + log.info("LogValuesVoidOp mem result: {any}", .{result.items(f32)}); + log.info("LogValuesVoidOp mem a: {any}", .{a.items(f32)}); + log.info("LogValuesVoidOp mem b: {any}", .{b.items(f32)}); + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + +/// Model definition +const Layer = struct { + // a = 40x128 bf16 ou f32 + pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + const a_ = zml.custom_call(LogResultOp, .{a}, &[_]zml.Shape{a.shape()}, a.getContext(), &.{0}); + const results = zml.custom_call(AddOp, .{ a_[0], b }, &[_]zml.Shape{a.shape()}, a.getContext(), &.{0}); + return results[0]; + // return zml.custom_call(LogResultOp, .{results[2]}, &[_]zml.Shape{a.shape()}); + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Arena allocator for BufferStore etc. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + // const shape = zml.Shape.init(.{ 40, 128 }, .bf16); + const shape = zml.Shape.init(.{}, .f32); + + // We manually produce a BufferStore. You would not normally do that. + // A BufferStore is usually created by loading model data from a file. + const buffers: zml.aio.BufferStore.Buffers = .{}; + + // the actual BufferStore + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + // A clone of our model, consisting of shapes. We only need shapes for compiling. + // We use the BufferStore to infer the shapes. + const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + + // Start compiling. This uses the inferred shapes from the BufferStore. + // The shape of the input tensor, we have to pass in manually. + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + + // Produce a bufferized weights struct from the fake BufferStore. + // This is like the inferred shapes, but with actual values. + // We will need to send those to the computation device later. + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); // for good practice + + // Wait for compilation to finish + const compiled = try compilation.awaitt(); + + // pass the model weights to the compiled module to create an executable module + var executable = compiled.prepare(model_weights).withExecutionContext(); + defer executable.deinit(); + + const add_op_ctx: AddOp = .{ .platform = platform }; + try executable.inner.attach(&add_op_ctx); + const log_result_op_ctx: LogResultOp = .{ .platform = platform }; + try executable.inner.attach(&log_result_op_ctx); + // const log_values_op_ctx: LogValuesVoidOp = .{ ._platform = &platform }; + // try executable.attach(&log_values_op_ctx); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + _ = random; // autofix + + for (0..2) |i| { + log.warn("Iteration {d}", .{i}); + + // prepare input buffers + // var input_buffer_a = try createRandomBuffer(allocator, platform, shape, random); + // _ = try input_buffer_a.awaitt(); + // defer input_buffer_a.deinit(); + // var input_buffer_b = try createRandomBuffer(allocator, platform, shape, random); + // _ = try input_buffer_b.awaitt(); + // defer input_buffer_b.deinit(); + + var input_a = [1]f32{1.0}; + var input_buffer_a = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_a)); + _ = try input_buffer_a.awaitt(); + defer input_buffer_a.deinit(); + var input_b = [1]f32{1.0}; + var input_buffer_b = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_b)); + _ = try input_buffer_b.awaitt(); + defer input_buffer_b.deinit(); + + var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + defer result.deinit(); + + // fetch the result to CPU memory + // const cpu_result = try result.toHostAlloc(arena); + // log.warn( + // "\nThe result of {d} + {d} = {d}\n", + // .{ &input_a, &input_b, cpu_result.items(f32) }, + // ); + + var cpu_result = try result.toHostAlloc(arena); + _ = try cpu_result.awaitt(); + log.warn( + "\nThe result is {d}\n", + .{cpu_result.items(f32)}, + ); + + std.time.sleep(1 * std.time.ns_per_s); + } +} + +fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const data = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = random.float(f64); + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return zml.Buffer.from(platform, host_buffer); +} diff --git a/examples/custom_call/playground.zig b/examples/custom_call/playground.zig new file mode 100644 index 00000000..4174fedd --- /dev/null +++ b/examples/custom_call/playground.zig @@ -0,0 +1,251 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +const cuda = zml.context.cuda; +const ffi = zml.ffi; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +const CuLaunchHostFunc = *const fn ( + hStream: *anyopaque, + fn_: *const fn (user_data: *const anyopaque) callconv(.c) void, + userData: *anyopaque, +) callconv(.c) c_int; + +const UserData = struct { + value: *const ffi.FFIBuffer, + platform: zml.Platform, +}; + +fn host_func(data: *const anyopaque) callconv(.c) void { + const user_data = @as(*const UserData, @alignCast(@ptrCast(data))); + const tracer = user_data.platform.tracer; + const frame = tracer.frameStart("HostFunc"); + defer tracer.frameEnd(frame, "HostFunc"); + log.warn("HostFunc data: {any}", .{user_data.value}); + std.time.sleep(10 * std.time.ns_per_ms); +} + +pub const AddOp = struct { + const Self = @This(); + + platform: zml.Platform, + allocator: std.mem.Allocator, + buffer: zml.Buffer, + cuLaunchHostFunc: CuLaunchHostFunc, + user_data: UserData = undefined, + + buffers: []*const ffi.FFIBuffer = undefined, + stream: *ffi.FFIStream = undefined, + + pub fn call(self: *Self, value: *const ffi.FFIBuffer) !void { + const tracer = self.platform.tracer; + _ = tracer; // autofix + + self.user_data = .{ .value = value, .platform = self.platform }; + log.warn("user_data: {*}", .{&self.user_data}); + + const res = self.cuLaunchHostFunc( + self.stream, + @ptrCast(&host_func), + @ptrCast(&self.user_data), + ); + + log.warn("cuLaunchHostFunc: {any}", .{res}); + + // std.time.sleep(1 * std.time.ns_per_ms); + + // const frame_as_view_of = tracer.frameStart("AddOp as view of"); + // const value_pinned = zml.Buffer.asViewOfDeviceBuffer(self.platform, ffi.getShape(value), null, value.data.asPtr()); + // log.warn("{s}@{*} value: {any}", .{ @typeName(Self), self, (try value_pinned.dataInMemory())[0..100] }); + // tracer.frameEnd(frame_as_view_of, "AddOp as view of"); + // const ptr = value.data.asPtr(); + // log.warn("{*} -> {*}", .{ ptr, self.buffers[0].data.asPtr() }); + // const frame_copy_to_memory = tracer.frameStart("AddOp copyToMemory"); + // const value_pinned = value_device.copyToMemory(self.platform, .host_pinned) catch unreachable; + // tracer.frameEnd(frame_copy_to_memory, "AddOp copyToMemory"); + + // const frame_await = tracer.frameStart("AddOp await"); + // _ = value_pinned.awaitt() catch unreachable; + // tracer.frameEnd(frame_await, "AddOp await"); + + // const frame_memcpy = tracer.frameStart("AddOp memcpy"); + // cuda.memcpyToDeviceAsync(self.buffers[0].data, value_pinned.asPinnedHostBuffer().data, self.stream); + // tracer.frameEnd(frame_memcpy, "AddOp memcpy"); + } +}; + +pub const AddOpAlreadyPinned = struct { + const Self = @This(); + + platform: zml.Platform, + buffer: zml.Buffer, + + buffers: []*const ffi.FFIBuffer = undefined, + stream: *ffi.FFIStream = undefined, + + pub fn call(self: *Self) !void { + const tracer = self.platform.tracer; + + const frame_await = tracer.frameStart("AddOpAlreadyPinned sleep 1 ms"); + std.time.sleep(1 * std.time.ns_per_ms); + tracer.frameEnd(frame_await, "AddOpAlreadyPinned 1 ms"); + + const frame_memcpy = tracer.frameStart("AddOpAlreadyPinned memcpy"); + cuda.memcpyToDeviceAsync(self.buffers[0].data, self.buffer.asPinnedHostBuffer().data, self.stream); + tracer.frameEnd(frame_memcpy, "AddOpAlreadyPinned memcpy"); + } +}; + +const Layer = struct { + pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + _ = b; // autofix + // const a_ = a.toMemory(.host_pinned); + const custom_call_result = zml.custom_call(AddOp, .{a}, &[_]zml.Shape{a.shape()}, a.getContext(), &.{0}); + // const temp = a.withTags(.{ .x, .y }).dynamicUpdateSlice(.{ .x = zml.Tensor.scalar(0, .i32), .y = zml.Tensor.scalar(0, .i32) }, a); + // var res = a.negate(); + // res = zml.Tensor.optimizationBarrier(&.{res})[0]; + + var res = custom_call_result[0]; + + for (0..10) |_| { + res = res.matmul(res); + } + res = res.matmul(res); + // res = zml.Tensor.optimizationBarrier(&.{res})[0]; + + // res = res.negate(); + + // const res2 = a.withTags(.{ .x, .y }).dynamicUpdateSlice(.{ .x = zml.Tensor.scalar(0, .i32), .y = zml.Tensor.scalar(0, .i32) }, res); + + // return res2.reuseBuffer(a); + + // const custom_call_result = zml.custom_call(AddOp, .{a}, &[_]zml.Shape{a.shape()}, a.getContext(), &.{}); + + return res.reuseBuffer(a); + + // // const custom_call_result = zml.custom_call(AddOpAlreadyPinned, .{}, &[_]zml.Shape{a.shape()}, a.getContext(), &.{0}); + // return b.matmul(custom_call_result[0]).toMemory(.host_pinned); + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const shape = zml.Shape.init(.{ 4096, 4096 }, .bf16); + + const buffers: zml.aio.BufferStore.Buffers = .{}; + + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); // for good practice + + const compiled = try compilation.awaitt(); + + var executable = compiled.prepare(model_weights).withExecutionContext(); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + var custom_call_buffer = try createRandomBuffer(allocator, platform, shape, random); + _ = try custom_call_buffer.awaitt(); + defer custom_call_buffer.deinit(); + + var libpjrt_cuda = std.DynLib.open("libpjrt_cuda.so") catch unreachable; + defer libpjrt_cuda.close(); + + const cuLaunchHostFunc = libpjrt_cuda.lookup(CuLaunchHostFunc, "cuLaunchHostFunc"); + + const add_op_ctx = try allocator.create(AddOp); + add_op_ctx.* = .{ .platform = platform, .allocator = allocator, .buffer = custom_call_buffer, .cuLaunchHostFunc = cuLaunchHostFunc.? }; + try executable.inner.attach(add_op_ctx); + + const add_op_already_pinned_ctx = try allocator.create(AddOpAlreadyPinned); + add_op_already_pinned_ctx.* = .{ .platform = platform, .buffer = custom_call_buffer }; + try executable.inner.attach(add_op_already_pinned_ctx); + + var input_buffer_a = try createRandomBuffer(allocator, platform, shape, random); + _ = try input_buffer_a.awaitt(); + defer input_buffer_a.deinit(); + var input_buffer_b = try createRandomBuffer(allocator, platform, shape, random); + _ = try input_buffer_b.awaitt(); + defer input_buffer_b.deinit(); + + var input_buffer_a_pinned = try input_buffer_a.copyToMemory(platform, .host_pinned); + defer input_buffer_a_pinned.deinit(); + _ = try input_buffer_a_pinned.awaitt(); + + var result: zml.Buffer = input_buffer_a_pinned; + defer result.deinit(); + + for (0..5) |i| { + log.warn("Iteration {d}", .{i}); + result = executable.call(.{ result, input_buffer_b }); + } + + log.warn("Result: {any}", .{result}); +} + +fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const data = try allocator.alloc(u8, shape.byteSize()); + errdefer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = random.float(f64); + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return zml.Buffer.from(platform, host_buffer); +} diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 0b121e65..c6e442bb 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -314,7 +314,10 @@ pub fn asyncMain() !void { const seed = res.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); const skip_llama3_encoding = res.args.@"no-llama3" orelse false; - const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); - // generated text will be printed token by token. - defer allocator.free(generated_text); + for (0..2) |i| { + _ = i; // autofix + const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); + // generated text will be printed token by token. + defer allocator.free(generated_text); + } } diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 96221ff1..16929474 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -2,14 +2,40 @@ const std = @import("std"); const asynk = @import("async"); const zml = @import("zml"); - +const pjrt = zml.pjrt; const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ .log_level = .info, + .log_scope_levels = &[_]std.log.ScopeLevel{ + .{ .scope = .@"zml/module", .level = .debug }, + }, .logFn = asynk.logFn(std.log.defaultLog), }; +pub const PrintOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn beforeCustomCall(input: zml.Tensor, another_input: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + log.info("Compiling PrintOp for target: {s}", .{@tagName(input.getContext().target())}); + return .{ input, another_input.add(another_input) }; + } + + pub fn call(self: *Self, input: zml.Buffer, another_input: zml.Buffer) !struct { zml.Buffer, zml.Buffer } { + var output = input; + log.info("Input 0: {any}", .{output.getMemory().kind(self._platform.pjrt_api)}); + output = try output.copyToMemory(self.getPlatform(), .host_pinned); + log.info("After copy 0: {any}", .{output.getMemory().kind(self._platform.pjrt_api)}); + return .{ try output.copyToMemory(self.getPlatform(), .device), another_input }; + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + /// Model definition const Mnist = struct { fc1: Layer, @@ -26,13 +52,36 @@ const Mnist = struct { /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); + log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); var x = input.flattenAll().convert(.f32); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { x = zml.call(layer, .forward, .{x}); } - return x.argMax(0).indices.convert(.u8); + return relu(x).argMax(0).indices.convert(.u8); + } + + pub fn relu(input: zml.Tensor) zml.Tensor { + return zml.ops.addHostCallback( + &reluCallback, + null, + &.{input}, + &.{input.shape()}, + .{}, + )[0]; + } + + fn reluCallback(_: ?*anyopaque, inputs: []const zml.HostBuffer, outputs: []const zml.HostBuffer) void { + const in = inputs[0].items(f32); + const out = outputs[0].mutItems(f32); + std.log.defaultLog(.info, .zml, "in: {*}[0..{}] = {e:.3}", .{ in.ptr, in.len, in }); + + var i: usize = 0; + for (in) |x| { + out[i] = @max(x, 0); + i += 1; + } + std.log.defaultLog(.info, .zml, "out: {*}[0..{}] = {e:.3}", .{ out.ptr, out.len, out }); } }; @@ -49,7 +98,7 @@ pub fn asyncMain() !void { // log.info("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); - // // Auto-select platform + // Auto-select platform const platform = context.autoPlatform(.{}); context.printAvailablePlatforms(platform); @@ -85,7 +134,7 @@ pub fn asyncMain() !void { const compiled_mnist = try compilation.awaitt(); log.info("✅ Compiled model in {d}ms", .{start_time.read() / std.time.ns_per_ms}); - const mnist = compiled_mnist.prepare(model_weights); + const mnist = compiled_mnist.prepare(model_weights).withExecutionContext(); defer mnist.deinit(); log.info("✅ Weights transferred in {d}ms", .{start_time.read() / std.time.ns_per_ms}); @@ -98,6 +147,10 @@ pub fn asyncMain() !void { // inference - can be looped { + const print_op_ctx: PrintOp = .{ ._platform = &platform }; + try mnist.attach(&print_op_ctx); + std.debug.print("Attached print op to model: {}\n", .{print_op_ctx}); + const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index 63283219..f194360e 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -731,6 +731,14 @@ pub fn convolution( }); } +pub fn optimization_barrier(ctx: mlir.Context, inputs: []const mlir.Value, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.optimization_barrier", .{ + .operands = inputs, + .results = res_types, + .location = location, + }); +} + pub const CustomCallOpts = struct { pub const ApiVersion = enum(i32) { original = 1, @@ -781,6 +789,8 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa for (opts.output_operand_aliases) |alias| { output_operand_aliases.appendAssumeCapacity( OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).as(mlir.Attribute), + // OutputOperandAliasAttribute.init(ctx, &.{0}, 0, &.{}).as(mlir.Attribute), + ); } attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 48c924ad..b7f81217 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -88,8 +88,6 @@ fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { }; } -pub const Stream = opaque {}; - pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; @@ -98,7 +96,6 @@ pub const Api = opaque { .ctx = if (context) |ctx| ctx.inner() else null, }); const result = self.inner().XLA_FFI_Stream_Get.?(&ret); - if (result) |ffi_error| { const err = Error.fromInner(ffi_error); defer err.destroy(self); @@ -166,54 +163,92 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - // pub fn attach(self: *ExecutionContext, api: *const Api, value: anytype) ApiError!void { - // // register type id ==> typeid - // const typename_ = "zml." ++ @typeName(@TypeOf(value)); + pub fn Context(comptime T: type) type { + return struct { + pub fn get(self: *const ExecutionContext, api: *const Api) ApiError!*T { + const type_id: TypeId = .{ .type_id = T.type_id }; + var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ + .ctx = @constCast(self.inner()), + .type_id = @constCast(&type_id.toCStruct()), + }); + const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); - // var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Register_Args{ - // .ctx = self.inner(), - // .handler = @ptrCast(@alignCast(handler)), - // }); - // const result = api.inner().XLA_FFI_ExecutionContext_Register.?(&ret); + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); - // var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Register_Args{ - // .ctx = self.inner(), - // .handler = @ptrCast(@alignCast(handler)), - // }); - // const result = api.inner().XLA_FFI_ExecutionContext_Register.?(&ret); + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } - // if (result) |ffi_error| { - // const err = Error.fromInner(ffi_error); - // defer err.destroy(api); - // log.err("[ExecutionContext.register] {s}", .{err.getMessage(api)}); + if (ret.data == null) return error.NotFound; + return @ptrCast(@alignCast(ret.data.?)); + } + }; + } - // // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - // return error.Unknown; - // } - // } - - pub fn get(self: *ExecutionContext, api: *const Api, type_id: *TypeId) ApiError!*anyopaque { - var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = self.inner(), - .type_id = @ptrCast(@alignCast(type_id)), + pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { + var ret = pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ + .ctx = @constCast(self.inner()), }); - const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(&ret); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); defer err.destroy(api); - log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + log.err("[ExecutionContext.getDeviceOrdinal] {s}", .{err.getMessage(api)}); // TODO(Corentin): Retrieve error code from Error when implemented in XLA. return error.Unknown; } - return ret.data.?; + return ret.device_ordinal; } - // TODO getDeviceOrdinal() + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { + var ret = pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ + .ctx = @constCast(self.inner()), + .task = @ptrCast(@alignCast(task)), + .data = @ptrCast(@alignCast(data)), + }); + + const result = api.inner().XLA_FFI_ThreadPool_Schedule.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + std.debug.print("error: {any} \n", .{err}); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } + + fn getTypeId(type_name: []const u8) TypeId { + const id: i64 = @bitCast(std.hash.Fnv1a_64.hash(type_name)); + + return .{ + .type_id = id, + }; + } }; +const TypeId = extern struct { + type_id: i64, + + pub fn toCStruct(self: TypeId) c.XLA_FFI_TypeId { + return c.XLA_FFI_TypeId{ + .type_id = self.type_id, + }; + } +}; + +const Task = fn (*anyopaque) void; + +pub const Stream = opaque {}; + const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -223,10 +258,6 @@ const ByteSpan = extern struct { } }; -pub const TypeId = extern struct { - type_id: i64, -}; - pub const DataType = enum(c.XLA_FFI_DataType) { invalid = c.XLA_FFI_DataType_INVALID, pred = c.XLA_FFI_DataType_PRED, @@ -283,7 +314,7 @@ pub const Buffer = extern struct { _ = fmt; _ = options; - try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data) }); + try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data.asPtr()) }); } }; @@ -360,6 +391,11 @@ pub const Attrs = extern struct { } }; + pub fn slice(self: Array, T: type) []const T { + const ptr: [*]const T = @alignCast(@ptrCast(self.data)); + return ptr[0..self.len]; + } + pub fn getByIndex(self: Attrs, comptime attr_type: AttrType, index: usize) ?*const @FieldType(Attr, @tagName(attr_type)) { const attr = self.ptr[0..self.len][index]; const actual_type = self.types[index]; @@ -450,7 +486,7 @@ pub const Error = opaque { pub const inner = TransmuteMixin(Error, c.XLA_FFI_Error).to; pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; - pub fn create(api: *const Api, error_code: ErrorCode, message: [:0]const u8) *Error { + pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret = pjrtStruct(c.XLA_FFI_Error_Create_Args{ .message = message.ptr, .errc = @intFromEnum(error_code), diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 6528eb89..02736647 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -160,9 +160,14 @@ pub const Api = struct { return state.str; } - pub fn customCallRegistry(api: *const Api) ?CustomCallRegistry { + pub fn createExecuteContext(api: *const Api) ApiError!*ExecuteContext { + const ret = try api.call(.PJRT_ExecuteContext_Create, .{}); + return @ptrCast(ret.context.?); + } + + pub fn ffi(api: *const Api) ?FFI { if (api.lookupExtension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { - return .{ .inner = ext.register_handler.? }; + return .{ .inner = ext }; } return null; } @@ -410,15 +415,26 @@ pub const Client = opaque { element_type: BufferType, layout: MemoryLayout, device: *const Device, + memory_kind: ?Memory.Kind, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.C) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.C) void {} }.call, on_delete_callback_arg: ?*anyopaque = null, - stream: ?*const ffi.Stream = null, + stream: ?*const anyopaque = null, }; pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { const layout = args.layout.toCStruct(); + var memory: ?*const Memory = null; + + if (args.memory_kind) |kind| { + for (try args.device.addressableMemories(api)) |m| { + if (m.kind(api) == kind) { + memory = m; + } + } + } + const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), .device_buffer_ptr = @ptrCast(@constCast(args.data)), @@ -427,6 +443,7 @@ pub const Client = opaque { .element_type = @intFromEnum(args.element_type), .layout = @ptrCast(@constCast(&layout)), .device = @ptrCast(@constCast(args.device)), + .memory = if (memory) |m| @ptrCast(@constCast(m)) else null, .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, .stream = @bitCast(@intFromPtr(args.stream)), @@ -444,20 +461,21 @@ pub const Client = opaque { return &.{}; } - pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!*Buffer { - const ret = try api.call(.PJRT_Client_DMA_Map, .{ + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + const args: Api.CallFnArgType(.PJRT_Client_DmaMap) = .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), .size = @intCast(data.len), - }); - return @ptrCast(ret.buffer.?); + }; + std.debug.print("DMA map {any} \n", .{args}); + _ = try api.call(.PJRT_Client_DmaMap, args); } - pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) void { - _ = api.call(.PJRT_Client_DMA_Unmap, .{ + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + _ = try api.call(.PJRT_Client_DmaUnmap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), - }) catch unreachable; + }); } pub const CreateBuffersForAsyncHostToDeviceArgs = struct { @@ -564,6 +582,14 @@ pub const SerializeResult = struct { } }; +pub const ExecuteContext = opaque { + pub fn deinit(self: *ExecuteContext, api: *const Api) void { + _ = api.call(.PJRT_ExecuteContext_Destroy, .{ + .context = @ptrCast(self), + }) catch {}; + } +}; + pub const Executable = opaque { const inner = InnerMixin(c.PJRT_Executable).inner; @@ -630,6 +656,7 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { var options = pjrtStruct(c.PJRT_ExecuteOptions{ @@ -640,6 +667,7 @@ pub const LoadedExecutable = opaque { .launch_id = 0, .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices.ptr), .num_non_donatable_input_indices = args.non_donatable_input_indices.len, + .context = @ptrCast(args.context), }); _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), @@ -1044,21 +1072,6 @@ pub const AsyncHostToDeviceTransferManager = opaque { } }; -pub const ExecutionContext = opaque { - const inner = InnerMixin(c.PJRT_ExecutionContext).inner; - - pub fn init(api: *const Api) ApiError!*ExecutionContext { - const ret = try api.call(.PJRT_ExecutionContext_Create, .{}); - return @ptrCast(ret.context.?); - } - - pub fn deinit(self: *ExecutionContext, api: *const Api) void { - _ = api.call(.PJRT_ExecutionContext_Destroy, .{ - .context = self.inner(), - }) catch unreachable; - } -}; - pub const NamedValue = extern struct { comptime { std.debug.assert(@sizeOf(NamedValue) == @sizeOf(c.PJRT_NamedValue)); @@ -1164,17 +1177,83 @@ pub const NamedValue = extern struct { } }; +pub const FFI = extern struct { + inner: *const c.PJRT_FFI, + + pub const UserData = extern struct { + type_id: i64, + user_data: *anyopaque, + + fn toCStruct(self: UserData) c.PJRT_FFI_UserData { + return .{ + .type_id = self.type_id, + .data = self.user_data, + }; + } + }; + + pub fn customCallRegistry(self: *const FFI) ?CustomCallRegistry { + return .{ .inner = self.inner.register_handler.? }; + } + + pub fn registerTypeId(self: *const FFI, api: *const Api, user_type: anytype) ApiError!i64 { + const type_name = @typeName(user_type); + var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ + .type_name = type_name.ptr, + .type_name_size = type_name.len, + .type_id = undefined, + }); + const result = self.inner.type_id_register.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + return pjrt_error.getCode(api).toApiError(); + } + + user_type.type_id = ret.type_id; + + return ret.type_id; + } + + pub fn addUserData(self: *const FFI, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { + var ret = pjrtStruct(c.PJRT_FFI_UserData_Add_Args{ + .context = @ptrCast(context), + .user_data = user_data.toCStruct(), + }); + const result = self.inner.user_data_add.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + log.err("[addUserData] {s}", .{pjrt_error.getMessage(api)}); + return pjrt_error.getCode(api).toApiError(); + } + } + + pub fn getTypeId(type_name: []const u8) i64 { + // Use FNV-1a 64-bit hash algorithm + return @bitCast(std.hash.Fnv1a_64.hash(type_name)); + } +}; + +pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { + command_buffer_compatible = 0, + _, +}; + // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub const CustomCallRegistry = extern struct { inner: *const c.PJRT_FFI_Register_Handler, + pub const RegisterFfiOptions = struct { + traits: RegisterHandlerTraits = @enumFromInt(1), + }; + pub fn registerFfi( self: *const CustomCallRegistry, api: *const Api, target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, + options: RegisterFfiOptions, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .api_version = 1, @@ -1183,6 +1262,7 @@ pub const CustomCallRegistry = extern struct { .handler = @ptrCast(@constCast(func)), .platform_name = platform_name.ptr, .platform_name_size = platform_name.len, + .traits = @intFromEnum(options.traits), }); const result = self.inner(&ret); if (result) |pjrt_c_error| { diff --git a/runtimes/cpu/cpu.bzl b/runtimes/cpu/cpu.bzl index d3613802..9e0b0374 100644 --- a/runtimes/cpu/cpu.bzl +++ b/runtimes/cpu/cpu.bzl @@ -25,22 +25,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "1cda1325095c12bd0019838d28ee92d811ac478d22ed3c08020d5a0cd2d9f34a", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "a799573bc238bfbfa18c415002280902e4547f7e12fb09a9a7e73e8b35b58b9b", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "35af82d9e5c70d16ac15f4c18024a2dd5ed2faefc89940eafe3d5350d2cbd9e7", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "31f6fcc1a5341ea7cd6eaf5c441189e5f6399a37dfafb4a45699edacf44c742c", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "da4deaf850d715997614768b2fc0283595ee8181133ab3243d65635e3439de69", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "91a18c0fb2c82f28953b5134833415638422925099ee6498987d583f178588b5", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index a7f8bf94..ed50ebf5 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -1,5 +1,11 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") +exports_files( + ["libpjrt_cuda.so"], + visibility = ["//visibility:public"], +) + cc_library( name = "zmlxcuda_lib", srcs = ["zmlxcuda.c"], diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index 82d1e448..2c116ef0 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -202,8 +202,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "64029cd3d68118b166198e3246877ed706ed35eb732b1770a9bf530b5b0a8ab4", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "9e2d1741de8531274c4c2ffe77f3a82003591827294e2a2773cc09f33cc2137e", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index 8881a016..b5b30f67 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -25,7 +25,8 @@ copy_to_directory( cc_import( name = "libpjrt_cuda", data = [":sandbox"], - shared_library = "libpjrt_cuda.so", + shared_library = "@zml//runtimes/cuda:libpjrt_cuda.so", + # shared_library = "libpjrt_cuda.so", add_needed = ["libzmlxcuda.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index c190a9ee..9d41c106 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -127,8 +127,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "13a2ced965a44a0e8ad0d752c8ac5aa99107b17b41bd850967d40b82e102ec50", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "4bdb7690f6e7ac0faa1c29299663269781c2b5d33fbe28c1ff4bc33ce9ec01ff", ) return mctx.extension_metadata( diff --git a/stdx/meta.zig b/stdx/meta.zig index 7aa90531..ee8d998d 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -164,6 +164,14 @@ pub fn FnResult(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } +pub fn FnResultNoError(comptime func: anytype) type { + const ReturnT = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(ReturnT)) { + .error_union => |info| info.payload, + else => ReturnT, + }; +} + pub fn Head(Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch index 58e3874d..bff748ec 100644 --- a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -31,10 +31,14 @@ diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_e index a33bd4aa9c..3309194538 100644 --- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h +++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); +@@ -66,12 +66,33 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); // Adds a user data to the execute context. typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); ++ typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ +struct PJRT_FFI_Register_Handler_Args { + size_t struct_size; + const char* target_name; @@ -43,8 +47,9 @@ index a33bd4aa9c..3309194538 100644 + void* handler; + const char* platform_name; + size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; +}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); + +// Registers an FFI call handler for a specific platform. +typedef PJRT_Error* PJRT_FFI_Register_Handler( diff --git a/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel b/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel new file mode 100644 index 00000000..50cafb9e --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel @@ -0,0 +1,37 @@ +module( + name = "xla", + version = "20250409.0-7167d71", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.11") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.20.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl", "python_version_repo") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "llvm-raw", + "stablehlo", +) + +llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") +llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel b/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/llvm.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/llvm.bzl new file mode 100644 index 00000000..57e77361 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/llvm.bzl @@ -0,0 +1,13 @@ +load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_configure_impl(mctx): + _llvm_configure(name = "llvm-project") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm_configure = module_extension( + implementation = _llvm_configure_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl new file mode 100644 index 00000000..192e06f8 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl @@ -0,0 +1,60 @@ +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//third_party/triton:workspace.bzl", triton = "repo") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + pybind11_bazel() + triton() + llvm("llvm-raw") + stablehlo() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "//third_party/grpc:generate_cc_env_fix.patch", + "//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", + "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..89cee870 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,137 @@ +From 490985facbe2fe6462b2006a99514b1c7e8a0775 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Wed, 5 Feb 2025 19:25:03 +0100 +Subject: [PATCH 1/7] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 ++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 +++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 33 ++++++++++++++++++++++++++- + 3 files changed, 53 insertions(+), 1 deletion(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index 8e8a4baa53..551c35b461 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,8 +69,13 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index d9f6d163d8..3dbd4df089 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -69,12 +69,33 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++ typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + size_t struct_size; + PJRT_Extension_Type type; + PJRT_Extension_Base* next; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 8898088a23..746861f697 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,16 +13,21 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" + #include "absl/strings/string_view.h" ++#include "absl/strings/str_format.h" ++#include "xla/ffi/api/c_api.h" ++#include "xla/ffi/api/ffi.h" + #include "xla/ffi/execution_context.h" + #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -68,6 +73,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, +@@ -75,6 +105,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + /*next=*/next, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.39.5 (Apple Git-154) diff --git a/third_party/modules/xla/20250409.0-7167d71/source.json b/third_party/modules/xla/20250409.0-7167d71/source.json new file mode 100644 index 00000000..9f9bf0a7 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/source.json @@ -0,0 +1,16 @@ +{ + "strip_prefix": "xla-7167d711e572e1da17d6c4a87ab1f3661c8f2ac0", + "url": "https://github.com/openxla/xla/archive/7167d711e572e1da17d6c4a87ab1f3661c8f2ac0.tar.gz", + "integrity": "sha256-QxwcMAFT4lQLHp+mHh3nhWo9qwbTf9+kiOMWmNuXoVI=", + "overlay": { + "llvm.bzl": "", + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" + } +} diff --git a/third_party/modules/xla/metadata.json b/third_party/modules/xla/metadata.json index d0304d21..01c87137 100644 --- a/third_party/modules/xla/metadata.json +++ b/third_party/modules/xla/metadata.json @@ -19,8 +19,7 @@ "20250123.0-cc075be", "20250204-0-6789523", "20250317.0-71c67e2", - "20250317.1-71c67e2", - "20250317.2-71c67e2" + "20250409.0-7167d71" ], "yanked_versions": {} } diff --git a/zml/buffer.zig b/zml/buffer.zig index b1919e94..e8cc1032 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -4,6 +4,8 @@ const testing = std.testing; const asynk = @import("async"); const stdx = @import("stdx"); +const ffi = @import("ffi.zig"); +const ops = @import("ops.zig"); const Context = @import("context.zig").Context; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -97,6 +99,39 @@ pub const Buffer = struct { return res; } + pub const FromExOpts = struct { + memory: Memory = .device, + }; + pub fn fromEx(platform: Platform, host_buffer: HostBuffer, opts: FromExOpts) !Buffer { + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._shape = host_buffer.shape(), + ._shards = .{}, + }; + + const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); + const byte_strides = host_buffer.strides(); + + const devices = platform.getDevices(); + for (devices) |device| { + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = host_buffer.data, + .buffer_type = buffer_type, + .dims = host_buffer.shape().dims(), + .byte_strides = byte_strides, + .device = device, + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .memory = @constCast(platform.pjrt_client.memoryByKind(platform.pjrt_api, opts.memory.toPjrtMemory())), + }); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } + res._shards.appendAssumeCapacity(pjrt_buffer); + } + + return res; + } + pub fn awaitt(self: Buffer) !Buffer { for (self._shards.constSlice()) |buffer| { if (buffer.getReadyEvent(self._api)) |ev| { @@ -225,24 +260,18 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. - pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const pjrt.ffi.Stream, device_data: *anyopaque) Buffer { - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var res: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - res[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk res; - }; - + pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const anyopaque, device_data: *anyopaque) Buffer { const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, .element_type = bufferTypeFromDtype(shape_.dtype()), .dims = shape_.dims(), // TODO: exposes sharding in the API. .device = platform.getDevices()[0], + // TODO: expose memory in the API. + .memory_kind = null, .layout = .{ .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], + .minor_to_major = ops.minorToMajor(shape_.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, @@ -271,29 +300,31 @@ pub const Buffer = struct { return res; } - pub fn copyToMemory(self: Buffer, platform: Platform, kind: Memory) !Buffer { - const device = try self._shards.get(0).getDevice(self._api); - const memories = try device.addressableMemories(self._api); - var selected_mem: *const pjrt.Memory = undefined; - for (memories) |mem| { - if (mem.kind(platform.pjrt_api) == kind.toPjrtMemory()) { - selected_mem = mem; - } - } - - var shards: Shards = .{}; - for (self._shards.constSlice()) |shard| { - const pjrt_buffer = try shard.copyToMemory(self._api, selected_mem); - shards.appendAssumeCapacity(pjrt_buffer); - } - - return .{ - ._api = platform.pjrt_api, - ._shape = self._shape, - ._shards = shards, - }; + pub fn setValue(self: Buffer, T: type, value: anytype) !void { + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + const shard_buffer = self._shards.get(0); + const opaqueDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); + const sizeInBytes = try shard_buffer.getOnDeviceSizeInBytes(self._api); + stdx.debug.assert(sizeInBytes == @sizeOf(T), "Buffer size {d} does not match type size {d}", .{ sizeInBytes, @sizeOf(T) }); + const data = @as(*T, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + data.* = value; } + pub fn getValueFromDataInMemory(self: Buffer, T: type) !T { + stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); + const data = try self.dataInMemory(); + const value = std.mem.bytesAsValue(T, @constCast(data)); + return value.*; + } + + pub fn dataInMemory(self: Buffer) ![]const u8 { + const shard_buffer = self._shards.get(0); + const opaqueDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); + const sizeInbytes = try shard_buffer.getOnDeviceSizeInBytes(self._api); + const data = @as([*]const u8, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + const end: usize = @intCast(sizeInbytes); + return data[0..end]; + } /// Copies the content of the Buffer back to host, in the given buffer, /// and return a new `HostBuffer` object with the same shape. /// The returned `HostBuffer` doesn't own the memory. @@ -310,7 +341,7 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); + var output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); output._event = maybe_event; @@ -373,6 +404,29 @@ pub const Buffer = struct { return shard.memory(self._api); } + pub fn copyToMemory(self: Buffer, platform: Platform, kind: Memory) !Buffer { + const device = try self._shards.get(0).getDevice(self._api); + const memories = try device.addressableMemories(self._api); + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == kind.toPjrtMemory()) { + selected_mem = mem; + } + } + + var shards: Shards = .{}; + for (self._shards.constSlice()) |shard| { + const pjrt_buffer = try shard.copyToMemory(self._api, selected_mem); + shards.appendAssumeCapacity(pjrt_buffer); + } + + return .{ + ._api = platform.pjrt_api, + ._shape = self._shape, + ._shards = shards, + }; + } + fn hasShardedAxis(self: Buffer) bool { if (self._shards.len == 1) return false; return @reduce(.Or, self._shape._sharding_info); diff --git a/zml/context.zig b/zml/context.zig index 7f39d404..39d7cfeb 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -183,8 +183,6 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); return p; } @@ -218,107 +216,13 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; }; -const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { - const registry = platform.pjrt_api.customCallRegistry(); - - if (registry) |reg| { - try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback); - } else { - stdx.debug.panic("Registering custom calls failed", .{}); - } - } - - fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - // TODO correctly pass the callback w/o embedding ptr in MLIR - const err = "malformed zmlHostBufferCallback custom call"; - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse @panic(err); - std.debug.assert(callback_attr.dtype == .u64); - const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); - - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse @panic(err); - std.debug.assert(user_ctx_ptr.dtype == .u64); - const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - - const tags_arr = call_frame.attrs.getByName(.array, "__tags") orelse @panic(err); - std.debug.assert(tags_arr.dtype == .i64); - const tags = tags_arr.slice([*:0]const u8); - - const host_inputs = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - const host_outputs = stdx.stackSlice(8, HostBuffer, call_frame.results.len); - - // TODO correctly pass the platform w/o global - const platform = _active_platform; - if (platform.target == .cpu) { - for (host_inputs, call_frame.args.buffers()) |*host, device| { - host.* = hostViewOfDeviceBuffer(device); - } - for (host_outputs, call_frame.results.buffers()) |*host, device| { - host.* = hostViewOfDeviceBuffer(device); - } - copyTags(tags, host_inputs, host_outputs); - - callback(user_ctx, host_inputs, host_outputs); - return null; - } - - if (platform.target != .cuda) @panic("hostBufferCallback are only supported on cpu and cuda for now"); - const stream = call_frame.*.api.stream(call_frame.ctx); - for (host_inputs, call_frame.args.buffers()) |*host, device| { - // Note: we are using the smp allocator here. - // Typically a given custom call will be called a lot of times with the same shape - // So having a per custom call arena would be great. - // We do need to be careful for the case where two callbacks are called at the same time though. - host.* = HostBuffer.empty(std.heap.smp_allocator, getShape(device)) catch @panic("OOM"); - // log.warn("Readind device memory from {*} to {*}", .{ device.data.asPtr(), host.data }); - cuda.memcpyToHostAsync(@constCast(host.data), device.data, stream); - } - - init_host_outputs: for (host_outputs, call_frame.results.buffers()) |*host, device| { - // TODO: in case of input/output aliasing we don't need to allocate here. - // This information can be inferred from the data pointers of the inputs. - for (host_inputs, call_frame.args.buffers()) |host_input, device_input| { - if (device_input.data == device.data) { - host.* = host_input; - std.debug.assert(host_input.shape().eql(getShape(device))); - // log.warn("Aliased host buffer for output: {*}", .{host.data}); - continue :init_host_outputs; - } - } - host.* = HostBuffer.empty(std.heap.smp_allocator, getShape(device)) catch @panic("OOM"); - // log.warn("Allocated host buffer for output: {*}", .{host.data}); - } - copyTags(tags, host_inputs, host_outputs); - - cuda.streamSynchronize(stream); - - // TODO we should return an pjrt.ffi.Future to allow the pjrt thread to schedule more work. - callback(user_ctx, host_inputs, host_outputs); - - for (host_outputs, call_frame.results.buffers()) |host, device| { - // log.warn("Writing from {*} to device memory {*}", .{ device.data, host.data.asPtr() }); - cuda.memcpyToDeviceAsync(device.data, host.data, stream); - } - - for (host_inputs) |*b| b.deinit(std.heap.smp_allocator); - // TODO we could return here and schedule the sync/free in another task. - cuda.streamSynchronize(stream); - for (host_outputs) |*b| b.deinit(std.heap.smp_allocator); - - return null; - } -}; - -const cuda = struct { +pub const cuda = struct { var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); - var _streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); + pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); + pub var cuLaunchHostFunc: CuLaunchHostFunc = @ptrFromInt(0xdeadc00da00); pub const MemcpyKind = enum(c_int) { host_to_host = 0, @@ -331,6 +235,7 @@ const cuda = struct { const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; const MemcpyBlocking = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind) callconv(.C) c_int; const StreamSynchronize = *const fn (stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + const CuLaunchHostFunc = *const fn (stream: *anyopaque, host_func: *const fn (user_data: *const anyopaque) callconv(.c) void, user_data: *anyopaque) callconv(.c) c_int; pub fn init() void { var cudart = std.DynLib.open("libcudart.so.12") catch { @@ -345,9 +250,12 @@ const cuda = struct { _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { @panic("cudaMemcpy not found"); }; - _streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { + streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { @panic("cudaStreamSynchronize not found"); }; + cuLaunchHostFunc = cudart.lookup(CuLaunchHostFunc, "cudaLaunchHostFunc") orelse { + @panic("cudaLaunchHostFunc not found"); + }; } pub fn memcpyToHostBlocking(dst: []u8, src: pjrt.ffi.DevicePtr) void { @@ -355,23 +263,18 @@ const cuda = struct { check(err); } - pub fn memcpyToHostAsync(dst: []u8, src: pjrt.ffi.DevicePtr, stream: *pjrt.ffi.Stream) void { - const err = _memcpyAsync(dst.ptr, src.asPtr(), dst.len, .device_to_host, stream); - check(err); - } - pub fn memcpyToDeviceBlocking(dst: pjrt.ffi.DevicePtr, src: []const u8) void { const err = _memcpyBlocking(dst.asPtr(), src.ptr, src.len, .host_to_device); check(err); } - pub fn memcpyToDeviceAsync(dst: pjrt.ffi.DevicePtr, src: []const u8, stream: *pjrt.ffi.Stream) void { + pub fn memcpyToDeviceAsync(dst: pjrt.ffi.DevicePtr, src: []const u8, stream: ?*pjrt.ffi.Stream) void { const err = _memcpyAsync(dst.asPtr(), src.ptr, src.len, .host_to_device, stream); check(err); } - pub fn streamSynchronize(stream: *pjrt.ffi.Stream) void { - const err = _streamSynchronize(stream); + pub fn memcpyToHostAsync(dst: []u8, src: pjrt.ffi.DevicePtr, stream: ?*pjrt.ffi.Stream) void { + const err = _memcpyAsync(dst.ptr, src.asPtr(), dst.len, .device_to_host, stream); check(err); } @@ -381,46 +284,3 @@ const cuda = struct { @panic("CUDA error"); } }; - -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { - // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { - .invalid => @panic("invalid ffi"), - .pred => .bool, - .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), - }; - return Shape.init(buffer_desc.dims(), dt); -} - -/// Create a Device from a ffi description of a buffer. -/// Normally the ffi describe device buffer but when platform == cpu it's the same addresspace -fn hostViewOfDeviceBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { - const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( - buffer_shape, - buffer_desc.data.asPtr()[0..buffer_shape.byteSize()], - ); -} - -fn deviceBuffer(platform: Platform, buffer_desc: *const pjrt.ffi.Buffer, stream: *const pjrt.ffi.Stream) Buffer { - const buffer_shape = getShape(buffer_desc); - return Buffer.asViewOfDeviceBuffer(platform, buffer_shape, stream, buffer_desc.data); -} - -fn copyTags(flat_tags: []const [*:0]const u8, in: []HostBuffer, out: []HostBuffer) void { - var tags = flat_tags.ptr; - var num_read: usize = 0; - for (in) |*b| { - const r = b.rank(); - b._shape._tags.buffer = tags[0..8].*; - tags = tags[r..]; - num_read += r; - } - for (out) |*b| { - const r = b.rank(); - b._shape._tags.buffer = tags[0..8].*; - tags = tags[r..]; - num_read += r; - } -} diff --git a/zml/exe.zig b/zml/exe.zig index 5528ce4f..63f9865f 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -12,7 +12,7 @@ const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; -const log = std.log.scoped(.zml); +const log = std.log.scoped(.@"zml/exe"); test { std.testing.refAllDecls(@This()); @@ -155,6 +155,8 @@ pub const BaseExe = struct { /// Allocator backing memory _arena: std.heap.ArenaAllocator, + _context: ?*pjrt.ExecuteContext = null, + pub fn init(parent_allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, args: struct { n_in: u32, result_shapes: []const Shape, n_devices: u8 }) !BaseExe { var arena = std.heap.ArenaAllocator.init(parent_allocator); errdefer arena.deinit(); @@ -188,6 +190,9 @@ pub const BaseExe = struct { } pub fn deinit(self: BaseExe) void { + if (self._context) |ctx| { + ctx.deinit(self.platform.pjrt_api); + } self._arena.deinit(); } @@ -210,16 +215,17 @@ pub const BaseExe = struct { // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, + .context = if (self._context) |ctx| ctx else null, }) catch unreachable; self.platform.tracer.frameEnd(trace, "_unsafeCall execute"); - const trace_events_await = self.platform.tracer.frameStart("_unsafeCall await events"); - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } - self.platform.tracer.frameEnd(trace_events_await, "_unsafeCall await events"); + // const trace_events_await = self.platform.tracer.frameStart("_unsafeCall await events"); + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.awaitt(self.platform.pjrt_api) catch unreachable; + // } + // } + // self.platform.tracer.frameEnd(trace_events_await, "_unsafeCall await events"); } pub fn serialize(self: BaseExe, writer: anytype) !void { @@ -240,6 +246,23 @@ pub const BaseExe = struct { self.ready_buffer_count += n; } + pub fn attach(self: BaseExe, type_id: i64, value: anytype) !void { + stdx.debug.assert(self._context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = self.platform.pjrt_api; + const ffi = pjrt_api.ffi().?; + const ValueT = @TypeOf(value); + const type_name = switch (@typeInfo(ValueT)) { + .pointer => @typeName(std.meta.Child(ValueT)), + else => @typeName(ValueT), // todo error + }; + + // const type_id = try ffi.registerTypeId(pjrt_api, type_name); + + const user_data: *anyopaque = @ptrCast(@constCast(value)); + log.info("Attached {s}@{x} with type id {d} on {any}", .{ type_name, user_data, type_id, self._context.? }); + try ffi.addUserData(pjrt_api, self._context.?, .{ .type_id = type_id, .user_data = user_data }); + } + pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { var shards: Buffer.Shards = .{}; for (self.output_per_device) |dev_out| { @@ -250,11 +273,13 @@ pub const BaseExe = struct { } pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { - return .init(parent_allocator, self.platform, self.exe, .{ + var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ .n_in = self.input_buffer_count, .result_shapes = self.result_shapes, .n_devices = self.num_devices, }); + exe._context = self._context; + return exe; } }; @@ -283,6 +308,15 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + pub fn withExecutionContext(self: Self) Self { + stdx.debug.assert(self.inner._context == null, "Exe already has an execution context", .{}); + var new: Self = .{ .inner = self.inner }; + const pjrt_execute_context = self.inner.platform.pjrt_api.createExecuteContext() catch unreachable; + new.inner._context = pjrt_execute_context; + log.info("Created context execution {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); + return new; + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } diff --git a/zml/ffi.zig b/zml/ffi.zig new file mode 100644 index 00000000..02af157c --- /dev/null +++ b/zml/ffi.zig @@ -0,0 +1,244 @@ +const std = @import("std"); +const testing = std.testing; +const builtin = @import("builtin"); + +const asynk = @import("async"); +const stdx = @import("stdx"); + +const context = @import("context.zig"); +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; +const dtype = @import("dtype.zig"); +const pjrt = @import("pjrtx.zig"); +const buffer = @import("buffer.zig"); +const hostbuffer = @import("hostbuffer.zig"); +const mlir = @import("mlir.zig"); +const module = @import("module.zig"); +const ops = @import("ops.zig"); +const platform = @import("platform.zig"); +const tensor = @import("tensor.zig"); +const shape = @import("shape.zig"); +const cuda = context.cuda; +const Buffer = buffer.Buffer; +const HostBuffer = hostbuffer.HostBuffer; +const CompilationContext = module.CompilationContext; +const DataType = dtype.DataType; +const Platform = platform.Platform; +const Shape = shape.Shape; +const Tensor = tensor.Tensor; + +const log = std.log.scoped(.@"zml/custom_call"); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub const FFIBuffer = pjrt.ffi.Buffer; +pub const FFIRets = pjrt.ffi.Rets; +pub const FFIStream = pjrt.ffi.Stream; +// todo: check deallocs + +pub fn CustomCallCompilationInputType(custom_op: type) type { + const ArgsT = stdx.meta.FnArgs(custom_op.compile); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + // todo check typeof first input then others args + return [@typeInfo(ArgsT).@"struct".fields.len]Tensor; +} + +pub fn CustomCallCompilationOutputType(custom_op: type) type { + const ReturnT = stdx.meta.FnResultNoError(custom_op.compile); + + if (@typeInfo(ReturnT) != .@"struct") { + @compileError("Expected struct type"); + } + + if (ReturnT == Buffer) { + return Tensor; + } + return [@typeInfo(ReturnT).@"struct".fields.len]Tensor; +} + +pub fn lenOutputBeforeCustomCall(custom_op: type) usize { + const ReturnT = stdx.meta.FnResultNoError(custom_op.compile); + + if (@typeInfo(ReturnT) != .@"struct") { + @compileError("Expected struct type"); + } + + if (ReturnT == Tensor) { + return 1; + } + return @typeInfo(ReturnT).@"struct".fields.len; +} + +pub fn CustomCallInputsType(custom_op: type) type { + const ArgsT = stdx.meta.FnArgs(custom_op.call); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + // todo check typeof first input then others args + return [@typeInfo(ArgsT).@"struct".fields.len - 1]Tensor; +} + +pub fn custom_call( + comptime custom_op: type, + inputs: CustomCallInputsType(custom_op), + res_shapes_: []const Shape, +) []Tensor { + stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); + const op_name = @typeName(custom_op); + + const ctx = CompilationContext.current(); + const allocator = ctx.allocator(); + const mlir_ctx = ctx.mlirCtx(); + const platform_ = ctx._platform; + const pjrt_api = platform_.pjrt_api; + const ffi = pjrt_api.ffi().?; + const registry = ffi.customCallRegistry().?; + const ffi_func = proxy(custom_op); + const target_name = "callback_" ++ op_name; + + registry.registerFfi(pjrt_api, target_name, @tagName(platform_.target), &ffi_func, .{}) catch unreachable; + log.info("{s} / Registered custom call with target_name \"{s}\" with proxy func {*}", .{ op_name, target_name, &ffi_func }); + + // var custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; + // var res_shapes = allocator.alloc(Shape, 8) catch unreachable; + // var res_types = allocator.alloc(mlir.Type, 8) catch unreachable; + + // if (@typeInfo(@TypeOf(custom_op.compile)) != .@"fn") { + // stdx.debug.panic("compile must be a function"); + // } + + // var args: std.meta.ArgsTuple(@TypeOf(custom_op.compile)) = undefined; + // inline for (0..args.len) |i| { + // args[i] = inputs[i]; + // } + + // const compile_ouputs = ctx.callFunc(@typeName(custom_op), custom_op.compile, args) catch |err| { + // stdx.debug.panic("Error in {any} beforeCustomCall func: {any}\n", .{ @typeName(custom_op), err }); + // }; + // var compile_ouputs_array = allocator.alloc(Shape, 8) catch unreachable; + + // if (@TypeOf(compile_ouputs) == Shape) { + // compile_ouputs_array[0] = compile_ouputs; + // } else { + // inline for (@typeInfo(@TypeOf(compile_ouputs)).@"struct".fields, 0..) |field, i| { + // compile_ouputs_array[i] = @field(compile_ouputs, field.name); + // } + // } + + // for (compile_ouputs_array[0..res_shapes_.len]) |*rest_t| { + // log.warn("compile_ouputs_array: {any}", .{rest_t}); + // } + + // for (inputs, 0..) |t, i| { + // custom_call_inputs[i] = t.value(); + // } + + // for (compile_ouputs_array[0..res_shapes_.len], 0..) |sh, i| { + // res_shapes[i] = sh; + // res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, sh).as(mlir.Type); + // } + + const custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; + for (inputs, 0..) |t, i| { + custom_call_inputs[i] = t.value(); + } + + const res_types = allocator.alloc(mlir.Type, 8) catch unreachable; + for (res_shapes_, 0..) |sh, i| { + res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, sh).as(mlir.Type); + } + + const frontend_attributes = mlir.Attribute.dict(mlir_ctx, &.{ + .{ "_xla_compute_type", .string(mlir_ctx, "host") }, + .{ "_xla_buffer_placement", .string(mlir_ctx, @tagName(Buffer.Memory.host_pinned.toPjrtMemory())) }, + }); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + custom_call_inputs[0..inputs.len], + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, + .has_side_effect = true, + .output_operand_aliases = &.{}, + }, + res_types[0..res_shapes_.len], + mlir_ctx.location(@src()), + ); + + // if (CustomCallCompilationOutputType(custom_op) == Tensor) { + // return Tensor._result(res_shapes_[0], op.result(0)); + // } + + var custom_call_outputs = allocator.alloc(Tensor, 8) catch unreachable; + + for (custom_call_outputs[0..res_shapes_.len], res_shapes_, 0..) |*t, sh, i| { + t.* = Tensor._result(sh, op.result(i)); + } + + return custom_call_outputs[0..res_shapes_.len]; +} + +fn proxy(comptime custom_op: type) pjrt.ffi.Handler { + const op_name = @typeName(custom_op); + _ = op_name; // autofix + + return struct { + const Self = @This(); + + pub fn proxy(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + const execution_context = call_frame.ctx.?; + // const device_id = execution_context.getDeviceOrdinal(call_frame.api) catch unreachable; + const user_ctx: *custom_op = pjrt.ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api) catch unreachable; + // const platform_: Platform = user_ctx.platform; + // const pjrt_api = platform_.pjrt_api; + // const pjrt_client = platform_.pjrt_client; + // const tracer = platform_.tracer; + // _ = tracer; // autofix + + // const device = pjrt_client.getAddressableDevices(pjrt_api)[@as(u32, @intCast(device_id))]; + // _ = device; // autofix + const ctx = call_frame.ctx; + const stream = call_frame.api.stream(@constCast(ctx)); + + var callback_args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; + callback_args[0] = user_ctx; + + inline for (1..callback_args.len) |i| { + const ffi_buffer = call_frame.args.buffers()[i - 1]; + callback_args[i] = ffi_buffer; + } + + user_ctx.*.results = call_frame.results.buffers(); + user_ctx.*.stream = stream; + + @call(.auto, custom_op.call, callback_args) catch |err| { + stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(custom_op), err }); + }; + + return null; + } + }.proxy; +} + +pub fn getShape(ffi_buffer: *const pjrt.ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .pred => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index a8577932..91c5b6c0 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -162,9 +162,18 @@ pub const HostBuffer = struct { /// Copies this HostBuffer to the given accelerator. pub fn toDevice(self: HostBuffer, platform_: Platform) !Buffer { + const frame = platform_.tracer.frameStart("HostBuffer toDevice"); + defer platform_.tracer.frameEnd(frame, "HostBuffer toDevice"); return try Buffer.from(platform_, self); } + pub const ToExOpts = Buffer.FromExOpts; + pub fn toEx(self: HostBuffer, platform_: Platform, opts: ToExOpts) !Buffer { + const frame = platform_.tracer.frameStart("HostBuffer toEx"); + defer platform_.tracer.frameEnd(frame, "HostBuffer toEx"); + return try Buffer.fromEx(platform_, self, opts); + } + /// Interpret the underlying data as a contiguous slice. /// WARNING: It's only valid if the buffer is contiguous. /// Strided buffers can't use this method. diff --git a/zml/module.zig b/zml/module.zig index 773cb274..b0da8c2d 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -196,6 +196,9 @@ pub const CompilationContext = struct { pjrt_location = try std.fs.path.joinZ(arena, &.{ module_dir.?, "module.pjrt" }); } + log.info("******** ZML generated MLIR ********", .{}); + log.info("{}", .{module.op().mlirFormatter(.{})}); + const loaded_executable: *pjrt.LoadedExecutable = blk: { if (pjrt_location) |pjrt_loc| { if (loadPjrtExecutable(arena, self._platform, pjrt_loc)) |exe| { @@ -220,9 +223,6 @@ pub const CompilationContext = struct { break :blk loaded_executable; }; - log.debug("******** ZML generated MLIR ********", .{}); - log.debug("{}", .{module.op().mlirFormatter(.{})}); - if (timer) |*t| { const time_ms = @divFloor(t.lap(), std.time.ns_per_ms); if (time_ms > 1000) log.info("Compilation took {d:.3}s", .{stdx.math.divFloat(f32, time_ms, 1000)}); @@ -396,6 +396,7 @@ pub const CompilationContext = struct { @memset(res_attrs, .{}); if (opts.kind == .main) { + self.addInputsMemoryKindAttributes(arg_attrs, res_attrs, input_shapes.items, fn_res_shapes); self.addDonationsAttributes(arg_attrs, fn_res_donations); self.addOutputMemoryKindAttributes(res_attrs, fn_res_output_memory_kind); if (self._platform.sharding().num_partitions > 1) { @@ -525,6 +526,27 @@ pub const CompilationContext = struct { } } + fn addInputsMemoryKindAttributes(self: CompilationContext, arg_attrs: []AttributeList, res_attrs: []AttributeList, input_shapes: []const Shape, output_shapes: []const Shape) void { + _ = res_attrs; // autofix + _ = output_shapes; // autofix + const ctx = self.mlirCtx(); + _ = ctx; // autofix + + const memory_kind = Buffer.Memory.host_pinned; + _ = memory_kind; // autofix + for (arg_attrs, input_shapes, 0..) |*attr, shape, i| { + _ = attr; // autofix + _ = shape; // autofix + if (i == 0) { + // attr.appendAssumeCapacity(.named( + // ctx, + // "mhlo.memory_kind", + // .string(ctx, memory_kind.pjrtName()), + // )); + } + } + } + fn addShardingAttributes(self: CompilationContext, arg_attrs: []AttributeList, res_attrs: []AttributeList, input_shapes: []const Shape, output_shapes: []const Shape) void { const ctx = self.mlirCtx(); if (!self._platform.compilation_options.sharding_enabled) return; @@ -723,6 +745,7 @@ pub const CompilationContext = struct { ._shape = shape, ._id = .{ .arg_id = arg_id.* }, ._donation = .input_buffer, + // ._output_memory_kind = .host_pinned, }; } }; @@ -1158,9 +1181,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1176,7 +1199,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/ops.zig b/zml/ops.zig index 14580d9d..3f70997b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -769,96 +769,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - const ctx = CompilationContext.current(); - var tags = std.ArrayListUnmanaged(i64).initCapacity(ctx.allocator(), (inputs.len + output_shapes.len) * Shape.MAX_RANK) catch @panic("OOM"); - defer tags.deinit(ctx.allocator()); - - var readable_tags = std.ArrayList(u8).initCapacity(ctx.allocator(), (inputs.len + output_shapes.len) * 32) catch @panic("OOM"); - defer readable_tags.deinit(); - - readable_tags.appendSlice("(") catch {}; - for (inputs) |input| { - readable_tags.appendSlice(".{") catch {}; - for (input.shape().tags()) |t| { - const ptr: u64 = @intFromPtr(t); - tags.appendAssumeCapacity(@bitCast(ptr)); - readable_tags.appendSlice(std.mem.span(t)) catch {}; - readable_tags.append(',') catch {}; - } - readable_tags.appendSlice("},") catch {}; - } - readable_tags.appendSlice(") -> (") catch {}; - for (output_shapes) |output| { - readable_tags.appendSlice(".{") catch {}; - for (output.tags()) |t| { - const ptr: u64 = @intFromPtr(t); - tags.appendAssumeCapacity(@bitCast(ptr)); - readable_tags.appendSlice(std.mem.span(t)) catch {}; - readable_tags.append(',') catch {}; - } - readable_tags.appendSlice("},") catch {}; - } - readable_tags.appendSlice(")") catch {}; - - const mlir_ctx = ctx.mlirCtx(); - const backend_config = mlir.Attribute.dict(mlir_ctx, &.{ - .{ "callback", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(callback))) }, - .{ "user_context", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(blkctx))) }, - .{ "tags", .string(mlir_ctx, readable_tags.items) }, - .{ "__tags", .dense(mlir_ctx, .i64, tags.items) }, - }); - - const values = stdx.stackSlice(8, mlir.Value, inputs.len); - const operands_layouts = stdx.stackSlice(8, []const usize, inputs.len); - for (inputs, 0..) |input, i| { - values[i] = ctx.getValue(input); - operands_layouts[i] = minorToMajor(input.rank()); - } - - const res_types = stdx.stackSlice(8, mlir.Type, output_shapes.len); - const results_layouts = stdx.stackSlice(8, []const usize, output_shapes.len); - for (output_shapes, 0..) |o, i| { - res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o).as(mlir.Type); - results_layouts[i] = minorToMajor(o.rank()); - } - - const loc = ctx.mlirCtx().location(@src()); - const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), - values, - .{ - .call_target_name = "zmlHostBufferCallback", - .api_version = .typed_ffi, - .backend_config = backend_config, - .has_side_effect = opts.has_side_effect, - .operand_layouts = operands_layouts, - .result_layouts = results_layouts, - .output_operand_aliases = opts.output_operand_aliases, - }, - res_types, - loc, - ); - - const res = ctx.allocator().alloc(Tensor, output_shapes.len) catch @panic("OOM"); - for (res, output_shapes, 0..) |*r, o, i| { - r.* = Tensor._result(o, op.result(i)); //.toMemory(.device); - } - - return res; -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, @@ -1291,13 +1201,13 @@ inline fn toI64(values: anytype) []i64 { } const _MINOR_TO_MAJOR = blk: { - var ret: [Shape.MAX_RANK]usize = undefined; + var ret: [Shape.MAX_RANK]i64 = undefined; for (0..Shape.MAX_RANK) |i| { ret[i] = @intCast(Shape.MAX_RANK - i - 1); } break :blk ret; }; -fn minorToMajor(rank: u8) []const usize { +pub fn minorToMajor(rank: u8) []const i64 { return _MINOR_TO_MAJOR[_MINOR_TO_MAJOR.len - rank ..]; } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 91126c72..aa40de12 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -8,6 +8,7 @@ pub const ffi = pjrt.ffi; pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; +pub const ExecuteContext = pjrt.ExecuteContext; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; @@ -62,6 +63,13 @@ pub const Client = opaque { return .{ @ptrCast(buffer), @ptrCast(event_) }; } + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaMap(api, data); + } + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaUnmap(api, data); + } + pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { return @ptrCast(try asynk.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); } @@ -116,7 +124,7 @@ pub const Client = opaque { return self.inner().addressableMemories(api); } - pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*Memory { + pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { for (self.addressableMemories(api)) |mem| { if (mem.kind(api) == kind) { return mem; @@ -201,6 +209,10 @@ pub const Event = opaque { return self.inner().getEventError(api); } + pub fn awaitt2(self: *Event, api: *const Api) !void { + return self.inner().await_(api); + } + pub const await_ = awaitt; pub fn awaitt(self: *Event, api: *const Api) !void { defer self.deinit(api); @@ -255,6 +267,7 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { @@ -264,6 +277,7 @@ pub const LoadedExecutable = opaque { .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, + .context = args.context, }); } diff --git a/zml/platform.zig b/zml/platform.zig index adf8d639..f71efbe5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -128,6 +128,7 @@ const _CreateOptions = struct { }; pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.fromBool("should_stage_host_to_device_transfers", false)); switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); diff --git a/zml/tensor.zig b/zml/tensor.zig index 5e48ba21..05558418 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -8,8 +8,10 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; +const custom_call = @import("ffi.zig").custom_call; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const Memory = @import("buffer.zig").Buffer.Memory; +const ffi = @import("ffi.zig"); const meta = @import("meta.zig"); const mlir = @import("mlir.zig"); const Location = mlir.Location; @@ -234,6 +236,39 @@ pub const Tensor = struct { }; } + pub fn optimizationBarrier(tensors: []const Tensor) []Tensor { + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const allocator = ctx.allocator(); + + var inputs: [16]mlir.Value = undefined; + var res_types: [16]mlir.Type = undefined; + + for (tensors, 0..) |t, i| { + inputs[i] = t.value(); + } + + for (tensors, 0..) |t, i| { + res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, t.shape()).as(mlir.Type); + } + + const res = allocator.alloc(Tensor, tensors.len) catch @panic("OOM"); + errdefer allocator.dealloc(res); + + const op = dialect.stablehlo.optimization_barrier( + mlir_ctx, + inputs[0..tensors.len], + res_types[0..tensors.len], + mlir_ctx.location(@src()), + ); + + for (res, 0..) |*t, i| { + t.* = _result(tensors[i]._shape, op.result(i)); + } + + return res; + } + /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { var res = self; @@ -3822,26 +3857,8 @@ pub const Tensor = struct { }.binaryOpHelper; } - /// Insert code that will print the content of the given buffer at runtime. - /// Only for debug purpose, it inserts device to host synchronization - /// so it will slow down the program execution. - pub fn print(input: Tensor) Tensor { - // TODO: find a way of doing print that doesn't involve a H2D copy. - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.log.defaultLog(.info, .zml, "Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); - // This is true because of the operand aliases. - // Since the result is already pointing to the input we don't need to modify the buffer. - std.debug.assert(host_buffer.data.ptr == outputs[0].data.ptr); + pub fn print(self: Tensor, print_op: type, another_tensor: Tensor) ffi.CustomCallOutputType(print_op) { + return custom_call(print_op, .{ self, another_tensor }); } }; diff --git a/zml/test_buffer.zig b/zml/test_buffer.zig new file mode 100644 index 00000000..2028e3ff --- /dev/null +++ b/zml/test_buffer.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const testing = std.testing; +const zml = @import("zml.zig"); +const Buffer = zml.Buffer; +const Shape = zml.Shape; +const DataType = zml.DataType; + +test "Buffer.fromSlice basic usage" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + const data = [_]f32{ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 }; + var buf = try Buffer.fromSlice(platform, .{ 2, 3 }, &data); + defer buf.deinit(); + + try testing.expectEqual(Shape.init(.{ 2, 3 }, .f32), buf.shape()); + + const result = try buf.getValue([6]f32); + try testing.expectEqualSlices(f32, &data, &result); +} + +test "Buffer.constant creation" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + // Test scalar constant + { + var buf = try Buffer.constant(platform, Shape.init(.{}, .f32), 42.0); + defer buf.deinit(); + + const val = try buf.getValue(f32); + try testing.expectEqual(@as(f32, 42.0), val); + } + + // Test multi-dimensional constant + { + var buf = try Buffer.constant(platform, Shape.init(.{ 2, 3 }, .i32), 7); + defer buf.deinit(); + + const result = try buf.getValue([6]i32); + try testing.expectEqualSlices(i32, &[_]i32{7} ** 6, &result); + } +} + +test "Buffer.fromArray" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + const arr = [2][3]u8{ + [_]u8{ 1, 2, 3 }, + [_]u8{ 4, 5, 6 }, + }; + + var buf = try Buffer.fromArray(platform, arr); + defer buf.deinit(); + + try testing.expectEqual(Shape.init(.{ 2, 3 }, .u8), buf.shape()); + + const result = try buf.getValue([6]u8); + try testing.expectEqualSlices(u8, &[_]u8{ 1, 2, 3, 4, 5, 6 }, &result); +} + +test "Buffer dtype and shape operations" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + var buf = try Buffer.constant(platform, Shape.init(.{ 2, 3, 4 }, .f32), 1.0); + defer buf.deinit(); + + try testing.expectEqual(DataType.f32, buf.dtype()); + try testing.expectEqual(@as(u4, 3), buf.rank()); + try testing.expectEqualSlices(i64, &[_]i64{ 2, 3, 4 }, buf.dims()); +} diff --git a/zml/zml.zig b/zml/zml.zig index 26d821db..40ec6069 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -32,6 +32,8 @@ pub const mlir = @import("mlir.zig"); pub const module = @import("module.zig"); pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); +pub const ffi = @import("ffi.zig"); +pub const custom_call = ffi.custom_call; pub const call = ops.call; pub const pjrt = @import("pjrtx.zig"); pub const Platform = @import("platform.zig").Platform;