diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 3be7dbcb..19eedf4e 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -113,7 +113,44 @@ pub fn main() !void { log.info("\t- {f}", .{device}); } - var compiled_model_result_future = io.async(compileModel, .{ allocator, io, platform, llama_model, llama_parameters }); + const topology_description = try platform.pjrt_client.topologyDescription(platform.pjrt_api); + _ = topology_description; // autofix + + const layout_extension = platform.layoutExtension() orelse return error.LayoutExtensionUnavailable; + + const LocalContext = struct { + layout_extension: *const zml.pjrt.layout.LayoutExtension, + allocator: std.mem.Allocator, + platform: zml.Platform, + }; + var context: LocalContext = .{ + .layout_extension = layout_extension, + .allocator = allocator, + .platform = platform, + }; + try zml.meta.visit(struct { + fn cb(context_: *LocalContext, t: *const Tensor) !void { + const dims = t.shape().dims(); + const buffer_type = zml.pjrtx.bufferTypeFromDtype(t.dtype()); + const client_default_layout = try context_.layout_extension.clientGetDefaultLayout(context_.platform.pjrt_api, .{ .client = context_.platform.pjrt_client, .type = buffer_type, .dims = dims }); + defer client_default_layout.deinit(context_.layout_extension, context_.platform.pjrt_api); + + const serialized_layout = try client_default_layout.serializeAlloc(allocator, context_.layout_extension, context_.platform.pjrt_api); + defer allocator.free(serialized_layout); + + std.log.info("Client default layout for {f}: {s}", .{ t.shape(), serialized_layout }); + } + }.cb, &context, &llama_model); + { + const s: zml.Shape = .init(.{ 128, 128 }, .f16); + const client_default_layout = try layout_extension.clientGetDefaultLayout(platform.pjrt_api, .{ .client = platform.pjrt_client, .type = zml.pjrtx.bufferTypeFromDtype(s.dtype()), .dims = s.dims() }); + const serialized_layout = try client_default_layout.serializeAlloc(allocator, layout_extension, platform.pjrt_api); + defer allocator.free(serialized_layout); + + std.log.info("Client default layout for: {s}", .{serialized_layout}); + } + + var compiled_model_result_future = io.async(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, layout_extension }); errdefer if (compiled_model_result_future.cancel(io)) |v| { defer v.prefill_exe.deinit(); defer v.decode_exe.deinit(); @@ -216,7 +253,7 @@ const CompileModelResult = struct { decode_exe: zml.Exe, }; -fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, llama_model: llama.LlamaLM, parameters: LlamaParameters) !CompileModelResult { +fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, llama_model: llama.LlamaLM, parameters: LlamaParameters, layout_extension: *const zml.pjrt.layout.LayoutExtension) !CompileModelResult { var timer = try std.time.Timer.start(); log.info("Compiling model", .{}); defer log.info("Compiled model [{D}]", .{timer.read()}); @@ -245,6 +282,32 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform const prefill_exe = try prefill_future.await(io); const decode_exe = try decode_future.await(io); + { + const exe: *zml.pjrt.Executable = try prefill_exe.exe.getExecutable(platform.pjrt_api); + defer exe.deinit(platform.pjrt_api); + + const output_layouts = try layout_extension.executableGetOutputLayouts(platform.pjrt_api, exe); + for (output_layouts, 0..) |layout, index| { + const serialized_layout = try layout.serializeAlloc(allocator, layout_extension, platform.pjrt_api); + defer allocator.free(serialized_layout); + + std.log.info("Prefill exe output #{d} layout: {s}", .{ index, serialized_layout }); + } + } + + { + const exe: *zml.pjrt.Executable = try decode_exe.exe.getExecutable(platform.pjrt_api); + defer exe.deinit(platform.pjrt_api); + + const output_layouts = try layout_extension.executableGetOutputLayouts(platform.pjrt_api, exe); + for (output_layouts, 0..) |layout, index| { + const serialized_layout = try layout.serializeAlloc(allocator, layout_extension, platform.pjrt_api); + defer allocator.free(serialized_layout); + + std.log.info("Decode exe output #{d} layout: {s}", .{ index, serialized_layout }); + } + } + return .{ .prefill_exe = prefill_exe, .decode_exe = decode_exe }; } diff --git a/examples/tiles/BUILD.bazel b/examples/tiles/BUILD.bazel new file mode 100644 index 00000000..6fc89f33 --- /dev/null +++ b/examples/tiles/BUILD.bazel @@ -0,0 +1,13 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "tiles", + main = "main.zig", + deps = ["//zml"], +) + +build_test( + name = "test", + targets = [":tiles"], +) diff --git a/examples/tiles/main.zig b/examples/tiles/main.zig new file mode 100644 index 00000000..efb0c313 --- /dev/null +++ b/examples/tiles/main.zig @@ -0,0 +1,121 @@ +const std = @import("std"); +const log = std.log; + +const zml = @import("zml"); +const stdx = zml.stdx; + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return a.dot(b, .k); +} + +pub fn main() !void { + const allocator = std.heap.smp_allocator; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const io = threaded.io(); + + zml.init(); + defer zml.deinit(); + + // Auto-select platform + const platform: zml.Platform = try .auto(io, .{}); + + const a: zml.Tensor = .init(.{ .m = 128, .k = 128 }, .f16); + const b: zml.Tensor = .init(.{ .k = 128, .n = 128 }, .f16); + + var exe = blk: { + log.info("⏱️ Compiling benchmark...", .{}); + var timer = try std.time.Timer.start(); + defer log.info("✅ Compiled benchmark [{D}]", .{timer.read()}); + break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }); + }; + defer exe.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + _ = random; // autofix + + var a_data: [128 * 128]f16 = undefined; + for (a_data[0..], 0..) |*d, i| d.* = @floatCast(@as(f64, @floatFromInt(i)) / a_data.len); + const a_slice: zml.ConstSlice = .init(a.shape(), std.mem.sliceAsBytes(&a_data)); + const a_buffer: zml.Buffer = try .fromSlice(io, platform, a_slice); + defer a_buffer.deinit(); + + var b_buffer: zml.Buffer = b: { + var transfer: zml.io.Transfer = try .init(allocator, &.{b.shape()}, platform); + defer transfer.deinit(platform); + + const buffer = try allocator.alloc(u8, 128 * 128 * 2); + defer allocator.free(buffer); + + var writer = try transfer.getWriter(io, 0, buffer); + var reader: std.Io.Reader = .fixed(std.mem.sliceAsBytes(&a_data)); + + _ = try reader.streamRemaining(&writer.interface); + try writer.interface.flush(); + + break :b try transfer.getBuffer(0); + }; + defer b_buffer.deinit(); + + var exe_args = try exe.args(allocator); + defer exe_args.deinit(allocator); + + var exe_results = try exe.results(allocator); + defer exe_results.deinit(allocator); + + exe_args.set(.{ a_buffer, a_buffer }); + + // call our executable module + exe.call(exe_args, &exe_results); + var result1 = exe_results.get(zml.Buffer); + _ = try result1.await(io); + defer result1.deinit(); + + const result1_slice = try result1.toSliceAlloc(allocator, io); + defer result1_slice.free(allocator); + + std.log.info("result1: {d}", .{result1_slice}); + + exe_args.set(.{ b_buffer, b_buffer }); + + // call our executable module + exe.call(exe_args, &exe_results); + var result2 = exe_results.get(zml.Buffer); + _ = try result2.await(io); + defer result2.deinit(); + + const result2_slice = try result2.toSliceAlloc(allocator, io); + defer result2_slice.free(allocator); + std.log.info("result2: {d}", .{result2_slice}); +} + +fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (slice.items(ZigType)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = random.float(f32); + for (slice.items(ZigType)) |*e| e.* = switch (ZigType) { + f64, f32 => value, + f16 => @floatCast(value), + inline else => |T| if (@hasDecl(T, "fromF32")) T.fromF32(value) else unreachable, + }; + }, + .complex => unreachable, + } + }, + } + + return .fromSlice(io, platform, slice); +} diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 173bea7e..7822f6ee 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -2,7 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "pjrt", - srcs = ["ffi.zig"], + srcs = ["ffi.zig", "layout.zig"], main = "pjrt.zig", visibility = ["//visibility:public"], deps = [ @@ -12,5 +12,6 @@ zig_library( "@xla//xla/pjrt/c:pjrt_c_api_gpu_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_triton_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_layouts_extension_hdrs", ], ) diff --git a/pjrt/layout.zig b/pjrt/layout.zig new file mode 100644 index 00000000..1b02dd89 --- /dev/null +++ b/pjrt/layout.zig @@ -0,0 +1,102 @@ +const std = @import("std"); + +const c = @import("c"); + +const pjrt = @import("pjrt.zig"); + +const log = std.log.scoped(.pjrt); + +pub const LayoutExtension = struct { + inner: c.PJRT_Layouts_Extension, + + pub const extension_id = c.PJRT_Extension_Type_Layouts; + const Funcs = std.meta.FieldEnum(c.PJRT_Layouts_Extension); + + fn CallFnArgType(comptime func: Funcs) type { + const fti = @typeInfo(@FieldType(c.PJRT_Layouts_Extension, @tagName(func))); + const fn_ptr = @typeInfo(fti.optional.child); + const fn_type_info = @typeInfo(fn_ptr.pointer.child); + const arg_array_type_info = @typeInfo(fn_type_info.@"fn".params[0].type.?); + return arg_array_type_info.pointer.child; + } + + inline fn call(self: *const LayoutExtension, api: *const pjrt.Api, comptime method: Funcs, arg: anytype) pjrt.ApiError!CallFnArgType(method) { + var ret = pjrt.pjrtStruct2(CallFnArgType(method), arg); + const fn_ptr = @field(&self.inner, @tagName(method)).?; + const result = fn_ptr(&ret); + if (@TypeOf(result) == void) { + return ret; + } + + if (result) |pjrt_c_error| { + const pjrt_error: *pjrt.Error = @ptrCast(pjrt_c_error); + log.err("[{s}] {s}", .{ @tagName(method), pjrt_error.getMessage(api) }); + return pjrt_error.getCode(api).toApiError(); + } + + return ret; + } + + pub const ClientGetDefaultLayoutArgs = struct { + client: *pjrt.Client, + type: pjrt.BufferType, + dims: []const i64, + }; + + pub fn clientGetDefaultLayout(extension: *const LayoutExtension, api: *const pjrt.Api, args: ClientGetDefaultLayoutArgs) pjrt.ApiError!*MemoryLayout { + const ret = try extension.call(api, .PJRT_Layouts_PJRT_Client_GetDefaultLayout, .{ + .client = args.client.inner(), + .type = @intFromEnum(args.type), + .dims = args.dims.ptr, + .num_dims = args.dims.len, + }); + return @ptrCast(ret.layout.?); + } + + pub const TopologyGetDefaultLayoutArgs = struct { + topology_description: *pjrt.TopologyDescription, + type: pjrt.BufferType, + dims: []const i64, + }; + + pub fn topologyGetDefaultLayout(extension: *const LayoutExtension, api: *const pjrt.Api, args: TopologyGetDefaultLayoutArgs) pjrt.ApiError!*MemoryLayout { + const ret = try extension.call(api, .PJRT_Layouts_PJRT_Topology_GetDefaultLayout, .{ + .topology_description = args.topology_description.inner(), + .type = @intFromEnum(args.type), + .dims = args.dims.ptr, + .num_dims = args.dims.len, + }); + return @ptrCast(ret.layout.?); + } + + pub fn executableGetOutputLayouts(extension: *const LayoutExtension, api: *const pjrt.Api, executable: *pjrt.Executable) pjrt.ApiError![]const *MemoryLayout { + const ret = try extension.call(api, .PJRT_Layouts_PJRT_Executable_GetOutputLayouts, .{ + .executable = executable.inner(), + }); + return @as([*]const *MemoryLayout, @ptrCast(ret.layouts))[0..ret.num_outputs]; + } +}; + +pub const MemoryLayout = opaque { + pub const inner = pjrt.InnerMixin(c.PJRT_Layouts_MemoryLayout).inner; + + pub fn deinit(self: *MemoryLayout, extension: *const LayoutExtension, api: *const pjrt.Api) void { + _ = extension.call(api, .PJRT_Layouts_MemoryLayout_Destroy, .{ + .layout = self.inner(), + }) catch {}; + } + + pub const SerializeAllocError = pjrt.ApiError || std.mem.Allocator.Error; + + pub fn serializeAlloc(self: *MemoryLayout, allocator: std.mem.Allocator, extension: *const LayoutExtension, api: *const pjrt.Api) SerializeAllocError![]const u8 { + const ret = try extension.call(api, .PJRT_Layouts_MemoryLayout_Serialize, .{ .layout = self.inner() }); + defer ret.serialized_layout_deleter.?(ret.serialized_layout); + + const out = try allocator.alloc(u8, ret.serialized_bytes_size); + errdefer allocator.free(out); + + @memcpy(out, ret.serialized_bytes[0..ret.serialized_bytes_size]); + + return out; + } +}; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index f2aa4830..00622e29 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -5,6 +5,7 @@ const c = @import("c"); const stdx = @import("stdx"); pub const ffi = @import("ffi.zig"); +pub const layout = @import("layout.zig"); const log = std.log.scoped(.pjrt); @@ -61,9 +62,9 @@ pub const ApiError = error{ Unauthenticated, }; -fn InnerMixin(comptime innerT: type) type { +pub fn InnerMixin(comptime innerT: type) type { return struct { - fn inner(self: anytype) *innerT { + pub fn inner(self: anytype) *innerT { return @ptrCast(@alignCast(@constCast(self))); } }; @@ -292,7 +293,7 @@ pub const ShapeSpec = extern struct { pub const Stream = opaque {}; pub const Client = opaque { - const inner = InnerMixin(c.PJRT_Client).inner; + pub const inner = InnerMixin(c.PJRT_Client).inner; pub const ProgramFormat = enum { hlo, @@ -424,14 +425,14 @@ pub const Client = opaque { }; pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { - const layout = args.layout.toCStruct(); + const layout_ = args.layout.toCStruct(); const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), .device_buffer_ptr = @constCast(args.data), .dims = args.dims.ptr, .num_dims = args.dims.len, .element_type = @intFromEnum(args.element_type), - .layout = @as([*c]c.PJRT_Buffer_MemoryLayout, @ptrCast(@constCast(&layout))), + .layout = @as([*c]c.PJRT_Buffer_MemoryLayout, @ptrCast(@constCast(&layout_))), .device = @as(?*c.PJRT_Device, @ptrCast(@constCast(args.device))), .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, @@ -467,16 +468,39 @@ pub const Client = opaque { pub const CreateBuffersForAsyncHostToDeviceArgs = struct { shape_specs: []const ShapeSpec, - device_layouts: ?[]*const MemoryLayout = null, + device_layouts: ?[]const []const MemoryLayout = null, memory: *const Memory, }; + const CreateBuffersForAsyncHostToDeviceError = ApiError || std.mem.Allocator.Error; + + pub fn createBuffersForAsyncHostToDevice(self: *const Client, allocator: std.mem.Allocator, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) CreateBuffersForAsyncHostToDeviceError!*AsyncHostToDeviceTransferManager { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const device_layouts: [*c][*c]c.PJRT_Buffer_MemoryLayout = if (args.device_layouts) |device_layouts| b: { + const device_count = device_layouts.len; + const layout_count = device_layouts[0].len; + const device_layouts_slice = try arena.allocator().alloc([*]const c.PJRT_Buffer_MemoryLayout, device_count); + + const temp = try arena.allocator().alloc(c.PJRT_Buffer_MemoryLayout, device_count * layout_count); + for (0..device_layouts.len) |device_index| { + std.debug.assert(device_layouts[device_index].len == layout_count); + + const per_device_layouts = temp[device_index * layout_count ..][0..layout_count]; + for (0..device_layouts[device_index].len) |i| { + temp[i] = device_layouts[device_index][i].toCStruct(); + } + device_layouts_slice[device_index] = per_device_layouts.ptr; + } + + break :b @ptrCast(device_layouts_slice); + } else null; - pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { const ret = try api.call(.PJRT_Client_CreateBuffersForAsyncHostToDevice, .{ .client = self.inner(), .shape_specs = @as([*c]c.PJRT_ShapeSpec, @ptrCast(@constCast(args.shape_specs.ptr))), .num_shape_specs = args.shape_specs.len, - .device_layouts = @as([*c][*c]c.PJRT_Buffer_MemoryLayout, if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts.ptr)) else null), + .device_layouts = device_layouts, .num_device_layouts = @as(usize, if (args.device_layouts) |layouts| @intCast(layouts.len) else 0), .memory = @as(?*c.PJRT_Memory, @ptrCast(@constCast(args.memory))), }); @@ -494,18 +518,26 @@ pub const Client = opaque { }; pub fn createUninitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { - var layout = args.layout.toCStruct(); + var layout_ = args.layout.toCStruct(); const ret = try api.call(.PJRT_Client_CreateUninitializedBuffer, .{ .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = @as(usize, @intCast(args.dims.len)), .shape_element_type = @intFromEnum(args.element_type), - .shape_layout = @as([*c]c.PJRT_Buffer_MemoryLayout, @ptrCast(&layout)), + .shape_layout = @as([*c]c.PJRT_Buffer_MemoryLayout, @ptrCast(&layout_)), .device = if (args.dst == .device) @as(?*c.PJRT_Device, @ptrCast(@constCast(args.dst.device))) else null, .memory = if (args.dst == .memory) @as(?*c.PJRT_Memory, @ptrCast(@constCast(args.dst.memory))) else null, }); return @ptrCast(ret.buffer.?); } + + pub fn topologyDescription(self: *const Client, api: *const Api) ApiError!*TopologyDescription { + const ret = try api.call(.PJRT_Client_TopologyDescription, .{ + .client = self.inner(), + }); + + return @ptrCast(ret.topology.?); + } }; pub const MemoryStats = struct { @@ -673,7 +705,7 @@ pub const ExecuteContext = opaque { }; pub const Executable = opaque { - const inner = InnerMixin(c.PJRT_Executable).inner; + pub const inner = InnerMixin(c.PJRT_Executable).inner; pub fn deinit(self: *Executable, api: *const Api) void { _ = api.call(.PJRT_Executable_Destroy, .{ @@ -1356,6 +1388,10 @@ pub const NamedValue = extern struct { } }; +pub const TopologyDescription = opaque { + pub const inner = InnerMixin(c.PJRT_TopologyDescription).inner; +}; + pub const Ffi = extern struct { inner: *const c.PJRT_FFI, diff --git a/runtimes/tpu/tpu.zig b/runtimes/tpu/tpu.zig index 83d6c4a9..7c65c88c 100644 --- a/runtimes/tpu/tpu.zig +++ b/runtimes/tpu/tpu.zig @@ -46,7 +46,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); defer arena.deinit(); - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { + var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { stdx.debug.panic("Unable to find runfiles", .{}); }; diff --git a/zml/io.zig b/zml/io.zig index 24454e33..91b88168 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -372,7 +372,20 @@ pub const Transfer = struct { const memory = pjrtx.Client.memoryByKind(platform.pjrt_client, platform.pjrt_api, .device).?; - const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); + const buffer_memory_layout: pjrt.MemoryLayout = .{ + .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 8, 128, 2, 1 }, + .tile_dims_sizes = &.{ 2, 2 }, + }, + }; + const device_layouts: []const []const pjrt.MemoryLayout = &.{&.{buffer_memory_layout}}; + + const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(allocator, platform.pjrt_api, .{ + .shape_specs = shape_specs, + .memory = memory, + .device_layouts = device_layouts, + }); errdefer transfer_manager.deinit(platform.pjrt_api); return .{ diff --git a/zml/platform.zig b/zml/platform.zig index e03783c1..2fb6a6b5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -240,6 +240,10 @@ pub const Platform = struct { return zml.module.compile(allocator, io, func, args, self); } + pub fn layoutExtension(self: Platform) ?*const pjrt.layout.LayoutExtension { + return self.pjrt_api.lookupExtension(pjrt.layout.LayoutExtension, pjrt.layout.LayoutExtension.extension_id); + } + pub fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.print("{s} {{ ", .{@tagName(self.target)}); const devices = self.getDevices(); diff --git a/zml/zml.zig b/zml/zml.zig index 7fd9e07e..fa109ab5 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -8,6 +8,7 @@ const std = @import("std"); const c = @import("c"); pub const pjrt = @import("pjrt"); +pub const pjrtx = @import("pjrtx.zig"); const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer");