diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index acfc51e9..077f5b34 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -4,10 +4,14 @@ load("@bazel_lib//tools/release:hashes.bzl", "hashes") load("@bazel_skylib//rules:run_binary.bzl", "run_binary") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") load("@tar.bzl//tar:mtree.bzl", "mtree_spec") load("@tar.bzl//tar:tar.bzl", "tar") +# zig libraries + +# smi + run_binary( name = "libnrt.stripped", srcs = ["@aws-neuronx-runtime-lib//:libnrt.patchelf"], @@ -57,7 +61,6 @@ copy_to_directory( "libnrt.so.1": "lib/libnrt.so.1", "libncfw.so.2": "lib/libncfw.so.2", }, - visibility = ["//bin/zml-smi:__subpackages__"], ) run_binary( @@ -75,29 +78,36 @@ run_binary( zig_binary( name = "zml-smi", - srcs = ["platforms.zig"], data = [":sandbox"], main = "main.zig", visibility = ["//visibility:public"], deps = [ - "//bin/zml-smi/smi", + "//bin/zml-smi/api", + "//bin/zml-smi/collector", + "//bin/zml-smi/csv", + "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/prometheus", "//bin/zml-smi/tui", + "//bin/zml-smi/utils:sysfs", "//stdx", + "@libvaxis//:vaxis", + "@rules_zig//zig/runfiles", ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "@platforms//os:linux": ["//bin/zml-smi/platforms/linux"], + }) + select({ "@llvm//platforms/config:linux_x86_64": [ "//bin/zml-smi/platforms/amdsmi", - "//bin/zml-smi/platforms/furiosa", "//bin/zml-smi/platforms/neuron", "//bin/zml-smi/platforms/nvml", "//bin/zml-smi/platforms/oneapi", "//bin/zml-smi/platforms/tpu", + "//platforms/neuron:libnrt_headers", ], "@llvm//platforms/config:linux_aarch64": [ "//bin/zml-smi/platforms/nvml", ], - "@llvm//platforms/config:macos_aarch64": [ - "//bin/zml-smi/platforms/metal", - ], "//conditions:default": [], }), ) diff --git a/bin/zml-smi/api/BUILD.bazel b/bin/zml-smi/api/BUILD.bazel new file mode 100644 index 00000000..1a09f597 --- /dev/null +++ b/bin/zml-smi/api/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "api", + import_name = "zml-smi/api", + main = "api.zig", + srcs = [ + "client.zig", + "server.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/utils:double_buffer", + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "//conditions:default": ["//bin/zml-smi/platforms/linux"], + }), +) diff --git a/bin/zml-smi/api/api.zig b/bin/zml-smi/api/api.zig new file mode 100644 index 00000000..fb39ddd8 --- /dev/null +++ b/bin/zml-smi/api/api.zig @@ -0,0 +1,2 @@ +pub const Server = @import("server.zig").Server; +pub const addRemotes = @import("client.zig").addRemotes; diff --git a/bin/zml-smi/api/client.zig b/bin/zml-smi/api/client.zig new file mode 100644 index 00000000..ddec6d2d --- /dev/null +++ b/bin/zml-smi/api/client.zig @@ -0,0 +1,138 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +fn setRemote(device: *DeviceInfo) void { + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (device.* == @field(smi_info.device_info.Target, field_name)) { + @field(device, field_name).values[0].remote = true; + @field(device, field_name).values[1].remote = true; + } + } +} + +pub fn addRemotes(collector: *Collector, hosts: []const u8) !void { + var it = std.mem.splitScalar(u8, hosts, ','); + while (it.next()) |host| { + addHost(collector, host) catch |err| { + std.log.err("{s}: {s}", .{ host, @errorName(err) }); + }; + } +} + +fn addHost(collector: *Collector, host: []const u8) !void { + const body = try httpGet(collector.gpa, collector.io, host); + defer collector.gpa.free(body); + + const parsed = try std.json.parseFromSlice(std.json.Value, collector.gpa, body, .{}); + defer parsed.deinit(); + + const root = parsed.value.object; + const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; + + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + var devices: std.ArrayList(*DeviceInfo) = .empty; + for (json_devices) |item| { + var device = std.json.parseFromValueLeaky(DeviceInfo, collector.arena, item, .{ .ignore_unknown_fields = true }) catch continue; + setRemote(&device); + const info = try collector.addDevice(device); + try devices.append(collector.arena, info); + } + + if (devices.items.len == 0) { + return; + } + + const processes = try collector.createProcessList(); + const str_arenas: [2]*std.heap.ArenaAllocator = .{ try collector.createPollArena(), try collector.createPollArena() }; + + const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; + const back_idx: usize = 1 - processes.current.load(.acquire); + const back = processes.back(); + for (json_processes) |item| { + var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, str_arenas[back_idx].allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; + proc.device_idx += dev_offset; + proc.remote = true; + back.append(collector.gpa, proc) catch continue; + } + processes.swap(); + + const url = try collector.arena.dupe(u8, host); + const poll_arena = try collector.createPollArena(); + try collector.spawnPoll(pollOnce, .{ poll_arena, collector.gpa, collector.io, url, devices.items, processes, dev_offset, str_arenas }); +} + +fn pollOnce(poll_arena: *std.heap.ArenaAllocator, gpa: std.mem.Allocator, io: std.Io, host: []const u8, devices: []const *DeviceInfo, processes: *ProcessDoubleBuffer, dev_offset: u16, str_arenas: [2]*std.heap.ArenaAllocator) void { + _ = poll_arena.reset(.retain_capacity); + + const body = httpGet(gpa, io, host) catch return; + defer gpa.free(body); + + const parsed = std.json.parseFromSlice(std.json.Value, gpa, body, .{}) catch return; + defer parsed.deinit(); + + const root = parsed.value.object; + const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; + + for (json_devices, 0..) |item, i| { + if (i >= devices.len) { + std.log.warn("{s}: remote has more devices than expected ({d} > {d})", .{ host, json_devices.len, devices.len }); + break; + } + var device = std.json.parseFromValueLeaky(DeviceInfo, poll_arena.allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; + setRemote(&device); + + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (devices[i].* == @field(smi_info.device_info.Target, field_name)) { + @field(devices[i], field_name).back().* = @field(device, field_name).front().*; + @field(devices[i], field_name).swap(); + } + } + } + + const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; + const back_idx: usize = 1 - processes.current.load(.acquire); + _ = str_arenas[back_idx].reset(.retain_capacity); + const alloc = str_arenas[back_idx].allocator(); + const back = processes.back(); + back.clearRetainingCapacity(); + for (json_processes) |item| { + var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, alloc, item, .{ .ignore_unknown_fields = true }) catch continue; + proc.device_idx += dev_offset; + proc.remote = true; + back.append(gpa, proc) catch continue; + } + processes.swap(); +} + +fn httpGet(allocator: std.mem.Allocator, io: std.Io, host: []const u8) ![]const u8 { + if (!std.mem.startsWith(u8, host, "http://") and !std.mem.startsWith(u8, host, "https://")) { + std.log.err("remote host must start with http:// or https://: {s}", .{host}); + return error.InvalidUrl; + } + + const url = try std.fmt.allocPrint(allocator, "{s}/metrics", .{host}); + defer allocator.free(url); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + var aw: std.Io.Writer.Allocating = .init(allocator); + errdefer aw.deinit(); + + const result = try client.fetch(.{ + .location = .{ .url = url }, + .response_writer = &aw.writer, + }); + + if (result.status != .ok) { + aw.deinit(); + return error.HttpError; + } + + return try aw.toOwnedSlice(); +} diff --git a/bin/zml-smi/api/server.zig b/bin/zml-smi/api/server.zig new file mode 100644 index 00000000..e05aa7fe --- /dev/null +++ b/bin/zml-smi/api/server.zig @@ -0,0 +1,105 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const ProcessInfo = smi_info.process_info.ProcessInfo; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(ProcessInfo)); +const json = @import("zml-smi/json"); +const ProcessEnricher = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos").process.ProcessEnricher +else + @import("zml-smi/platforms/linux").process.ProcessEnricher; + +pub const Server = struct { + allocator: std.mem.Allocator, + io: std.Io, + devices: []const *DeviceInfo, + process_lists: []const *const ProcessDoubleBuffer, + enricher: *ProcessEnricher, + tcp: std.Io.net.Server, + connection_group: std.Io.Group = .init, + + pub fn init(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) !Server { + const address: std.Io.net.IpAddress = .{ .ip4 = .{ + .bytes = .{ 0, 0, 0, 0 }, + .port = port, + } }; + var tcp = try address.listen(io, .{ .reuse_address = true }); + errdefer tcp.deinit(io); + + return .{ + .allocator = allocator, + .io = io, + .devices = devices, + .process_lists = process_lists, + .enricher = enricher, + .tcp = tcp, + }; + } + + pub fn run(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) void { + var self = init(allocator, io, port, devices, process_lists, enricher) catch |err| { + std.log.err("api server failed to start: {s}", .{@errorName(err)}); + return; + }; + defer self.deinit(); + self.serve(); + } + + pub fn deinit(self: *Server) void { + self.connection_group.await(self.io) catch {}; + self.tcp.deinit(self.io); + } + + pub fn serve(self: *Server) void { + while (true) { + const stream = self.tcp.accept(self.io) catch break; + self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; + } + } + + fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { + defer stream.close(self.io); + self.handleConn(stream) catch {}; + } + + fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { + var read_buf: [8192]u8 = undefined; + var reader = stream.reader(self.io, &read_buf); + + var write_buf: [4096]u8 = undefined; + var writer = stream.writer(self.io, &write_buf); + + var http_server: std.http.Server = .init(&reader.interface, &writer.interface); + + var procs: std.ArrayList(ProcessInfo) = .empty; + defer procs.deinit(self.allocator); + + while (true) { + var request = http_server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => return err, + }; + + procs.clearRetainingCapacity(); + for (self.process_lists) |pl| { + procs.appendSlice(self.allocator, pl.front().items) catch {}; + } + self.enricher.enrich(self.io, procs.items); + + var body_buf: [4096]u8 = undefined; + var body_writer = try request.respondStreaming(&body_buf, .{ + .respond_options = .{ + .status = .ok, + .extra_headers = &.{ + .{ .name = "content-type", .value = "application/json" }, + .{ .name = "access-control-allow-origin", .value = "*" }, + }, + }, + }); + + try json.write(&body_writer.writer, self.devices, procs.items); + try body_writer.end(); + } + } +}; diff --git a/bin/zml-smi/collector/BUILD.bazel b/bin/zml-smi/collector/BUILD.bazel new file mode 100644 index 00000000..0bf5fa8e --- /dev/null +++ b/bin/zml-smi/collector/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "collector", + import_name = "zml-smi/collector", + main = "collector.zig", + srcs = ["worker.zig"], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + ], +) diff --git a/bin/zml-smi/collector/collector.zig b/bin/zml-smi/collector/collector.zig new file mode 100644 index 00000000..d98b204d --- /dev/null +++ b/bin/zml-smi/collector/collector.zig @@ -0,0 +1,93 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Worker = @import("worker.zig").Worker; + +pub const Collector = struct { + device_infos: std.ArrayList(*DeviceInfo) = .empty, + process_lists: std.ArrayList(*ProcessDoubleBuffer) = .empty, + poll_arenas: std.ArrayList(*std.heap.ArenaAllocator) = .empty, + arena: std.mem.Allocator, + gpa: std.mem.Allocator, + worker: Worker, + io: std.Io, + poll_only: bool, + + pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, options: struct { poll_interval_ms: u16, poll_only: bool = false }) Collector { + return .{ + .arena = arena, + .gpa = gpa, + .io = io, + .worker = .{ .poll_interval_ms = options.poll_interval_ms }, + .poll_only = options.poll_only, + }; + } + + pub fn addDevice(self: *Collector, initial: DeviceInfo) !*DeviceInfo { + const info = try self.arena.create(DeviceInfo); + info.* = initial; + try self.device_infos.append(self.arena, info); + + return info; + } + + pub fn createProcessList(self: *Collector) !*ProcessDoubleBuffer { + const list = try self.arena.create(ProcessDoubleBuffer); + list.* = .{ .values = .{ .empty, .empty } }; + try self.process_lists.append(self.arena, list); + + return list; + } + + pub fn createPollArena(self: *Collector) !*std.heap.ArenaAllocator { + const poll_arena = try self.arena.create(std.heap.ArenaAllocator); + poll_arena.* = std.heap.ArenaAllocator.init(self.gpa); + try self.poll_arenas.append(self.arena, poll_arena); + + return poll_arena; + } + + pub fn spawnPoll(self: *Collector, comptime pollOnce: anytype, args: std.meta.ArgsTuple(@TypeOf(pollOnce))) !void { + @call(.auto, pollOnce, args); + if (self.poll_only) return; + + const Args = std.meta.ArgsTuple(@TypeOf(pollOnce)); + try self.worker.spawn(self.io, struct { + fn f(io: std.Io, w: *const Worker, a: Args) void { + const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); + while (w.isRunning()) { + const start: std.Io.Timestamp = .now(io, .awake); + + @call(.auto, pollOnce, a); + + const elapsed = start.untilNow(io, .awake); + if (elapsed.nanoseconds < interval.nanoseconds) { + io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + } + } + } + }.f, .{ self.io, &self.worker, args }); + } + + pub fn deinit(self: *Collector) void { + self.worker.shutdown(self.io); + + for (self.device_infos.items) |info| { + self.arena.destroy(info); + } + self.device_infos.deinit(self.arena); + + for (self.process_lists.items) |list| { + list.values[0].deinit(self.gpa); + list.values[1].deinit(self.gpa); + } + self.process_lists.deinit(self.arena); + + for (self.poll_arenas.items) |poll_arena| { + poll_arena.deinit(); + } + self.poll_arenas.deinit(self.arena); + } +}; diff --git a/bin/zml-smi/collector/worker.zig b/bin/zml-smi/collector/worker.zig new file mode 100644 index 00000000..125440c5 --- /dev/null +++ b/bin/zml-smi/collector/worker.zig @@ -0,0 +1,25 @@ +const std = @import("std"); + +pub const Worker = struct { + poll_interval_ms: u16, + should_stop: std.atomic.Value(bool) = .init(false), + group: std.Io.Group = .init, + + pub fn shutdown(self: *Worker, io: std.Io) void { + self.should_stop.store(true, .release); + self.group.await(io) catch {}; + } + + pub fn isRunning(self: *const Worker) bool { + return !self.should_stop.load(.acquire); + } + + pub fn spawn( + self: *Worker, + io: std.Io, + comptime runFn: anytype, + args: std.meta.ArgsTuple(@TypeOf(runFn)), + ) !void { + try self.group.concurrent(io, runFn, args); + } +}; diff --git a/bin/zml-smi/csv/BUILD.bazel b/bin/zml-smi/csv/BUILD.bazel new file mode 100644 index 00000000..a0cb3422 --- /dev/null +++ b/bin/zml-smi/csv/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "csv", + import_name = "zml-smi/csv", + main = "csv.zig", + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig new file mode 100644 index 00000000..dbd80dd9 --- /dev/null +++ b/bin/zml-smi/csv/csv.zig @@ -0,0 +1,46 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { + const info = @typeInfo(DeviceInfo).@"union"; + inline for (info.field_names, info.field_types) |type_name, DeviceType| { + const tag = @field(smi_info.Target, type_name); + var header_printed = false; + + for (devices, 0..) |dev, i| { + switch (dev.*) { + tag => |*db| { + if (!header_printed) { + try writer.writeAll("index,type"); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { + try writer.writeAll("," ++ field_name); + } + try writer.writeAll("\n"); + + header_printed = true; + } + + const val = db.front().*; + + try writer.print("{d},{s}", .{ i, type_name }); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { + try writer.writeAll(","); + try writeValue(writer, @field(val, field_name)); + } + try writer.writeAll("\n"); + }, + else => {}, + } + } + } +} + +fn writeValue(writer: *std.Io.Writer, field: anytype) !void { + if (@typeInfo(@TypeOf(field)) == .optional) { + if (field) |v| return writeValue(writer, v); + } else switch (@typeInfo(@TypeOf(field))) { + .pointer => return writer.writeAll(field), + else => return writer.print("{any}", .{field}), + } +} diff --git a/bin/zml-smi/info/BUILD.bazel b/bin/zml-smi/info/BUILD.bazel new file mode 100644 index 00000000..d035b161 --- /dev/null +++ b/bin/zml-smi/info/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "info", + import_name = "zml-smi/info", + main = "info.zig", + srcs = [ + "device_info.zig", + "host_info.zig", + "poll_metrics.zig", + "process_info.zig", + ], + visibility = ["//visibility:public"], + deps = ["//bin/zml-smi/utils:double_buffer"], +) diff --git a/bin/zml-smi/info/device_info.zig b/bin/zml-smi/info/device_info.zig new file mode 100644 index 00000000..4b01c728 --- /dev/null +++ b/bin/zml-smi/info/device_info.zig @@ -0,0 +1,95 @@ +const std = @import("std"); +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; + +pub const Target = enum { + cuda, + rocm, + oneapi, + neuron, + tpu, +}; + +pub const DeviceInfo = union(Target) { + cuda: DoubleBuffer(GpuInfo), + rocm: DoubleBuffer(GpuInfo), + oneapi: DoubleBuffer(GpuInfo), + neuron: DoubleBuffer(NeuronInfo), + tpu: DoubleBuffer(TpuInfo), + + pub fn isRemote(self: *const DeviceInfo) bool { + switch (self.*) { + inline else => |*db| return db.front().remote, + } + } +}; + +pub const GpuInfo = struct { + remote: bool = false, + name: ?[]const u8 = null, + driver_version: ?[]const u8 = null, + cuda_driver_version: ?[]const u8 = null, + + // Utilization + util_percent: ?u64 = null, + encoder_util_percent: ?u64 = null, + decoder_util_percent: ?u64 = null, + + // Power + power_mw: ?u64 = null, + power_limit_mw: ?u64 = null, + // Thermal + temperature: ?u64 = null, + fan_speed_percent: ?u64 = null, + + // Clocks + clock_graphics_mhz: ?u64 = null, + clock_sm_mhz: ?u64 = null, + clock_soc_mhz: ?u64 = null, + clock_mem_mhz: ?u64 = null, + clock_graphics_max_mhz: ?u64 = null, + clock_mem_max_mhz: ?u64 = null, + + // Memory + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + mem_bus_width: ?u64 = null, + + // PCIe + pcie_tx_kbps: ?u64 = null, + pcie_rx_kbps: ?u64 = null, + pcie_bandwidth_mbps: ?u64 = null, + pcie_link_gen: ?u64 = null, + pcie_link_width: ?u64 = null, +}; + +pub const NeuronInfo = struct { + remote: bool = false, + name: ?[]const u8 = null, + driver_version: ?[]const u8 = null, + + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + + // Neuron per-core HBM breakdown (bytes) + nc_tensors: ?u64 = null, + nc_constants: ?u64 = null, + nc_model_code: ?u64 = null, + nc_shared_scratchpad: ?u64 = null, + nc_nonshared_scratchpad: ?u64 = null, + nc_runtime: ?u64 = null, + nc_driver: ?u64 = null, + nc_dma_rings: ?u64 = null, + nc_collectives: ?u64 = null, + nc_notifications: ?u64 = null, + nc_uncategorized: ?u64 = null, +}; + +pub const TpuInfo = struct { + remote: bool = false, + name: ?[]const u8 = null, + + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, +}; diff --git a/bin/zml-smi/info/host_info.zig b/bin/zml-smi/info/host_info.zig new file mode 100644 index 00000000..f33f6fd8 --- /dev/null +++ b/bin/zml-smi/info/host_info.zig @@ -0,0 +1,16 @@ +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; + +pub const HostInfo = DoubleBuffer(HostData); + +pub const HostData = struct { + hostname: ?[]const u8 = null, + kernel: ?[]const u8 = null, + cpu_name: ?[]const u8 = null, + cpu_cores: ?u64 = null, + mem_total_kib: ?u64 = null, + mem_available_kib: ?u64 = null, + load_1: ?f32 = null, + load_5: ?f32 = null, + load_15: ?f32 = null, + uptime_seconds: ?u64 = null, +}; diff --git a/bin/zml-smi/info/info.zig b/bin/zml-smi/info/info.zig new file mode 100644 index 00000000..42e1b1b3 --- /dev/null +++ b/bin/zml-smi/info/info.zig @@ -0,0 +1,9 @@ +const std = @import("std"); + +pub const device_info = @import("device_info.zig"); +pub const host_info = @import("host_info.zig"); +pub const poll_metrics = @import("poll_metrics.zig"); +pub const process_info = @import("process_info.zig"); + +pub const Target = device_info.Target; +pub const Targets = std.EnumSet(Target); diff --git a/bin/zml-smi/info/poll_metrics.zig b/bin/zml-smi/info/poll_metrics.zig new file mode 100644 index 00000000..62ed94a7 --- /dev/null +++ b/bin/zml-smi/info/poll_metrics.zig @@ -0,0 +1,20 @@ +const std = @import("std"); + +pub fn apply(comptime table: anytype, dest: anytype, src: anytype) void { + inline for (table) |m| { + @field(dest, m.field) = m.query(src) catch null; + } +} + +pub fn poll(comptime DB: type, comptime Dev: type, comptime table: anytype) fn (?*std.heap.ArenaAllocator, DB, Dev) void { + return struct { + fn f(arena: ?*std.heap.ArenaAllocator, db: DB, dev: Dev) void { + if (arena) |a| _ = a.reset(.retain_capacity); + + const back = db.back(); + back.* = db.front().*; + apply(table, back, dev); + db.swap(); + } + }.f; +} diff --git a/bin/zml-smi/info/process_info.zig b/bin/zml-smi/info/process_info.zig new file mode 100644 index 00000000..73504656 --- /dev/null +++ b/bin/zml-smi/info/process_info.zig @@ -0,0 +1,12 @@ +pub const ProcessInfo = struct { + pid: u32 = 0, + device_idx: u16 = 0, + dev_util_percent: ?u16 = null, + dev_mem_kib: ?u64 = null, + uid: u32 = 0, + username: []const u8 = "", + comm: []const u8 = "", + cpu_percent: u16 = 0, // *10, e.g. 125 = 12.5% + rss_kib: u64 = 0, + remote: bool = false, +}; diff --git a/bin/zml-smi/json/BUILD.bazel b/bin/zml-smi/json/BUILD.bazel new file mode 100644 index 00000000..decc8ca1 --- /dev/null +++ b/bin/zml-smi/json/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "json", + import_name = "zml-smi/json", + main = "json.zig", + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/json/json.zig b/bin/zml-smi/json/json.zig new file mode 100644 index 00000000..c32a16cf --- /dev/null +++ b/bin/zml-smi/json/json.zig @@ -0,0 +1,15 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const ProcessInfo = smi_info.process_info.ProcessInfo; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, processes: []const ProcessInfo) !void { + var jw: std.json.Stringify = .{ .writer = writer }; + try jw.write(Response{ .devices = devices, .processes = processes }); + try writer.writeAll("\n"); +} + +const Response = struct { + devices: []const *DeviceInfo, + processes: []const ProcessInfo, +}; diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index 7de518ca..f066dd34 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -1,21 +1,33 @@ const std = @import("std"); +const builtin = @import("builtin"); const stdx = @import("stdx"); -const smi = @import("zml-smi"); -const tui = @import("zml-smi/tui"); - -const platforms = @import("platforms.zig"); pub const std_options: std.Options = .{ .log_scope_levels = &.{ .{ .scope = .vaxis, .level = .warn }, }, - // A /proc file can vanish between its open and its read, which std reports - // as an unexpected errno, with a stack trace in debug builds. - .unexpected_error_tracing = false, }; -const log = std.log; +const Collector = @import("zml-smi/collector").Collector; +const smi_info = @import("zml-smi/info"); +const HostInfo = smi_info.host_info.HostInfo; +const platform = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos") +else + @import("zml-smi/platforms/linux"); +const ProcessEnricher = platform.process.ProcessEnricher; +const host = platform.metrics; +const c = @import("c"); +const csv = @import("zml-smi/csv"); +const json = @import("zml-smi/json"); +const api = @import("zml-smi/api"); +const prometheus = @import("zml-smi/prometheus"); +const sysfs = @import("zml-smi/sysfs"); +const smi_tui = @import("zml-smi/tui"); +const tui = smi_tui.top; +const static_print = smi_tui.print; +const data = smi_tui.data; const CliArgs = struct { top: bool = false, @@ -47,83 +59,183 @@ const CliArgs = struct { // \\ --remotes Add devices from remote hosts (comma-separated URLs) }; +const device_backends = switch (builtin.os.tag) { + .macos => .{}, + .linux => switch (builtin.cpu.arch) { + .x86_64 => .{ + .{ .cuda, @import("zml-smi/platforms/nvml") }, + .{ .rocm, @import("zml-smi/platforms/amdsmi") }, + .{ .oneapi, @import("zml-smi/platforms/oneapi") }, + .{ .neuron, @import("zml-smi/platforms/neuron") }, + .{ .tpu, @import("zml-smi/platforms/tpu") }, + }, + .aarch64 => .{ + .{ .cuda, @import("zml-smi/platforms/nvml") }, + }, + else => unreachable, + }, + else => unreachable, +}; + pub fn main(init: std.process.Init) !void { const gpa = init.gpa; const arena = init.arena.allocator(); const io = init.io; const args = stdx.flags.parse(init.minimal.args, CliArgs); - const interval: std.Io.Duration = .fromMilliseconds(args.poll_interval); - var collector: smi.Collector = .init(arena, gpa, io, .{ .poll_interval_ms = args.poll_interval }); + const targets = detect(gpa, io); + + var collector: Collector = .init(arena, gpa, io, .{ + .poll_interval_ms = args.poll_interval, + .poll_only = !args.top and args.prometheus_listen == null and !args.api, + }); defer collector.deinit(); - inline for (platforms.all) |Platform| { - if (Platform.detect(io)) { - const platform = try arena.create(Platform); - if (Platform.init(arena, gpa, io)) |value| { - platform.* = value; - try collector.add(&platform.platform); - } else |err| log.warn("{s} skipped: {t}", .{ @typeName(Platform), err }); + var host_info: HostInfo = .{ .values = .{ .{}, .{} } }; + try host.init(&collector, &host_info); + + var enricher: ProcessEnricher = try .init(gpa, io); + defer enricher.deinit(); + + if (args.api) { + std.log.info("serving api metrics on port {d}", .{args.api_port}); + } + + if (args.prometheus_listen) |listen| { + std.log.info("serving prometheus metrics on {s}", .{listen}); + } + + inline for (device_backends) |entry| { + const target, const backend = entry; + if (targets.contains(target)) { + backend.start(&collector) catch |err| { + std.log.err("{s} skipped: {s}", .{ @tagName(target), @errorName(err) }); + }; } } + var api_group: std.Io.Group = .init; + if (args.remotes) |hosts| { - var it = std.mem.splitScalar(u8, hosts, ','); - while (it.next()) |host| { - const remote = try arena.create(smi.Remote); - if (smi.Remote.open(arena, gpa, io, host, interval)) |value| { - remote.* = value; - try collector.add(&remote.platform); - } else |err| log.warn("{s} skipped: {t}", .{ host, err }); - } + try api.addRemotes(&collector, hosts); } - const serving = args.prometheus_listen != null or args.api; - if (serving or args.top) { - try collector.start(); - } else { - collector.round(); - if (collector.two_readings) { - try io.sleep(interval, .awake); - collector.round(); - } - } - - if (serving) { - var servers: std.Io.Group = .init; - defer servers.cancel(io); - if (args.prometheus_listen) |listen| { - log.info("serving prometheus metrics on {s}", .{listen}); - try servers.concurrent(io, smi.api.serve, .{ gpa, io, try .parseLiteral(listen), &collector, .prometheus }); - } - if (args.api) { - log.info("serving api metrics on port {d}", .{args.api_port}); - const address: std.Io.net.IpAddress = .{ .ip4 = .{ .bytes = .{ 0, 0, 0, 0 }, .port = args.api_port } }; - try servers.concurrent(io, smi.api.serve, .{ gpa, io, address, &collector, .json }); - } - try servers.await(io); - return; - } - - if (args.csv or args.json) { - var buffer: [4096]u8 = undefined; - var stdout = std.Io.File.stdout().writer(io, &buffer); - try if (args.csv) smi.csv.write(&stdout.interface, &collector) else smi.json.write(&stdout.interface, &collector); - return stdout.interface.flush(); - } - - var state: tui.data.SystemState = try .init(.{ - .collector = &collector, + var state = try data.SystemState.init(.{ + .devices = collector.device_infos.items, + .host = &host_info, + .targets = targets, .tui_refresh_rate = args.tui_refresh_rate, + .process_lists = collector.process_lists.items, + .enricher = &enricher, .gpa = gpa, .arena = arena, .io = io, }); defer state.deinit(arena); - if (args.top) { - try tui.top.run(gpa, io, init.environ_map, &state); + if (args.prometheus_listen) |listen| { + try api_group.concurrent(io, prometheus.Server.run, .{ io, listen, collector.device_infos.items, &host_info }); + } + + if (args.api) { + try api_group.concurrent(io, api.Server.run, .{ gpa, io, args.api_port, collector.device_infos.items, collector.process_lists.items, &enricher }); + } + + if (args.prometheus_listen != null or args.api) { + try api_group.await(io); + } else if (args.csv) { + var csv_buf: [4096]u8 = undefined; + var csv_writer = std.Io.File.stdout().writer(io, &csv_buf); + + try csv.write(&csv_writer.interface, collector.device_infos.items); + try csv_writer.flush(); + } else if (args.json) { + var procs: std.ArrayList(smi_info.process_info.ProcessInfo) = .empty; + defer procs.deinit(gpa); + for (collector.process_lists.items) |pl| { + try procs.appendSlice(gpa, pl.front().items); + } + enricher.enrich(io, procs.items); + + var json_buf: [4096]u8 = undefined; + var json_writer = std.Io.File.stdout().writer(io, &json_buf); + + try json.write(&json_writer.interface, collector.device_infos.items, procs.items); + try json_writer.flush(); + } else if (args.top) { + try tui.run(gpa, io, init.environ_map, &state); } else { - try tui.print.run(arena, io, init.environ_map, &state); + try static_print.run(arena, io, init.environ_map, &state); } } + +fn detect(allocator: std.mem.Allocator, io: std.Io) smi_info.Targets { + if (comptime builtin.os.tag == .macos) { + return .{}; + } + + var targets: smi_info.Targets = .{}; + + switch (builtin.os.tag) { + .macos => {}, + .linux => { + switch (builtin.cpu.arch) { + .x86_64 => { + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } + + if (hasDevice(io, "/dev/kfd")) { + targets.insert(.rocm); + } + + if (hasDrmRenderDevice(allocator, io, "0x8086")) { + targets.insert(.oneapi); + } + + if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { + targets.insert(.neuron); + } + + if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { + targets.insert(.tpu); + } + }, + .aarch64 => { + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } + }, + else => unreachable, + } + }, + else => unreachable, + } + + return targets; +} + +fn hasDevice(io: std.Io, path: []const u8) bool { + std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; + return true; +} + +fn hasDrmRenderDevice(allocator: std.mem.Allocator, io: std.Io, comptime pci_vendor_id: []const u8) bool { + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (!std.mem.startsWith(u8, entry.name, "renderD")) continue; + + var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = std.fmt.bufPrint(&vendor_path_buf, "/sys/class/drm/{s}/device/vendor", .{entry.name}) catch continue; + const vendor = sysfs.readString(allocator, io, vendor_path) catch continue; + defer allocator.free(vendor); + if (std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), pci_vendor_id)) { + return true; + } + } + + return false; +} diff --git a/bin/zml-smi/platforms.zig b/bin/zml-smi/platforms.zig deleted file mode 100644 index e21c8a19..00000000 --- a/bin/zml-smi/platforms.zig +++ /dev/null @@ -1,26 +0,0 @@ -//! The platforms of this OS and architecture. Each one is a file whose struct holds an `smi.Platform`. -const builtin = @import("builtin"); - -pub const all = switch (builtin.os.tag) { - .linux => switch (builtin.cpu.arch) { - .x86_64 => .{ - @import("zml-smi/platforms/nvml"), - @import("zml-smi/platforms/amdsmi"), - @import("zml-smi/platforms/oneapi"), - @import("zml-smi/platforms/neuron"), - @import("zml-smi/platforms/tpu"), - @import("zml-smi/platforms/furiosa"), - }, - .aarch64 => .{ - @import("zml-smi/platforms/nvml"), - }, - else => .{}, - }, - .macos => switch (builtin.cpu.arch) { - .aarch64 => .{ - @import("zml-smi/platforms/metal"), - }, - else => .{}, - }, - else => .{}, -}; diff --git a/bin/zml-smi/platforms/amdsmi/AmdSmi.zig b/bin/zml-smi/platforms/amdsmi/AmdSmi.zig deleted file mode 100644 index b809005e..00000000 --- a/bin/zml-smi/platforms/amdsmi/AmdSmi.zig +++ /dev/null @@ -1,252 +0,0 @@ -//! AMD GPUs, through the bundled libamd_smi, with the memory of each process from its fdinfo files. -const std = @import("std"); -const c = @import("c"); -const smi = @import("zml-smi"); - -const AmdSmi = @This(); -const log = std.log.scoped(.@"zml-smi/amdsmi"); - -platform: smi.Platform, -io: std.Io, -api: Api, -gpus: []const Gpu, -proc: ?smi.sysfs.Dir, -/// Graphics plus encode nanoseconds of each process on each GPU. -busy: smi.Counters(struct { pid: u32, device: u16 }), - -const Gpu = struct { - handle: c.amdsmi_processor_handle, - /// The PCI address fdinfo files name the GPU by: "0000:03:00.0". - address: ?[12]u8, -}; - -const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit }; - -const Api = struct { - amdsmi_init: Checked(c.amdsmi_init), - amdsmi_status_code_to_string: Checked(c.amdsmi_status_code_to_string), - amdsmi_get_socket_handles: Checked(c.amdsmi_get_socket_handles), - amdsmi_get_processor_handles: Checked(c.amdsmi_get_processor_handles), - amdsmi_get_gpu_asic_info: Checked(c.amdsmi_get_gpu_asic_info), - amdsmi_get_gpu_driver_info: Checked(c.amdsmi_get_gpu_driver_info), - amdsmi_get_gpu_bdf_id: Checked(c.amdsmi_get_gpu_bdf_id), - amdsmi_get_gpu_metrics_info: Checked(c.amdsmi_get_gpu_metrics_info), - amdsmi_get_power_info: Checked(c.amdsmi_get_power_info), - amdsmi_get_temp_metric: Checked(c.amdsmi_get_temp_metric), - amdsmi_get_gpu_fan_speed: Checked(c.amdsmi_get_gpu_fan_speed), - amdsmi_get_gpu_activity: Checked(c.amdsmi_get_gpu_activity), - amdsmi_get_clock_info: Checked(c.amdsmi_get_clock_info), - amdsmi_get_gpu_memory_total: Checked(c.amdsmi_get_gpu_memory_total), - amdsmi_get_gpu_memory_usage: Checked(c.amdsmi_get_gpu_memory_usage), - amdsmi_get_pcie_info: Checked(c.amdsmi_get_pcie_info), - amdsmi_get_gpu_process_list: Checked(c.amdsmi_get_gpu_process_list), - - fn Checked(comptime function: anytype) type { - return smi.dynlib.Checked(function, check); - } - - fn check(status: c.amdsmi_status_t) error{ Unsupported, AmdSmiError }!void { - return switch (status) { - c.AMDSMI_STATUS_SUCCESS => {}, - c.AMDSMI_STATUS_NOT_SUPPORTED, c.AMDSMI_STATUS_NOT_YET_IMPLEMENTED, c.AMDSMI_STATUS_NO_DATA => error.Unsupported, - else => error.AmdSmiError, - }; - } -}; - -pub fn detect(io: std.Io) bool { - return smi.sysfs.exists(io, "/dev/kfd"); -} - -pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io) !AmdSmi { - const api = try smi.dynlib.openBundled(Api, io, "libamd_smi.so.27"); - - // The first call says why the library cannot be used. - const status = api.amdsmi_init.function(c.AMDSMI_INIT_AMD_GPUS); - Api.check(status) catch |err| { - const message: [*c]const u8 = api.amdsmi_status_code_to_string.get(.{status}) catch "unknown status"; - log.warn("amdsmi_init: {s}", .{std.mem.span(message)}); - return err; - }; - - var socket_count: u32 = 0; - try api.amdsmi_get_socket_handles.call(.{ &socket_count, null }); - const sockets = try gpa.alloc(c.amdsmi_socket_handle, socket_count); - defer gpa.free(sockets); - try api.amdsmi_get_socket_handles.call(.{ &socket_count, sockets.ptr }); - - var gpus: std.ArrayList(Gpu) = .empty; - var devices: std.ArrayList(smi.Device) = .empty; - for (sockets[0..socket_count]) |socket| { - var count: u32 = 0; - try api.amdsmi_get_processor_handles.call(.{ socket, &count, null }); - const handles = try gpa.alloc(c.amdsmi_processor_handle, count); - defer gpa.free(handles); - try api.amdsmi_get_processor_handles.call(.{ socket, &count, handles.ptr }); - - for (handles[0..count]) |handle| { - try gpus.append(arena, .{ .handle = handle, .address = if (api.amdsmi_get_gpu_bdf_id.get(.{handle})) |bdf| pciAddress(bdf) else |_| null }); - try devices.append(arena, .{ - .target = .rocm, - .name = if (api.amdsmi_get_gpu_asic_info.get(.{handle})) |info| try arena.dupe(u8, std.mem.sliceTo(&info.market_name, 0)) else |_| null, - .driver_version = if (api.amdsmi_get_gpu_driver_info.get(.{handle})) |info| try arena.dupe(u8, std.mem.sliceTo(&info.driver_version, 0)) else |_| null, - }); - } - } - - return .{ - .platform = .{ .devices = devices.items, .two_readings = true, .vtable = &vtable }, - .io = io, - .api = api, - .gpus = gpus.items, - .proc = smi.sysfs.Dir.open(io, "/proc") catch null, - .busy = .init(gpa), - }; -} - -fn deinit(platform: *smi.Platform) void { - const self: *AmdSmi = @alignCast(@fieldParentPtr("platform", platform)); - if (self.proc) |proc| proc.close(); - self.busy.deinit(); -} - -fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { - const self: *AmdSmi = @alignCast(@fieldParentPtr("platform", platform)); - const api = &self.api; - - for (self.gpus, samples) |gpu, *sample| { - const fields = &sample.rocm; - const handle = gpu.handle; - - // Watts: the current power where the GPU has it, else the average. - if (api.amdsmi_get_gpu_metrics_info.get(.{handle})) |metrics| { - if (known(metrics.current_socket_power) orelse known(metrics.average_socket_power)) |watts| fields.power_mw = watts * 1000; - } else |_| {} - if (api.amdsmi_get_power_info.get(.{handle})) |info| { - if (known(info.power_limit)) |microwatts| fields.power_limit_mw = smi.units.scale(microwatts, .micro, .milli); - } else |_| {} - if (api.amdsmi_get_temp_metric.get(.{ handle, c.AMDSMI_TEMPERATURE_TYPE_EDGE, c.AMDSMI_TEMP_CURRENT })) |celsius| { - fields.temperature = std.math.cast(u64, celsius); - } else |_| {} - if (api.amdsmi_get_gpu_fan_speed.get(.{ handle, 0 })) |speed| { - if (std.math.cast(u64, speed)) |pwm| fields.fan_speed_percent = smi.hwmon.pwmPercent(pwm); - } else |_| {} - if (api.amdsmi_get_gpu_activity.get(.{handle})) |activity| { - // Not a percentage when unsupported: 0xFFFF. - if (activity.gfx_activity <= 100) fields.util_percent = activity.gfx_activity; - } else |_| {} - - if (api.amdsmi_get_clock_info.get(.{ handle, c.AMDSMI_CLK_TYPE_SYS })) |clock| { - fields.clock_graphics_mhz = known(clock.clk); - fields.clock_graphics_max_mhz = known(clock.max_clk); - } else |_| {} - if (api.amdsmi_get_clock_info.get(.{ handle, c.AMDSMI_CLK_TYPE_SOC })) |clock| { - fields.clock_soc_mhz = known(clock.clk); - } else |_| {} - if (api.amdsmi_get_clock_info.get(.{ handle, c.AMDSMI_CLK_TYPE_MEM })) |clock| { - fields.clock_mem_mhz = known(clock.clk); - fields.clock_mem_max_mhz = known(clock.max_clk); - } else |_| {} - - fields.mem_used_bytes = api.amdsmi_get_gpu_memory_usage.get(.{ handle, c.AMDSMI_MEM_TYPE_VRAM }) catch null; - fields.mem_total_bytes = api.amdsmi_get_gpu_memory_total.get(.{ handle, c.AMDSMI_MEM_TYPE_VRAM }) catch null; - - if (api.amdsmi_get_pcie_info.get(.{handle})) |pcie| { - const link: smi.pci.Link = .{ .gen = known(pcie.pcie_static.pcie_interface_version), .width = known(pcie.pcie_metric.pcie_width) }; - fields.pcie_link_gen = link.gen; - fields.pcie_link_width = link.width; - // The rate comes from the generation: the speed amdsmi reports stays at 2.5 GT/s on an R9700 whose link runs at 32. - fields.pcie_bandwidth_mbps = link.mbps(); - } else |_| {} - } -} - -fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { - const self: *AmdSmi = @alignCast(@fieldParentPtr("platform", platform)); - defer self.busy.sweep(); - var read_any = self.gpus.len == 0; - - for (self.gpus, 0..) |gpu, index| { - const device: u16 = @intCast(index); - const list = self.processList(scratch, gpu.handle) catch continue; - read_any = true; - for (list) |process| { - const busy = try self.busy.advance(.{ .pid = process.pid, .device = device }, self.io, process.engine_usage.gfx +| process.engine_usage.enc); - try out.append(.{ - .pid = process.pid, - .device_idx = device, - .dev_util_percent = if (busy) |growth| growth.percentOfTime() else null, - .dev_mem_kib = if (gpu.address) |address| self.vramKib(process.pid, &address) else null, - }); - } - } - if (!read_any) return error.Unavailable; -} - -/// The processes on a GPU, from a query sized by a first call. -fn processList(self: *AmdSmi, scratch: std.mem.Allocator, handle: c.amdsmi_processor_handle) ![]const c.amdsmi_proc_info_t { - var count: u32 = 0; - try Api.check(self.listQuietly(handle, &count, null)); - if (count == 0) return &.{}; - - const list = try scratch.alloc(c.amdsmi_proc_info_t, count); - const status = self.listQuietly(handle, &count, list.ptr); - // The list is full, and the processes that started since are left out. - if (status != c.AMDSMI_STATUS_OUT_OF_RESOURCES) try Api.check(status); - return list[0..@min(count, list.len)]; -} - -/// amdsmi prints while it lists processes, so stderr points at the null device for the length of the call. -fn listQuietly(self: *AmdSmi, handle: c.amdsmi_processor_handle, count: *u32, list: [*c]c.amdsmi_proc_info_t) c.amdsmi_status_t { - const saved = std.c.dup(std.c.STDERR_FILENO); - if (saved >= 0) { - const null_device = std.c.open("/dev/null", .{ .ACCMODE = .WRONLY }); - if (null_device >= 0) { - _ = std.c.dup2(null_device, std.c.STDERR_FILENO); - _ = std.c.close(null_device); - } - } - defer if (saved >= 0) { - _ = std.c.dup2(saved, std.c.STDERR_FILENO); - _ = std.c.close(saved); - }; - return self.api.amdsmi_get_gpu_process_list.function(handle, count, list); -} - -/// The VRAM a process holds on a GPU, summed over its fdinfo files that name the GPU. -fn vramKib(self: *AmdSmi, pid: u32, address: []const u8) ?u64 { - const proc = self.proc orelse return null; - var path: [24]u8 = undefined; - const files = proc.openDir(std.fmt.bufPrint(&path, "{d}/fdinfo", .{pid}) catch return null) catch return null; - defer files.close(); - - var bytes: u64 = 0; - var buffer: [8192]u8 = undefined; - var it = files.iterate(); - while (it.next(self.io) catch null) |entry| { - const info = smi.fdinfo.read(files, entry.name, &buffer) catch continue; - if (!std.mem.eql(u8, info.pdev orelse continue, address)) continue; - for (info.regions()) |region| { - if (std.mem.eql(u8, region.name, "vram")) bytes += region.resident orelse 0; - } - } - return bytes / 1024; -} - -/// amdsmi reports a value it cannot read as the largest of its type. -fn known(value: anytype) ?u64 { - return if (value == std.math.maxInt(@TypeOf(value))) null else value; -} - -/// The PCI address of an amdsmi BDF id, whose domain is in the upper 32 bits. -fn pciAddress(bdf: u64) ?[12]u8 { - var address: [12]u8 = undefined; - _ = std.fmt.bufPrint(&address, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ bdf >> 32, (bdf >> 8) & 0xff, (bdf >> 3) & 0x1f, bdf & 0x7 }) catch return null; - return address; -} - -test pciAddress { - try std.testing.expectEqualStrings("0000:03:00.0", &pciAddress(0x0300).?); - // Bits 28 to 31 hold a partition id, which is not part of the address. - try std.testing.expectEqualStrings("0001:c1:00.1", &pciAddress(0x1_1000_c101).?); -} diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel index a285ca7a..dac6b935 100644 --- a/bin/zml-smi/platforms/amdsmi/BUILD.bazel +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -1,18 +1,22 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "amdsmi", + srcs = [ + "amdsmi.zig", + "process.zig", + ], import_name = "zml-smi/platforms/amdsmi", - main = "AmdSmi.zig", + main = "metrics.zig", target_compatible_with = ["@platforms//os:linux"], - visibility = ["//bin/zml-smi:__subpackages__"], + visibility = ["//visibility:public"], deps = [ - "//bin/zml-smi/smi", + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + "//bin/zml-smi/utils:sandbox", + "//stdx", "@amdrocm-amdsmi//:amdsmi", ], ) - -zig_test( - name = "test", - deps = [":amdsmi"], -) diff --git a/bin/zml-smi/platforms/amdsmi/amdsmi.zig b/bin/zml-smi/platforms/amdsmi/amdsmi.zig new file mode 100644 index 00000000..2d89bc7c --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/amdsmi.zig @@ -0,0 +1,295 @@ +const std = @import("std"); +const c = @import("c"); +const stdx = @import("stdx"); +const DynLib = @import("zml-smi/dynlib"); +const sandbox = @import("zml-smi/sandbox"); + +const AmdSmi = @This(); + +pub const Error = ReturnError || error{AmdSmiUnavailable}; +pub const Handle = c.amdsmi_processor_handle; +pub const ProcInfo = c.amdsmi_proc_info_t; + +lib: Fns, +gpu_handles: []c.amdsmi_processor_handle, + +const Fns = struct { + amdsmi_init: *const @TypeOf(c.amdsmi_init), + amdsmi_get_socket_handles: *const @TypeOf(c.amdsmi_get_socket_handles), + amdsmi_get_processor_handles: *const @TypeOf(c.amdsmi_get_processor_handles), + amdsmi_get_gpu_asic_info: *const @TypeOf(c.amdsmi_get_gpu_asic_info), + amdsmi_get_gpu_metrics_info: *const @TypeOf(c.amdsmi_get_gpu_metrics_info), + amdsmi_get_power_info: *const @TypeOf(c.amdsmi_get_power_info), + amdsmi_get_temp_metric: *const @TypeOf(c.amdsmi_get_temp_metric), + amdsmi_get_gpu_fan_speed: *const @TypeOf(c.amdsmi_get_gpu_fan_speed), + amdsmi_get_gpu_activity: *const @TypeOf(c.amdsmi_get_gpu_activity), + amdsmi_get_clock_info: *const @TypeOf(c.amdsmi_get_clock_info), + amdsmi_get_gpu_memory_total: *const @TypeOf(c.amdsmi_get_gpu_memory_total), + amdsmi_get_gpu_memory_usage: *const @TypeOf(c.amdsmi_get_gpu_memory_usage), + amdsmi_get_pcie_info: *const @TypeOf(c.amdsmi_get_pcie_info), + amdsmi_get_gpu_bdf_id: *const @TypeOf(c.amdsmi_get_gpu_bdf_id), + amdsmi_get_gpu_process_list: *const @TypeOf(c.amdsmi_get_gpu_process_list), + amdsmi_get_gpu_driver_info: *const @TypeOf(c.amdsmi_get_gpu_driver_info), +}; + +pub fn init(allocator: std.mem.Allocator) !AmdSmi { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = sandbox.path(&path_buf) orelse return error.AmdSmiUnavailable; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libamd_smi.so.27" }); + + var dynlib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("amdsmi: dlopen: {s}", .{err}); + return error.AmdSmiUnavailable; + }, + } }; + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.AmdSmiUnavailable; + + try check(fns.amdsmi_init(c.AMDSMI_INIT_AMD_GPUS)); + + var socket_count: u32 = 0; + try check(fns.amdsmi_get_socket_handles(&socket_count, null)); + + const sockets = try allocator.alloc(c.amdsmi_socket_handle, socket_count); + defer allocator.free(sockets); + try check(fns.amdsmi_get_socket_handles(&socket_count, @ptrCast(sockets.ptr))); + + var handle_list: std.ArrayList(c.amdsmi_processor_handle) = .empty; + defer handle_list.deinit(allocator); + + for (sockets[0..socket_count]) |socket| { + var proc_count: u32 = 0; + try check(fns.amdsmi_get_processor_handles(socket, &proc_count, null)); + + const procs = try allocator.alloc(c.amdsmi_processor_handle, proc_count); + defer allocator.free(procs); + + try check(fns.amdsmi_get_processor_handles(socket, &proc_count, @ptrCast(procs.ptr))); + + for (procs[0..proc_count]) |proc| { + try handle_list.append(allocator, proc); + } + } + + return .{ .lib = fns, .gpu_handles = try handle_list.toOwnedSlice(allocator) }; +} + +pub fn handleByIndex(self: AmdSmi, device_id: u32) Error!Handle { + if (device_id >= self.gpu_handles.len) { + return error.not_found; + } + return self.gpu_handles[device_id]; +} + +pub fn deviceCount(self: AmdSmi) u32 { + return @intCast(self.gpu_handles.len); +} + +pub const name_buf_len = c.AMDSMI_MAX_STRING_LENGTH; + +pub fn name(self: AmdSmi, handle: Handle, buf: *[c.AMDSMI_MAX_STRING_LENGTH]u8) Error![:0]const u8 { + var info: c.amdsmi_asic_info_t = undefined; + try check(self.lib.amdsmi_get_gpu_asic_info(handle, &info)); + @memcpy(buf, &info.market_name); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn powerUsage(self: AmdSmi, handle: Handle) Error!u32 { + var metrics: c.amdsmi_gpu_metrics_t = undefined; + try check(self.lib.amdsmi_get_gpu_metrics_info(handle, &metrics)); + const unsupported16 = std.math.maxInt(u16); + if (metrics.current_socket_power != unsupported16) { + return metrics.current_socket_power; + } + if (metrics.average_socket_power != unsupported16) { + return metrics.average_socket_power; + } + return error.not_supported; +} + +pub fn powerLimit(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_power_info_t = undefined; + try check(self.lib.amdsmi_get_power_info(handle, &info)); + return info.power_limit; +} + +pub fn temperature(self: AmdSmi, handle: Handle) Error!i64 { + var temp: i64 = 0; + try check(self.lib.amdsmi_get_temp_metric(handle, c.AMDSMI_TEMPERATURE_TYPE_EDGE, c.AMDSMI_TEMP_CURRENT, &temp)); + return temp; +} + +pub fn fanSpeed(self: AmdSmi, handle: Handle) Error!i64 { + var speed: i64 = 0; + try check(self.lib.amdsmi_get_gpu_fan_speed(handle, @as(u32, 0), &speed)); + return speed; +} + +pub fn gpuUtil(self: AmdSmi, handle: Handle) Error!u32 { + var usage: c.amdsmi_engine_usage_t = undefined; + try check(self.lib.amdsmi_get_gpu_activity(handle, &usage)); + return usage.gfx_activity; +} + +pub fn clockGraphics(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); + return info.clk; +} + +pub fn maxClockGraphics(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); + return info.max_clk; +} + +pub fn clockMem(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); + return info.clk; +} + +pub fn maxClockMem(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); + return info.max_clk; +} + +pub fn clockSoc(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SOC, &info)); + return info.clk; +} + +pub fn memTotal(self: AmdSmi, handle: Handle) Error!u64 { + var total: u64 = 0; + try check(self.lib.amdsmi_get_gpu_memory_total(handle, c.AMDSMI_MEM_TYPE_VRAM, &total)); + return total; +} + +pub fn memUsed(self: AmdSmi, handle: Handle) Error!u64 { + var used: u64 = 0; + try check(self.lib.amdsmi_get_gpu_memory_usage(handle, c.AMDSMI_MEM_TYPE_VRAM, &used)); + return used; +} + +pub fn pcieWidth(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return @intCast(info.pcie_metric.pcie_width); +} + +pub fn pcieBandwidth(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return info.pcie_metric.pcie_bandwidth; +} + +pub fn pcieLinkGen(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return info.pcie_static.pcie_interface_version; +} + +pub fn bdfId(self: AmdSmi, handle: Handle) Error!u64 { + var bdf_id: u64 = 0; + try check(self.lib.amdsmi_get_gpu_bdf_id(handle, &bdf_id)); + return bdf_id; +} + +pub const driver_version_buf_len = @typeInfo(@TypeOf(@as(c.amdsmi_driver_info_t, undefined).driver_version)).array.len; + +pub fn driverVersion(self: AmdSmi, handle: Handle, buf: *[driver_version_buf_len]u8) Error![:0]const u8 { + var info: c.amdsmi_driver_info_t = std.mem.zeroes(c.amdsmi_driver_info_t); + try check(self.lib.amdsmi_get_gpu_driver_info(handle, &info)); + @memcpy(buf, &info.driver_version); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn processList(self: AmdSmi, allocator: std.mem.Allocator, handle: Handle) (Error || error{OutOfMemory})![]const ProcInfo { + var count: u32 = 0; + try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, null)); + if (count == 0) { + return &.{}; + } + const procs = try allocator.alloc(ProcInfo, count); + errdefer allocator.free(procs); + try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, @ptrCast(procs.ptr))); + return procs; +} + +// Error handling + +pub const ReturnError = error{ + inval, + not_supported, + not_yet_implemented, + fail_load_module, + fail_load_symbol, + drm_error, + api_failed, + timeout, + retry, + no_perm, + interrupt, + io_error, + address_fault, + file_error, + out_of_resources, + internal_exception, + input_out_of_bounds, + init_error, + refcount_overflow, + directory_not_found, + busy, + not_found, + not_init, + no_slot, + driver_not_loaded, + no_data, + insufficient_size, + unexpected_size, + unexpected_data, + unknown_error, +}; + +fn check(ret: c.amdsmi_status_t) Error!void { + if (ret == c.AMDSMI_STATUS_SUCCESS) { + return; + } + + return switch (ret) { + c.AMDSMI_STATUS_INVAL => error.inval, + c.AMDSMI_STATUS_NOT_SUPPORTED => error.not_supported, + c.AMDSMI_STATUS_NOT_YET_IMPLEMENTED => error.not_yet_implemented, + c.AMDSMI_STATUS_FAIL_LOAD_MODULE => error.fail_load_module, + c.AMDSMI_STATUS_FAIL_LOAD_SYMBOL => error.fail_load_symbol, + c.AMDSMI_STATUS_DRM_ERROR => error.drm_error, + c.AMDSMI_STATUS_API_FAILED => error.api_failed, + c.AMDSMI_STATUS_TIMEOUT => error.timeout, + c.AMDSMI_STATUS_RETRY => error.retry, + c.AMDSMI_STATUS_NO_PERM => error.no_perm, + c.AMDSMI_STATUS_INTERRUPT => error.interrupt, + c.AMDSMI_STATUS_IO => error.io_error, + c.AMDSMI_STATUS_ADDRESS_FAULT => error.address_fault, + c.AMDSMI_STATUS_FILE_ERROR => error.file_error, + c.AMDSMI_STATUS_OUT_OF_RESOURCES => error.out_of_resources, + c.AMDSMI_STATUS_INTERNAL_EXCEPTION => error.internal_exception, + c.AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS => error.input_out_of_bounds, + c.AMDSMI_STATUS_INIT_ERROR => error.init_error, + c.AMDSMI_STATUS_REFCOUNT_OVERFLOW => error.refcount_overflow, + c.AMDSMI_STATUS_DIRECTORY_NOT_FOUND => error.directory_not_found, + c.AMDSMI_STATUS_BUSY => error.busy, + c.AMDSMI_STATUS_NOT_FOUND => error.not_found, + c.AMDSMI_STATUS_NOT_INIT => error.not_init, + c.AMDSMI_STATUS_NO_SLOT => error.no_slot, + c.AMDSMI_STATUS_DRIVER_NOT_LOADED => error.driver_not_loaded, + c.AMDSMI_STATUS_NO_DATA => error.no_data, + c.AMDSMI_STATUS_INSUFFICIENT_SIZE => error.insufficient_size, + c.AMDSMI_STATUS_UNEXPECTED_SIZE => error.unexpected_size, + c.AMDSMI_STATUS_UNEXPECTED_DATA => error.unexpected_data, + else => error.unknown_error, + }; +} diff --git a/bin/zml-smi/platforms/amdsmi/metrics.zig b/bin/zml-smi/platforms/amdsmi/metrics.zig new file mode 100644 index 00000000..8aa0487c --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/metrics.zig @@ -0,0 +1,151 @@ +const std = @import("std"); +const AmdSmi = @import("amdsmi.zig"); +const device_info = @import("zml-smi/info").device_info; +const DeviceInfo = device_info.DeviceInfo; +const GpuInfo = device_info.GpuInfo; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const process = @import("process.zig"); + +pub fn start(collector: *Collector) !void { + const amdsmi = try collector.arena.create(AmdSmi); + amdsmi.* = try AmdSmi.init(collector.arena); + + const count = amdsmi.deviceCount(); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + for (0..count) |i| { + const dev = Device.open(amdsmi, @intCast(i)) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + }; + const info = try collector.addDevice(.{ .rocm = .{ .values = .{ initial, initial } } }); + try collector.spawnPoll(pollOnce, .{ null, &info.rocm, dev }); + } + + const processes = try collector.createProcessList(); + try process.init(collector, processes, amdsmi, dev_offset); +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); + +const Device = struct { + amdsmi: *const AmdSmi, + handle: AmdSmi.Handle, + + pub fn open(amdsmi: *const AmdSmi, index: u32) !Device { + return .{ .amdsmi = amdsmi, .handle = try amdsmi.handleByIndex(index) }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [AmdSmi.name_buf_len]u8 = undefined; + const slice = try self.amdsmi.name(self.handle, &buf); + + return try arena.dupe(u8, slice); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [AmdSmi.driver_version_buf_len]u8 = undefined; + const slice = try self.amdsmi.driverVersion(self.handle, &buf); + return try arena.dupe(u8, slice); + } + + // Power + pub fn powerUsage(self: Device) !u64 { + const pw = try self.amdsmi.powerUsage(self.handle); + return @as(u64, pw) * 1000; + } + + pub fn powerLimit(self: Device) !u64 { + const limit = try self.amdsmi.powerLimit(self.handle); + if (limit == std.math.maxInt(u32)) { + return error.not_supported; + } + + return @as(u64, limit) / 1000; + } + + // Thermal + pub fn temperature(self: Device) !u64 { + const temp = try self.amdsmi.temperature(self.handle); + return @intCast(temp); + } + + pub fn fanSpeed(self: Device) !u64 { + const speed = try self.amdsmi.fanSpeed(self.handle); + return @intCast(@divTrunc(speed * 100, 255)); + } + + // Utilization + pub fn gpuUtil(self: Device) !u64 { + return try self.amdsmi.gpuUtil(self.handle); + } + + // Clocks + pub fn clockGraphics(self: Device) !u64 { + return try self.amdsmi.clockGraphics(self.handle); + } + + pub fn clockSoc(self: Device) !u64 { + return try self.amdsmi.clockSoc(self.handle); + } + + pub fn clockMem(self: Device) !u64 { + return try self.amdsmi.clockMem(self.handle); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return try self.amdsmi.maxClockGraphics(self.handle); + } + + pub fn maxClockMem(self: Device) !u64 { + return try self.amdsmi.maxClockMem(self.handle); + } + + // Memory + pub fn memUsed(self: Device) !u64 { + return self.amdsmi.memUsed(self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.amdsmi.memTotal(self.handle); + } + + // PCIe + pub fn pcieBandwidth(self: Device) !u64 { + const bw = try self.amdsmi.pcieBandwidth(self.handle); + if (bw == std.math.maxInt(u32)) { + return error.not_supported; + } + + return bw; + } + + pub fn pcieLinkGen(self: Device) !u64 { + return try self.amdsmi.pcieLinkGen(self.handle); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return try self.amdsmi.pcieWidth(self.handle); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_soc_mhz", .query = Device.clockSoc }, + .{ .field = "clock_mem_mhz", .query = Device.clockMem }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; diff --git a/bin/zml-smi/platforms/amdsmi/process.zig b/bin/zml-smi/platforms/amdsmi/process.zig new file mode 100644 index 00000000..8d1fdd01 --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/process.zig @@ -0,0 +1,139 @@ +const std = @import("std"); +const c = std.c; +const AmdSmi = @import("amdsmi.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +const bdf_len = "0000:00:00.0".len; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16) !void { + const device_count = amdsmi.deviceCount(); + const pci_slots = try collector.arena.alloc(?[bdf_len]u8, device_count); + + for (pci_slots, 0..) |*slot, i| { + const handle = amdsmi.handleByIndex(@intCast(i)) catch { + slot.* = null; + continue; + }; + + slot.* = formatBdf(amdsmi.bdfId(handle) catch { + slot.* = null; + continue; + }); + } + + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, amdsmi, dev_offset, device_count, pci_slots }); +} + +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16, device_count: u32, pci_slots: []?[bdf_len]u8) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (0..device_count) |dev_idx| { + const handle = amdsmi.handleByIndex(@intCast(dev_idx)) catch continue; + + const procs = blk: { + const saved = c.dup(c.STDERR_FILENO); + const devnull = c.open("/dev/null", .{ .ACCMODE = .WRONLY }); + + if (devnull >= 0) { + if (saved >= 0) { + _ = c.dup2(devnull, c.STDERR_FILENO); + } + + _ = c.close(devnull); + } + + defer if (saved >= 0) { + _ = c.dup2(saved, c.STDERR_FILENO); + _ = c.close(saved); + }; + + break :blk amdsmi.processList(allocator, handle) catch continue; + }; + defer { + if (procs.len > 0) { + allocator.free(procs); + } + } + + const pci_slot = if (dev_idx < pci_slots.len) &(pci_slots[dev_idx] orelse continue) else continue; + + for (procs) |proc| { + back.append(allocator, .{ + .pid = proc.pid, + .device_idx = @intCast(dev_idx + dev_offset), + .dev_util_percent = @intCast(proc.engine_usage.gfx + proc.engine_usage.enc), + .dev_mem_kib = readProcessGpuVram(io, proc.pid, pci_slot), + }) catch break; + } + } + + list.swap(); +} + +/// Reads per-GPU VRAM from /proc//fdinfo/, filtered by PCI slot. +/// amdsmi reports global VRAM across all GPUs; fdinfo gives per-GPU values. +fn readProcessGpuVram(io: std.Io, pid: u32, pci_slot: *const [bdf_len]u8) u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const dir_path = std.fmt.bufPrint(&path_buf, "/proc/{d}/fdinfo", .{pid}) catch return 0; + + var dir = std.Io.Dir.openDirAbsolute(io, dir_path, .{ .iterate = true }) catch return 0; + defer dir.close(io); + + var total_vram_kib: u64 = 0; + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + var file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const file_path = std.fmt.bufPrint(&file_buf, "{s}/{s}", .{ dir_path, entry.name }) catch continue; + + total_vram_kib += parseFdinfoVram(io, file_path, pci_slot); + } + + return total_vram_kib; +} + +fn parseFdinfoVram(io: std.Io, path: []const u8, pci_slot: *const [bdf_len]u8) u64 { + var file = std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }) catch return 0; + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + + var found_pci = false; + var vram_kib: ?u64 = null; + + while (true) { + const line = reader.interface.takeDelimiterExclusive('\n') catch break; + reader.interface.toss(1); + + if (std.mem.indexOf(u8, line, pci_slot) != null) found_pci = true; + + // Support both new and old fdinfo keys (see amdgpu_fdinfo.c) + inline for ([_][]const u8{ "drm-memory-vram:\t", "vram mem:\t" }) |key| { + if (std.mem.startsWith(u8, line, key)) { + const rest = line[key.len..]; + const end = std.mem.indexOfAny(u8, rest, " \t\n") orelse rest.len; + vram_kib = std.fmt.parseInt(u64, rest[0..end], 10) catch 0; + } + } + + if (found_pci and vram_kib != null) return vram_kib.?; + } + + return 0; +} + +fn formatBdf(bdf_id: u64) [bdf_len]u8 { + var buf: [bdf_len]u8 = undefined; + _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ + (bdf_id >> 16) & 0xFFFF, + (bdf_id >> 8) & 0xFF, + (bdf_id >> 3) & 0x1F, + bdf_id & 0x7, + }) catch unreachable; + + return buf; +} diff --git a/bin/zml-smi/platforms/furiosa/BUILD.bazel b/bin/zml-smi/platforms/furiosa/BUILD.bazel deleted file mode 100644 index 818b2d25..00000000 --- a/bin/zml-smi/platforms/furiosa/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "furiosa", - import_name = "zml-smi/platforms/furiosa", - main = "Furiosa.zig", - target_compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - visibility = ["//bin/zml-smi:__subpackages__"], - deps = ["//bin/zml-smi/smi"], -) - -zig_test( - name = "test", - deps = [":furiosa"], -) diff --git a/bin/zml-smi/platforms/furiosa/Furiosa.zig b/bin/zml-smi/platforms/furiosa/Furiosa.zig deleted file mode 100644 index 135b3403..00000000 --- a/bin/zml-smi/platforms/furiosa/Furiosa.zig +++ /dev/null @@ -1,448 +0,0 @@ -//! FuriosaAI RNGD chips, through sysfs. Paths, units and formulas follow furiosa-smi-go 2026.4.0. -const std = @import("std"); -const smi = @import("zml-smi"); - -const Furiosa = @This(); - -platform: smi.Platform, -io: std.Io, -/// /sys/class/rngd_mgmt: rngd!npumgmt for chip N, and rngd!npupe for its core C. -class: smi.sysfs.Dir, -chips: []Chip, - -/// The cores (PEs) of a chip. -const cores = 8; -const Cores = std.bit_set.Integer(cores); - -const Chip = struct { - /// N in /dev/rngd/npu. - node: u8, - model: *const Model, - mgmt: smi.sysfs.Dir, - hwmon: ?Hwmon, - /// The cores this host may use: those a node of /dev/rngd names. - usable: Cores, - cycles: [cores]Cycles = @splat(.{}), - /// This round: the utilization of each core, and the memory in use of the group each core heads. - util: [cores]u8 = @splat(0), - mem_used: [cores]u64 = @splat(0), - - fn lessThan(_: void, a: Chip, b: Chip) bool { - if (a.model.device != b.model.device) return a.model.device < b.model.device; - return a.node < b.node; - } - - /// The row of a process: the mean utilization of the cores it holds, and the memory their groups use. - fn row(chip: *const Chip, pid: u32, device: u16, held: Cores, sample: smi.Furiosa) smi.ProcessInfo { - const mine = held.intersectWith(chip.usable); - var util: u64 = 0; - var mem_used: u64 = 0; - var it = mine.iterator(.{}); - while (it.next()) |core| { - util += chip.util[core]; - mem_used += chip.mem_used[core]; - } - return .{ - .pid = pid, - .device_idx = device, - .dev_util_percent = if (sample.util_percent != null) @intCast(util / @max(@as(u64, mine.count()), 1)) else null, - .dev_mem_kib = if (sample.mem_used_bytes != null) mem_used / 1024 else null, - }; - } -}; - -/// The running totals of a core: its cycles, and those it spent on tasks. -const Cycles = struct { all: smi.Counter = .{}, task: smi.Counter = .{} }; - -const Hwmon = struct { - dir: smi.sysfs.Dir, - temperature: ?[]const u8, - ambient: ?[]const u8, - power: ?[]const u8, - - fn open(arena: std.mem.Allocator, device: smi.sysfs.Dir) ?Hwmon { - const dir = smi.hwmon.find(device, null) catch return null; - return .{ - .dir = dir, - .temperature = smi.hwmon.resolve(arena, dir, &.{.{ .prefix = "temp", .label = "PEAK" }}), - .ambient = smi.hwmon.resolve(arena, dir, &.{.{ .prefix = "temp", .label = "AMBIENT" }}), - .power = smi.hwmon.resolve(arena, dir, &.{.{ .prefix = "power", .label = "RMS_TOTAL", .suffix = "average" }}), - }; - } -}; - -/// A device type. Its firmware reserves 512 MiB of its DRAM, and its cores share the rest evenly. -const Model = struct { - device: u16, - name: []const u8, - dram_bytes: u64, - - fn memoryBytes(model: Model) u64 { - return model.dram_bytes - 512 * 1024 * 1024; - } -}; - -const models = [_]Model{ - .{ .device = 0x0001, .name = "RNGD", .dram_bytes = 48 << 30 }, - .{ .device = 0x0002, .name = "RNGD+", .dram_bytes = 72 << 30 }, -}; - -const vendor = 0x1ed2; - -const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit }; - -pub fn detect(io: std.Io) bool { - return smi.sysfs.exists(io, "/dev/rngd"); -} - -pub fn init(arena: std.mem.Allocator, _: std.mem.Allocator, io: std.Io) !Furiosa { - const usable = try usableCores(io); - const class = try smi.sysfs.Dir.open(io, "/sys/class/rngd_mgmt"); - - var chips: std.ArrayList(Chip) = .empty; - var it = class.iterate(); - while (try it.next(io)) |entry| { - const node = mgmtNode(entry.name) orelse continue; - // A chip with no node in /dev/rngd is not this host's. - if (usable[node].count() == 0) continue; - const mgmt = class.openDir(entry.name) catch continue; - const chip = openChip(arena, mgmt, node, usable[node]) catch { - mgmt.close(); - continue; - }; - try chips.append(arena, chip); - } - std.mem.sort(Chip, chips.items, {}, Chip.lessThan); - - const devices = try arena.alloc(smi.Device, chips.items.len); - for (devices, chips.items) |*device, chip| { - var details: std.ArrayList(smi.Device.Detail) = .empty; - if (string(arena, chip.mgmt, "fw_version")) |value| try details.append(arena, .{ .key = "firmware_version", .label = "Firmware", .value = value }); - if (string(arena, chip.mgmt, "device_sn")) |value| try details.append(arena, .{ .key = "serial", .label = "Serial", .value = value }); - device.* = .{ - .target = .furiosa, - .name = chip.model.name, - .driver_version = string(arena, chip.mgmt, "version"), - .details = details.items, - }; - } - - return .{ - .platform = .{ .devices = devices, .two_readings = true, .vtable = &vtable }, - .io = io, - .class = class, - .chips = chips.items, - }; -} - -/// The cores of each chip that the nodes of /dev/rngd name. -fn usableCores(io: std.Io) ![256]Cores { - var usable: [256]Cores = @splat(.empty); - const dev = try smi.sysfs.Dir.open(io, "/dev/rngd"); - defer dev.close(); - var it = dev.iterate(); - while (try it.next(io)) |entry| { - if (entry.kind != .character_device) continue; - const node = Node.parse(entry.name) orelse continue; - usable[node.chip].setUnion(node.cores); - } - return usable; -} - -fn openChip(arena: std.mem.Allocator, mgmt: smi.sysfs.Dir, node: u8, usable: Cores) !Chip { - var buffer: [64]u8 = undefined; - var path: [64]u8 = undefined; - const pci = try smi.sysfs.Dir.open(mgmt.io, try std.fmt.bufPrint(&path, "/sys/bus/pci/devices/{s}", .{try mgmt.read("busname", &buffer)})); - defer pci.close(); - - if (try pci.int(u16, "vendor") != vendor) return error.NotFuriosa; - const device = try pci.int(u16, "device"); - const model = for (&models) |*model| { - if (model.device == device) break model; - } else return error.UnknownDevice; - - return .{ .node = node, .model = model, .mgmt = mgmt, .hwmon = .open(arena, pci), .usable = usable }; -} - -fn deinit(platform: *smi.Platform) void { - const self: *Furiosa = @alignCast(@fieldParentPtr("platform", platform)); - for (self.chips) |chip| { - chip.mgmt.close(); - if (chip.hwmon) |hwmon| hwmon.dir.close(); - } - self.class.close(); -} - -fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { - const self: *Furiosa = @alignCast(@fieldParentPtr("platform", platform)); - - for (self.chips, samples) |*chip, *sample| { - const fields = &sample.furiosa; - fields.mem_total_bytes = chip.model.memoryBytes(); - fields.governor = governor(chip.mgmt); - fields.liveness = liveness(chip.mgmt); - fields.throttle_reasons = throttleReasons(chip.mgmt); - - chip.util = @splat(0); - chip.mem_used = @splat(0); - if (occupancy(chip.mgmt)) |in_use| { - fields.util_percent = self.utilization(chip, in_use); - fields.mem_used_bytes = self.memoryInUse(chip, in_use); - } - - const hwmon = chip.hwmon orelse continue; - // hwmon reports millidegrees and microwatts. - if (hwmon.temperature) |file| fields.temperature = hwmon.dir.scaled(file, .milli, .one) catch null; - if (hwmon.ambient) |file| fields.ambient_temperature = hwmon.dir.scaled(file, .milli, .one) catch null; - if (hwmon.power) |file| fields.power_mw = hwmon.dir.scaled(file, .micro, .milli) catch null; - fields.power_limit_mw = hwmon.dir.scaled("power1_max", .micro, .milli) catch null; - } -} - -/// The mean over the usable cores of the share of its cycles each spent on tasks. -fn utilization(self: *const Furiosa, chip: *Chip, in_use: Cores) ?u64 { - var sum: u64 = 0; - var it = chip.usable.iterator(.{}); - while (it.next()) |core| { - var buffer: [1024]u8 = undefined; - const text = if (in_use.isSet(core)) (self.coreFile(chip.node, core, "metrics", &buffer) catch return null) else null; - // A core not in use reads 0 cycles, as in furiosa-smi-go. - const all = if (text) |metrics| (hexField(metrics, "CycleCount") orelse return null) else 0; - const task = if (text) |metrics| (hexField(metrics, "TaskCycleCount") orelse return null) else 0; - const all_growth = chip.cycles[core].all.advance(self.io, all); - const task_growth = chip.cycles[core].task.advance(self.io, task); - chip.util[core] = if (all_growth != null and task_growth != null) task_growth.?.percentOf(all_growth.?) else 0; - sum += chip.util[core]; - } - return sum / @max(@as(u64, chip.usable.count()), 1); -} - -/// Memory in use, summed over the groups of fused cores. Every core of a group reports the whole -/// group, so a group is read once, from its first core. A core not in use is a group of its own. -fn memoryInUse(self: *const Furiosa, chip: *Chip, in_use: Cores) ?u64 { - const share = chip.model.memoryBytes() / cores; - var used: u64 = 0; - var left = chip.usable.count(); - var it = chip.usable.iterator(.{}); - while (it.next()) |head| { - var size: usize = 1; - var buffer: [1024]u8 = undefined; - const text = if (in_use.isSet(head)) (self.coreFile(chip.node, head, "alloc_status", &buffer) catch return null) else null; - if (text) |status| { - const capacity = (hexField(status, "dram_capacity") orelse return null) + (hexField(status, "shared_capacity") orelse return null); - const usage = (hexField(status, "dram_usage") orelse return null) + (hexField(status, "shared_usage") orelse return null); - if (capacity > 0) { - size = @intCast(@max(capacity / share, 1)); - chip.mem_used[head] = usage; - used += usage; - } - } - if (size > left) return null; - left -= size; - for (1..size) |_| _ = it.next(); - } - return used; -} - -/// A file of a core, or null when the core is not in use: the file is then empty or missing. -fn coreFile(self: *const Furiosa, node: u8, core: usize, name: []const u8, buffer: []u8) !?[]const u8 { - var path: [48]u8 = undefined; - const text = self.class.read(try std.fmt.bufPrint(&path, "rngd!npu{d}pe{d}/{s}", .{ node, core, name }), buffer) catch |err| switch (err) { - error.FileNotFound => return null, - else => |e| return e, - }; - return if (text.len > 0) text else null; -} - -/// The cores in use: one line per core, 1 when it is in use and 0 when it is not. -fn occupancy(mgmt: smi.sysfs.Dir) ?Cores { - var buffer: [64]u8 = undefined; - const text = mgmt.read("pe_occupancy", &buffer) catch return null; - var in_use: Cores = .empty; - var lines = std.mem.splitScalar(u8, text, '\n'); - var core: usize = 0; - while (lines.next()) |line| : (core += 1) { - const state = std.mem.trim(u8, line, &std.ascii.whitespace); - if (std.mem.eql(u8, state, "1")) { - if (core < cores) in_use.set(core); - } else if (!std.mem.eql(u8, state, "0")) return null; - } - return in_use; -} - -/// The governor whose name ends the first line. -fn governor(mgmt: smi.sysfs.Dir) ?u64 { - var buffer: [256]u8 = undefined; - const text = mgmt.read("npu_governor", &buffer) catch return null; - const line = std.mem.trimEnd(u8, std.mem.sliceTo(text, '\n'), &std.ascii.whitespace); - for (std.enums.values(smi.Furiosa.Governor)) |profile| { - if (std.mem.endsWith(u8, line, @tagName(profile))) return @backingInt(profile); - } - return null; -} - -/// Alive when the device state is 0. -fn liveness(mgmt: smi.sysfs.Dir) ?u64 { - var buffer: [64]u8 = undefined; - const state = mgmt.read("device_state", &buffer) catch return null; - return @backingInt(if (std.mem.eql(u8, state, "0")) smi.Furiosa.Liveness.alive else .unknown); -} - -/// The bitmask that ends the file, in hex: "Throttle reason = 0x00000000". -fn throttleReasons(mgmt: smi.sysfs.Dir) ?u64 { - var buffer: [128]u8 = undefined; - const text = mgmt.read("throttle_reason", &buffer) catch return null; - const start = if (std.mem.findLastAny(u8, text, &std.ascii.whitespace)) |space| space + 1 else 0; - return hex(text[start..]); -} - -/// The hex number of `key` in a file of "key: value" lines. -fn hexField(text: []const u8, key: []const u8) ?u64 { - var lines = std.mem.tokenizeScalar(u8, text, '\n'); - while (lines.next()) |line| { - if (smi.sysfs.fieldValue(std.mem.trim(u8, line, " \t"), key)) |value| return hex(value); - } - return null; -} - -/// A number in hex, with or without a 0x prefix. -fn hex(text: []const u8) ?u64 { - const digits = std.mem.cutPrefix(u8, text, "0x") orelse std.mem.cutPrefix(u8, text, "0X") orelse text; - return std.fmt.parseInt(u64, digits, 16) catch null; -} - -fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { - const self: *Furiosa = @alignCast(@fieldParentPtr("platform", platform)); - // The cores each process holds on each chip, from the chip and core nodes it has open. - var held: std.AutoArrayHashMapUnmanaged(struct { u32, u16 }, Cores) = .empty; - var holders: smi.proc.FdHolders = .init(self.io, &.{"/dev/rngd/"}); - defer holders.deinit(); - while (holders.next()) |holder| { - const node = Node.parse(holder.rest) orelse continue; - const device = self.deviceOf(node.chip) orelse continue; - const entry = try held.getOrPutValue(scratch, .{ holder.pid, device }, .empty); - entry.value_ptr.setUnion(node.cores); - } - for (held.keys(), held.values()) |key, set| { - const pid, const device = key; - try out.append(self.chips[device].row(pid, device, set, samples[device].furiosa)); - } -} - -fn deviceOf(self: *const Furiosa, node: u8) ?u16 { - for (self.chips, 0..) |chip, index| { - if (chip.node == node) return @intCast(index); - } - return null; -} - -/// The chip and cores a node of /dev/rngd names: npu, npupe or npupe-. -/// A bare npu is core 0, as furiosa-smi-go reads it. Other nodes, such as npuch0, name no core. -const Node = struct { - chip: u8, - cores: Cores, - - fn parse(name: []const u8) ?Node { - const rest = std.mem.cutPrefix(u8, name, "npu") orelse return null; - const end = std.mem.findNone(u8, rest, "0123456789") orelse rest.len; - const chip = decimal(rest[0..end]) orelse return null; - const range = if (end == rest.len) "0" else std.mem.cutPrefix(u8, rest[end..], "pe") orelse return null; - const first, const last = std.mem.cutScalar(u8, range, '-') orelse .{ range, range }; - const start = decimal(first) orelse return null; - const stop = decimal(last) orelse return null; - if (start > stop or stop >= cores) return null; - var set: Cores = .empty; - set.setRangeValue(.{ .start = start, .end = @as(usize, stop) + 1 }, true); - return .{ .chip = chip, .cores = set }; - } -}; - -/// N in a management node name, rngd!npumgmt. -fn mgmtNode(name: []const u8) ?u8 { - const rest = std.mem.cutPrefix(u8, name, "rngd!npu") orelse return null; - return decimal(std.mem.cutSuffix(u8, rest, "mgmt") orelse return null); -} - -fn decimal(text: []const u8) ?u8 { - if (std.mem.findNone(u8, text, "0123456789") != null) return null; - return std.fmt.parseInt(u8, text, 10) catch null; -} - -/// A string of a sysfs file, copied into `arena`, or null when it is missing or empty. -fn string(arena: std.mem.Allocator, dir: smi.sysfs.Dir, file: []const u8) ?[]const u8 { - var buffer: [128]u8 = undefined; - const text = dir.read(file, &buffer) catch return null; - if (text.len == 0) return null; - return arena.dupe(u8, text) catch null; -} - -test Node { - try std.testing.expectEqual(Node{ .chip = 3, .cores = .{ .mask = 0x0f } }, Node.parse("npu3pe0-3").?); - try std.testing.expectEqual(Node{ .chip = 0, .cores = .{ .mask = 0x20 } }, Node.parse("npu0pe5").?); - try std.testing.expectEqual(Node{ .chip = 12, .cores = .{ .mask = 0x01 } }, Node.parse("npu12").?); - for ([_][]const u8{ "npu0ch0", "npu0ch0r", "npu0mgmt", "npu0bar4", "npu0dmar", "npu0pe3-1", "npu0pe8", "npu0pe", "npu" }) |name| { - try std.testing.expectEqual(null, Node.parse(name)); - } - try std.testing.expectEqual(7, mgmtNode("rngd!npu7mgmt")); - try std.testing.expectEqual(null, mgmtNode("rngd!npu7pe0")); -} - -test "utilization and memory of fused cores" { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); - defer tmp.cleanup(); - const gib: u64 = 1 << 30; - const share = models[0].memoryBytes() / cores; - - // Cores 0 to 3 are one group, core 5 is alone, and core 6 was released after the occupancy was read: - // the files of a core not in use are empty. - try tmp.dir.createDirPath(io, "rngd!npu0mgmt"); - try tmp.dir.createDirPath(io, "rngd!npu0pe6"); - try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0pe6/metrics", .data = "" }); - try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0pe6/alloc_status", .data = "" }); - try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0mgmt/pe_occupancy", .data = "1\n1\n1\n1\n0\n1\n1\n0\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0mgmt/npu_governor", .data = "powersave\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0mgmt/device_state", .data = "0\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0mgmt/throttle_reason", .data = "Throttle reason = 0x00000003\n" }); - var text: [128]u8 = undefined; - var path: [32]u8 = undefined; - for ([_]u8{ 0, 1, 2, 3, 5 }) |core| { - const fused = core < 4; - try tmp.dir.createDirPath(io, try std.fmt.bufPrint(&path, "rngd!npu0pe{d}", .{core})); - try tmp.dir.writeFile(io, .{ - .sub_path = try std.fmt.bufPrint(&path, "rngd!npu0pe{d}/alloc_status", .{core}), - .data = try std.fmt.bufPrint(&text, "dram_capacity: 0x{x}\ndram_usage: 0x{x}\nshared_capacity: 0x0\nshared_usage: 0x0\n", .{ - if (fused) 4 * share else share, if (fused) 3 * gib else gib, - }), - }); - } - - const class: smi.sysfs.Dir = .{ .handle = tmp.dir, .io = io }; - var chips = [_]Chip{.{ .node = 0, .model = &models[0], .mgmt = try class.openDir("rngd!npu0mgmt"), .hwmon = null, .usable = .full }}; - defer chips[0].mgmt.close(); - var furiosa: Furiosa = .{ .platform = .{ .devices = &.{.{ .target = .furiosa }}, .vtable = &vtable }, .io = io, .class = class, .chips = &chips }; - - // Between the polls, the group spends half its cycles on tasks and core 5 all of them. - for ([_]u64{ 0, 37 }, [_]u64{ 1000, 2000 }) |util, cycles| { - for ([_]u8{ 0, 1, 2, 3, 5 }) |core| { - const task = if (core == 5) cycles else cycles / 2 + 250; - try tmp.dir.writeFile(io, .{ - .sub_path = try std.fmt.bufPrint(&path, "rngd!npu0pe{d}/metrics", .{core}), - .data = try std.fmt.bufPrint(&text, "CycleCount: 0x{x}\nTaskCycleCount: 0x{x}\n", .{ cycles, task }), - }); - } - var samples = [_]smi.Sample{.init(.furiosa)}; - poll(&furiosa.platform, std.testing.allocator, &samples); - const chip = samples[0].furiosa; - try std.testing.expectEqual(util, chip.util_percent); - try std.testing.expectEqual(4 * gib, chip.mem_used_bytes); - try std.testing.expectEqual(@backingInt(smi.Furiosa.Governor.powersave), chip.governor); - try std.testing.expectEqual(@backingInt(smi.Furiosa.Liveness.alive), chip.liveness); - try std.testing.expectEqual(0b11, chip.throttle_reasons); - } - - // A process that holds the group sees its share. - const row = chips[0].row(7, 0, .{ .mask = 0x0f }, .{ .util_percent = 37, .mem_used_bytes = 4 * gib }); - try std.testing.expectEqual(50, row.dev_util_percent); - try std.testing.expectEqual(3 * gib / 1024, row.dev_mem_kib); -} diff --git a/bin/zml-smi/platforms/linux/BUILD.bazel b/bin/zml-smi/platforms/linux/BUILD.bazel new file mode 100644 index 00000000..57bb96d7 --- /dev/null +++ b/bin/zml-smi/platforms/linux/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "linux", + import_name = "zml-smi/platforms/linux", + main = "linux.zig", + srcs = [ + "metrics.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:sysfs", + ], +) diff --git a/bin/zml-smi/platforms/linux/linux.zig b/bin/zml-smi/platforms/linux/linux.zig new file mode 100644 index 00000000..ffab3ec4 --- /dev/null +++ b/bin/zml-smi/platforms/linux/linux.zig @@ -0,0 +1,2 @@ +pub const metrics = @import("metrics.zig"); +pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/linux/metrics.zig b/bin/zml-smi/platforms/linux/metrics.zig new file mode 100644 index 00000000..5a69663a --- /dev/null +++ b/bin/zml-smi/platforms/linux/metrics.zig @@ -0,0 +1,97 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const host_info = @import("zml-smi/info").host_info; +const HostInfo = host_info.HostInfo; +const HostData = host_info.HostData; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, info: *HostInfo) !void { + const poll_arena = try collector.createPollArena(); + const host: Host = .{ .io = collector.io, .arena = poll_arena }; + + // read once metrics + var initial = info.front().*; + initial.hostname = host.hostname(collector.arena) catch null; + initial.kernel = host.kernel(collector.arena) catch null; + initial.cpu_name = host.cpuName(collector.arena) catch null; + initial.cpu_cores = host.cpuCores() catch null; + initial.mem_total_kib = host.memTotal() catch null; + info.back().* = initial; + info.swap(); + + try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); +} + +const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); + +const Host = struct { + io: std.Io, + arena: *std.heap.ArenaAllocator, + + pub fn hostname(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, "/proc/sys/kernel/hostname"); + } + + pub fn kernel(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, "/proc/sys/kernel/osrelease"); + } + + pub fn cpuName(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readFieldString(allocator, self.io, "/proc/cpuinfo", "model name"); + } + + pub fn cpuCores(self: Host) !u64 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/sys/devices/system/cpu/present"); + const dash = std.mem.indexOfScalar(u8, data, '-') orelse return 1; + + return (try std.fmt.parseInt(u64, data[dash + 1 ..], 10)) + 1; + } + + pub fn memTotal(self: Host) !u64 { + return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemTotal"); + } + + pub fn memAvailable(self: Host) !u64 { + return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemAvailable"); + } + + fn parseLoadAvg(self: Host, comptime n: usize) !f32 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/loadavg"); + var iter = std.mem.splitScalar(u8, data, ' '); + + var i: usize = 0; + while (iter.next()) |tok| : (i += 1) { + if (i == n) return std.fmt.parseFloat(f32, tok); + } + + return error.NotFound; + } + + pub fn load1(self: Host) !f32 { + return self.parseLoadAvg(0); + } + + pub fn load5(self: Host) !f32 { + return self.parseLoadAvg(1); + } + + pub fn load15(self: Host) !f32 { + return self.parseLoadAvg(2); + } + + pub fn uptime(self: Host) !u64 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/uptime"); + const dot = std.mem.indexOfScalar(u8, data, '.') orelse data.len; + + return std.fmt.parseInt(u64, data[0..dot], 10); + } +}; + +const metrics = .{ + .{ .field = "mem_available_kib", .query = Host.memAvailable }, + .{ .field = "load_1", .query = Host.load1 }, + .{ .field = "load_5", .query = Host.load5 }, + .{ .field = "load_15", .query = Host.load15 }, + .{ .field = "uptime_seconds", .query = Host.uptime }, +}; diff --git a/bin/zml-smi/platforms/linux/process.zig b/bin/zml-smi/platforms/linux/process.zig new file mode 100644 index 00000000..c57d1725 --- /dev/null +++ b/bin/zml-smi/platforms/linux/process.zig @@ -0,0 +1,118 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const pi = @import("zml-smi/info").process_info; + +const ProcessInfo = pi.ProcessInfo; + +pub const ProcessEnricher = struct { + gpa: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + prev_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}, + prev_total_ticks: u64 = 0, + + pub fn init(gpa: std.mem.Allocator, io: std.Io) !ProcessEnricher { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + + const prev_total_ticks = readTotalCpuTicks(arena.allocator(), io); + var enricher: ProcessEnricher = .{ + .gpa = gpa, + .arena = arena, + .prev_total_ticks = prev_total_ticks, + .prev_ticks = .{}, + }; + + try enricher.prev_ticks.ensureTotalCapacity(gpa, 64); + + return enricher; + } + + pub fn deinit(self: *ProcessEnricher) void { + self.prev_ticks.deinit(self.gpa); + self.arena.deinit(); + } + + pub fn enrich(self: *ProcessEnricher, io: std.Io, procs: []ProcessInfo) void { + _ = self.arena.reset(.retain_capacity); + const alloc = self.arena.allocator(); + + const curr_total = readTotalCpuTicks(alloc, io); + const delta_total = curr_total -| self.prev_total_ticks; + + var curr_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}; + curr_ticks.ensureTotalCapacity(self.gpa, @intCast(procs.len)) catch return; + + for (procs) |*info| { + if (info.remote) continue; + // Read Uid, VmRSS from /proc/{pid}/status + const status_path = std.fmt.allocPrint(alloc, "/proc/{d}/status", .{info.pid}) catch continue; + const uid: u32 = @intCast(sysfs.readFieldInt(alloc, io, status_path, "Uid") catch 0); + info.uid = uid; + info.username = lookupUsername(alloc, uid) catch continue; + info.rss_kib = sysfs.readFieldInt(alloc, io, status_path, "VmRSS") catch 0; + + // Read cmdline + const cmdline_path = std.fmt.allocPrint(alloc, "/proc/{d}/cmdline", .{info.pid}) catch continue; + info.comm = readCmdline(alloc, io, cmdline_path) catch continue; + + // CPU% delta + const ticks = readProcTicks(alloc, io, info.pid) orelse continue; + if (delta_total > 0) { + if (self.prev_ticks.get(info.pid)) |prev| { + const delta_proc = ticks -| prev; + info.cpu_percent = @intCast(@min(delta_proc *| 1000 / delta_total, std.math.maxInt(u16))); + } + } + + curr_ticks.getOrPutAssumeCapacity(info.pid).value_ptr.* = ticks; + } + + self.prev_ticks.deinit(self.gpa); + self.prev_ticks = curr_ticks; + self.prev_total_ticks = curr_total; + } +}; + +fn lookupUsername(allocator: std.mem.Allocator, uid: u32) ![]const u8 { + const pw = std.c.getpwuid(uid) orelse return error.NoUser; + const name = std.mem.span(pw.name orelse return error.NoName); + + return try allocator.dupe(u8, name); +} + +/// Extract utime + stime from /proc/{pid}/stat. +fn readProcTicks(allocator: std.mem.Allocator, io: std.Io, pid: u32) ?u64 { + const path = std.fmt.allocPrint(allocator, "/proc/{d}/stat", .{pid}) catch return null; + const data = sysfs.readFirstLine(allocator, io, path) catch return null; + + // Fields after ") ": state=0, ppid=1, ..., utime=11, stime=12 + const rest = data[(std.mem.lastIndexOfScalar(u8, data, ')') orelse return null) + 2 ..]; + + return sysfs.nthTokenInt(rest, 11) + sysfs.nthTokenInt(rest, 12); +} + +fn readCmdline(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { + const data = try sysfs.readFirstLine(allocator, io, path); + + for (data) |*c| { + if (c.* == 0) { + c.* = ' '; + } + } + + return std.mem.trimEnd(u8, data, &std.ascii.whitespace); +} + +fn readTotalCpuTicks(allocator: std.mem.Allocator, io: std.Io) u64 { + const data = sysfs.readFirstLine(allocator, io, "/proc/stat") catch return 0; + + var total: u64 = 0; + var iter = std.mem.tokenizeAny(u8, data, " \t"); + + _ = iter.next(); // skip "cpu" + while (iter.next()) |tok| { + total += std.fmt.parseInt(u64, tok, 10) catch 0; + } + + return total; +} diff --git a/bin/zml-smi/platforms/macos/BUILD.bazel b/bin/zml-smi/platforms/macos/BUILD.bazel new file mode 100644 index 00000000..a92e298e --- /dev/null +++ b/bin/zml-smi/platforms/macos/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "macos", + import_name = "zml-smi/platforms/macos", + main = "macos.zig", + srcs = [ + "metrics.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/platforms/macos/macos.zig b/bin/zml-smi/platforms/macos/macos.zig new file mode 100644 index 00000000..ffab3ec4 --- /dev/null +++ b/bin/zml-smi/platforms/macos/macos.zig @@ -0,0 +1,2 @@ +pub const metrics = @import("metrics.zig"); +pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/macos/metrics.zig b/bin/zml-smi/platforms/macos/metrics.zig new file mode 100644 index 00000000..616168cd --- /dev/null +++ b/bin/zml-smi/platforms/macos/metrics.zig @@ -0,0 +1,161 @@ +const std = @import("std"); +const host_info = @import("zml-smi/info").host_info; +const HostInfo = host_info.HostInfo; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Collector = @import("zml-smi/collector").Collector; + +const c = std.c; + +const vm_statistics64 = extern struct { + free_count: c_uint, + active_count: c_uint, + inactive_count: c_uint, + wire_count: c_uint, + zero_fill_count: u64, + reactivations: u64, + pageins: u64, + pageouts: u64, + faults: u64, + cow_faults: u64, + lookups: u64, + hits: u64, + purges: u64, + purgeable_count: c_uint, + speculative_count: c_uint, + decompressions: u64, + compressions: u64, + swapins: u64, + swapouts: u64, + compressor_page_count: c_uint, + throttled_count: c_uint, + external_page_count: c_uint, + internal_page_count: c_uint, + total_uncompressed_pages_in_compressor: u64, +}; + +const HOST_VM_INFO64: c_int = 4; +const HOST_VM_INFO64_COUNT: c_uint = @sizeOf(vm_statistics64) / @sizeOf(c_int); +const KERN_SUCCESS: c_int = 0; + +extern "c" fn host_statistics64(host: c.mach_port_t, flavor: c_int, info: *anyopaque, count: *c_uint) c.kern_return_t; +extern "c" fn getloadavg(loadavg: *[3]f64, nelem: c_int) c_int; + +pub fn init(collector: *Collector, info: *HostInfo) !void { + const poll_arena = try collector.createPollArena(); + const host: Host = .{}; + + var initial = info.front().*; + initial.hostname = sysctlString(collector.arena, "kern.hostname") catch null; + initial.kernel = sysctlString(collector.arena, "kern.osrelease") catch null; + initial.cpu_name = sysctlString(collector.arena, "machdep.cpu.brand_string") catch null; + initial.cpu_cores = host.cpuCores() catch null; + initial.mem_total_kib = host.memTotal() catch null; + info.back().* = initial; + info.swap(); + + try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); +} + +const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); + +const Host = struct { + pub fn cpuCores(_: Host) !u64 { + return @intCast(try sysctlU32("hw.logicalcpu")); + } + + pub fn memTotal(_: Host) !u64 { + return (try sysctlU64("hw.memsize")) / 1024; + } + + pub fn memAvailable(_: Host) !u64 { + var vm_stat: vm_statistics64 = undefined; + var count: c_uint = HOST_VM_INFO64_COUNT; + + if (host_statistics64(c.mach_host_self(), HOST_VM_INFO64, &vm_stat, &count) != KERN_SUCCESS) { + return error.HostStatsFailed; + } + + var page_size: u64 = 0; + if (c._host_page_size(c.mach_host_self(), &page_size) != KERN_SUCCESS) { + return error.PageSizeFailed; + } + + const available_pages: u64 = @as(u64, vm_stat.free_count) + + @as(u64, vm_stat.inactive_count) + + @as(u64, vm_stat.purgeable_count); + + return (available_pages * page_size) / 1024; + } + + fn loadAvg() ![3]f64 { + var avg: [3]f64 = undefined; + if (getloadavg(&avg, 3) != 3) { + return error.LoadAvgFailed; + } + + return avg; + } + + pub fn load1(_: Host) !f32 { + return @floatCast((try loadAvg())[0]); + } + + pub fn load5(_: Host) !f32 { + return @floatCast((try loadAvg())[1]); + } + + pub fn load15(_: Host) !f32 { + return @floatCast((try loadAvg())[2]); + } + + pub fn uptime(_: Host) !u64 { + var boottime: c.timeval = undefined; + var len: usize = @sizeOf(c.timeval); + if (c.sysctlbyname("kern.boottime", std.mem.asBytes(&boottime), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + var now: c.timeval = undefined; + _ = c.gettimeofday(&now, null); + + return @intCast(now.sec - boottime.sec); + } +}; + +fn sysctlString(allocator: std.mem.Allocator, name: [*:0]const u8) ![]const u8 { + var buf: [256]u8 = undefined; + var len: usize = buf.len; + if (c.sysctlbyname(name, &buf, &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return allocator.dupe(u8, std.mem.span(@as([*:0]const u8, @ptrCast(buf[0..len])))); +} + +fn sysctlU64(name: [*:0]const u8) !u64 { + var value: u64 = 0; + var len: usize = @sizeOf(u64); + if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return value; +} + +fn sysctlU32(name: [*:0]const u8) !u32 { + var value: u32 = 0; + var len: usize = @sizeOf(u32); + if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return value; +} + +const metrics = .{ + .{ .field = "mem_available_kib", .query = Host.memAvailable }, + .{ .field = "load_1", .query = Host.load1 }, + .{ .field = "load_5", .query = Host.load5 }, + .{ .field = "load_15", .query = Host.load15 }, + .{ .field = "uptime_seconds", .query = Host.uptime }, +}; diff --git a/bin/zml-smi/platforms/macos/process.zig b/bin/zml-smi/platforms/macos/process.zig new file mode 100644 index 00000000..a48e9831 --- /dev/null +++ b/bin/zml-smi/platforms/macos/process.zig @@ -0,0 +1,12 @@ +const std = @import("std"); +const pi = @import("zml-smi/info").process_info; + +pub const ProcessEnricher = struct { + pub fn init(_: std.mem.Allocator, _: std.Io) !ProcessEnricher { + return .{}; + } + + pub fn deinit(_: *ProcessEnricher) void {} + + pub fn enrich(_: *ProcessEnricher, _: std.Io, _: []pi.ProcessInfo) void {} +}; diff --git a/bin/zml-smi/platforms/metal/BUILD.bazel b/bin/zml-smi/platforms/metal/BUILD.bazel deleted file mode 100644 index 5e1c68f8..00000000 --- a/bin/zml-smi/platforms/metal/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "metal", - srcs = [ - "Smc.zig", - "iokit.zig", - ], - import_name = "zml-smi/platforms/metal", - main = "Metal.zig", - target_compatible_with = ["@platforms//os:macos"], - visibility = ["//bin/zml-smi:__subpackages__"], - deps = ["//bin/zml-smi/smi"], -) - -zig_test( - name = "test", - deps = [":metal"], -) diff --git a/bin/zml-smi/platforms/metal/Metal.zig b/bin/zml-smi/platforms/metal/Metal.zig deleted file mode 100644 index e9d5eb8b..00000000 --- a/bin/zml-smi/platforms/metal/Metal.zig +++ /dev/null @@ -1,643 +0,0 @@ -//! Apple GPUs: utilization and memory from the IORegistry, power and clock from IOReport, -//! temperature and fans from the SMC. It also measures the power of the CPU and of the whole machine. -//! The IOReport readers and the GPU time of each client are ported from mactop v2.1.6 (8dbfaae): -//! internal/app/ioreport.m, internal/app/pmp_names.h and internal/app/native_stats.go. - -// MIT License -// -// Copyright (c) 2024-2026 Carsen Klock -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -const std = @import("std"); -const smi = @import("zml-smi"); -const iokit = @import("iokit.zig"); -const Smc = @import("Smc.zig"); - -const Metal = @This(); -const log = std.log.scoped(.@"zml-smi/metal"); - -platform: smi.Platform, -io: std.Io, -cf: CoreFoundation, -io_kit: iokit.Api, -ioreport: IoReport, -keys: std.EnumArray(Key, *const String), -accelerator: iokit.Object, -smc: ?Smc, -/// The GPU shares the host's memory. -mem_total_bytes: ?u64, -/// The frequency of each active GPU performance state, in MHz, in order. -frequencies: []const u64, -channels: *Dictionary, -subscription: *const Subscription, -/// The sample the next poll makes its delta against. -last: ?Reading = null, -/// macOS 27 reads 0 for the CPU in the Energy Model. Once it does while the PMP -/// histograms show power, they give the CPU power for the rest of the run. -cpu_from_pmp: bool = false, -/// What this round measured of the host, for `host`. -cpu_power_mw: ?u64 = null, -system_power_mw: ?u64 = null, -/// GPU nanoseconds of each process. -busy: smi.Counters(u32), - -const Reading = struct { sample: *const Dictionary, at: std.Io.Timestamp }; - -const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit, .host = host }; - -/// The keys Metal looks up, named by their text. Made once in `init`. -const Key = enum { - IOReportChannels, - PerformanceStatistics, - @"Device Utilization %", - @"In use system memory", - IOUserClientCreator, - AppUsage, - accumulatedGPUTime, - model, - IOSourceVersion, - @"voltage-states9-sram", - @"voltage-states9", -}; - -/// Every Apple silicon Mac has a GPU, which `init` finds. -pub fn detect(_: std.Io) bool { - return true; -} - -pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io) !Metal { - var self: Metal = .{ - .platform = undefined, - .io = io, - .cf = try smi.dynlib.open(CoreFoundation, &.{"/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation"}), - .io_kit = try iokit.open(), - .ioreport = try smi.dynlib.open(IoReport, &.{"/usr/lib/libIOReport.dylib"}), - .keys = undefined, - .accelerator = 0, - .smc = null, - .mem_total_bytes = memoryTotal(), - .frequencies = &.{}, - .channels = undefined, - .subscription = undefined, - .busy = .init(gpa), - }; - - self.accelerator = self.io_kit.IOServiceGetMatchingService(iokit.main_port, self.io_kit.IOServiceMatching("AGXAccelerator")); - if (self.accelerator == 0) { - log.warn("no Apple GPU", .{}); - return error.GpuNotFound; - } - errdefer _ = self.io_kit.IOObjectRelease(self.accelerator); - - for (std.enums.values(Key), 0..) |key, made| { - self.keys.set(key, self.cf.CFStringCreateWithCString(null, @tagName(key), utf8) orelse { - for (std.enums.values(Key)[0..made]) |done| self.cf.CFRelease(self.keys.get(done)); - return error.OutOfMemory; - }); - } - errdefer for (self.keys.values) |string| self.cf.CFRelease(string); - - self.channels = try self.copyChannels(); - errdefer self.cf.CFRelease(self.channels); - var subscribed: ?*Dictionary = null; - self.subscription = self.ioreport.IOReportCreateSubscription(null, self.channels, &subscribed, 0, null) orelse { - log.warn("IOReport refused the subscription", .{}); - return error.IoReportUnavailable; - }; - if (subscribed) |channels| self.cf.CFRelease(channels); - errdefer self.cf.CFRelease(self.subscription); - - self.frequencies = try self.gpuFrequencies(arena); - self.smc = Smc.open(arena, self.io_kit) catch |err| smc: { - log.warn("no SMC, so no temperature, fan or system power: {t}", .{err}); - break :smc null; - }; - errdefer if (self.smc) |smc| smc.close(); - - const devices = try arena.alloc(smi.Device, 1); - devices[0] = .{ - .target = .metal, - .name = try self.propertyText(arena, .model), - .driver_version = try self.propertyText(arena, .IOSourceVersion), - }; - self.platform = .{ .devices = devices, .two_readings = true, .vtable = &vtable }; - return self; -} - -fn deinit(platform: *smi.Platform) void { - const self: *Metal = @alignCast(@fieldParentPtr("platform", platform)); - if (self.last) |last| self.cf.CFRelease(last.sample); - self.cf.CFRelease(self.subscription); - self.cf.CFRelease(self.channels); - for (self.keys.values) |string| self.cf.CFRelease(string); - if (self.smc) |smc| smc.close(); - _ = self.io_kit.IOObjectRelease(self.accelerator); - self.busy.deinit(); -} - -fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { - const self: *Metal = @alignCast(@fieldParentPtr("platform", platform)); - const gpu = &samples[0].metal; - - gpu.mem_total_bytes = self.mem_total_bytes; - gpu.clock_graphics_max_mhz = if (self.frequencies.len > 0) std.mem.max(u64, self.frequencies) else null; - if (self.property(self.accelerator, .PerformanceStatistics)) |statistics| { - defer self.cf.CFRelease(statistics); - if (self.as(Dictionary, statistics)) |values| { - gpu.util_percent = self.number(self.cf.CFDictionaryGetValue(values, self.keys.get(.@"Device Utilization %"))); - gpu.mem_used_bytes = self.number(self.cf.CFDictionaryGetValue(values, self.keys.get(.@"In use system memory"))); - } - } - - self.cpu_power_mw = null; - self.readReport(gpu); - - self.system_power_mw = null; - if (self.smc) |smc| { - gpu.temperature = smc.temperature(); - gpu.fan_speed_percent = smc.fanPercent(); - self.system_power_mw = smc.systemPower(); - } -} - -fn host(platform: *smi.Platform, data: *smi.HostData) void { - const self: *Metal = @alignCast(@fieldParentPtr("platform", platform)); - data.cpu_power_mw = self.cpu_power_mw; - data.system_power_mw = self.system_power_mw; -} - -/// One row per process with GPU time, which has no GPU memory of its own. -fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { - const self: *Metal = @alignCast(@fieldParentPtr("platform", platform)); - defer self.busy.sweep(); - const children = self.io_kit.IORegistryEntryGetChildIterator.get(.{ self.accelerator, "IOService" }) catch return error.Unavailable; - defer _ = self.io_kit.IOObjectRelease(children); - - // A process may have several clients. - var times: std.AutoArrayHashMapUnmanaged(u32, u64) = .empty; - while (true) { - const client = self.io_kit.IOIteratorNext(children); - if (client == 0) break; - defer _ = self.io_kit.IOObjectRelease(client); - if (self.io_kit.IOObjectConformsTo(client, "AGXDeviceUserClient") == 0) continue; - const pid = self.clientPid(client) orelse continue; - const time = try times.getOrPutValue(scratch, pid, 0); - time.value_ptr.* +|= self.gpuTime(client); - } - - const growths = try scratch.alloc(?smi.Growth, times.count()); - var total: u64 = 0; - for (times.keys(), times.values(), growths) |pid, nanoseconds, *growth| { - growth.* = if (nanoseconds == 0) null else try self.busy.advance(pid, self.io, nanoseconds); - if (growth.*) |busy| total +|= busy.amount; - } - const device = samples[0].metal.util_percent; - for (times.keys(), times.values(), growths) |pid, nanoseconds, growth| { - if (nanoseconds == 0) continue; - try out.append(.{ .pid = pid, .dev_util_percent = if (growth) |busy| share(busy, total, device) else null }); - } -} - -/// A process's part of the device's utilization, in proportion to its GPU time, as mactop does: -/// the GPU time of the clients adds up to less than the device's utilization. -/// Its own share of the elapsed time when the device's utilization is unknown or 0. -fn share(busy: smi.Growth, total: u64, device: ?u64) u16 { - const percent = device orelse 0; - if (percent == 0 or total == 0) return busy.percentOfTime(); - return @intCast(@as(u128, @min(percent, 100)) * busy.amount / total); -} - -/// The process that opened a GPU client, from its creator: "pid 682, WindowServer". -fn clientPid(self: *Metal, client: iokit.Object) ?u32 { - const creator = self.property(client, .IOUserClientCreator) orelse return null; - defer self.cf.CFRelease(creator); - var buffer: [128]u8 = undefined; - const creator_text = self.text(self.as(String, creator), &buffer); - if (!std.mem.startsWith(u8, creator_text, "pid ")) return null; - const end = std.mem.indexOfScalar(u8, creator_text, ',') orelse return null; - return std.fmt.parseInt(u32, creator_text[4..end], 10) catch null; -} - -/// The GPU nanoseconds of a client, summed over the APIs it used. -fn gpuTime(self: *Metal, client: iokit.Object) u64 { - const usage = self.property(client, .AppUsage) orelse return 0; - defer self.cf.CFRelease(usage); - const uses = self.as(Array, usage) orelse return 0; - var total: u64 = 0; - for (0..@intCast(@max(0, self.cf.CFArrayGetCount(uses)))) |index| { - const use = self.as(Dictionary, self.cf.CFArrayGetValueAtIndex(uses, @intCast(index))) orelse continue; - total +|= self.number(self.cf.CFDictionaryGetValue(use, self.keys.get(.accumulatedGPUTime))) orelse 0; - } - return total; -} - -/// Takes a sample and reads its delta against the last one: GPU, ANE and CPU power, and the GPU clock. -fn readReport(self: *Metal, gpu: *smi.Metal) void { - const sample = self.ioreport.IOReportCreateSamples(self.subscription, self.channels, null) orelse return; - const reading: Reading = .{ .sample = sample, .at = .now(self.io, .awake) }; - const last = self.last orelse { - self.last = reading; - return; - }; - self.last = reading; - defer self.cf.CFRelease(last.sample); - - const delta = self.ioreport.IOReportCreateSamplesDelta(last.sample, sample, null) orelse return; - defer self.cf.CFRelease(delta); - const elapsed = last.at.durationTo(reading.at); - if (elapsed.nanoseconds <= 0) return; - const channels = self.as(Array, self.cf.CFDictionaryGetValue(delta, self.keys.get(.IOReportChannels))) orelse return; - - var energy: std.EnumArray(Energy, u64) = .initFill(0); - var pmp_cpu_mw: u64 = 0; - for (0..@intCast(@max(0, self.cf.CFArrayGetCount(channels)))) |index| { - const channel = self.as(Dictionary, self.cf.CFArrayGetValueAtIndex(channels, @intCast(index))) orelse continue; - var group_buffer: [64]u8 = undefined; - var name_buffer: [128]u8 = undefined; - const group = self.text(self.ioreport.IOReportChannelGetGroup(channel), &group_buffer); - const name = self.text(self.ioreport.IOReportChannelGetChannelName(channel), &name_buffer); - - if (std.mem.eql(u8, group, "Energy Model")) { - const kind = Energy.of(name) orelse continue; - // Negative is an invalid value. - const amount = std.math.cast(u64, self.ioreport.IOReportSimpleGetIntegerValue(channel, 0)) orelse continue; - var unit_buffer: [16]u8 = undefined; - const unit = self.text(self.ioreport.IOReportChannelGetUnitLabel(channel), &unit_buffer); - energy.getPtr(kind).* +|= milliwatts(amount, unit, elapsed); - } else if (std.mem.eql(u8, group, "GPU Stats") and std.mem.eql(u8, name, "GPUPH")) { - gpu.clock_graphics_mhz = self.clock(channel); - } else if (isPmpGroup(group) and isPmpCpuPower(name)) { - pmp_cpu_mw +|= self.histogramMilliwatts(channel); - } - } - - // Whole-CPU totals, else the clusters: an OS with both would count them twice. - const cpu = if (energy.get(.cpu) > 0) energy.get(.cpu) else energy.get(.cpu_cluster); - if (cpu == 0 and pmp_cpu_mw > 0) self.cpu_from_pmp = true; - self.cpu_power_mw = if (self.cpu_from_pmp) pmp_cpu_mw else cpu; - gpu.power_mw = if (energy.get(.gpu) > 0) energy.get(.gpu) else energy.get(.gpu_alias); - gpu.ane_power_mw = if (energy.get(.ane_block) > 0) energy.get(.ane_block) else energy.get(.ane); -} - -/// The Energy Model channels Metal reads, by name. Where an OS has two names for one -/// thing, the first of each pair is read, and the second only when the first reads 0. -const Energy = enum { - cpu, - cpu_cluster, - gpu, - gpu_alias, - ane_block, - ane, - - fn of(name: []const u8) ?Energy { - const clusters = [_][]const u8{ "ECPU Energy", "PCPU Energy", "MCPU Energy", "eCPUs Energy", "pCPUs Energy", "mCPUs Energy" }; - for (clusters) |cluster| { - if (std.mem.indexOf(u8, name, cluster) != null) return .cpu_cluster; - } - if (std.mem.indexOf(u8, name, "CPU Energy") != null) return .cpu; - if (std.mem.eql(u8, name, "GPU Energy")) return .gpu; - if (std.mem.eql(u8, name, "GPU")) return .gpu_alias; - for ([_][]const u8{ "ANE", "NPU", "Neural", "ane" }) |engine| { - if (std.mem.indexOf(u8, name, engine) == null) continue; - return if (std.mem.indexOf(u8, name, "Energy") != null) .ane else .ane_block; - } - return null; - } -}; - -/// Energy over the time it took, in milliwatts. mactop reads a unit that is not mJ or nJ as µJ. -fn milliwatts(amount: u64, unit: []const u8, elapsed: std.Io.Duration) u64 { - const symbol = unit[0 .. std.mem.indexOfScalar(u8, unit, ' ') orelse unit.len]; - const prefix: smi.units.Prefix = if (std.mem.eql(u8, symbol, "mJ")) .milli else if (std.mem.eql(u8, symbol, "nJ")) .nano else .micro; - const growth: smi.Growth = .{ .amount = amount, .elapsed = elapsed }; - return smi.units.scale(growth.perSecond(), prefix, .milli); -} - -/// The mean frequency of the active GPU performance states, weighted by the time spent in each. -/// 0 when the GPU was off or idle all along. -fn clock(self: *Metal, channel: *const Dictionary) ?u64 { - if (self.frequencies.len == 0) return null; - var active: u64 = 0; - var weighted: u128 = 0; - var state_index: usize = 0; - for (0..@intCast(@max(0, self.ioreport.IOReportStateGetCount(channel)))) |state| { - var buffer: [32]u8 = undefined; - const name = self.text(self.ioreport.IOReportStateGetNameForIndex(channel, @intCast(state)), &buffer); - if (std.mem.eql(u8, name, "OFF") or std.mem.eql(u8, name, "IDLE") or std.mem.eql(u8, name, "DOWN")) continue; - const residency = std.math.cast(u64, self.ioreport.IOReportStateGetResidency(channel, @intCast(state))) orelse 0; - active += residency; - if (state_index < self.frequencies.len) weighted += @as(u128, self.frequencies[state_index]) * residency; - state_index += 1; - } - return if (active > 0) @intCast(weighted / active) else 0; -} - -/// The power of a CPU cluster from its histogram, whose states are named by watts (" 2W", " 0.250W"): -/// the mean of the states weighted by the time spent in each, the lowest reading 0, where an idle cluster sits. -fn histogramMilliwatts(self: *Metal, channel: *const Dictionary) u64 { - var total: u64 = 0; - var weighted: f64 = 0; - for (0..@intCast(@max(0, self.ioreport.IOReportStateGetCount(channel)))) |state| { - const residency = std.math.cast(u64, self.ioreport.IOReportStateGetResidency(channel, @intCast(state))) orelse 0; - total += residency; - if (state == 0) continue; - var buffer: [32]u8 = undefined; - weighted += stateWatts(self.text(self.ioreport.IOReportStateGetNameForIndex(channel, @intCast(state)), &buffer)) * @as(f64, @floatFromInt(residency)); - } - if (total == 0) return 0; - return @intFromFloat(@round(weighted / @as(f64, @floatFromInt(total)) * 1000)); -} - -fn stateWatts(name: []const u8) f64 { - return std.fmt.parseFloat(f64, std.mem.trimEnd(u8, std.mem.trim(u8, name, " "), "W")) catch 0; -} - -/// "PMP" on older chips, "PMP0", "PMP1"... one per die on an M5 Max. -fn isPmpGroup(group: []const u8) bool { - if (!std.mem.startsWith(u8, group, "PMP")) return false; - for (group[3..]) |char| if (!std.ascii.isDigit(char)) return false; - return true; -} - -/// A CPU cluster's power histogram: "PACC", "MACC1", "PACC0 SRAM", but not "AGX", the GPU. -fn isPmpCpuPower(name: []const u8) bool { - if (name.len < 4 or !std.mem.eql(u8, name[1..4], "ACC")) return false; - var rest = name[4..]; - while (rest.len > 0 and std.ascii.isDigit(rest[0])) rest = rest[1..]; - return rest.len == 0 or std.mem.eql(u8, rest, " SRAM"); -} - -/// The channels Metal subscribes to: the Energy Model, the GPU performance states, and the -/// power histograms of PMP, the CPU clusters' among them. -fn copyChannels(self: *Metal) !*Dictionary { - const energy = self.channelsIn("Energy Model", null) orelse { - log.warn("IOReport has no Energy Model", .{}); - return error.IoReportUnavailable; - }; - defer self.cf.CFRelease(energy); - const channels = self.cf.CFDictionaryCreateMutableCopy(null, 0, energy) orelse return error.OutOfMemory; - - if (self.channelsIn("GPU Stats", "GPU Performance States")) |gpu| { - defer self.cf.CFRelease(gpu); - self.ioreport.IOReportMergeChannels(channels, gpu, null); - } - // The dies are numbered contiguously, so the first without channels ends the search. - for (0..9) |die| { - var buffer: [8]u8 = undefined; - const group = if (die == 0) "PMP" else std.fmt.bufPrintSentinel(&buffer, "PMP{d}", .{die - 1}, 0) catch unreachable; - const pmp = self.channelsIn(group, "Energy") orelse if (die == 0) continue else break; - defer self.cf.CFRelease(pmp); - self.ioreport.IOReportMergeChannels(channels, pmp, null); - } - return channels; -} - -/// The channels of an IOReport group, or null when it has none. -fn channelsIn(self: *Metal, group: [:0]const u8, subgroup: ?[:0]const u8) ?*const Dictionary { - const group_string = self.cf.CFStringCreateWithCString(null, group, utf8) orelse return null; - defer self.cf.CFRelease(group_string); - const subgroup_string = if (subgroup) |name| (self.cf.CFStringCreateWithCString(null, name, utf8) orelse return null) else null; - defer if (subgroup_string) |string| self.cf.CFRelease(string); - - const channels = self.ioreport.IOReportCopyChannelsInGroup(group_string, subgroup_string, 0, 0, 0) orelse return null; - const list = self.as(Array, self.cf.CFDictionaryGetValue(channels, self.keys.get(.IOReportChannels))); - if (list == null or self.cf.CFArrayGetCount(list.?) == 0) { - self.cf.CFRelease(channels); - return null; - } - return channels; -} - -/// The frequency of each GPU performance state, from the power manager's voltage states: -/// pairs of a frequency and a voltage, in Hz up to the M4 and for the M5 GPU, in kHz for the M5 CPU. -fn gpuFrequencies(self: *Metal, arena: std.mem.Allocator) ![]const u64 { - const manager = self.io_kit.IOServiceGetMatchingService(iokit.main_port, self.io_kit.IOServiceNameMatching("pmgr")); - if (manager == 0) return &.{}; - defer _ = self.io_kit.IOObjectRelease(manager); - - for ([_]Key{ .@"voltage-states9-sram", .@"voltage-states9" }) |key| { - const states = self.property(manager, key) orelse continue; - defer self.cf.CFRelease(states); - const data = self.as(Data, states) orelse continue; - const bytes = self.cf.CFDataGetBytePtr(data)[0..@intCast(@max(0, self.cf.CFDataGetLength(data)))]; - - var frequencies: std.ArrayList(u64) = .empty; - var pairs = std.mem.window(u8, bytes, 8, 8); - while (pairs.next()) |pair| { - if (pair.len < 4) break; - const frequency = std.mem.readInt(u32, pair[0..4], .little); - const mhz = if (frequency >= 100_000_000) frequency / 1_000_000 else if (frequency >= 100_000) frequency / 1000 else 0; - if (mhz > 0) try frequencies.append(arena, mhz); - } - return frequencies.items; - } - log.warn("no GPU frequencies, so no clock", .{}); - return &.{}; -} - -/// A string property of the GPU, copied into `arena`. -fn propertyText(self: *Metal, arena: std.mem.Allocator, key: Key) !?[]const u8 { - const value = self.property(self.accelerator, key) orelse return null; - defer self.cf.CFRelease(value); - var buffer: [128]u8 = undefined; - const value_text = self.text(self.as(String, value), &buffer); - return if (value_text.len > 0) try arena.dupe(u8, value_text) else null; -} - -/// A property of a registry entry, which the caller releases. -fn property(self: *Metal, entry: iokit.Object, key: Key) ?*const anyopaque { - return self.io_kit.IORegistryEntryCreateCFProperty(entry, self.keys.get(key), null, 0); -} - -/// `object` as a T, or null when it is another kind of object. -fn as(self: *Metal, comptime T: type, object: ?*const anyopaque) ?*const T { - const value = object orelse return null; - const id = switch (T) { - String => self.cf.CFStringGetTypeID(), - Number => self.cf.CFNumberGetTypeID(), - Array => self.cf.CFArrayGetTypeID(), - Dictionary => self.cf.CFDictionaryGetTypeID(), - Data => self.cf.CFDataGetTypeID(), - else => @compileError("not a CoreFoundation type"), - }; - return if (self.cf.CFGetTypeID(value) == id) @ptrCast(value) else null; -} - -fn number(self: *Metal, object: ?*const anyopaque) ?u64 { - const value = self.as(Number, object) orelse return null; - var result: i64 = 0; - if (self.cf.CFNumberGetValue(value, number_sint64, &result) == 0) return null; - return std.math.cast(u64, result); -} - -/// The text of a string, in `buffer`. Empty when there is none or it does not fit. -fn text(self: *Metal, string: ?*const String, buffer: []u8) []const u8 { - if (self.cf.CFStringGetCString(string orelse return "", buffer.ptr, @intCast(buffer.len), utf8) == 0) return ""; - return std.mem.sliceTo(buffer, 0); -} - -fn memoryTotal() ?u64 { - var bytes: u64 = 0; - var size: usize = @sizeOf(u64); - if (std.c.sysctlbyname("hw.memsize", &bytes, &size, null, 0) != 0) return null; - return bytes; -} - -const String = opaque {}; -const Number = opaque {}; -const Array = opaque {}; -const Dictionary = opaque {}; -const Data = opaque {}; -const Subscription = opaque {}; - -/// kCFStringEncodingUTF8. -const utf8: u32 = 0x08000100; -/// kCFNumberSInt64Type. -const number_sint64: isize = 4; - -const CoreFoundation = struct { - CFRelease: *const @TypeOf(decls.CFRelease), - CFGetTypeID: *const @TypeOf(decls.CFGetTypeID), - CFStringGetTypeID: *const @TypeOf(decls.CFStringGetTypeID), - CFNumberGetTypeID: *const @TypeOf(decls.CFNumberGetTypeID), - CFArrayGetTypeID: *const @TypeOf(decls.CFArrayGetTypeID), - CFDictionaryGetTypeID: *const @TypeOf(decls.CFDictionaryGetTypeID), - CFDataGetTypeID: *const @TypeOf(decls.CFDataGetTypeID), - CFStringCreateWithCString: *const @TypeOf(decls.CFStringCreateWithCString), - CFStringGetCString: *const @TypeOf(decls.CFStringGetCString), - CFNumberGetValue: *const @TypeOf(decls.CFNumberGetValue), - CFArrayGetCount: *const @TypeOf(decls.CFArrayGetCount), - CFArrayGetValueAtIndex: *const @TypeOf(decls.CFArrayGetValueAtIndex), - CFDictionaryGetValue: *const @TypeOf(decls.CFDictionaryGetValue), - CFDictionaryCreateMutableCopy: *const @TypeOf(decls.CFDictionaryCreateMutableCopy), - CFDataGetLength: *const @TypeOf(decls.CFDataGetLength), - CFDataGetBytePtr: *const @TypeOf(decls.CFDataGetBytePtr), -}; - -/// The private library behind powermetrics. -const IoReport = struct { - IOReportCopyChannelsInGroup: *const @TypeOf(decls.IOReportCopyChannelsInGroup), - IOReportMergeChannels: *const @TypeOf(decls.IOReportMergeChannels), - IOReportCreateSubscription: *const @TypeOf(decls.IOReportCreateSubscription), - IOReportCreateSamples: *const @TypeOf(decls.IOReportCreateSamples), - IOReportCreateSamplesDelta: *const @TypeOf(decls.IOReportCreateSamplesDelta), - IOReportChannelGetGroup: *const @TypeOf(decls.IOReportChannelGetGroup), - IOReportChannelGetChannelName: *const @TypeOf(decls.IOReportChannelGetChannelName), - IOReportChannelGetUnitLabel: *const @TypeOf(decls.IOReportChannelGetUnitLabel), - IOReportSimpleGetIntegerValue: *const @TypeOf(decls.IOReportSimpleGetIntegerValue), - IOReportStateGetCount: *const @TypeOf(decls.IOReportStateGetCount), - IOReportStateGetNameForIndex: *const @TypeOf(decls.IOReportStateGetNameForIndex), - IOReportStateGetResidency: *const @TypeOf(decls.IOReportStateGetResidency), -}; - -/// Never called directly: `CoreFoundation` and `IoReport` take their types. -/// The functions named Create or Copy return an object the caller releases; Get functions do not. -const decls = struct { - extern fn CFRelease(object: *const anyopaque) void; - extern fn CFGetTypeID(object: *const anyopaque) usize; - extern fn CFStringGetTypeID() usize; - extern fn CFNumberGetTypeID() usize; - extern fn CFArrayGetTypeID() usize; - extern fn CFDictionaryGetTypeID() usize; - extern fn CFDataGetTypeID() usize; - extern fn CFStringCreateWithCString(allocator: ?*const anyopaque, text: [*:0]const u8, encoding: u32) ?*const String; - extern fn CFStringGetCString(string: *const String, buffer: [*]u8, size: isize, encoding: u32) u8; - extern fn CFNumberGetValue(number: *const Number, kind: isize, value: *anyopaque) u8; - extern fn CFArrayGetCount(array: *const Array) isize; - extern fn CFArrayGetValueAtIndex(array: *const Array, index: isize) ?*const anyopaque; - extern fn CFDictionaryGetValue(dictionary: *const Dictionary, key: *const anyopaque) ?*const anyopaque; - extern fn CFDictionaryCreateMutableCopy(allocator: ?*const anyopaque, capacity: isize, dictionary: *const Dictionary) ?*Dictionary; - extern fn CFDataGetLength(data: *const Data) isize; - extern fn CFDataGetBytePtr(data: *const Data) [*]const u8; - - extern fn IOReportCopyChannelsInGroup(group: ?*const String, subgroup: ?*const String, a: u64, b: u64, c: u64) ?*const Dictionary; - /// Adds the channels of `from` to `into`. - extern fn IOReportMergeChannels(into: *Dictionary, from: *const Dictionary, unused: ?*const anyopaque) void; - extern fn IOReportCreateSubscription(a: ?*const anyopaque, channels: *Dictionary, subscribed: *?*Dictionary, d: u64, e: ?*const anyopaque) ?*const Subscription; - extern fn IOReportCreateSamples(subscription: *const Subscription, channels: *Dictionary, unused: ?*const anyopaque) ?*const Dictionary; - extern fn IOReportCreateSamplesDelta(previous: *const Dictionary, current: *const Dictionary, unused: ?*const anyopaque) ?*const Dictionary; - extern fn IOReportChannelGetGroup(channel: *const Dictionary) ?*const String; - extern fn IOReportChannelGetChannelName(channel: *const Dictionary) ?*const String; - extern fn IOReportChannelGetUnitLabel(channel: *const Dictionary) ?*const String; - extern fn IOReportSimpleGetIntegerValue(channel: *const Dictionary, index: i32) i64; - extern fn IOReportStateGetCount(channel: *const Dictionary) i32; - extern fn IOReportStateGetNameForIndex(channel: *const Dictionary, index: i32) ?*const String; - extern fn IOReportStateGetResidency(channel: *const Dictionary, index: i32) i64; -}; - -test "channel names" { - try std.testing.expectEqual(.cpu, Energy.of("CPU Energy")); - try std.testing.expectEqual(.cpu, Energy.of("DIE_0_CPU Energy")); - try std.testing.expectEqual(.cpu_cluster, Energy.of("PCPU Energy")); - try std.testing.expectEqual(.gpu, Energy.of("GPU Energy")); - try std.testing.expectEqual(null, Energy.of("GPU0")); - try std.testing.expectEqual(.ane_block, Energy.of("ANE0")); - try std.testing.expectEqual(.ane, Energy.of("ANE Energy")); - try std.testing.expectEqual(null, Energy.of("PACC_0")); - - try std.testing.expect(isPmpGroup("PMP") and isPmpGroup("PMP1") and !isPmpGroup("PMPX")); - try std.testing.expect(isPmpCpuPower("PACC") and isPmpCpuPower("MACC1") and isPmpCpuPower("PACC0 SRAM")); - try std.testing.expect(!isPmpCpuPower("AGX") and !isPmpCpuPower("AGX AFR") and !isPmpCpuPower("PACC SRAM2")); - try std.testing.expectEqual(0.25, stateWatts(" 0.250W")); - try std.testing.expectEqual(64, stateWatts(" 64W")); - - // 1500 mJ in 500 ms. - try std.testing.expectEqual(3000, milliwatts(1500, "mJ", .fromMilliseconds(500))); - try std.testing.expectEqual(450, milliwatts(450_000_000, "nJ", .fromSeconds(1))); -} - -test share { - const second: std.Io.Duration = .fromSeconds(1); - // Two processes with 300 ms and 100 ms of GPU time share a device at 80%. - try std.testing.expectEqual(60, share(.{ .amount = 300_000_000, .elapsed = second }, 400_000_000, 80)); - try std.testing.expectEqual(20, share(.{ .amount = 100_000_000, .elapsed = second }, 400_000_000, 80)); - try std.testing.expectEqual(30, share(.{ .amount = 300_000_000, .elapsed = second }, 400_000_000, null)); -} - -test "a GPU of this Mac" { - const io = std.testing.io; - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - var metal = Metal.init(arena.allocator(), std.testing.allocator, io) catch return error.SkipZigTest; - defer deinit(&metal.platform); - - var scratch: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer scratch.deinit(); - var list: smi.ProcessList = .init(std.testing.allocator); - defer list.deinit(); - var data: smi.HostData = .{}; - - var samples: [1]smi.Sample = .{.init(.metal)}; - for (0..2) |round| { - samples = .{.init(.metal)}; - list.clear(); - if (round > 0) try io.sleep(.fromMilliseconds(100), .awake); - poll(&metal.platform, scratch.allocator(), &samples); - try processes(&metal.platform, scratch.allocator(), &samples, &list); - host(&metal.platform, &data); - } - const gpu = samples[0].metal; - try std.testing.expect(gpu.util_percent.? <= 100); - try std.testing.expect(gpu.mem_used_bytes.? < gpu.mem_total_bytes.?); - try std.testing.expect(gpu.clock_graphics_mhz.? <= gpu.clock_graphics_max_mhz.?); - try std.testing.expect(gpu.power_mw != null and data.cpu_power_mw != null); - // WindowServer draws the screen with the GPU. - try std.testing.expect(list.rows.items.len > 0); -} diff --git a/bin/zml-smi/platforms/metal/Smc.zig b/bin/zml-smi/platforms/metal/Smc.zig deleted file mode 100644 index 0e23923e..00000000 --- a/bin/zml-smi/platforms/metal/Smc.zig +++ /dev/null @@ -1,196 +0,0 @@ -//! The System Management Controller of an Apple silicon Mac: GPU temperature, fans and system power. -//! Ported from mactop v2.1.6 (8dbfaae): internal/app/smc.c and the SMC readers of internal/app/ioreport.m. - -// MIT License -// -// Copyright (c) 2024-2026 Carsen Klock -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -const std = @import("std"); -const iokit = @import("iokit.zig"); - -const Smc = @This(); - -api: iokit.Api, -connection: iokit.Object, -/// The GPU sensors. -temperatures: []const Key, -fans: []const Fan, -/// Whole-machine power: the system total (laptops), the DC-in total, then the DC-in rail, -/// which is the power supply's output on a Mac Studio. The first that reads a value wins. -power: []const Key, - -/// A key and the type and size of its value. -const Key = struct { code: u32, info: Info }; - -const Fan = struct { speed: Key, min_rpm: f64, max_rpm: f64 }; - -const Info = extern struct { size: u32 = 0, type: u32 = 0, attributes: u8 = 0 }; - -/// SMCKeyData_t: what each call sends and receives. -const Call = extern struct { - key: u32 = 0, - version: extern struct { major: u8 = 0, minor: u8 = 0, build: u8 = 0, reserved: u8 = 0, release: u16 = 0 } = .{}, - power_limits: extern struct { version: u16 = 0, length: u16 = 0, cpu: u32 = 0, gpu: u32 = 0, memory: u32 = 0 } = .{}, - info: Info = .{}, - result: u8 = 0, - status: u8 = 0, - command: Command = .read_bytes, - index: u32 = 0, - bytes: [32]u8 = @splat(0), - - const Command = enum(u8) { read_bytes = 5, read_index = 8, read_info = 9 }; -}; - -comptime { - std.debug.assert(@sizeOf(Call) == 80); -} - -/// A GPU sensor at or below this has not started: it reads about 2 °C right after boot. -const min_celsius = 10; -const max_celsius = 150; -const max_watts = 1000; - -pub fn open(arena: std.mem.Allocator, api: iokit.Api) !Smc { - const service = api.IOServiceGetMatchingService(iokit.main_port, api.IOServiceMatching("AppleSMC")); - if (service == 0) return error.SmcNotFound; - defer _ = api.IOObjectRelease(service); - - var smc: Smc = .{ - .api = api, - .connection = try api.IOServiceOpen.get(.{ service, std.c.mach_task_self(), 0 }), - .temperatures = &.{}, - .fans = &.{}, - .power = &.{}, - }; - errdefer smc.close(); - smc.temperatures = try smc.gpuSensors(arena); - smc.fans = try smc.findFans(arena); - smc.power = try smc.powerKeys(arena); - return smc; -} - -pub fn close(self: Smc) void { - _ = self.api.IOServiceClose(self.connection); -} - -/// The hottest GPU sensor, in degrees Celsius. -pub fn temperature(self: Smc) ?u64 { - var hottest: ?f64 = null; - for (self.temperatures) |sensor| { - const celsius = self.read(sensor) catch continue; - if (celsius <= min_celsius or celsius > max_celsius) continue; - hottest = @max(hottest orelse celsius, celsius); - } - return if (hottest) |celsius| @intFromFloat(@round(celsius)) else null; -} - -/// The fastest fan, as a share of its top speed. -pub fn fanPercent(self: Smc) ?u64 { - var fastest: ?u64 = null; - for (self.fans) |fan| { - const rpm = self.read(fan.speed) catch continue; - // A fan that cannot stop and reads 0 hides its speed from this user. - if (rpm <= 0 and fan.min_rpm > 0) continue; - const percent: u64 = @intFromFloat(@round(std.math.clamp(rpm / fan.max_rpm * 100, 0, 100))); - fastest = @max(fastest orelse percent, percent); - } - return fastest; -} - -/// The power of the whole machine, in milliwatts. -pub fn systemPower(self: Smc) ?u64 { - for (self.power) |key| { - const watts = self.read(key) catch continue; - if (watts > 0 and watts <= max_watts) return @intFromFloat(@round(watts * 1000)); - } - return null; -} - -/// The GPU sensors, "Tg" keys that hold a float. Finding them reads the name of every key, about 3800 on an M5 Max. -fn gpuSensors(self: Smc, arena: std.mem.Allocator) ![]const Key { - const count = self.read(self.find(code("#KEY".*)) catch return &.{}) catch return &.{}; - var sensors: std.ArrayList(Key) = .empty; - for (0..@intFromFloat(count)) |index| { - const name = (self.call(.{ .command = .read_index, .index = @intCast(index) }) catch continue).key; - if (name >> 16 != code("Tg\x00\x00".*) >> 16) continue; - const key = self.find(name) catch continue; - if (key.info.type == code("flt ".*)) try sensors.append(arena, key); - } - return sensors.items; -} - -fn findFans(self: Smc, arena: std.mem.Allocator) ![]const Fan { - const count = self.read(self.find(code("FNum".*)) catch return &.{}) catch return &.{}; - var fans: std.ArrayList(Fan) = .empty; - for (0..@intFromFloat(@min(count, 10))) |index| { - const max_rpm = self.read(self.find(fanKey(index, "Mx".*)) catch continue) catch continue; - if (max_rpm <= 0) continue; - try fans.append(arena, .{ - .speed = self.find(fanKey(index, "Ac".*)) catch continue, - .min_rpm = if (self.find(fanKey(index, "Mn".*))) |key| self.read(key) catch 0 else |_| 0, - .max_rpm = max_rpm, - }); - } - return fans.items; -} - -fn powerKeys(self: Smc, arena: std.mem.Allocator) ![]const Key { - var keys: std.ArrayList(Key) = .empty; - for ([_][4]u8{ "PSTR".*, "PDTR".*, "PD0R".* }) |name| { - const key = self.find(code(name)) catch continue; - if (key.info.type == code("flt ".*)) try keys.append(arena, key); - } - return keys.items; -} - -fn call(self: Smc, input: Call) !Call { - var output: Call = .{}; - var size: usize = @sizeOf(Call); - try self.api.IOConnectCallStructMethod.call(.{ self.connection, 2, &input, @sizeOf(Call), &output, &size }); - if (output.result != 0) return error.SmcError; - return output; -} - -fn find(self: Smc, name: u32) !Key { - return .{ .code = name, .info = (try self.call(.{ .key = name, .command = .read_info })).info }; -} - -/// The value of a key that holds a float or an unsigned integer. -fn read(self: Smc, key: Key) !f64 { - const bytes = (try self.call(.{ .key = key.code, .info = .{ .size = key.info.size }, .command = .read_bytes })).bytes; - return switch (key.info.type) { - // Native byte order, unlike the integers. - code("flt ".*) => @as(f32, @bitCast(bytes[0..4].*)), - code("ui8 ".*) => @floatFromInt(bytes[0]), - code("ui16".*) => @floatFromInt(std.mem.readInt(u16, bytes[0..2], .big)), - code("ui32".*) => @floatFromInt(std.mem.readInt(u32, bytes[0..4], .big)), - else => error.UnknownType, - }; -} - -fn code(name: [4]u8) u32 { - return std.mem.readInt(u32, &name, .big); -} - -/// "F0Ac": the actual speed of fan 0. -fn fanKey(index: usize, suffix: [2]u8) u32 { - return code(.{ 'F', '0' + @as(u8, @intCast(index)), suffix[0], suffix[1] }); -} diff --git a/bin/zml-smi/platforms/metal/iokit.zig b/bin/zml-smi/platforms/metal/iokit.zig deleted file mode 100644 index 6081c56e..00000000 --- a/bin/zml-smi/platforms/metal/iokit.zig +++ /dev/null @@ -1,53 +0,0 @@ -//! The IOKit functions Metal calls. The hermetic toolchain has no IOKit to link -//! against and no headers for it, so they are declared here and resolved by install name. -const std = @import("std"); -const smi = @import("zml-smi"); - -/// An io_object_t: a service, an iterator or a connection. 0 is none. -pub const Object = std.c.mach_port_t; - -/// kIOMainPortDefault. -pub const main_port: std.c.mach_port_t = 0; - -pub const Api = struct { - IOServiceMatching: *const @TypeOf(decls.IOServiceMatching), - IOServiceNameMatching: *const @TypeOf(decls.IOServiceNameMatching), - IOServiceGetMatchingService: *const @TypeOf(decls.IOServiceGetMatchingService), - IOServiceOpen: Checked(decls.IOServiceOpen), - IOServiceClose: *const @TypeOf(decls.IOServiceClose), - IOConnectCallStructMethod: Checked(decls.IOConnectCallStructMethod), - IORegistryEntryGetChildIterator: Checked(decls.IORegistryEntryGetChildIterator), - IORegistryEntryCreateCFProperty: *const @TypeOf(decls.IORegistryEntryCreateCFProperty), - IOIteratorNext: *const @TypeOf(decls.IOIteratorNext), - IOObjectConformsTo: *const @TypeOf(decls.IOObjectConformsTo), - IOObjectRelease: *const @TypeOf(decls.IOObjectRelease), -}; - -pub fn open() !Api { - return smi.dynlib.open(Api, &.{"/System/Library/Frameworks/IOKit.framework/IOKit"}); -} - -fn Checked(comptime function: anytype) type { - return smi.dynlib.Checked(function, check); -} - -fn check(status: std.c.kern_return_t) error{IoKitError}!void { - if (status != 0) return error.IoKitError; -} - -/// Never called directly: `Api` takes their types. -const decls = struct { - /// Returns a matching dictionary, which the lookup consumes. - extern fn IOServiceMatching(name: [*:0]const u8) ?*anyopaque; - extern fn IOServiceNameMatching(name: [*:0]const u8) ?*anyopaque; - extern fn IOServiceGetMatchingService(port: std.c.mach_port_t, matching: ?*anyopaque) Object; - extern fn IOServiceOpen(service: Object, task: std.c.mach_port_t, kind: u32, connection: *Object) std.c.kern_return_t; - extern fn IOServiceClose(connection: Object) std.c.kern_return_t; - extern fn IOConnectCallStructMethod(connection: Object, selector: u32, input: *const anyopaque, input_size: usize, output: *anyopaque, output_size: *usize) std.c.kern_return_t; - extern fn IORegistryEntryGetChildIterator(entry: Object, plane: [*:0]const u8, iterator: *Object) std.c.kern_return_t; - /// Returns a CoreFoundation object the caller releases. - extern fn IORegistryEntryCreateCFProperty(entry: Object, key: *const anyopaque, allocator: ?*const anyopaque, options: u32) ?*const anyopaque; - extern fn IOIteratorNext(iterator: Object) Object; - extern fn IOObjectConformsTo(object: Object, class: [*:0]const u8) c_int; - extern fn IOObjectRelease(object: Object) std.c.kern_return_t; -}; diff --git a/bin/zml-smi/platforms/neuron/BUILD.bazel b/bin/zml-smi/platforms/neuron/BUILD.bazel index 845caeed..c7129ac7 100644 --- a/bin/zml-smi/platforms/neuron/BUILD.bazel +++ b/bin/zml-smi/platforms/neuron/BUILD.bazel @@ -1,21 +1,22 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "neuron", import_name = "zml-smi/platforms/neuron", - main = "Neuron.zig", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["//bin/zml-smi:__subpackages__"], + main = "metrics.zig", + srcs = [ + "nrt.zig", + "process.zig", + ], + visibility = ["//visibility:public"], deps = [ - "//bin/zml-smi/smi", + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + "//bin/zml-smi/utils:sandbox", + "//bin/zml-smi/utils:sysfs", "//platforms/neuron:libnrt_headers", + "//stdx", ], ) - -zig_test( - name = "test", - # The test opens the libnrt of the sandbox, which only linux-amd64 ships. - data = ["//bin/zml-smi:sandbox"], - target_compatible_with = ["@platforms//cpu:x86_64"], - deps = [":neuron"], -) diff --git a/bin/zml-smi/platforms/neuron/Neuron.zig b/bin/zml-smi/platforms/neuron/Neuron.zig deleted file mode 100644 index 845bbf6e..00000000 --- a/bin/zml-smi/platforms/neuron/Neuron.zig +++ /dev/null @@ -1,349 +0,0 @@ -//! AWS Neuron cores, through the bundled libnrt and sysfs. -const std = @import("std"); -const c = @import("c"); -const smi = @import("zml-smi"); - -const Neuron = @This(); -const log = std.log.scoped(.@"zml-smi/neuron"); - -platform: smi.Platform, -io: std.Io, -nrt: Private, -chips: []const Chip, -/// Busy nanoseconds of each process on each core. -busy: smi.Counters(Key), -/// The rows of this round's scan, in scratch. Null when no chip could be read. -rows: ?[]const smi.ProcessInfo = null, - -/// A Neuron device. Each of its cores is a device of the platform. -const Chip = struct { - handle: *c.ndl_device_t, - name: ?[]const u8 = null, - mem_total_bytes: ?u64, - /// stats/memory_usage/device_mem of each core. - memory: []const ?smi.sysfs.Dir, -}; - -/// A process and the position of a core in `platform.devices`. -const Key = struct { pid: u32, core: u16 }; - -const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit }; - -const Api = struct { - nrt_get_total_nc_count: Checked(c.nrt_get_total_nc_count), - nrt_get_version: Checked(c.nrt_get_version), -}; - -/// The functions libnrt does not export, found in its symbol table. -const Private = struct { - ndl_available_devices: *const @TypeOf(c.ndl_available_devices), - ndl_open_device: Checked(c.ndl_open_device), - ndl_get_all_apps_info: Checked(c.ndl_get_all_apps_info), - nds_open: Checked(c.nds_open), - nds_close: *const @TypeOf(c.nds_close), - nds_get_nc_counter: Checked(c.nds_get_nc_counter), -}; - -fn Checked(comptime function: anytype) type { - return smi.dynlib.Checked(function, check); -} - -/// The libnrt functions that report a status, exported or not, return 0 on success. -fn check(status: i64) error{NrtError}!void { - if (status != 0) return error.NrtError; -} - -pub fn detect(io: std.Io) bool { - return smi.sysfs.exists(io, c.NEURON_DEVICE_PREFIX ++ "0"); -} - -pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io) !Neuron { - const api = try smi.dynlib.openBundled(Api, io, "libnrt.so.1"); - const nrt = try findPrivate(io, api.nrt_get_version.function); - const core_count = try api.nrt_get_total_nc_count.get(.{}); - - var version = std.mem.zeroes(c.nrt_version_t); - const driver_version = if (api.nrt_get_version.call(.{ &version, @sizeOf(c.nrt_version_t) })) |_| - try arena.dupe(u8, std.mem.sliceTo(&version.rt_detail, 0)) - else |_| - null; - - var indices: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; - const count: usize = @intCast(std.math.clamp(nrt.ndl_available_devices(&indices, indices.len), 0, indices.len)); - var opened: std.ArrayList(struct { c_int, *c.ndl_device_t }) = try .initCapacity(arena, count); - for (indices[0..count]) |index| { - var params: c.ndl_device_init_param_t = .{ .initialize_device = false, .num_dram_regions = 0, .map_hbm = false, .skip_copy_bufs = true }; - // A device that cannot be opened is skipped. - const handle: ?*c.ndl_device_t = nrt.ndl_open_device.get(.{ index, ¶ms }) catch continue; - opened.appendAssumeCapacity(.{ index, handle orelse continue }); - } - - // The devices share the cores evenly. - const cores_per_chip = core_count / @max(1, opened.items.len); - const chips = try arena.alloc(Chip, opened.items.len); - var devices: std.ArrayList(smi.Device) = try .initCapacity(arena, chips.len * cores_per_chip); - for (chips, opened.items) |*chip, entry| { - const index, const handle = entry; - chip.* = try openChip(arena, io, index, handle, cores_per_chip); - for (chip.memory) |_| devices.appendAssumeCapacity(.{ .target = .neuron, .name = chip.name, .driver_version = driver_version }); - } - - return .{ - .platform = .{ .devices = devices.items, .two_readings = true, .vtable = &vtable }, - .io = io, - .nrt = nrt, - .chips = chips, - .busy = .init(gpa), - }; -} - -/// A chip's memory total from its device type, and its name and the memory directory of each core from sysfs. -fn openChip(arena: std.mem.Allocator, io: std.Io, index: c_int, handle: *c.ndl_device_t, cores: usize) !Chip { - const memory = try arena.alloc(?smi.sysfs.Dir, cores); - @memset(memory, null); - var chip: Chip = .{ .handle = handle, .mem_total_bytes = memoryTotal(handle.device_type), .memory = memory }; - - var path: [64]u8 = undefined; - const dir = smi.sysfs.Dir.open(io, try std.fmt.bufPrint(&path, "/sys/devices/virtual/neuron_device/neuron{d}", .{index})) catch return chip; - defer dir.close(); - for (memory, 0..) |*core, core_index| { - core.* = dir.openDir(try std.fmt.bufPrint(&path, "neuron_core{d}/stats/memory_usage/device_mem", .{core_index})) catch null; - } - var buffer: [128]u8 = undefined; - chip.name = try arena.dupe(u8, dir.read("info/architecture/device_name", &buffer) catch return chip); - return chip; -} - -/// Device memory by device type: Inferentia, then Inferentia2 and Trainium, Trainium2, Trainium3. -fn memoryTotal(device_type: u8) ?u64 { - const gib: u64 = 1 << 30; - return switch (device_type) { - 1 => 8 * gib, - 2 => 32 * gib, - 3 => 96 * gib, - 4 => 144 * gib, - else => null, - }; -} - -fn deinit(platform: *smi.Platform) void { - const self: *Neuron = @alignCast(@fieldParentPtr("platform", platform)); - for (self.chips) |chip| { - for (chip.memory) |memory| if (memory) |dir| dir.close(); - } - self.busy.deinit(); -} - -fn poll(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []smi.Sample) void { - const self: *Neuron = @alignCast(@fieldParentPtr("platform", platform)); - defer self.busy.sweep(); - var rows: std.ArrayList(smi.ProcessInfo) = .empty; - var read_any = false; - - var first: usize = 0; - for (self.chips) |chip| { - const cores = samples[first..][0..chip.memory.len]; - defer first += cores.len; - - // The cores of a chip share its memory: each shows what all of them use. - const mem_used = memoryInUse(chip.memory); - for (cores, chip.memory) |*sample, memory| { - const core = &sample.neuron; - core.mem_used_bytes = mem_used; - core.mem_total_bytes = chip.mem_total_bytes; - const dir = memory orelse continue; - core.nc_tensors = dir.int(u64, "tensors/total") catch null; - core.nc_constants = dir.int(u64, "constants/total") catch null; - core.nc_model_code = dir.int(u64, "model_code/total") catch null; - core.nc_shared_scratchpad = dir.int(u64, "model_shared_scratchpad/total") catch null; - core.nc_nonshared_scratchpad = dir.int(u64, "nonshared_scratchpad/total") catch null; - core.nc_runtime = dir.int(u64, "runtime_memory/total") catch null; - core.nc_driver = dir.int(u64, "driver_memory/total") catch null; - core.nc_dma_rings = dir.int(u64, "dma_rings/total") catch null; - core.nc_collectives = dir.int(u64, "collectives/total") catch null; - core.nc_notifications = dir.int(u64, "notifications/total") catch null; - core.nc_uncategorized = dir.int(u64, "uncategorized/total") catch null; - } - - self.scan(scratch, chip.handle, @intCast(first), cores, &rows) catch continue; - read_any = true; - } - self.rows = if (read_any) rows.items else null; -} - -fn processes(platform: *smi.Platform, _: std.mem.Allocator, samples: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { - const self: *Neuron = @alignCast(@fieldParentPtr("platform", platform)); - for (self.rows orelse return error.Unavailable) |row| { - var copy = row; - if (samples[row.device_idx].neuron.util_percent) |util| copy.dev_util_percent = @intCast(util); - try out.append(copy); - } -} - -fn memoryInUse(memory: []const ?smi.sysfs.Dir) ?u64 { - var total: u64 = 0; - for (memory) |dir| total += (dir orelse return null).int(u64, "total") catch return null; - return total; -} - -/// Reads the busy time of every process on one chip: the utilization of each core, -/// and a row per process and core it has used, if it holds device memory. -fn scan(self: *Neuron, scratch: std.mem.Allocator, handle: *c.ndl_device_t, first: u16, cores: []smi.Sample, rows: *std.ArrayList(smi.ProcessInfo)) !void { - var list: ?[*]c.struct_neuron_app_info = null; - var count: usize = 0; - try self.nrt.ndl_get_all_apps_info.call(.{ handle, &list, &count, c.APP_INFO_ALL }); - defer std.c.free(list); - const apps: []const c.struct_neuron_app_info = if (list) |items| items[0..count] else &.{}; - - const busy = try scratch.alloc(?smi.Growth, cores.len); - @memset(busy, null); - for (apps) |app| { - if (app.pid <= 0) continue; - const nds = (self.nrt.nds_open.get(.{ handle, app.pid }) catch continue) orelse continue; - defer _ = self.nrt.nds_close(nds); - - for (busy, 0..) |*sum, core| { - const micros = self.nrt.nds_get_nc_counter.get(.{ nds, @intCast(core), c.NDS_NC_COUNTER_TIME_IN_USE }) catch continue; - const key: Key = .{ .pid = @intCast(app.pid), .core = first + @as(u16, @intCast(core)) }; - if (try self.busy.advance(key, self.io, smi.units.scale(micros, .micro, .nano))) |growth| smi.Growth.add(sum, growth); - - const mem_kib = app.device_mem_size / 1024; - if (micros > 0 and mem_kib > 0) try rows.append(scratch, .{ .pid = key.pid, .device_idx = key.core, .dev_mem_kib = mem_kib }); - } - } - for (cores, busy) |*sample, sum| sample.neuron.util_percent = if (sum) |growth| growth.percentOfTime() else 0; -} - -/// Finds the functions of `Private` in the symbol table of the library that holds `known`. -fn findPrivate(io: std.Io, known: *const anyopaque) !Private { - var library: DlInfo = undefined; - if (dladdr(known, &library) == 0) return error.LibraryNotFound; - const file = try std.Io.Dir.cwd().openFile(io, std.mem.span(library.fname), .{}); - defer file.close(io); - var map = try file.createMemoryMap(io, .{ .len = @intCast((try file.stat(io)).size), .protection = .{ .read = true }, .populate = false }); - defer map.destroy(io); - - const elf = std.elf.ElfN; - const image = map.memory; - const header = std.mem.bytesAsValue(elf.Ehdr, image[0..@sizeOf(elf.Ehdr)]); - const sections = std.mem.bytesAsSlice(elf.Shdr, image[header.shoff..][0 .. @as(usize, header.shnum) * @sizeOf(elf.Shdr)]); - - const info = @typeInfo(Private).@"struct"; - var private: Private = undefined; - var found: [info.field_names.len]bool = @splat(false); - for (sections) |section| { - if (section.type != .SYMTAB) continue; - const strings = image[sections[section.link].offset..][0..sections[section.link].size]; - for (std.mem.bytesAsSlice(elf.Sym, image[section.offset..][0..section.size])) |symbol| { - if (symbol.value == 0) continue; - const name = std.mem.sliceTo(strings[symbol.name..], 0); - inline for (info.field_names, info.field_types, &found) |field, Field, *seen| { - if (!seen.* and std.mem.eql(u8, name, field)) { - seen.* = true; - const address = @intFromPtr(library.fbase) + symbol.value; - @field(private, field) = if (@typeInfo(Field) == .@"struct") .{ .function = @ptrFromInt(address) } else @ptrFromInt(address); - } - } - } - } - inline for (info.field_names, found) |field, seen| { - if (!seen) { - log.warn("libnrt has no symbol {s}", .{field}); - return error.MissingSymbol; - } - } - return private; -} - -const DlInfo = extern struct { - fname: [*:0]const u8, - fbase: *anyopaque, - sname: ?[*:0]const u8, - saddr: ?*anyopaque, -}; - -extern fn dladdr(address: *const anyopaque, info: *DlInfo) c_int; - -test findPrivate { - const io = std.testing.io; - const api = try smi.dynlib.openBundled(Api, io, "libnrt.so.1"); - const nrt = try findPrivate(io, api.nrt_get_version.function); - // Listing devices works without any, so the call checks the address. - var indices: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; - try std.testing.expect(nrt.ndl_available_devices(&indices, indices.len) >= 0); -} - -test "utilization and process rows" { - const fake = struct { - var busy_us: u64 = 1000; - - fn apps(device: [*c]c.ndl_device_t, info: [*c][*c]c.struct_neuron_app_info, count: [*c]usize, _: u16) callconv(.c) c_int { - if (@intFromPtr(device) == 0x1000) return -1; - const list: [*]c.struct_neuron_app_info = @ptrCast(@alignCast(std.c.malloc(3 * @sizeOf(c.struct_neuron_app_info)).?)); - // pid 0 is skipped, and pid 8 holds less than 1 KiB. - for (list[0..3], [_]i32{ 0, 7, 8 }, [_]usize{ 1 << 20, 1 << 20, 512 }) |*app, pid, bytes| { - app.* = std.mem.zeroes(c.struct_neuron_app_info); - app.pid = pid; - app.device_mem_size = bytes; - } - info.* = list; - count.* = 3; - return 0; - } - - fn open(_: [*c]c.ndl_device_t, pid: c.pid_t, instance: [*c]?*c.nds_instance_t) callconv(.c) c_int { - instance.* = @ptrFromInt(@as(usize, @intCast(pid))); - return 0; - } - - fn close(_: ?*c.nds_instance_t) callconv(.c) c_int { - return 0; - } - - /// Every process is busy on the first core of a chip. - fn counter(_: ?*c.nds_instance_t, core: c_int, _: u32, value: [*c]u64) callconv(.c) c_int { - value.* = if (core == 0) busy_us else 0; - return 0; - } - }; - - const devices: [4]smi.Device = @splat(.{ .target = .neuron }); - var neuron: Neuron = .{ - .platform = .{ .devices = &devices, .vtable = &vtable }, - .io = std.testing.io, - .nrt = .{ - .ndl_available_devices = undefined, - .ndl_open_device = undefined, - .ndl_get_all_apps_info = .{ .function = &fake.apps }, - .nds_open = .{ .function = &fake.open }, - .nds_close = &fake.close, - .nds_get_nc_counter = .{ .function = &fake.counter }, - }, - // The first chip cannot be read. - .chips = &.{ - .{ .handle = @ptrFromInt(0x1000), .mem_total_bytes = null, .memory = &.{ null, null } }, - .{ .handle = @ptrFromInt(0x2000), .mem_total_bytes = null, .memory = &.{ null, null } }, - }, - .busy = .init(std.testing.allocator), - }; - defer neuron.busy.deinit(); - var scratch: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer scratch.deinit(); - var list: smi.ProcessList = .init(std.testing.allocator); - defer list.deinit(); - - // The first reading has no growth; the second grows by far more than the time between them. - for ([_]u8{ 0, 100 }) |util| { - var samples: [4]smi.Sample = @splat(.init(.neuron)); - list.clear(); - poll(&neuron.platform, scratch.allocator(), &samples); - try processes(&neuron.platform, scratch.allocator(), &samples, &list); - - try std.testing.expectEqual(null, samples[0].neuron.util_percent); - try std.testing.expectEqual(util, samples[2].neuron.util_percent); - try std.testing.expectEqual(0, samples[3].neuron.util_percent); - const row: smi.ProcessInfo = .{ .pid = 7, .device_idx = 2, .dev_util_percent = util, .dev_mem_kib = 1024 }; - try std.testing.expectEqualDeep(@as([]const smi.ProcessInfo, &.{row}), list.rows.items); - fake.busy_us += 1_000_000_000; - } -} diff --git a/bin/zml-smi/platforms/neuron/metrics.zig b/bin/zml-smi/platforms/neuron/metrics.zig new file mode 100644 index 00000000..0b5a5ddd --- /dev/null +++ b/bin/zml-smi/platforms/neuron/metrics.zig @@ -0,0 +1,201 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const device_info = @import("zml-smi/info").device_info; +const DeviceInfo = device_info.DeviceInfo; +const NeuronInfo = device_info.NeuronInfo; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Nrt = @import("nrt.zig"); +const process = @import("process.zig"); + +const base_path = "/sys/devices/virtual/neuron_device"; + +pub const UtilAtomic = std.atomic.Value(u64); + +pub fn start(collector: *Collector) !void { + const nrt = try collector.arena.create(Nrt); + nrt.* = try Nrt.init(collector.arena, collector.io); + + if (nrt.handles.len == 0) { + return; + } + + const total_nc = nrt.totalNcCount() catch return; + const nc_per_device = total_nc / @as(u32, @intCast(nrt.handles.len)); + + const total_cores = @as(usize, nrt.handles.len) * nc_per_device; + const util_buf = try collector.arena.alloc(UtilAtomic, total_cores); + for (util_buf) |*u| u.* = .init(0); + + const GiB: u64 = 1024 * 1024 * 1024; + var neuron_infos: std.ArrayList(*DeviceInfo) = .empty; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + var ver_buf: [Nrt.version_buf_len]u8 = undefined; + const nrt_ver: ?[]const u8 = if (nrt.version(&ver_buf)) |v| + collector.arena.dupe(u8, v) catch null + else |_| + null; + + var flat_core: usize = 0; + for (nrt.handles, nrt.device_indexes) |dev_handle, device_idx| { + const device_type = Nrt.deviceType(dev_handle); + const mem_per_device: u64 = switch (device_type) { + .inf1 => 8 * GiB, + .inf2_trn1 => 32 * GiB, + .trn2 => 96 * GiB, + .trn3 => 144 * GiB, + else => 0, + }; + + for (0..nc_per_device) |ci| { + const poll_arena = try collector.createPollArena(); + + const dev: Device = .{ + .io = collector.io, + .arena = poll_arena, + .device_idx = @intCast(device_idx), + .core_idx = @intCast(ci), + .mem_per_device = mem_per_device, + .nc_per_device = nc_per_device, + .util_src = &util_buf[flat_core], + }; + + const initial: NeuronInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = nrt_ver, + }; + const info = try collector.addDevice(.{ .neuron = .{ .values = .{ initial, initial } } }); + try neuron_infos.append(collector.arena, info); + try collector.spawnPoll(pollOnce, .{ poll_arena, &info.neuron, dev }); + flat_core += 1; + } + } + + if (neuron_infos.items.len > 0) { + const processes = try collector.createProcessList(); + const warmup = collector.poll_only; + try process.init(collector, processes, nrt, nc_per_device, util_buf, dev_offset, warmup); + + if (warmup) { + for (neuron_infos.items, util_buf) |info, *u| { + const back = info.neuron.back(); + back.* = info.neuron.front().*; + back.util_percent = u.load(.acquire); + info.neuron.swap(); + } + } + } +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(NeuronInfo), Device, metrics); + +const Device = struct { + io: std.Io, + arena: *std.heap.ArenaAllocator, + device_idx: u32, + core_idx: u32, + mem_per_device: u64 = 0, + nc_per_device: u32 = 1, + util_src: *const UtilAtomic, + + pub fn utilPercent(self: Device) !u64 { + return self.util_src.load(.acquire); + } + + fn devicePath(self: Device, sub_path: []const u8) ![]const u8 { + return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/{s}", .{ self.device_idx, sub_path }); + } + + fn corePath(self: Device, sub_path: []const u8) ![]const u8 { + return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/{s}", .{ self.device_idx, self.core_idx, sub_path }); + } + + fn name(self: Device, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, try self.devicePath("info/architecture/device_name")); + } + + fn readCoreMem(self: Device, comptime subdir: []const u8) !u64 { + return sysfs.readInt(self.arena.allocator(), self.io, try self.corePath("stats/memory_usage/device_mem/" ++ subdir ++ "/total")); + } + + pub fn memTotal(self: Device) !u64 { + if (self.mem_per_device == 0) { + return error.NrtUnavailable; + } + return self.mem_per_device; + } + + // Sysfs-based metrics — RAM is shared across cores, so sum all cores on the device. + pub fn memUsed(self: Device) !u64 { + var total: u64 = 0; + for (0..self.nc_per_device) |ci| { + const path = try std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/stats/memory_usage/device_mem/total", .{ self.device_idx, ci }); + total += try sysfs.readInt(self.arena.allocator(), self.io, path); + } + + return total; + } + + pub fn tensors(self: Device) !u64 { + return self.readCoreMem("tensors"); + } + + pub fn constants(self: Device) !u64 { + return self.readCoreMem("constants"); + } + + pub fn modelCode(self: Device) !u64 { + return self.readCoreMem("model_code"); + } + + pub fn sharedScratchpad(self: Device) !u64 { + return self.readCoreMem("model_shared_scratchpad"); + } + + pub fn nonsharedScratchpad(self: Device) !u64 { + return self.readCoreMem("nonshared_scratchpad"); + } + + pub fn runtime(self: Device) !u64 { + return self.readCoreMem("runtime_memory"); + } + + pub fn driver(self: Device) !u64 { + return self.readCoreMem("driver_memory"); + } + + pub fn dmaRings(self: Device) !u64 { + return self.readCoreMem("dma_rings"); + } + + pub fn collectives(self: Device) !u64 { + return self.readCoreMem("collectives"); + } + + pub fn notifications(self: Device) !u64 { + return self.readCoreMem("notifications"); + } + + pub fn uncategorized(self: Device) !u64 { + return self.readCoreMem("uncategorized"); + } +}; + +const metrics = .{ + .{ .field = "util_percent", .query = Device.utilPercent }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "nc_tensors", .query = Device.tensors }, + .{ .field = "nc_constants", .query = Device.constants }, + .{ .field = "nc_model_code", .query = Device.modelCode }, + .{ .field = "nc_shared_scratchpad", .query = Device.sharedScratchpad }, + .{ .field = "nc_nonshared_scratchpad", .query = Device.nonsharedScratchpad }, + .{ .field = "nc_runtime", .query = Device.runtime }, + .{ .field = "nc_driver", .query = Device.driver }, + .{ .field = "nc_dma_rings", .query = Device.dmaRings }, + .{ .field = "nc_collectives", .query = Device.collectives }, + .{ .field = "nc_notifications", .query = Device.notifications }, + .{ .field = "nc_uncategorized", .query = Device.uncategorized }, +}; diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig new file mode 100644 index 00000000..cd77b3c0 --- /dev/null +++ b/bin/zml-smi/platforms/neuron/nrt.zig @@ -0,0 +1,230 @@ +const std = @import("std"); +const c = @import("c"); +const elf = std.elf; +const stdx = @import("stdx"); +const DynLib = @import("zml-smi/dynlib"); +const sandbox = @import("zml-smi/sandbox"); + +const Nrt = @This(); + +pub const Error = error{ nrt_error, NrtUnavailable }; +pub const AppInfo = c.struct_neuron_app_info; +pub const DeviceType = enum(c_int) { + inf1 = 1, + inf2_trn1 = 2, + trn2 = 3, + trn3 = 4, + _, +}; + +lib: Fns, +private_lib: PrivateFns, +handles: []const *c.ndl_device_t, +device_indexes: []const c_int, + +const Fns = struct { + nrt_get_total_nc_count: *const @TypeOf(c.nrt_get_total_nc_count), + nrt_get_version: *const @TypeOf(c.nrt_get_version), +}; + +const PrivateFns = struct { + nds_open: *const @TypeOf(c.nds_open), + ndl_available_devices: *const @TypeOf(c.ndl_available_devices), + ndl_open_device: *const @TypeOf(c.ndl_open_device), + ndl_close_device: *const @TypeOf(c.ndl_close_device), + ndl_get_all_apps_info: *const @TypeOf(c.ndl_get_all_apps_info), + nds_close: *const @TypeOf(c.nds_close), + nds_get_nc_counter: *const @TypeOf(c.nds_get_nc_counter), +}; + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !Nrt { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = sandbox.path(&path_buf) orelse { + std.log.err("neuron: sandbox path unavailable", .{}); + return error.NrtUnavailable; + }; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnrt.so.1" }) catch { + std.log.err("neuron: failed to construct libnrt.so.1 path", .{}); + return error.NrtUnavailable; + }; + + const dl_handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("neuron: dlopen: {s}", .{err}); + return error.NrtUnavailable; + }; + var dynlib: std.DynLib = .{ .inner = .{ .handle = dl_handle } }; + + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NrtUnavailable; + const private_fns = resolveFromElf(dl_handle, io, "nrt_init") catch |err| { + std.log.err("neuron: ELF resolution failed: {s}", .{@errorName(err)}); + return error.NrtUnavailable; + }; + + var dev_index_buf: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; + const count: usize = @intCast(@max(0, private_fns.ndl_available_devices(&dev_index_buf, c.MAX_NEURON_DEVICE_COUNT))); + + var handle_list: std.ArrayList(*c.ndl_device_t) = .empty; + var index_list: std.ArrayList(c_int) = .empty; + + for (dev_index_buf[0..count]) |device_idx| { + var dev: ?*c.ndl_device_t = null; + var t: c.struct_ndl_device_init_param = .{ + .initialize_device = false, + .num_dram_regions = 0, + .map_hbm = false, + .skip_copy_bufs = true, + }; + + if (private_fns.ndl_open_device(device_idx, &t, &dev) == 0) { + if (dev) |d| { + errdefer _ = private_fns.ndl_close_device(d); + try handle_list.append(allocator, d); + try index_list.append(allocator, device_idx); + } + } + } + + return .{ + .lib = fns, + .private_lib = private_fns, + .handles = try handle_list.toOwnedSlice(allocator), + .device_indexes = try index_list.toOwnedSlice(allocator), + }; +} + +pub fn deviceCount(self: Nrt) u32 { + return @intCast(self.handles.len); +} + +pub fn allAppsInfo(self: Nrt, dev: *c.ndl_device_t) Error!struct { ptr: ?[*]AppInfo, count: usize } { + var info: ?[*]AppInfo = null; + var count: usize = 0; + if (self.private_lib.ndl_get_all_apps_info(dev, &info, &count, c.APP_INFO_ALL) != 0) { + return error.nrt_error; + } + return .{ .ptr = info, .count = count }; +} + +pub fn ndsOpen(self: Nrt, dev: *c.ndl_device_t, pid: c.pid_t) Error!*c.nds_instance_t { + var inst: ?*c.nds_instance_t = null; + if (self.private_lib.nds_open(dev, pid, &inst) != 0) { + return error.nrt_error; + } + return inst orelse error.nrt_error; +} + +pub fn ndsClose(self: Nrt, inst: *c.nds_instance_t) void { + _ = self.private_lib.nds_close(inst); +} + +pub fn ncCounter(self: Nrt, inst: *c.nds_instance_t, pnc_index: c_int, counter_index: u32) Error!u64 { + var value: u64 = 0; + if (self.private_lib.nds_get_nc_counter(inst, pnc_index, counter_index, &value) != 0) { + return error.nrt_error; + } + return value; +} + +pub const version_buf_len = c.RT_VERSION_DETAIL_LEN; + +pub fn version(self: Nrt, buf: *[version_buf_len]u8) Error![:0]const u8 { + var ver: c.nrt_version_t = std.mem.zeroes(c.nrt_version_t); + if (self.lib.nrt_get_version(&ver, @sizeOf(c.nrt_version_t)) != 0) { + return error.nrt_error; + } + @memcpy(buf, &ver.rt_detail); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn totalNcCount(self: Nrt) Error!u32 { + var count: u32 = 0; + if (self.lib.nrt_get_total_nc_count(&count) != 0) { + return error.nrt_error; + } + return count; +} + +pub fn hbmSize(dev: *c.ndl_device_t) usize { + return dev.hbm_size; +} + +pub fn deviceType(dev: *c.ndl_device_t) DeviceType { + return @fromBackingInt(@intCast(dev.device_type)); +} + +fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const u8) !PrivateFns { + const field_names = comptime std.meta.fieldNames(PrivateFns); + + const sym_addr = std.c.dlsym(handle, known_sym) orelse + return error.SymbolNotFound; + + var dl_info: DlInfo = undefined; + if (dladdr(sym_addr, &dl_info) == 0) return error.DladdrFailed; + + const elf_base = @intFromPtr(dl_info.dli_fbase); + const elf_path = std.mem.span(dl_info.dli_fname); + + const file = try std.Io.Dir.openFileAbsolute(io, elf_path, .{}); + defer file.close(io); + + const stat = try file.stat(io); + const mmap = try std.posix.mmap( + null, + @intCast(stat.size), + .{ .READ = true }, + .{ .TYPE = .PRIVATE }, + file.handle, + 0, + ); + defer std.posix.munmap(mmap); + + var result: PrivateFns = undefined; + var remaining: usize = field_names.len; + + const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(mmap)); + const shdrs: []const elf.ElfN.Shdr = @as([*]const elf.ElfN.Shdr, @ptrCast(@alignCast(mmap[ehdr.shoff..])))[0..ehdr.shnum]; + + for (shdrs) |sh| { + if (sh.type != .SYMTAB) { + continue; + } + + const strtab_sh = shdrs[sh.link]; + const strtab: []const u8 = mmap[strtab_sh.offset..][0..strtab_sh.size]; + + for (std.mem.bytesAsSlice(elf.ElfN.Sym, mmap[sh.offset..][0..sh.size])) |s| { + if (s.value == 0) { + continue; + } + + const sym_name = std.mem.span(@as([*c]const u8, @ptrCast(strtab[@as(usize, s.name)..]))); + + inline for (field_names) |field_name| { + if (std.mem.eql(u8, sym_name, field_name)) { + @field(result, field_name) = @ptrFromInt(elf_base + @as(usize, s.value)); + remaining -= 1; + } + } + + if (remaining == 0) { + return result; + } + } + } + + if (remaining > 0) { + return error.SymbolResolutionFailed; + } + return result; +} + +const DlInfo = extern struct { + dli_fname: [*:0]const u8, + dli_fbase: *anyopaque, + dli_sname: [*c]const u8, + dli_saddr: ?*anyopaque, +}; + +extern fn dladdr(addr: *anyopaque, info: *DlInfo) c_int; diff --git a/bin/zml-smi/platforms/neuron/process.zig b/bin/zml-smi/platforms/neuron/process.zig new file mode 100644 index 00000000..2dbf363c --- /dev/null +++ b/bin/zml-smi/platforms/neuron/process.zig @@ -0,0 +1,108 @@ +const std = @import("std"); +const c = @import("c"); +const Nrt = @import("nrt.zig"); +const smi_info = @import("zml-smi/info"); +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; +const UtilAtomic = @import("metrics.zig").UtilAtomic; + +const DeltaState = struct { + prev_total_us: u64 = 0, + prev_timestamp: ?std.Io.Timestamp = null, +}; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, warmup: bool) !void { + if (nrt.handles.len == 0) return; + const delta_states = try collector.arena.alloc(DeltaState, util_buf.len); + @memset(delta_states, .{}); + const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.gpa, collector.io, list, nrt, nc_per_device, util_buf, dev_offset, delta_states }; + if (warmup) { + @call(.auto, pollOnce, args); + collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; + @call(.auto, pollOnce, args); + collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; + } + + try collector.spawnPoll(pollOnce, args); +} + +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, delta_states: []DeltaState) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (nrt.handles, 0..) |handle, dev_i| { + const apps_result = nrt.allAppsInfo(handle) catch continue; + defer { + if (apps_result.ptr) |ptr| { + std.c.free(@ptrCast(ptr)); + } + } + + const apps = if (apps_result.ptr) |ptr| ptr[0..apps_result.count] else continue; + + var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = @splat(0); + + for (apps) |*app| { + if (app.pid <= 0) { + continue; + } + + const nds = nrt.ndsOpen(handle, app.pid) catch continue; + defer nrt.ndsClose(nds); + + for (0..nc_per_device) |ci| { + const time_in_use = nrt.ncCounter(nds, @intCast(ci), c.NDS_NC_COUNTER_TIME_IN_USE) catch continue; + total_us_per_core[ci] += time_in_use; + + if (time_in_use == 0) { + continue; + } + + const dev_idx: u32 = @intCast(dev_i * nc_per_device + ci); + + const pid = std.math.cast(u32, app.pid) orelse continue; + var info: pi.ProcessInfo = .{ + .pid = pid, + .device_idx = @intCast(dev_idx + dev_offset), + .dev_mem_kib = @intCast(app.device_mem_size / 1024), + }; + + if (dev_idx < util_buf.len) { + info.dev_util_percent = @intCast(util_buf[dev_idx].load(.acquire)); + } + + if (info.dev_mem_kib != null and info.dev_mem_kib.? > 0) + back.append(allocator, info) catch break; + } + } + + const now: std.Io.Timestamp = .now(io, .awake); + + for (0..nc_per_device) |ci| { + const dev_idx = dev_i * nc_per_device + ci; + if (dev_idx >= delta_states.len) { + break; + } + + const ds = &delta_states[dev_idx]; + const prev_us = ds.prev_total_us; + ds.prev_total_us = total_us_per_core[ci]; + + if (ds.prev_timestamp) |prev| { + const elapsed = prev.untilNow(io, .awake); + const elapsed_us: u64 = @intCast(@divFloor(elapsed.nanoseconds, std.time.ns_per_us)); + + if (elapsed_us > 0) { + const delta_us = total_us_per_core[ci] -| prev_us; + util_buf[dev_idx].store(@min(100, delta_us * 100 / elapsed_us), .release); + } + } + + ds.prev_timestamp = now; + } + } + + list.swap(); +} diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel index 54f68d15..0ab0fae1 100644 --- a/bin/zml-smi/platforms/nvml/BUILD.bazel +++ b/bin/zml-smi/platforms/nvml/BUILD.bazel @@ -1,18 +1,22 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "nvml", + srcs = [ + "nvml.zig", + "process.zig", + ], import_name = "zml-smi/platforms/nvml", - main = "Nvml.zig", + main = "metrics.zig", target_compatible_with = ["@platforms//os:linux"], - visibility = ["//bin/zml-smi:__subpackages__"], - deps = ["//bin/zml-smi/smi"] + select({ + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + ] + select({ "@llvm//platforms/config:linux_x86_64": ["@cuda_nvml_dev_linux_x86_64//:nvml"], "@llvm//platforms/config:linux_aarch64": ["@cuda_nvml_dev_linux_sbsa//:nvml"], }), ) - -zig_test( - name = "test", - deps = [":nvml"], -) diff --git a/bin/zml-smi/platforms/nvml/Nvml.zig b/bin/zml-smi/platforms/nvml/Nvml.zig deleted file mode 100644 index bc971f70..00000000 --- a/bin/zml-smi/platforms/nvml/Nvml.zig +++ /dev/null @@ -1,216 +0,0 @@ -//! NVIDIA GPUs, through libnvidia-ml. -const std = @import("std"); -const c = @import("c"); -const smi = @import("zml-smi"); - -const Nvml = @This(); -const log = std.log.scoped(.@"zml-smi/nvml"); - -platform: smi.Platform, -api: Api, -gpus: []Gpu, -tick: u8 = 0, - -const Gpu = struct { - handle: c.nvmlDevice_t, - /// PCIe throughput is read every fourth poll and kept in between. - pcie_tx_kbps: ?u64 = null, - pcie_rx_kbps: ?u64 = null, - /// The newest process utilization the GPU reported, so that a poll reads only newer ones. - last_seen: u64 = 0, -}; - -const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes }; - -const Api = struct { - nvmlInit_v2: Checked(c.nvmlInit_v2), - nvmlErrorString: *const @TypeOf(c.nvmlErrorString), - nvmlSystemGetDriverVersion: Checked(c.nvmlSystemGetDriverVersion), - nvmlSystemGetCudaDriverVersion_v2: Checked(c.nvmlSystemGetCudaDriverVersion_v2), - nvmlDeviceGetCount_v2: Checked(c.nvmlDeviceGetCount_v2), - nvmlDeviceGetHandleByIndex_v2: Checked(c.nvmlDeviceGetHandleByIndex_v2), - nvmlDeviceGetName: Checked(c.nvmlDeviceGetName), - nvmlDeviceGetPowerUsage: Checked(c.nvmlDeviceGetPowerUsage), - nvmlDeviceGetEnforcedPowerLimit: Checked(c.nvmlDeviceGetEnforcedPowerLimit), - nvmlDeviceGetTemperature: Checked(c.nvmlDeviceGetTemperature), - nvmlDeviceGetFanSpeed: Checked(c.nvmlDeviceGetFanSpeed), - nvmlDeviceGetUtilizationRates: Checked(c.nvmlDeviceGetUtilizationRates), - nvmlDeviceGetEncoderUtilization: Checked(c.nvmlDeviceGetEncoderUtilization), - nvmlDeviceGetDecoderUtilization: Checked(c.nvmlDeviceGetDecoderUtilization), - nvmlDeviceGetClockInfo: Checked(c.nvmlDeviceGetClockInfo), - nvmlDeviceGetMaxClockInfo: Checked(c.nvmlDeviceGetMaxClockInfo), - nvmlDeviceGetMemoryInfo: Checked(c.nvmlDeviceGetMemoryInfo), - nvmlDeviceGetMemoryBusWidth: Checked(c.nvmlDeviceGetMemoryBusWidth), - nvmlDeviceGetPcieThroughput: Checked(c.nvmlDeviceGetPcieThroughput), - nvmlDeviceGetPcieSpeed: Checked(c.nvmlDeviceGetPcieSpeed), - nvmlDeviceGetCurrPcieLinkGeneration: Checked(c.nvmlDeviceGetCurrPcieLinkGeneration), - nvmlDeviceGetCurrPcieLinkWidth: Checked(c.nvmlDeviceGetCurrPcieLinkWidth), - nvmlDeviceGetComputeRunningProcesses_v3: Checked(c.nvmlDeviceGetComputeRunningProcesses_v3), - nvmlDeviceGetGraphicsRunningProcesses_v3: Checked(c.nvmlDeviceGetGraphicsRunningProcesses_v3), - nvmlDeviceGetProcessUtilization: Checked(c.nvmlDeviceGetProcessUtilization), - - fn Checked(comptime function: anytype) type { - return smi.dynlib.Checked(function, check); - } - - fn check(status: c.nvmlReturn_t) error{ Unsupported, NvmlError }!void { - return switch (status) { - c.NVML_SUCCESS => {}, - c.NVML_ERROR_NOT_SUPPORTED, c.NVML_ERROR_NO_DATA => error.Unsupported, - else => error.NvmlError, - }; - } -}; - -pub fn detect(io: std.Io) bool { - return smi.sysfs.exists(io, "/dev/nvidiactl"); -} - -pub fn init(arena: std.mem.Allocator, _: std.mem.Allocator, _: std.Io) !Nvml { - const api = try smi.dynlib.open(Api, &.{ "libnvidia-ml.so.1", "libnvidia-ml.so" }); - - // The first call says why the library cannot be used. - const status = api.nvmlInit_v2.function(); - Api.check(status) catch |err| { - log.warn("nvmlInit: {s}", .{std.mem.span(api.nvmlErrorString(status))}); - return err; - }; - - var version: [c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE]u8 = undefined; - const driver_version = if (api.nvmlSystemGetDriverVersion.call(.{ &version, version.len })) |_| - try arena.dupe(u8, std.mem.sliceTo(&version, 0)) - else |_| - null; - const details: []const smi.Device.Detail = if (api.nvmlSystemGetCudaDriverVersion_v2.get(.{})) |cuda| try arena.dupe(smi.Device.Detail, &.{.{ - .key = "cuda_version", - .label = "CUDA", - .value = try std.fmt.allocPrint(arena, "{d}.{d}", .{ @divTrunc(cuda, 1000), @divTrunc(@mod(cuda, 1000), 10) }), - .in_title = true, - }}) else |_| &.{}; - - const count = try api.nvmlDeviceGetCount_v2.get(.{}); - var gpus: std.ArrayList(Gpu) = try .initCapacity(arena, count); - var devices: std.ArrayList(smi.Device) = try .initCapacity(arena, count); - for (0..count) |index| { - // A GPU that fell off the bus is skipped; the others still show. - const handle = api.nvmlDeviceGetHandleByIndex_v2.get(.{@intCast(index)}) catch continue; - var name: [c.NVML_DEVICE_NAME_V2_BUFFER_SIZE]u8 = undefined; - gpus.appendAssumeCapacity(.{ .handle = handle }); - devices.appendAssumeCapacity(.{ - .target = .cuda, - .name = if (api.nvmlDeviceGetName.call(.{ handle, &name, name.len })) |_| try arena.dupe(u8, std.mem.sliceTo(&name, 0)) else |_| null, - .driver_version = driver_version, - .details = details, - }); - } - - return .{ - .platform = .{ .devices = devices.items, .vtable = &vtable }, - .api = api, - .gpus = gpus.items, - }; -} - -fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { - const self: *Nvml = @alignCast(@fieldParentPtr("platform", platform)); - const api = &self.api; - // PCIe throughput blocks while it measures (#766). - const read_pcie = self.tick % 4 == 0; - self.tick +%= 1; - - for (self.gpus, samples) |*gpu, *sample| { - const fields = &sample.cuda; - const handle = gpu.handle; - - fields.power_mw = api.nvmlDeviceGetPowerUsage.get(.{handle}) catch null; - fields.power_limit_mw = api.nvmlDeviceGetEnforcedPowerLimit.get(.{handle}) catch null; - fields.temperature = api.nvmlDeviceGetTemperature.get(.{ handle, c.NVML_TEMPERATURE_GPU }) catch null; - fields.fan_speed_percent = api.nvmlDeviceGetFanSpeed.get(.{handle}) catch null; - fields.util_percent = if (api.nvmlDeviceGetUtilizationRates.get(.{handle})) |rates| rates.gpu else |_| null; - - // Utilization and its sampling period. - var util: c_uint, var period: c_uint = .{ 0, 0 }; - fields.encoder_util_percent = if (api.nvmlDeviceGetEncoderUtilization.call(.{ handle, &util, &period })) |_| util else |_| null; - fields.decoder_util_percent = if (api.nvmlDeviceGetDecoderUtilization.call(.{ handle, &util, &period })) |_| util else |_| null; - - fields.clock_graphics_mhz = api.nvmlDeviceGetClockInfo.get(.{ handle, c.NVML_CLOCK_GRAPHICS }) catch null; - fields.clock_sm_mhz = api.nvmlDeviceGetClockInfo.get(.{ handle, c.NVML_CLOCK_SM }) catch null; - fields.clock_mem_mhz = api.nvmlDeviceGetClockInfo.get(.{ handle, c.NVML_CLOCK_MEM }) catch null; - fields.clock_graphics_max_mhz = api.nvmlDeviceGetMaxClockInfo.get(.{ handle, c.NVML_CLOCK_GRAPHICS }) catch null; - fields.clock_mem_max_mhz = api.nvmlDeviceGetMaxClockInfo.get(.{ handle, c.NVML_CLOCK_MEM }) catch null; - - // One call fills two fields. On failure both stay null. - if (api.nvmlDeviceGetMemoryInfo.get(.{handle})) |memory| { - fields.mem_used_bytes = memory.used; - fields.mem_total_bytes = memory.total; - } else |_| {} - fields.mem_bus_width = api.nvmlDeviceGetMemoryBusWidth.get(.{handle}) catch null; - - fields.pcie_link_gen = api.nvmlDeviceGetCurrPcieLinkGeneration.get(.{handle}) catch null; - fields.pcie_link_width = api.nvmlDeviceGetCurrPcieLinkWidth.get(.{handle}) catch null; - // NVML gives the speed of one lane. - if (api.nvmlDeviceGetPcieSpeed.get(.{handle})) |lane_mbps| { - if (fields.pcie_link_width) |width| fields.pcie_bandwidth_mbps = lane_mbps * width; - } else |_| {} - - if (read_pcie) { - gpu.pcie_tx_kbps = api.nvmlDeviceGetPcieThroughput.get(.{ handle, c.NVML_PCIE_UTIL_TX_BYTES }) catch null; - gpu.pcie_rx_kbps = api.nvmlDeviceGetPcieThroughput.get(.{ handle, c.NVML_PCIE_UTIL_RX_BYTES }) catch null; - } - fields.pcie_tx_kbps = gpu.pcie_tx_kbps; - fields.pcie_rx_kbps = gpu.pcie_rx_kbps; - } -} - -fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { - const self: *Nvml = @alignCast(@fieldParentPtr("platform", platform)); - var read_any = self.gpus.len == 0; - - for (self.gpus, 0..) |*gpu, device| { - const handle = gpu.handle; - const first_row = out.rows.items.len; - inline for (.{ self.api.nvmlDeviceGetComputeRunningProcesses_v3, self.api.nvmlDeviceGetGraphicsRunningProcesses_v3 }) |query| { - if (running(scratch, query, handle)) |list| { - read_any = true; - for (list) |process| { - try out.append(.{ .pid = process.pid, .device_idx = @intCast(device), .dev_mem_kib = process.usedGpuMemory / 1024 }); - } - } else |_| {} - } - - const since = gpu.last_seen; - const utilization = self.processUtilization(scratch, handle, since) catch continue; - for (utilization) |util| { - if (util.smUtil > 100 or util.timeStamp <= since) continue; - gpu.last_seen = @max(gpu.last_seen, util.timeStamp); - for (out.rows.items[first_row..]) |*row| { - if (row.pid == util.pid) row.dev_util_percent = @intCast(util.smUtil); - } - } - } - if (!read_any) return error.Unavailable; -} - -/// The processes of a GPU, from a query sized by a first call. -fn running(scratch: std.mem.Allocator, query: anytype, handle: c.nvmlDevice_t) ![]const c.nvmlProcessInfo_t { - var count: c_uint = 0; - const status = query.function(handle, &count, null); - if (status == c.NVML_SUCCESS) return &.{}; - if (status != c.NVML_ERROR_INSUFFICIENT_SIZE) try Api.check(status); - - const list = try scratch.alloc(c.nvmlProcessInfo_t, count); - try query.call(.{ handle, &count, list.ptr }); - return list[0..count]; -} - -/// The utilization samples newer than `since`, from a query sized by a first call. -fn processUtilization(self: *Nvml, scratch: std.mem.Allocator, handle: c.nvmlDevice_t, since: u64) ![]const c.nvmlProcessUtilizationSample_t { - var count: c_uint = 0; - const status = self.api.nvmlDeviceGetProcessUtilization.function(handle, null, &count, since); - if (status == c.NVML_SUCCESS) return &.{}; - if (status != c.NVML_ERROR_INSUFFICIENT_SIZE) try Api.check(status); - - const samples = try scratch.alloc(c.nvmlProcessUtilizationSample_t, count); - try self.api.nvmlDeviceGetProcessUtilization.call(.{ handle, samples.ptr, &count, since }); - return samples[0..count]; -} diff --git a/bin/zml-smi/platforms/nvml/metrics.zig b/bin/zml-smi/platforms/nvml/metrics.zig new file mode 100644 index 00000000..213dd253 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/metrics.zig @@ -0,0 +1,222 @@ +const std = @import("std"); +const Nvml = @import("nvml.zig"); +const device_info = @import("zml-smi/info").device_info; +const GpuInfo = device_info.GpuInfo; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const process = @import("process.zig"); + +const pcie_every: u8 = 4; + +const Slot = struct { + info: *DoubleBuffer(GpuInfo), + dev: Device, +}; + +const Ctx = struct { + gpa: std.mem.Allocator, + nvml: *const Nvml, + slots: []Slot, + processes: *process.ProcessDoubleBuffer, + last_seen_ts: []u64, + dev_offset: u16, + device_count: u32, + tick: u8 = 0, +}; + +pub fn start(collector: *Collector) !void { + const nvml = try collector.arena.create(Nvml); + nvml.* = try Nvml.init(); + + const count = try nvml.deviceCount(); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + var slots: std.ArrayList(Slot) = try .initCapacity(collector.arena, count); + for (0..count) |i| { + const dev = Device.open(nvml, @intCast(i)) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + .cuda_driver_version = dev.cudaDriverVersion(collector.arena) catch null, + }; + const info = try collector.addDevice(.{ .cuda = .{ .values = .{ initial, initial } } }); + slots.appendAssumeCapacity(.{ .info = &info.cuda, .dev = dev }); + } + + const last_seen_ts = try collector.arena.alloc(u64, count); + @memset(last_seen_ts, 0); + + const ctx = try collector.arena.create(Ctx); + ctx.* = .{ + .gpa = collector.gpa, + .nvml = nvml, + .slots = slots.items, + .processes = try collector.createProcessList(), + .last_seen_ts = last_seen_ts, + .dev_offset = dev_offset, + .device_count = count, + }; + try collector.spawnPoll(pollOnce, .{ctx}); +} + +fn pollOnce(ctx: *Ctx) void { + const pcie = ctx.tick % pcie_every == 0; + ctx.tick +%= 1; + + for (ctx.slots) |slot| { + pollDevice(slot.info, slot.dev, pcie); + } + process.pollOnce(ctx.gpa, ctx.processes, ctx.nvml, ctx.dev_offset, ctx.device_count, ctx.last_seen_ts); +} + +fn pollDevice(db: *DoubleBuffer(GpuInfo), dev: Device, pcie: bool) void { + const back = db.back(); + back.* = db.front().*; + poll_metrics.apply(metrics, back, dev); + if (pcie) poll_metrics.apply(pcie_metrics, back, dev); + db.swap(); +} + +const Device = struct { + nvml: *const Nvml, + handle: Nvml.Handle, + + pub fn open(nvml: *const Nvml, index: u32) !Device { + return .{ .nvml = nvml, .handle = try nvml.handleByIndex(index) }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [Nvml.name_buf_len]u8 = undefined; + const slice = try self.nvml.name(self.handle, &buf); + + return try arena.dupe(u8, slice); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [Nvml.driver_version_buf_size]u8 = undefined; + const slice = try self.nvml.driverVersion(&buf); + + return try arena.dupe(u8, slice); + } + + fn cudaDriverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + const v = try self.nvml.cudaDriverVersion(); + return try std.fmt.allocPrint(arena, "{}.{}", .{ + Nvml.cudaDriverVersionMajor(v), + Nvml.cudaDriverVersionMinor(v), + }); + } + + // Power + pub fn powerUsage(self: Device) !u64 { + return @intCast(try self.nvml.powerUsage(self.handle)); + } + + pub fn powerLimit(self: Device) !u64 { + return @intCast(try self.nvml.powerLimit(self.handle)); + } + + // Thermal + pub fn temperature(self: Device) !u64 { + return @intCast(try self.nvml.temperature(self.handle)); + } + + pub fn fanSpeed(self: Device) !u64 { + return @intCast(try self.nvml.fanSpeed(self.handle)); + } + + // Utilization + pub fn gpuUtil(self: Device) !u64 { + return @intCast(try self.nvml.utilizationGpu(self.handle)); + } + + pub fn encoderUtil(self: Device) !u64 { + return @intCast(try self.nvml.encoderUtil(self.handle)); + } + + pub fn decoderUtil(self: Device) !u64 { + return @intCast(try self.nvml.decoderUtil(self.handle)); + } + + // Clocks + pub fn clockGraphics(self: Device) !u64 { + return @intCast(try self.nvml.clockGraphics(self.handle)); + } + + pub fn clockSm(self: Device) !u64 { + return @intCast(try self.nvml.clockSm(self.handle)); + } + + pub fn clockMem(self: Device) !u64 { + return @intCast(try self.nvml.clockMem(self.handle)); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return @intCast(try self.nvml.maxClockGraphics(self.handle)); + } + + pub fn maxClockMem(self: Device) !u64 { + return @intCast(try self.nvml.maxClockMem(self.handle)); + } + + // Memory + pub fn memUsed(self: Device) !u64 { + return self.nvml.memUsed(self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.nvml.memTotal(self.handle); + } + + pub fn memBusWidth(self: Device) !u64 { + return @intCast(try self.nvml.memBusWidth(self.handle)); + } + + // PCIe + pub fn pcieTx(self: Device) !u64 { + return @intCast(try self.nvml.pcieTxKBps(self.handle)); + } + + pub fn pcieRx(self: Device) !u64 { + return @intCast(try self.nvml.pcieRxKBps(self.handle)); + } + + pub fn pcieBandwidth(self: Device) !u64 { + return @intCast(try self.nvml.pcieSpeed(self.handle)); + } + + pub fn pcieLinkGen(self: Device) !u64 { + return @intCast(try self.nvml.pcieLinkGen(self.handle)); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return @intCast(try self.nvml.pcieLinkWidth(self.handle)); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, + .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_sm_mhz", .query = Device.clockSm }, + .{ .field = "clock_mem_mhz", .query = Device.clockMem }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "mem_bus_width", .query = Device.memBusWidth }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; + +const pcie_metrics = .{ + .{ .field = "pcie_tx_kbps", .query = Device.pcieTx }, + .{ .field = "pcie_rx_kbps", .query = Device.pcieRx }, +}; diff --git a/bin/zml-smi/platforms/nvml/nvml.zig b/bin/zml-smi/platforms/nvml/nvml.zig new file mode 100644 index 00000000..e2b36982 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/nvml.zig @@ -0,0 +1,349 @@ +const std = @import("std"); +const c = @import("c"); +const DynLib = @import("zml-smi/dynlib"); + +const Nvml = @This(); + +pub const driver_version_buf_size = c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE; +pub const Error = ReturnError || error{NvmlUnavailable}; +pub const Handle = c.nvmlDevice_t; +pub const ProcessInfo_t = c.nvmlProcessInfo_t; + +lib: Fns, + +const Fns = struct { + nvmlInit_v2: *const @TypeOf(c.nvmlInit_v2), + nvmlDeviceGetHandleByIndex_v2: *const @TypeOf(c.nvmlDeviceGetHandleByIndex_v2), + nvmlDeviceGetCount_v2: *const @TypeOf(c.nvmlDeviceGetCount_v2), + nvmlDeviceGetName: *const @TypeOf(c.nvmlDeviceGetName), + nvmlDeviceGetPowerUsage: *const @TypeOf(c.nvmlDeviceGetPowerUsage), + nvmlDeviceGetTemperature: *const @TypeOf(c.nvmlDeviceGetTemperature), + nvmlDeviceGetUtilizationRates: *const @TypeOf(c.nvmlDeviceGetUtilizationRates), + nvmlDeviceGetClockInfo: *const @TypeOf(c.nvmlDeviceGetClockInfo), + nvmlDeviceGetMaxClockInfo: *const @TypeOf(c.nvmlDeviceGetMaxClockInfo), + nvmlDeviceGetMemoryInfo: *const @TypeOf(c.nvmlDeviceGetMemoryInfo), + nvmlDeviceGetFanSpeed: *const @TypeOf(c.nvmlDeviceGetFanSpeed), + nvmlDeviceGetEnforcedPowerLimit: *const @TypeOf(c.nvmlDeviceGetEnforcedPowerLimit), + nvmlDeviceGetPcieThroughput: *const @TypeOf(c.nvmlDeviceGetPcieThroughput), + nvmlDeviceGetEncoderUtilization: *const @TypeOf(c.nvmlDeviceGetEncoderUtilization), + nvmlDeviceGetDecoderUtilization: *const @TypeOf(c.nvmlDeviceGetDecoderUtilization), + nvmlDeviceGetCurrPcieLinkGeneration: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkGeneration), + nvmlDeviceGetCurrPcieLinkWidth: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkWidth), + nvmlDeviceGetMemoryBusWidth: *const @TypeOf(c.nvmlDeviceGetMemoryBusWidth), + nvmlDeviceGetPcieSpeed: *const @TypeOf(c.nvmlDeviceGetPcieSpeed), + nvmlDeviceGetComputeRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetComputeRunningProcesses_v3), + nvmlDeviceGetGraphicsRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetGraphicsRunningProcesses_v3), + nvmlDeviceGetProcessUtilization: *const @TypeOf(c.nvmlDeviceGetProcessUtilization), + nvmlSystemGetCudaDriverVersion_v2: *const @TypeOf(c.nvmlSystemGetCudaDriverVersion_v2), + nvmlSystemGetDriverVersion: *const @TypeOf(c.nvmlSystemGetDriverVersion), +}; + +pub fn init() Error!Nvml { + var dynlib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen("libnvidia-ml.so.1", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse + std.c.dlopen("libnvidia-ml.so", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("nvml: dlopen: {s}", .{err}); + return error.NvmlUnavailable; + }, + } }; + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NvmlUnavailable; + try check(fns.nvmlInit_v2()); + return .{ .lib = fns }; +} + +pub fn handleByIndex(self: Nvml, device_id: u32) Error!c.nvmlDevice_t { + var handle: c.nvmlDevice_t = undefined; + try check(self.lib.nvmlDeviceGetHandleByIndex_v2(device_id, &handle)); + return handle; +} + +pub fn deviceCount(self: Nvml) Error!u32 { + var count: c_uint = 0; + try check(self.lib.nvmlDeviceGetCount_v2(&count)); + return @intCast(count); +} + +pub const name_buf_len = c.NVML_DEVICE_NAME_V2_BUFFER_SIZE; + +pub fn name(self: Nvml, handle: c.nvmlDevice_t, buf: *[c.NVML_DEVICE_NAME_V2_BUFFER_SIZE]u8) Error![:0]const u8 { + try check(self.lib.nvmlDeviceGetName(handle, buf, buf.len)); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn powerUsage(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var power_mw: c_uint = 0; + try check(self.lib.nvmlDeviceGetPowerUsage(handle, &power_mw)); + return power_mw; +} + +pub fn temperature(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var temp: c_uint = 0; + try check(self.lib.nvmlDeviceGetTemperature(handle, c.NVML_TEMPERATURE_GPU, &temp)); + return temp; +} + +pub fn utilizationGpu(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c.nvmlUtilization_t = undefined; + try check(self.lib.nvmlDeviceGetUtilizationRates(handle, &util)); + return util.gpu; +} + +pub fn clockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); + return clock; +} + +pub fn clockSm(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_SM, &clock)); + return clock; +} + +pub fn clockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); + return clock; +} + +pub fn maxClockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); + return clock; +} + +pub fn maxClockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); + return clock; +} + +pub fn memTotal(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { + var mem: c.nvmlMemory_t = undefined; + try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); + return @intCast(mem.total); +} + +pub fn memUsed(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { + var mem: c.nvmlMemory_t = undefined; + try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); + return @intCast(mem.used); +} + +pub fn fanSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var speed: c_uint = 0; + try check(self.lib.nvmlDeviceGetFanSpeed(handle, &speed)); + return speed; +} + +pub fn powerLimit(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var limit: c_uint = 0; + try check(self.lib.nvmlDeviceGetEnforcedPowerLimit(handle, &limit)); + return limit; +} + +pub fn pcieTxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var value: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_TX_BYTES, &value)); + return value; +} + +pub fn pcieRxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var value: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_RX_BYTES, &value)); + return value; +} + +pub fn encoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c_uint = 0; + var sampling: c_uint = 0; + try check(self.lib.nvmlDeviceGetEncoderUtilization(handle, &util, &sampling)); + return util; +} + +pub fn decoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c_uint = 0; + var sampling: c_uint = 0; + try check(self.lib.nvmlDeviceGetDecoderUtilization(handle, &util, &sampling)); + return util; +} + +pub fn pcieLinkGen(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var gen: c_uint = 0; + try check(self.lib.nvmlDeviceGetCurrPcieLinkGeneration(handle, &gen)); + return gen; +} + +pub fn pcieLinkWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var width: c_uint = 0; + try check(self.lib.nvmlDeviceGetCurrPcieLinkWidth(handle, &width)); + return width; +} + +pub fn pcieSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var speed: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieSpeed(handle, &speed)); + return speed; +} + +pub fn driverVersion(self: Nvml, buf: *[c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE]u8) Error![:0]const u8 { + try check(self.lib.nvmlSystemGetDriverVersion(buf, buf.len)); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn cudaDriverVersionMajor(v: c_int) c_int { + return @divTrunc(v, 1000); +} + +pub fn cudaDriverVersionMinor(v: c_int) c_int { + return @divTrunc(@mod(v, 1000), 10); +} + +pub fn cudaDriverVersion(self: Nvml) Error!c_int { + var version: c_int = 0; + try check(self.lib.nvmlSystemGetCudaDriverVersion_v2(&version)); + return version; +} + +pub fn memBusWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var width: c_uint = 0; + try check(self.lib.nvmlDeviceGetMemoryBusWidth(handle, &width)); + return width; +} + +const max_query = 4096; + +fn sizedCount(count: c_uint) ?c_uint { + if (count == 0 or count > max_query) return null; + return count; +} + +pub fn computeRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]c.nvmlProcessInfo_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, null); + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + count = sizedCount(count) orelse return &.{}; + const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); + errdefer allocator.free(infos); + try check(self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); + return infos; +} + +pub fn graphicsRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]const c.nvmlProcessInfo_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, null); + + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + + count = sizedCount(count) orelse return &.{}; + const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); + errdefer allocator.free(infos); + + try check(self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); + return infos; +} + +pub fn processUtilization(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t, last_seen: u64) (Error || error{OutOfMemory})![]const c.nvmlProcessUtilizationSample_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetProcessUtilization(handle, null, &count, last_seen); + + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + + count = sizedCount(count) orelse return &.{}; + const samples = try allocator.alloc(c.nvmlProcessUtilizationSample_t, count); + errdefer allocator.free(samples); + + try check(self.lib.nvmlDeviceGetProcessUtilization(handle, @ptrCast(samples.ptr), &count, last_seen)); + return samples; +} + +// Error handling + +pub const ReturnError = error{ + error_uninitialized, + error_invalid_argument, + error_not_supported, + error_no_permission, + error_already_initialized, + error_not_found, + error_insufficient_size, + error_insufficient_power, + error_driver_not_loaded, + error_timeout, + error_irq_issue, + error_library_not_found, + error_function_not_found, + error_corrupted_inforom, + error_gpu_is_lost, + error_reset_required, + error_operating_system, + error_lib_rm_version_mismatch, + error_in_use, + error_memory, + error_no_data, + error_vgpu_ecc_not_supported, + error_insufficient_resources, + error_freq_not_supported, + error_argument_version_mismatch, + error_deprecated, + error_not_ready, + error_gpu_not_found, + error_invalid_state, + error_reset_type_not_supported, + error_unknown, +}; + +fn check(ret: c_uint) Error!void { + if (ret == c.NVML_SUCCESS) { + return; + } + return switch (ret) { + c.NVML_ERROR_UNINITIALIZED => error.error_uninitialized, + c.NVML_ERROR_INVALID_ARGUMENT => error.error_invalid_argument, + c.NVML_ERROR_NOT_SUPPORTED => error.error_not_supported, + c.NVML_ERROR_NO_PERMISSION => error.error_no_permission, + c.NVML_ERROR_ALREADY_INITIALIZED => error.error_already_initialized, + c.NVML_ERROR_NOT_FOUND => error.error_not_found, + c.NVML_ERROR_INSUFFICIENT_SIZE => error.error_insufficient_size, + c.NVML_ERROR_INSUFFICIENT_POWER => error.error_insufficient_power, + c.NVML_ERROR_DRIVER_NOT_LOADED => error.error_driver_not_loaded, + c.NVML_ERROR_TIMEOUT => error.error_timeout, + c.NVML_ERROR_IRQ_ISSUE => error.error_irq_issue, + c.NVML_ERROR_LIBRARY_NOT_FOUND => error.error_library_not_found, + c.NVML_ERROR_FUNCTION_NOT_FOUND => error.error_function_not_found, + c.NVML_ERROR_CORRUPTED_INFOROM => error.error_corrupted_inforom, + c.NVML_ERROR_GPU_IS_LOST => error.error_gpu_is_lost, + c.NVML_ERROR_RESET_REQUIRED => error.error_reset_required, + c.NVML_ERROR_OPERATING_SYSTEM => error.error_operating_system, + c.NVML_ERROR_LIB_RM_VERSION_MISMATCH => error.error_lib_rm_version_mismatch, + c.NVML_ERROR_IN_USE => error.error_in_use, + c.NVML_ERROR_MEMORY => error.error_memory, + c.NVML_ERROR_NO_DATA => error.error_no_data, + c.NVML_ERROR_VGPU_ECC_NOT_SUPPORTED => error.error_vgpu_ecc_not_supported, + c.NVML_ERROR_INSUFFICIENT_RESOURCES => error.error_insufficient_resources, + c.NVML_ERROR_FREQ_NOT_SUPPORTED => error.error_freq_not_supported, + c.NVML_ERROR_ARGUMENT_VERSION_MISMATCH => error.error_argument_version_mismatch, + c.NVML_ERROR_DEPRECATED => error.error_deprecated, + c.NVML_ERROR_NOT_READY => error.error_not_ready, + c.NVML_ERROR_GPU_NOT_FOUND => error.error_gpu_not_found, + c.NVML_ERROR_INVALID_STATE => error.error_invalid_state, + c.NVML_ERROR_RESET_TYPE_NOT_SUPPORTED => error.error_reset_type_not_supported, + c.NVML_ERROR_UNKNOWN => error.error_unknown, + else => error.error_unknown, + }; +} diff --git a/bin/zml-smi/platforms/nvml/process.zig b/bin/zml-smi/platforms/nvml/process.zig new file mode 100644 index 00000000..5e18cdcb --- /dev/null +++ b/bin/zml-smi/platforms/nvml/process.zig @@ -0,0 +1,54 @@ +const std = @import("std"); +const Nvml = @import("nvml.zig"); +const pi = @import("zml-smi/info").process_info; + +pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); + +pub fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16, device_count: u32, last_seen_ts: []u64) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (0..device_count) |dev_idx| { + const handle = nvml.handleByIndex(@intCast(dev_idx)) catch continue; + const idx: u16 = @intCast(dev_idx + dev_offset); + + const compute = nvml.computeRunningProcesses(allocator, handle) catch &.{}; + defer if (compute.len > 0) allocator.free(compute); + collectFromQuery(allocator, back, idx, compute); + + const graphics = nvml.graphicsRunningProcesses(allocator, handle) catch &.{}; + defer if (graphics.len > 0) allocator.free(graphics); + collectFromQuery(allocator, back, idx, graphics); + + const last_ts = last_seen_ts[dev_idx]; + const utils = nvml.processUtilization(allocator, handle, last_ts) catch &.{}; + defer if (utils.len > 0) allocator.free(utils); + + for (utils) |sample| { + if (sample.smUtil > 100 or sample.timeStamp <= last_ts) { + continue; + } + + last_seen_ts[dev_idx] = @max(last_seen_ts[dev_idx], sample.timeStamp); + + for (back.items) |*entry| { + if (entry.pid == sample.pid and entry.device_idx == idx) { + entry.dev_util_percent = @intCast(sample.smUtil); + } + } + } + } + + list.swap(); +} + +fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u16, procs: []const Nvml.ProcessInfo_t) void { + for (procs) |gp| { + back.append(allocator, .{ + .pid = gp.pid, + .device_idx = dev_idx, + .dev_mem_kib = @intCast(gp.usedGpuMemory / 1024), + }) catch return; + } +} diff --git a/bin/zml-smi/platforms/oneapi/BUILD.bazel b/bin/zml-smi/platforms/oneapi/BUILD.bazel index c9096766..c0b27d84 100644 --- a/bin/zml-smi/platforms/oneapi/BUILD.bazel +++ b/bin/zml-smi/platforms/oneapi/BUILD.bazel @@ -3,9 +3,19 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "oneapi", import_name = "zml-smi/platforms/oneapi", - main = "OneApi.zig", - visibility = ["//bin/zml-smi:__subpackages__"], - deps = ["//bin/zml-smi/smi"], + main = "metrics.zig", + srcs = [ + "monitor.zig", + "oneapi.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:sysfs", + ], ) zig_test( diff --git a/bin/zml-smi/platforms/oneapi/OneApi.zig b/bin/zml-smi/platforms/oneapi/OneApi.zig deleted file mode 100644 index af8a1601..00000000 --- a/bin/zml-smi/platforms/oneapi/OneApi.zig +++ /dev/null @@ -1,481 +0,0 @@ -//! Intel GPUs, through sysfs: drm, hwmon, and the fdinfo of every DRM client. -const std = @import("std"); -const smi = @import("zml-smi"); - -const OneApi = @This(); - -platform: smi.Platform, -io: std.Io, -gpus: []Gpu, -proc: ?smi.sysfs.Dir, -/// Busy nanoseconds of each process, device and engine class (i915), or busy cycles of each process and device (xe). -busy: smi.Counters(EngineKey), -/// Total cycles of each process and device (xe). -cycles: smi.Counters(EngineKey), -/// The rows of this round's scan, in scratch. Null when the scan failed. -rows: ?[]const smi.ProcessInfo = null, - -const Gpu = struct { - /// The PCI device, /sys/class/drm/renderD/device. - dir: smi.sysfs.Dir, - /// The device or the bridge above it that carries its link. - link: smi.sysfs.Dir, - address: [12]u8, - hwmon: ?Hwmon, - energy: smi.Counter = .{}, - mem_total_bytes: ?u64, -}; - -/// The files chosen in init, read every poll. -const Hwmon = struct { - dir: smi.sysfs.Dir, - temperature: ?[]const u8, - energy: ?[]const u8, - power_limit: ?[]const u8, - fan: ?Fan, -}; - -/// A fan PWM, else a fan speed and its maximum. -const Fan = union(enum) { - pwm: []const u8, - speed: struct { input: []const u8, max: []const u8 }, - - /// The first PWM that reads, else the first speed that reads with a maximum above 0. - fn find(arena: std.mem.Allocator, dir: smi.sysfs.Dir) ?Fan { - var input: [16]u8 = undefined; - var max: [16]u8 = undefined; - for (1..16) |index| { - const pwm = std.fmt.bufPrint(&input, "pwm{d}", .{index}) catch unreachable; - _ = dir.int(u64, pwm) catch continue; - return .{ .pwm = arena.dupe(u8, pwm) catch return null }; - } - for (1..16) |index| { - const speed = std.fmt.bufPrint(&input, "fan{d}_input", .{index}) catch unreachable; - const limit = std.fmt.bufPrint(&max, "fan{d}_max", .{index}) catch unreachable; - _ = dir.int(u64, speed) catch continue; - if ((dir.int(u64, limit) catch 0) == 0) continue; - return .{ .speed = .{ .input = arena.dupe(u8, speed) catch return null, .max = arena.dupe(u8, limit) catch return null } }; - } - return null; - } - - fn percent(fan: Fan, dir: smi.sysfs.Dir) ?u64 { - switch (fan) { - .pwm => |file| return smi.hwmon.pwmPercent(dir.int(u64, file) catch return null), - .speed => |files| { - const input = dir.int(u64, files.input) catch return null; - const max = dir.int(u64, files.max) catch return null; - return if (max > 0) @min(input * 100 / max, 100) else null; - }, - } - } -}; - -const EngineKey = struct { pid: u32, device: u16, class: u64 }; - -const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit }; - -const intel = 0x8086; - -pub fn detect(io: std.Io) bool { - const drm = smi.sysfs.Dir.open(io, "/sys/class/drm") catch return false; - defer drm.close(); - var it = drm.iterate(); - while (it.next(io) catch null) |entry| { - const node = renderNode(entry.name) orelse continue; - const dir = openDevice(drm, node) catch continue; - defer dir.close(); - if ((dir.int(u16, "vendor") catch 0) == intel) return true; - } - return false; -} - -pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io) !OneApi { - const drm = try smi.sysfs.Dir.open(io, "/sys/class/drm"); - defer drm.close(); - - var nodes: std.ArrayList(u32) = .empty; - var it = drm.iterate(); - while (try it.next(io)) |entry| { - if (renderNode(entry.name)) |node| try nodes.append(arena, node); - } - std.mem.sort(u32, nodes.items, {}, std.sort.asc(u32)); - - const driver_version = driverVersion(arena, io); - var gpus: std.ArrayList(Gpu) = .empty; - var devices: std.ArrayList(smi.Device) = .empty; - for (nodes.items) |node| { - const dir = openDevice(drm, node) catch continue; - const gpu = openGpu(arena, dir) catch { - dir.close(); - continue; - }; - try gpus.append(arena, gpu); - try devices.append(arena, .{ .target = .oneapi, .name = try name(arena, io, gpu), .driver_version = driver_version }); - } - - return .{ - .platform = .{ .devices = devices.items, .two_readings = true, .vtable = &vtable }, - .io = io, - .gpus = gpus.items, - .proc = smi.sysfs.Dir.open(io, "/proc") catch null, - .busy = .init(gpa), - .cycles = .init(gpa), - }; -} - -fn renderNode(entry: []const u8) ?u32 { - if (!std.mem.startsWith(u8, entry, "renderD")) return null; - return std.fmt.parseInt(u32, entry["renderD".len..], 10) catch null; -} - -fn openDevice(drm: smi.sysfs.Dir, node: u32) !smi.sysfs.Dir { - var path: [32]u8 = undefined; - return drm.openDir(try std.fmt.bufPrint(&path, "renderD{d}/device", .{node})); -} - -fn openGpu(arena: std.mem.Allocator, dir: smi.sysfs.Dir) !Gpu { - if (try dir.int(u16, "vendor") != intel) return error.NotIntel; - - var buffer: [2048]u8 = undefined; - const slot = try dir.field("uevent", "PCI_SLOT_NAME", &buffer); - var address: [12]u8 = undefined; - if (slot.len != address.len) return error.NotFound; - @memcpy(&address, slot); - - return .{ - .dir = dir, - .link = try smi.pci.upstream(dir), - .address = address, - .hwmon = openHwmon(arena, dir), - .mem_total_bytes = largestBar(dir.read("resource", &buffer) catch ""), - }; -} - -fn openHwmon(arena: std.mem.Allocator, device: smi.sysfs.Dir) ?Hwmon { - var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const driver = device.readLink("driver", &buffer) catch null; - const dir = smi.hwmon.find(device, if (driver) |path| std.fs.path.basename(path) else null) catch return null; - return .{ - .dir = dir, - .temperature = smi.hwmon.resolve(arena, dir, &.{ - .{ .prefix = "temp", .label = "pkg" }, - .{ .prefix = "temp", .label = "gpu" }, - .{ .prefix = "temp2" }, - .{ .prefix = "temp1" }, - }), - .energy = smi.hwmon.resolve(arena, dir, &.{ - .{ .prefix = "energy", .label = "card" }, - .{ .prefix = "energy", .label = "pkg" }, - .{ .prefix = "energy1" }, - }), - .power_limit = smi.hwmon.resolve(arena, dir, &.{ .{ .prefix = "power1", .suffix = "cap" }, .{ .prefix = "power1", .suffix = "max" } }), - .fan = .find(arena, dir), - }; -} - -fn deinit(platform: *smi.Platform) void { - const self: *OneApi = @alignCast(@fieldParentPtr("platform", platform)); - for (self.gpus) |gpu| { - gpu.dir.close(); - gpu.link.close(); - if (gpu.hwmon) |hwmon| hwmon.dir.close(); - } - if (self.proc) |proc| proc.close(); - self.busy.deinit(); - self.cycles.deinit(); -} - -fn poll(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []smi.Sample) void { - const self: *OneApi = @alignCast(@fieldParentPtr("platform", platform)); - - for (self.gpus, samples) |*gpu, *sample| { - const fields = &sample.oneapi; - - fields.clock_graphics_mhz = gpu.dir.int(u64, "tile0/gt0/freq0/act_freq") catch gpu.dir.int(u64, "tile0/gt0/freq0/cur_freq") catch null; - fields.clock_graphics_max_mhz = gpu.dir.int(u64, "tile0/gt0/freq0/max_freq") catch null; - fields.mem_total_bytes = gpu.mem_total_bytes; - - const link = smi.pci.link(gpu.link, .current); - fields.pcie_link_gen = link.gen; - fields.pcie_link_width = link.width; - fields.pcie_bandwidth_mbps = link.mbps(); - - const hwmon = gpu.hwmon orelse continue; - // hwmon reports millidegrees, microwatts and microjoules. - if (hwmon.temperature) |file| fields.temperature = hwmon.dir.scaled(file, .milli, .one) catch null; - if (hwmon.power_limit) |file| fields.power_limit_mw = hwmon.dir.scaled(file, .micro, .milli) catch null; - if (hwmon.energy) |file| { - // Energy is a running total: its growth per second is microwatts. - if (hwmon.dir.int(u64, file)) |microjoules| { - if (gpu.energy.advance(self.io, microjoules)) |growth| fields.power_mw = smi.units.scale(growth.perSecond(), .micro, .milli); - } else |_| {} - } - if (hwmon.fan) |fan| fields.fan_speed_percent = fan.percent(hwmon.dir); - } - - self.scan(scratch, samples); -} - -fn processes(platform: *smi.Platform, _: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { - const self: *OneApi = @alignCast(@fieldParentPtr("platform", platform)); - for (self.rows orelse return error.Unavailable) |row| try out.append(row); -} - -/// One process on one device: its memory, and the growth of its busy time this round. -const Usage = struct { - row: smi.ProcessInfo, - busy: ?smi.Growth = null, - /// xe: the growth of the total cycles the busy cycles are a share of. - cycles: ?smi.Growth = null, -}; - -/// The busy time of every process on one device. -const DeviceUsage = struct { - busy: ?smi.Growth = null, - cycles: ?smi.Growth = null, - encoder: ?smi.Growth = null, - decoder: ?smi.Growth = null, - has_encoder: bool = false, - has_decoder: bool = false, -}; - -/// Reads every DRM client once: memory in use, and utilization through one counter per process. -fn scan(self: *OneApi, scratch: std.mem.Allocator, samples: []smi.Sample) void { - defer self.busy.sweep(); - defer self.cycles.sweep(); - self.rows = null; - const proc = self.proc orelse return; - - var usages: std.AutoArrayHashMapUnmanaged(struct { u32, u16 }, Usage) = .empty; - var engines: std.AutoArrayHashMapUnmanaged(EngineKey, Engine) = .empty; - // A client shared through dup() or fork() is counted once. - var clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .empty; - - var holders: smi.proc.FdHolders = .init(self.io, &.{"/dev/dri/"}); - defer holders.deinit(); - var buffer: [8192]u8 = undefined; - while (holders.next()) |holder| { - var path: [48]u8 = undefined; - const fdinfo_path = std.fmt.bufPrint(&path, "{d}/fdinfo/{d}", .{ holder.pid, holder.fd }) catch continue; - // The process may have closed it since: such a file is skipped. - const info = smi.fdinfo.read(proc, fdinfo_path, &buffer) catch continue; - const device = self.deviceOf(info.pdev orelse continue) orelse continue; - if (info.client_id) |id| { - if ((clients.getOrPut(scratch, .{ device, id }) catch continue).found_existing) continue; - } - - const usage = usages.getOrPutValue(scratch, .{ holder.pid, device }, .{ .row = .{ .pid = holder.pid, .device_idx = device } }) catch continue; - if (memoryBytes(&info)) |bytes| usage.value_ptr.row.dev_mem_kib = (usage.value_ptr.row.dev_mem_kib orelse 0) + bytes / 1024; - for (info.engines()) |engine| { - const xe = engine.total_cycles != null; - const key: EngineKey = .{ .pid = holder.pid, .device = device, .class = if (xe) 0 else std.hash.Wyhash.hash(0, engine.class) }; - const totals = (engines.getOrPutValue(scratch, key, .{ .kind = .of(engine.class) }) catch continue).value_ptr; - if (engine.total_cycles) |total| { - totals.busy += engine.cycles orelse 0; - totals.total = @max(totals.total orelse 0, total); - } else { - totals.busy += engine.busy_ns orelse 0; - } - } - } - - const devices = scratch.alloc(DeviceUsage, self.gpus.len) catch return; - @memset(devices, .{}); - var it = engines.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const usage = usages.getPtr(.{ key.pid, key.device }).?; - const device = &devices[key.device]; - device.has_encoder = device.has_encoder or entry.value_ptr.kind == .encoder; - device.has_decoder = device.has_decoder or entry.value_ptr.kind == .decoder; - - // Both counters advance at every reading, even the first. - const busy = self.busy.advance(key, self.io, entry.value_ptr.busy) catch continue; - const cycles = if (entry.value_ptr.total) |total| self.cycles.advance(key, self.io, total) catch continue else null; - const growth = busy orelse continue; - if (entry.value_ptr.total != null and cycles == null) continue; - smi.Growth.add(&usage.busy, growth); - smi.Growth.add(&device.busy, growth); - switch (entry.value_ptr.kind) { - .encoder => smi.Growth.add(&device.encoder, growth), - .decoder => smi.Growth.add(&device.decoder, growth), - .other => {}, - } - if (cycles) |total| { - usage.cycles = total; - if (device.cycles == null or total.amount > device.cycles.?.amount) device.cycles = total; - } - } - - const rows = scratch.alloc(smi.ProcessInfo, usages.count()) catch return; - for (rows, usages.values()) |*row, usage| { - row.* = usage.row; - row.dev_util_percent = percent(usage.busy, usage.cycles); - } - self.rows = rows; - - for (samples, devices, 0..) |*sample, device, index| { - const fields = &sample.oneapi; - fields.util_percent = percent(device.busy, device.cycles) orelse 0; - fields.encoder_util_percent = if (device.has_encoder) percent(device.encoder, null) orelse 0 else null; - fields.decoder_util_percent = if (device.has_decoder) percent(device.decoder, null) orelse 0 else null; - var mem_kib: u64 = 0; - for (rows) |row| { - if (row.device_idx == index) mem_kib += row.dev_mem_kib orelse 0; - } - fields.mem_used_bytes = mem_kib * 1024; - } -} - -/// The busy time of one engine class, summed over the clients of a process. -const Engine = struct { - busy: u64 = 0, - /// xe: the total cycles the busy cycles are counted against. - total: ?u64 = null, - kind: Kind, - - /// Encoder and decoder utilization count only the classes named so. - const Kind = enum { - encoder, - decoder, - other, - - fn of(class: []const u8) Kind { - if (std.ascii.findIgnoreCase(class, "encode") != null) return .encoder; - if (std.ascii.findIgnoreCase(class, "decode") != null) return .decoder; - return .other; - } - }; -}; - -/// Busy nanoseconds over the elapsed time (i915), or busy cycles over total cycles (xe). -fn percent(busy: ?smi.Growth, cycles: ?smi.Growth) ?u16 { - const growth = busy orelse return null; - return if (cycles) |total| growth.percentOf(total) else growth.percentOfTime(); -} - -fn deviceOf(self: *const OneApi, pdev: []const u8) ?u16 { - for (self.gpus, 0..) |gpu, index| { - if (std.mem.eql(u8, &gpu.address, pdev)) return @intCast(index); - } - return null; -} - -/// The memory of one client: its resident VRAM, else its VRAM, else what it holds resident, else in total. -fn memoryBytes(info: *const smi.fdinfo.Fdinfo) ?u64 { - var sums: [4]u64 = @splat(0); - for (info.regions()) |region| { - if (std.mem.startsWith(u8, region.name, "vram")) { - sums[0] += region.resident orelse 0; - sums[1] += region.total orelse 0; - } - sums[2] += region.resident orelse 0; - sums[3] += region.total orelse 0; - } - for (sums) |sum| { - if (sum > 0) return sum; - } - return null; -} - -/// Device memory: the largest BAR of the PCI resource file, if it is at least 256 MiB. -fn largestBar(resource: []const u8) ?u64 { - var largest: u64 = 0; - var lines = std.mem.tokenizeScalar(u8, resource, '\n'); - while (lines.next()) |line| { - var words = std.mem.tokenizeAny(u8, line, " \t"); - const start = smi.sysfs.parseInt(u64, words.next() orelse continue) catch continue; - const end = smi.sysfs.parseInt(u64, words.next() orelse continue) catch continue; - if (end > start) largest = @max(largest, end - start + 1); - } - return if (largest >= 256 * 1024 * 1024) largest else null; -} - -fn driverVersion(arena: std.mem.Allocator, io: std.Io) ?[]const u8 { - const module = smi.sysfs.Dir.open(io, "/sys/module") catch return null; - defer module.close(); - var buffer: [128]u8 = undefined; - for ([_][]const u8{ "xe/version", "xe/srcversion", "i915/version", "i915/srcversion" }) |file| { - const version = module.read(file, &buffer) catch continue; - if (version.len > 0) return arena.dupe(u8, version) catch null; - } - return null; -} - -/// The name lspci gives, else the device id. -fn name(arena: std.mem.Allocator, io: std.Io, gpu: Gpu) ![]const u8 { - if (std.process.run(arena, io, .{ - .argv = &.{ "lspci", "-Dnn", "-s", &gpu.address }, - .stdout_limit = .limited(4096), - .stderr_limit = .limited(1024), - })) |result| { - if (result.term == .exited and result.term.exited == 0) { - if (parseLspciName(result.stdout)) |lspci| return arena.dupe(u8, lspci); - } - } else |_| {} - - var buffer: [16]u8 = undefined; - return std.fmt.allocPrint(arena, "Intel GPU {s}", .{gpu.dir.read("device", &buffer) catch "unknown"}); -} - -fn parseLspciName(output: []const u8) ?[]const u8 { - var lines = std.mem.tokenizeScalar(u8, output, '\n'); - while (lines.next()) |line| { - const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace); - const header_end = std.mem.indexOf(u8, trimmed, "]: ") orelse continue; - const header = trimmed[0 .. header_end + 1]; - if (std.ascii.findIgnoreCase(header, "VGA") == null and std.ascii.findIgnoreCase(header, "Display") == null) continue; - - const rest = trimmed[header_end + "]: ".len ..]; - const end = std.mem.lastIndexOf(u8, rest, " [8086:") orelse std.mem.indexOf(u8, rest, " (rev ") orelse rest.len; - const device_name = std.mem.trim(u8, rest[0..end], &std.ascii.whitespace); - if (device_name.len == 0 or std.mem.startsWith(u8, device_name, "Intel Corporation Device ")) return null; - return device_name; - } - return null; -} - -test parseLspciName { - try std.testing.expectEqualStrings( - "Intel Corporation Battlemage G31 [Arc Pro B70]", - parseLspciName("0000:84:00.0 VGA compatible controller [0300]: Intel Corporation Battlemage G31 [Arc Pro B70] [8086:e223] (rev 04)\n").?, - ); - // A generic name is no better than the device id. - try std.testing.expectEqual(null, parseLspciName("0000:84:00.0 Display controller [0380]: Intel Corporation Device e223 [8086:e223] (rev 04)\n")); -} - -test largestBar { - try std.testing.expectEqual(34_359_738_368, largestBar( - \\0x0000048800000000 0x0000048fffffffff 0x000000000014220c - \\0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200 - \\0x0000000000000000 0x0000000000000000 0x0000000000000000 - )); - try std.testing.expectEqual(null, largestBar("0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200\n")); -} - -test memoryBytes { - const xe = smi.fdinfo.parse("drm-total-system:\t184 MiB\ndrm-total-vram0:\t4096 KiB\ndrm-resident-vram0:\t3072 KiB\n"); - try std.testing.expectEqual(3072 * 1024, memoryBytes(&xe)); - const system = smi.fdinfo.parse("drm-total-system0:\t2 MiB\n"); - try std.testing.expectEqual(2 * 1024 * 1024, memoryBytes(&system)); -} - -test Fan { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - const dir: smi.sysfs.Dir = .{ .handle = tmp.dir, .io = io }; - - // fan1 has no maximum, so the speed is fan2's. - try tmp.dir.writeFile(io, .{ .sub_path = "fan1_input", .data = "900\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "fan2_input", .data = "1500\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "fan2_max", .data = "3000\n" }); - try std.testing.expectEqual(50, Fan.find(arena.allocator(), dir).?.percent(dir)); - - // A PWM comes first, whatever its index. - try tmp.dir.writeFile(io, .{ .sub_path = "pwm3", .data = "255\n" }); - try std.testing.expectEqual(100, Fan.find(arena.allocator(), dir).?.percent(dir)); -} diff --git a/bin/zml-smi/platforms/oneapi/metrics.zig b/bin/zml-smi/platforms/oneapi/metrics.zig new file mode 100644 index 00000000..ae2a86a3 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/metrics.zig @@ -0,0 +1,134 @@ +const std = @import("std"); +const OneApi = @import("oneapi.zig"); +const device_info = @import("zml-smi/info").device_info; +const GpuInfo = device_info.GpuInfo; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const process = @import("process.zig"); + +pub fn start(collector: *Collector) !void { + const oneapi = try collector.arena.create(OneApi); + oneapi.* = try OneApi.init(collector.arena, collector.io); + + const count = oneapi.monitor.devices.len; + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + for (0..count) |i| { + const poll_arena = try collector.createPollArena(); + const dev = Device.open(oneapi, i, poll_arena) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + .power_limit_mw = dev.powerLimit() catch null, + .clock_graphics_max_mhz = dev.maxClockGraphics() catch null, + .mem_total_bytes = dev.memTotal() catch null, + .pcie_link_gen = dev.pcieLinkGen() catch null, + .pcie_link_width = dev.pcieLinkWidth() catch null, + .pcie_bandwidth_mbps = dev.pcieBandwidth() catch null, + }; + const info = try collector.addDevice(.{ .oneapi = .{ .values = .{ initial, initial } } }); + + try collector.spawnPoll(pollOnce, .{ poll_arena, &info.oneapi, dev }); + } + + const processes = try collector.createProcessList(); + try process.init(collector, processes, oneapi, dev_offset); +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); + +const Device = struct { + oneapi: *OneApi, + arena: *std.heap.ArenaAllocator, + handle: OneApi.Handle, + + fn open(oneapi: *OneApi, index: usize, arena: *std.heap.ArenaAllocator) !Device { + return .{ + .oneapi = oneapi, + .arena = arena, + .handle = try oneapi.handleByIndex(index), + }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + return self.oneapi.name(arena, self.handle); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + return self.oneapi.driverVersion(arena); + } + + pub fn powerUsage(self: Device) !u64 { + return self.oneapi.powerUsage(self.arena.allocator(), self.handle); + } + + pub fn temperature(self: Device) !u64 { + return self.oneapi.temperature(self.arena.allocator(), self.handle); + } + + pub fn fanSpeed(self: Device) !u64 { + return self.oneapi.fanSpeed(self.arena.allocator(), self.handle); + } + + pub fn gpuUtil(self: Device) !u64 { + return self.oneapi.gpuUtil(self.arena.allocator(), self.handle); + } + + pub fn encoderUtil(self: Device) !u64 { + return self.oneapi.encoderUtil(self.arena.allocator(), self.handle); + } + + pub fn decoderUtil(self: Device) !u64 { + return self.oneapi.decoderUtil(self.arena.allocator(), self.handle); + } + + pub fn powerLimit(self: Device) !u64 { + return self.oneapi.powerLimit(self.arena.allocator(), self.handle); + } + + pub fn clockGraphics(self: Device) !u64 { + return self.oneapi.clockGraphics(self.arena.allocator(), self.handle); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return self.oneapi.maxClockGraphics(self.arena.allocator(), self.handle); + } + + pub fn memUsed(self: Device) !u64 { + return self.oneapi.memUsed(self.arena.allocator(), self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.oneapi.memTotal(self.arena.allocator(), self.handle); + } + + pub fn pcieLinkGen(self: Device) !u64 { + return self.oneapi.pcieLinkGen(self.arena.allocator(), self.handle); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return self.oneapi.pcieLinkWidth(self.arena.allocator(), self.handle); + } + + pub fn pcieBandwidth(self: Device) !u64 { + return self.oneapi.pcieBandwidth(self.arena.allocator(), self.handle); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, + .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig new file mode 100644 index 00000000..dd5f97a9 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/monitor.zig @@ -0,0 +1,877 @@ +const std = @import("std"); +const smi_sysfs = @import("zml-smi/sysfs"); +const Monitor = @This(); + +pub const bus_device_identifier_len = "0000:00:00.0".len; +pub const Handle = enum(u32) { _ }; + +const Device = struct { + render_idx: usize, + drm_path: []const u8, + dev_path: []const u8, + pcie_path: []const u8, + hwmon_path: ?[]const u8, + bus_device_identifier: ?[bus_device_identifier_len]u8, + activity_prev: ?EngineSample = null, + encoder_prev: ?EngineSample = null, + decoder_prev: ?EngineSample = null, + energy_prev: ?EnergySample = null, +}; + +const EnergySample = struct { + micro_joules: u64, + timestamp_ns: u64, +}; + +pub const EngineSample = struct { + engine_ns: u64, + timestamp_ns: u64, +}; + +const DeviceSample = struct { + mem_kib: u64 = 0, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +pub const ProcessSample = struct { + pid: u32, + device_idx: u16, + mem_kib: ?u64 = null, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +pub const ProcessUsage = std.AutoHashMapUnmanaged(u64, ProcessSample); +pub const ProcessPrevious = std.AutoHashMapUnmanaged(u64, EngineSample); +const DeviceUsage = std.AutoHashMapUnmanaged(u16, DeviceSample); + +allocator: std.mem.Allocator, +io: std.Io, +devices: []Device, + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !Monitor { + var monitor: Monitor = .{ + .allocator = allocator, + .io = io, + .devices = try discoverIntelDevices(allocator, io), + }; + try monitor.seedDeviceMetrics(); + return monitor; +} + +pub fn deviceId(self: *const Monitor, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { + const dev = try self.device(handle); + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/device", .{dev.dev_path}); + const raw = try smi_sysfs.readString(allocator, self.io, path); + return std.mem.trim(u8, raw, &std.ascii.whitespace); +} + +inline fn seedDeviceMetrics(self: *Monitor) !void { + var device_usage = collectDeviceUsage(self.allocator, self.io, self.devices) catch DeviceUsage{}; + defer device_usage.deinit(self.allocator); + + for (self.devices, 0..) |*dev, idx| { + dev.energy_prev = readEnergy(self.allocator, self.io, dev) catch null; + if (device_usage.get(@intCast(idx))) |sample| { + dev.activity_prev = sample.engine; + dev.encoder_prev = sample.encoder; + dev.decoder_prev = sample.decoder; + } + } +} + +pub fn saveProcessPrevious(allocator: std.mem.Allocator, previous: *ProcessPrevious, usage: *const ProcessUsage) !void { + previous.clearRetainingCapacity(); + try previous.ensureTotalCapacity(allocator, @intCast(usage.count())); + + var it = usage.iterator(); + while (it.next()) |entry| { + if (entry.value_ptr.engine) |engine| { + previous.putAssumeCapacity(entry.key_ptr.*, engine); + } + } +} + +inline fn discoverIntelDevices(allocator: std.mem.Allocator, io: std.Io) ![]Device { + var devices: std.ArrayList(Device) = .empty; + errdefer devices.deinit(allocator); + + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch |err| switch (err) { + error.FileNotFound => return &.{}, + else => |e| return e, + }; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + const render_idx = parseRenderIndex(entry.name) orelse continue; + const dev = openDevice(allocator, io, render_idx) catch continue; + try devices.append(allocator, dev); + } + + std.mem.sort(Device, devices.items, {}, struct { + fn lessThan(_: void, lhs: Device, rhs: Device) bool { + return lhs.render_idx < rhs.render_idx; + } + }.lessThan); + return devices.toOwnedSlice(allocator); +} + +inline fn openDevice(allocator: std.mem.Allocator, io: std.Io, render_idx: usize) !Device { + const drm_path = try std.fmt.allocPrint(allocator, "/sys/class/drm/renderD{d}", .{render_idx}); + errdefer allocator.free(drm_path); + + const dev_path = try std.fmt.allocPrint(allocator, "{s}/device", .{drm_path}); + errdefer allocator.free(dev_path); + + var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = try std.fmt.bufPrint(&vendor_path_buf, "{s}/vendor", .{dev_path}); + const vendor = try smi_sysfs.readString(allocator, io, vendor_path); + if (!std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), "0x8086")) { + return error.not_intel; + } + + const pcie_path = try findPcieLinkPath(allocator, io, dev_path); + errdefer if (pcie_path.ptr != dev_path.ptr) allocator.free(pcie_path); + + return .{ + .render_idx = render_idx, + .drm_path = drm_path, + .dev_path = dev_path, + .pcie_path = pcie_path, + .hwmon_path = findHwmon(allocator, io, dev_path) catch null, + .bus_device_identifier = readBusDeviceId(allocator, io, dev_path) catch null, + }; +} + +pub fn device(self: *const Monitor, handle: Handle) !*const Device { + const idx = @backingInt(handle); + if (idx >= self.devices.len) return error.NotFound; + return &self.devices[idx]; +} + +pub fn readTemperature(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const milli_c = readHwmonInput(allocator, io, hwmon, "temp", "pkg") catch + readHwmonInput(allocator, io, hwmon, "temp", "gpu") catch + readNumberedInput(allocator, io, hwmon, "temp", 2) catch + try readNumberedInput(allocator, io, hwmon, "temp", 1); + return (milli_c + 500) / 1000; +} + +pub fn readEnergy(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !EnergySample { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const micro_joules = readHwmonInput(allocator, io, hwmon, "energy", "card") catch + readHwmonInput(allocator, io, hwmon, "energy", "pkg") catch + try readNumberedInput(allocator, io, hwmon, "energy", 1); + return .{ + .micro_joules = micro_joules, + .timestamp_ns = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds), + }; +} +pub fn readPowerLimit(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const microwatts = readNumberedSuffix(allocator, io, hwmon, "power", 1, "cap") catch + try readNumberedSuffix(allocator, io, hwmon, "power", 1, "max"); + return microwatts / 1000; +} + +pub fn readFanSpeed(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + return readPwmFanSpeed(allocator, io, hwmon) catch readRpmFanSpeedPercent(allocator, io, hwmon); +} + +pub fn readClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + return readFreq(allocator, io, dev, "act_freq") catch readFreq(allocator, io, dev, "cur_freq"); +} + +pub fn readMaxClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + return readFreq(allocator, io, dev, "max_freq"); +} + +pub fn readMemTotal(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/resource", .{dev.dev_path}); + return readLargestResource(allocator, io, path); +} + +pub fn readPcieLinkGen(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_speed", .{dev.pcie_path}); + const raw = try smi_sysfs.readString(allocator, io, path); + return pcieGenFromSpeed(raw); +} + +pub fn readPcieLinkWidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_width", .{dev.pcie_path}); + return smi_sysfs.readInt(allocator, io, path); +} + +pub fn readPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const gen = try readPcieLinkGen(allocator, io, dev); + const width = try readPcieLinkWidth(allocator, io, dev); + return pcieBandwidthFromLink(gen, width) orelse error.NotFound; +} + +pub fn collectDeviceUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !DeviceUsage { + var processes = try collectProcessUsage(allocator, io, devices); + errdefer processes.deinit(allocator); + + var result: DeviceUsage = .{}; + errdefer result.deinit(allocator); + + var it = processes.iterator(); + while (it.next()) |entry| { + const sample = entry.value_ptr.*; + const gop = try result.getOrPut(allocator, sample.device_idx); + if (!gop.found_existing) gop.value_ptr.* = .{}; + if (sample.mem_kib) |mem| gop.value_ptr.mem_kib += mem; + addEngineSample(&gop.value_ptr.engine, sample.engine); + addEngineSample(&gop.value_ptr.encoder, sample.encoder); + addEngineSample(&gop.value_ptr.decoder, sample.decoder); + } + + return result; +} + +pub fn collectProcessUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !ProcessUsage { + var result: ProcessUsage = .{}; + errdefer result.deinit(allocator); + + var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; + defer seen_device_clients.deinit(allocator); + + var proc_dir = try std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }); + defer proc_dir.close(io); + + const now_ns: u64 = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); + var proc_it = proc_dir.iterate(); + while (proc_it.next(io) catch null) |proc_entry| { + const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; + + var fdinfo_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const fdinfo_path = std.fmt.bufPrint(&fdinfo_path_buf, "/proc/{d}/fdinfo", .{pid}) catch continue; + var fdinfo_dir = std.Io.Dir.openDirAbsolute(io, fdinfo_path, .{ .iterate = true }) catch continue; + defer fdinfo_dir.close(io); + + var fd_it = fdinfo_dir.iterate(); + while (fd_it.next(io) catch null) |fd_entry| { + var file_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const file_path = std.fmt.bufPrint(&file_path_buf, "{s}/{s}", .{ fdinfo_path, fd_entry.name }) catch continue; + const parsed = parseFdinfoFile(io, file_path, now_ns) catch continue; + const bus_device_identifier = parsed.bus_device_identifier orelse continue; + const dev_idx = matchDevice(devices, bus_device_identifier) orelse continue; + + // Skip fd if DRM client has been seen for this device. + // Summing after dup() / fork() over-reports memory and compute. + if (parsed.drm_client_id) |cid| { + if ((try seen_device_clients.getOrPut(allocator, .{ dev_idx, cid })).found_existing) continue; + } + + const key = processKey(pid, dev_idx); + const gop = try result.getOrPut(allocator, key); + if (!gop.found_existing) { + gop.value_ptr.* = .{ .pid = pid, .device_idx = dev_idx }; + } + mergeProcessSample(gop.value_ptr, parsed.sample); + } + } + + return result; +} + +pub fn processUtil(previous: ?EngineSample, current: ?EngineSample) ?u16 { + const prev = previous orelse return null; + const cur = current orelse return null; + if (cur.timestamp_ns <= prev.timestamp_ns or cur.engine_ns < prev.engine_ns) return null; + const engine_delta = cur.engine_ns - prev.engine_ns; + const time_delta = cur.timestamp_ns - prev.timestamp_ns; + if (time_delta == 0) return null; + const pct = @min(engine_delta * 100 / time_delta, 100); + return @intCast(if (pct == 0 and engine_delta > 0) 0 else pct); +} + +pub fn powerMilliwatts(previous: ?EnergySample, current: ?EnergySample) ?u64 { + const prev = previous orelse return null; + const cur = current orelse return null; + return milliwattsFromEnergyDelta(prev.micro_joules, cur.micro_joules, prev.timestamp_ns, cur.timestamp_ns); +} + +inline fn milliwattsFromEnergyDelta(prev_uj: u64, cur_uj: u64, prev_ns: u64, cur_ns: u64) ?u64 { + if (cur_uj < prev_uj or cur_ns <= prev_ns) return null; + const energy_delta = cur_uj - prev_uj; + const time_delta = cur_ns - prev_ns; + if (time_delta == 0) return null; + return energy_delta * 1_000_000 / time_delta; +} + +inline fn readFreq(allocator: std.mem.Allocator, io: std.Io, dev: *const Device, file: []const u8) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/tile0/gt0/freq0/{s}", .{ dev.dev_path, file }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn parseRenderIndex(name: []const u8) ?usize { + if (!std.mem.startsWith(u8, name, "renderD")) return null; + return std.fmt.parseInt(usize, name["renderD".len..], 10) catch null; +} + +inline fn findHwmon(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !?[]const u8 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const hwmon_root = try std.fmt.bufPrint(&path_buf, "{s}/hwmon", .{dev_path}); + var dir = std.Io.Dir.openDirAbsolute(io, hwmon_root, .{ .iterate = true }) catch return null; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (!std.mem.startsWith(u8, entry.name, "hwmon")) continue; + const candidate = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ hwmon_root, entry.name }); + var name_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const name_path = try std.fmt.bufPrint(&name_buf, "{s}/name", .{candidate}); + const name = smi_sysfs.readString(allocator, io, name_path) catch return candidate; + const trimmed = std.mem.trim(u8, name, &std.ascii.whitespace); + if (std.mem.eql(u8, trimmed, "xe") or std.mem.eql(u8, trimmed, "i915")) return candidate; + } + return null; +} + +inline fn readBusDeviceId(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![bus_device_identifier_len]u8 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/uevent", .{dev_path}); + const raw = try smi_sysfs.readFieldString(allocator, io, path, "PCI_SLOT_NAME="); + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (trimmed.len != bus_device_identifier_len) return error.NotFound; + var out: [bus_device_identifier_len]u8 = undefined; + @memcpy(&out, trimmed); + return out; +} + +inline fn findPcieLinkPath(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![]const u8 { + var candidate = dev_path; + var best_path: ?[]const u8 = null; + + var best_bandwidth: u64 = 0; + var depth: usize = 0; + while (depth < 8) : (depth += 1) { + if (!try isIntelPciDevice(allocator, io, candidate)) break; + + const bandwidth = readMaxPcieBandwidth(allocator, io, candidate) catch 0; + if (bandwidth > best_bandwidth) { + best_path = candidate; + best_bandwidth = bandwidth; + } + + const parent = try std.fmt.allocPrint(allocator, "{s}/..", .{candidate}); + candidate = parent; + } + + return best_path orelse dev_path; +} + +inline fn isIntelPciDevice(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !bool { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = try std.fmt.bufPrint(&path_buf, "{s}/vendor", .{dev_path}); + const vendor_raw = smi_sysfs.readString(allocator, io, vendor_path) catch return false; + + const vendor = std.mem.trim(u8, vendor_raw, &std.ascii.whitespace); + return std.mem.eql(u8, vendor, "0x8086"); +} + +inline fn readMaxPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !u64 { + var speed_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const speed_path = try std.fmt.bufPrint(&speed_path_buf, "{s}/max_link_speed", .{dev_path}); + const raw_speed = try smi_sysfs.readString(allocator, io, speed_path); + + var width_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const width_path = try std.fmt.bufPrint(&width_path_buf, "{s}/max_link_width", .{dev_path}); + const width = try smi_sysfs.readInt(allocator, io, width_path); + const gen = try pcieGenFromSpeed(raw_speed); + return pcieBandwidthFromLink(gen, width) orelse error.NotFound; +} + +inline fn readHwmonInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, label: []const u8) !u64 { + for (1..16) |idx| { + var label_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const label_path = try std.fmt.bufPrint(&label_path_buf, "{s}/{s}{d}_label", .{ hwmon, prefix, idx }); + const raw = smi_sysfs.readString(allocator, io, label_path) catch continue; + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (!std.ascii.eqlIgnoreCase(trimmed, label)) continue; + return readNumberedInput(allocator, io, hwmon, prefix, idx); + } + return error.NotFound; +} + +inline fn readNumberedInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { + return readNumberedSuffix(allocator, io, hwmon, prefix, idx, "input"); +} + +inline fn readNumberedSuffix(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize, suffix: []const u8) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}_{s}", .{ hwmon, prefix, idx, suffix }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn readNumberedRaw(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}", .{ hwmon, prefix, idx }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn readPwmFanSpeed(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { + for (1..16) |idx| { + const pwm = readNumberedRaw(allocator, io, hwmon, "pwm", idx) catch continue; + return pwmToPercent(pwm); + } + return error.NotFound; +} + +inline fn readRpmFanSpeedPercent(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { + for (1..16) |idx| { + const input = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "input") catch continue; + const max = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "max") catch continue; + if (max == 0) continue; + return @min(input * 100 / max, 100); + } + return error.NotFound; +} + +inline fn pwmToPercent(pwm: u64) u64 { + return @min((@min(pwm, 255) * 100 + 127) / 255, 100); +} + +inline fn readLargestResource(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + var line_writer: std.Io.Writer.Allocating = .init(allocator); + defer line_writer.deinit(); + + var largest: u64 = 0; + while (true) { + line_writer.clearRetainingCapacity(); + _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch |err| switch (err) { + error.EndOfStream => break, + else => |e| return e, + }; + reader.interface.toss(1); + largest = @max(largest, resourceLineSize(line_writer.written()) orelse 0); + } + if (largest == 0) return error.NotFound; + return largest; +} + +inline fn resourceLineSize(line: []const u8) ?u64 { + var it = std.mem.tokenizeAny(u8, line, " \t"); + const base = parseHex(it.next() orelse return null) orelse return null; + const end = parseHex(it.next() orelse return null) orelse return null; + if (end < base) return null; + const size = end - base + 1; + return if (size >= 256 * 1024 * 1024) size else null; +} + +inline fn parseHex(raw: []const u8) ?u64 { + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + const digits = if (std.mem.startsWith(u8, trimmed, "0x")) trimmed[2..] else trimmed; + return std.fmt.parseInt(u64, digits, 16) catch null; +} + +inline fn pcieGenFromSpeed(raw: []const u8) !u64 { + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (std.mem.startsWith(u8, trimmed, "2.5")) return 1; + if (std.mem.startsWith(u8, trimmed, "5.0") or std.mem.startsWith(u8, trimmed, "5 ")) return 2; + if (std.mem.startsWith(u8, trimmed, "8.0") or std.mem.startsWith(u8, trimmed, "8 ")) return 3; + if (std.mem.startsWith(u8, trimmed, "16.0") or std.mem.startsWith(u8, trimmed, "16 ")) return 4; + if (std.mem.startsWith(u8, trimmed, "32.0") or std.mem.startsWith(u8, trimmed, "32 ")) return 5; + if (std.mem.startsWith(u8, trimmed, "64.0") or std.mem.startsWith(u8, trimmed, "64 ")) return 6; + return error.NotFound; +} + +inline fn pcieBandwidthFromLink(gen: u64, width: u64) ?u64 { + const lane_mb_s: u64 = switch (gen) { + 1 => 250, + 2 => 500, + 3 => 985, + 4 => 1969, + 5 => 3938, + 6 => 7563, + else => return null, + }; + return lane_mb_s * width; +} + +const FdinfoSample = struct { + mem_kib: ?u64 = null, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +const ParsedFdinfo = struct { + bus_device_identifier: ?[bus_device_identifier_len]u8 = null, + drm_client_id: ?u64 = null, + sample: FdinfoSample = .{}, + resident_vram_kib: u64 = 0, + total_vram_kib: u64 = 0, + resident_memory_kib: u64 = 0, + total_memory_kib: u64 = 0, + cycle_counter: u64 = 0, + total_cycle_counter: u64 = 0, + engine_time_seen: bool = false, +}; + +inline fn parseFdinfoFile(io: std.Io, path: []const u8, timestamp_ns: u64) !ParsedFdinfo { + var read_buf: [8192]u8 = undefined; + const data = std.Io.Dir.readFile(.cwd(), io, path, &read_buf) catch return error.FileUnavailable; + + var parsed: ParsedFdinfo = .{}; + + var lines = std.mem.tokenizeScalar(u8, data, '\n'); + while (lines.next()) |line| { + parseFdinfoLine(&parsed, line, timestamp_ns); + } + + finishFdinfo(&parsed); + return parsed; +} + +inline fn finishFdinfo(self: *ParsedFdinfo) void { + if (self.resident_vram_kib > 0) { + self.sample.mem_kib = self.resident_vram_kib; + } else if (self.total_vram_kib > 0) { + self.sample.mem_kib = self.total_vram_kib; + } else if (self.resident_memory_kib > 0) { + self.sample.mem_kib = self.resident_memory_kib; + } else if (self.total_memory_kib > 0) { + self.sample.mem_kib = self.total_memory_kib; + } + + if (!self.engine_time_seen and self.total_cycle_counter > 0) { + self.sample.engine = .{ + .engine_ns = self.cycle_counter, + .timestamp_ns = self.total_cycle_counter, + }; + } +} + +inline fn parseFdinfoLine(parsed: *ParsedFdinfo, line: []const u8, timestamp_ns: u64) void { + if (std.mem.startsWith(u8, line, "drm-pdev:")) { + parsed.bus_device_identifier = parseBdf(line["drm-pdev:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-client-id:")) { + parsed.drm_client_id = firstInt(line["drm-client-id:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "vram mem:")) { + parsed.total_vram_kib += memoryKiB(line["vram mem:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-cycles-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + parsed.cycle_counter += firstInt(line[colon + 1 ..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-total-cycles-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + parsed.total_cycle_counter = @max(parsed.total_cycle_counter, firstInt(line[colon + 1 ..])); + return; + } + + if (std.mem.startsWith(u8, line, "drm-engine-capacity-")) { + return; + } + + if (std.mem.startsWith(u8, line, "drm-engine-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + const engine_label = line["drm-engine-".len..colon]; + const value = firstInt(line[colon + 1 ..]); + parsed.engine_time_seen = true; + addEngineTime(&parsed.sample.engine, value, timestamp_ns); + if (containsAsciiIgnoreCase(engine_label, "encode")) { + addEngineTime(&parsed.sample.encoder, value, timestamp_ns); + } else if (containsAsciiIgnoreCase(engine_label, "decode")) { + addEngineTime(&parsed.sample.decoder, value, timestamp_ns); + } + return; + } + + if (std.mem.startsWith(u8, line, "drm-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + const key = line[0..colon]; + const value = memoryKiB(line[colon + 1 ..]); + if (value == 0) return; + + if (std.mem.startsWith(u8, key, "drm-resident-vram")) { + parsed.resident_vram_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-total-vram") or std.mem.startsWith(u8, key, "drm-memory-vram")) { + parsed.total_vram_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-resident-")) { + parsed.resident_memory_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-total-")) { + parsed.total_memory_kib += value; + return; + } + if (std.mem.eql(u8, key, "drm-resident-memory")) { + parsed.resident_memory_kib += value; + return; + } + if (std.mem.eql(u8, key, "drm-total-memory")) { + parsed.total_memory_kib += value; + return; + } + } +} + +fn mergeProcessSample(dst: *ProcessSample, src: FdinfoSample) void { + if (src.mem_kib) |mem| dst.mem_kib = (dst.mem_kib orelse 0) + mem; + addEngineSample(&dst.engine, src.engine); + addEngineSample(&dst.encoder, src.encoder); + addEngineSample(&dst.decoder, src.decoder); +} + +fn addEngineSample(dst: *?EngineSample, src: ?EngineSample) void { + if (src) |engine| addEngineTime(dst, engine.engine_ns, engine.timestamp_ns); +} + +fn addEngineTime(dst: *?EngineSample, engine_ns: u64, timestamp_ns: u64) void { + const current = dst.* orelse EngineSample{ .engine_ns = 0, .timestamp_ns = timestamp_ns }; + dst.* = .{ + .engine_ns = current.engine_ns + engine_ns, + .timestamp_ns = timestamp_ns, + }; +} + +fn matchDevice(devices: []const Device, bus_device_identifier: [bus_device_identifier_len]u8) ?u16 { + for (devices, 0..) |dev, idx| { + if (dev.bus_device_identifier) |device_bus_device_identifier| { + if (std.mem.eql(u8, &device_bus_device_identifier, &bus_device_identifier)) return @intCast(idx); + } + } + return null; +} + +fn processKey(pid: u32, device_idx: u16) u64 { + return (@as(u64, device_idx) << 32) | pid; +} + +pub const BdfParts = struct { + domain: u32, + bus: u32, + device: u32, + function: u32, +}; + +pub fn formatBdf(parts: BdfParts) [bus_device_identifier_len]u8 { + var buf: [bus_device_identifier_len]u8 = undefined; + _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ + parts.domain, + parts.bus, + parts.device, + parts.function, + }) catch unreachable; + return buf; +} + +inline fn parseBdf(raw: []const u8) ?[bus_device_identifier_len]u8 { + const trimmed = std.mem.trim(u8, raw, " \t\n"); + var out: [bus_device_identifier_len]u8 = undefined; + if (trimmed.len == bus_device_identifier_len) { + @memcpy(&out, trimmed[0..bus_device_identifier_len]); + return out; + } + if (trimmed.len == "00:00.0".len) { + _ = std.fmt.bufPrint(&out, "0000:{s}", .{trimmed}) catch return null; + return out; + } + return null; +} + +inline fn firstInt(raw: []const u8) u64 { + var iter = std.mem.tokenizeAny(u8, raw, " \t"); + return std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch 0; +} + +inline fn memoryKiB(raw: []const u8) u64 { + var iter = std.mem.tokenizeAny(u8, raw, " \t"); + const value = std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch return 0; + const unit = iter.next() orelse return value; + if (std.ascii.eqlIgnoreCase(unit, "KiB")) return value; + if (std.ascii.eqlIgnoreCase(unit, "MiB")) return value * 1024; + if (std.ascii.eqlIgnoreCase(unit, "GiB")) return value * 1024 * 1024; + if (std.ascii.eqlIgnoreCase(unit, "B")) return value / 1024; + return value; +} + +inline fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { + if (needle.len == 0) return true; + if (needle.len > haystack.len) return false; + for (0..haystack.len - needle.len + 1) |i| { + if (std.ascii.eqlIgnoreCase(haystack[i .. i + needle.len], needle)) return true; + } + return false; +} + +test "oneAPI fdinfo parser" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-client-id:\t42", 10); + parseFdinfoLine(&parsed, "drm-total-memory:\t2048 KiB", 10); + parseFdinfoLine(&parsed, "drm-engine-render:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-compute:\t50 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expect(parsed.bus_device_identifier != null); + try std.testing.expectEqualSlices(u8, "0000:03:00.0", &parsed.bus_device_identifier.?); + try std.testing.expectEqual(@as(?u64, 42), parsed.drm_client_id); + try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); + try std.testing.expectEqual(@as(u64, 150), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); +} + +test "oneAPI fdinfo parser classifies explicit media engines" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-engine-video-decode:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-media-encode:\t50 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-VIDEO-DECODE1:\t25 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(u64, 50), parsed.sample.encoder.?.engine_ns); + try std.testing.expectEqual(@as(u64, 125), parsed.sample.decoder.?.engine_ns); +} + +test "oneAPI fdinfo parser leaves generic video engines unclassified" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-engine-video:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-vcs0:\t50 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-vecs0:\t25 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); +} + +test "oneAPI fdinfo parser cycle counters" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-cycles-rcs:\t10", 10); + parseFdinfoLine(&parsed, "drm-total-cycles-rcs:\t100", 10); + parseFdinfoLine(&parsed, "drm-engine-capacity-vcs:\t2", 10); + parseFdinfoLine(&parsed, "drm-cycles-ccs:\t30", 10); + parseFdinfoLine(&parsed, "drm-total-cycles-ccs:\t100", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 40), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(u64, 100), parsed.sample.engine.?.timestamp_ns); +} + +test "oneAPI fdinfo parser xe vram regions" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-total-system:\t184 MiB", 10); + parseFdinfoLine(&parsed, "drm-total-vram0:\t4096 KiB", 10); + parseFdinfoLine(&parsed, "drm-resident-vram0:\t3072 KiB", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(?u64, 3072), parsed.sample.mem_kib); +} + +test "oneAPI fdinfo parser falls back to total vram" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-total-vram0:\t2 MiB", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); +} + +test "oneAPI DRM clients dedup per device" { + var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; + defer seen_device_clients.deinit(std.testing.allocator); + + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); + try std.testing.expectEqual(true, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 1, 42 })).found_existing); + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 43 })).found_existing); +} + +test "oneAPI process utilization delta" { + try std.testing.expectEqual(@as(?u16, 50), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 600, .timestamp_ns = 2000 })); + try std.testing.expectEqual(@as(?u16, 1), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 101, .timestamp_ns = 2000 })); + try std.testing.expectEqual(@as(?u16, null), processUtil(null, .{ .engine_ns = 600, .timestamp_ns = 2000 })); +} + +test "oneAPI PWM fan speed conversion" { + try std.testing.expectEqual(@as(u64, 0), pwmToPercent(0)); + try std.testing.expectEqual(@as(u64, 50), pwmToPercent(128)); + try std.testing.expectEqual(@as(u64, 100), pwmToPercent(255)); + try std.testing.expectEqual(@as(u64, 100), pwmToPercent(300)); +} + +test "oneAPI process previous keeps engine samples" { + var usage: ProcessUsage = .{}; + defer usage.deinit(std.testing.allocator); + + const engine_key = processKey(10, 1); + const mem_only_key = processKey(20, 1); + try usage.put(std.testing.allocator, engine_key, .{ + .pid = 10, + .device_idx = 1, + .engine = .{ .engine_ns = 100, .timestamp_ns = 1000 }, + }); + try usage.put(std.testing.allocator, mem_only_key, .{ + .pid = 20, + .device_idx = 1, + .mem_kib = 2048, + }); + + var previous: ProcessPrevious = .{}; + defer previous.deinit(std.testing.allocator); + + try saveProcessPrevious(std.testing.allocator, &previous, &usage); + + try std.testing.expectEqual(@as(u32, 1), previous.count()); + try std.testing.expectEqual(@as(?EngineSample, .{ .engine_ns = 100, .timestamp_ns = 1000 }), previous.get(engine_key)); + try std.testing.expectEqual(@as(?EngineSample, null), previous.get(mem_only_key)); +} + +test "oneAPI power delta from hwmon energy" { + try std.testing.expectEqual(@as(?u64, 20_000), milliwattsFromEnergyDelta(1_000, 11_000, 1_000, 501_000)); + try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(11_000, 1_000, 1_000, 501_000)); + try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(1_000, 11_000, 501_000, 1_000)); +} + +test "oneAPI resource line chooses large memory BARs" { + try std.testing.expectEqual(@as(?u64, 34_359_738_368), resourceLineSize("0x0000048800000000 0x0000048fffffffff 0x000000000014220c")); + try std.testing.expectEqual(@as(?u64, null), resourceLineSize("0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200")); +} + +test "oneAPI PCIe speed maps to generation" { + try std.testing.expectEqual(@as(u64, 1), try pcieGenFromSpeed("2.5 GT/s PCIe")); + try std.testing.expectEqual(@as(u64, 4), try pcieGenFromSpeed("16.0 GT/s PCIe")); + try std.testing.expectError(error.NotFound, pcieGenFromSpeed("Unknown")); +} + +test "oneAPI PCIe bandwidth derives from generation and width" { + try std.testing.expectEqual(@as(?u64, 250), pcieBandwidthFromLink(1, 1)); + try std.testing.expectEqual(@as(?u64, 63_008), pcieBandwidthFromLink(5, 16)); + try std.testing.expectEqual(@as(?u64, null), pcieBandwidthFromLink(0, 16)); +} diff --git a/bin/zml-smi/platforms/oneapi/oneapi.zig b/bin/zml-smi/platforms/oneapi/oneapi.zig new file mode 100644 index 00000000..2e1e5deb --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/oneapi.zig @@ -0,0 +1,248 @@ +const std = @import("std"); +const Monitor = @import("monitor.zig"); +const smi_sysfs = @import("zml-smi/sysfs"); + +const OneApi = @This(); + +pub const Handle = Monitor.Handle; + +monitor: Monitor, +io: std.Io, + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !OneApi { + return .{ + .monitor = try Monitor.init(allocator, io), + .io = io, + }; +} + +pub fn handleByIndex(self: *const OneApi, device_id: usize) !Handle { + if (device_id >= self.monitor.devices.len) return error.NotFound; + return @fromBackingInt(@intCast(@as(u32, @intCast(device_id)))); +} + +pub fn name(self: *const OneApi, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { + if (self.monitor.device(handle)) |dev| { + if (dev.bus_device_identifier) |bdf| { + if (try lspciName(allocator, self.io, bdf[0..])) |display_name| { + return display_name; + } + } + } else |_| {} + + const id = try self.monitor.deviceId(allocator, handle); + return try std.fmt.allocPrint(allocator, "Intel GPU {s}", .{id}); +} + +fn lspciName(allocator: std.mem.Allocator, io: std.Io, bdf: []const u8) !?[]const u8 { + const result = std.process.run(allocator, io, .{ + .argv = &.{ "lspci", "-Dnn", "-s", bdf }, + .stdout_limit = .limited(4096), + .stderr_limit = .limited(1024), + .reserve_amount = 256, + }) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return null, + }; + defer allocator.free(result.stdout); + defer allocator.free(result.stderr); + + switch (result.term) { + .exited => |code| if (code != 0) return null, + else => return null, + } + + return parseLspciName(allocator, result.stdout); +} + +fn parseLspciName(allocator: std.mem.Allocator, output: []const u8) !?[]const u8 { + var lines = std.mem.tokenizeScalar(u8, output, '\n'); + while (lines.next()) |line| { + const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace); + const header_end = std.mem.indexOf(u8, trimmed, "]: ") orelse continue; + const header = trimmed[0 .. header_end + 1]; + if (!containsAsciiIgnoreCase(header, "VGA") and !containsAsciiIgnoreCase(header, "Display")) continue; + + const name_start = header_end + "]: ".len; + const name_end = std.mem.lastIndexOf(u8, trimmed[name_start..], " [8086:") orelse + std.mem.indexOf(u8, trimmed[name_start..], " (rev ") orelse + trimmed[name_start..].len; + const raw_name = std.mem.trim(u8, trimmed[name_start .. name_start + name_end], &std.ascii.whitespace); + if (raw_name.len == 0 or std.mem.startsWith(u8, raw_name, "Intel Corporation Device ")) return null; + return try allocator.dupe(u8, raw_name); + } + return null; +} + +fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { + if (needle.len == 0) return true; + if (needle.len > haystack.len) return false; + for (0..haystack.len - needle.len + 1) |idx| { + if (std.ascii.eqlIgnoreCase(haystack[idx .. idx + needle.len], needle)) return true; + } + return false; +} + +test "parse lspci display name" { + const output = + \\0000:84:00.0 VGA compatible controller [0300]: Intel Corporation Battlemage G31 [Arc Pro B70] [8086:e223] (rev 04) + ; + const name_ = (try parseLspciName(std.testing.allocator, output)).?; + defer std.testing.allocator.free(name_); + try std.testing.expectEqualStrings("Intel Corporation Battlemage G31 [Arc Pro B70]", name_); +} + +test "parse lspci display name skips generic device ids" { + const output = + \\0000:84:00.0 Display controller [0380]: Intel Corporation Device e223 [8086:e223] (rev 04) + ; + try std.testing.expect(try parseLspciName(std.testing.allocator, output) == null); +} + +pub fn driverVersion(self: *const OneApi, allocator: std.mem.Allocator) ![]const u8 { + const version = smi_sysfs.readString(allocator, self.io, "/sys/module/xe/version") catch + smi_sysfs.readString(allocator, self.io, "/sys/module/xe/srcversion") catch + smi_sysfs.readString(allocator, self.io, "/sys/module/i915/version") catch + try smi_sysfs.readString(allocator, self.io, "/sys/module/i915/srcversion"); + const trimmed = std.mem.trim(u8, version, &std.ascii.whitespace); + if (trimmed.len == 0) return error.NotFound; + return trimmed; +} + +pub fn powerUsage(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + const current = Monitor.readEnergy(allocator, self.io, dev) catch |err| { + dev.energy_prev = null; + return err; + }; + const power = Monitor.powerMilliwatts(dev.energy_prev, current) orelse { + dev.energy_prev = current; + return error.NotFound; + }; + dev.energy_prev = current; + return power; +} + +pub fn powerLimit(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPowerLimit(allocator, self.io, dev); +} + +pub fn temperature(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readTemperature(allocator, self.io, dev); +} + +pub fn fanSpeed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readFanSpeed(allocator, self.io, dev); +} + +pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch { + dev.activity_prev = null; + return 0; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@backingInt(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.engine else null; + const util = Monitor.processUtil(dev.activity_prev, current) orelse 0; + dev.activity_prev = current; + return util; +} + +pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { + dev.encoder_prev = null; + return err; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@backingInt(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.encoder else null; + const util = Monitor.processUtil(dev.encoder_prev, current) orelse { + dev.encoder_prev = current; + return error.NotFound; + }; + dev.encoder_prev = current; + return util; +} + +pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { + dev.decoder_prev = null; + return err; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@backingInt(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.decoder else null; + const util = Monitor.processUtil(dev.decoder_prev, current) orelse { + dev.decoder_prev = current; + return error.NotFound; + }; + dev.decoder_prev = current; + return util; +} + +pub fn clockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readClockGraphics(allocator, self.io, dev); +} + +pub fn maxClockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readMaxClockGraphics(allocator, self.io, dev); +} + +pub fn memUsed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev_idx: u16 = @intCast(@backingInt(handle)); + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch return 0; + defer usage.deinit(allocator); + + return if (usage.get(dev_idx)) |sample| sample.mem_kib * 1024 else 0; +} + +pub fn memTotal(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readMemTotal(allocator, self.io, dev); +} + +pub fn pcieLinkGen(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieLinkGen(allocator, self.io, dev); +} + +pub fn pcieLinkWidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieLinkWidth(allocator, self.io, dev); +} + +pub fn pcieBandwidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieBandwidth(allocator, self.io, dev); +} diff --git a/bin/zml-smi/platforms/oneapi/process.zig b/bin/zml-smi/platforms/oneapi/process.zig new file mode 100644 index 00000000..c4d0c719 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/process.zig @@ -0,0 +1,45 @@ +const std = @import("std"); +const OneApi = @import("oneapi.zig"); +const Monitor = @import("monitor.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16) !void { + const state = try collector.arena.create(Monitor.ProcessPrevious); + state.* = .{}; + + const previous_usage: ?Monitor.ProcessUsage = Monitor.collectProcessUsage(collector.gpa, collector.io, oneapi.monitor.devices) catch null; + if (previous_usage) |usage_value| { + var usage = usage_value; + defer usage.deinit(collector.gpa); + Monitor.saveProcessPrevious(collector.arena, state, &usage) catch {}; + } + + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.arena, collector.io, list, oneapi, dev_offset, state }); +} + +fn pollOnce(allocator: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16, state: *Monitor.ProcessPrevious) void { + const back = list.back(); + back.clearRetainingCapacity(); + + var usage = Monitor.collectProcessUsage(allocator, io, oneapi.monitor.devices) catch { + list.swap(); + return; + }; + defer usage.deinit(allocator); + + var it = usage.iterator(); + while (it.next()) |entry| { + const sample = entry.value_ptr.*; + back.append(allocator, .{ + .pid = sample.pid, + .device_idx = @intCast(sample.device_idx + dev_offset), + .dev_mem_kib = sample.mem_kib, + .dev_util_percent = Monitor.processUtil(state.get(entry.key_ptr.*), sample.engine), + }) catch break; + } + + Monitor.saveProcessPrevious(arena, state, &usage) catch {}; + list.swap(); +} diff --git a/bin/zml-smi/platforms/tpu/BUILD.bazel b/bin/zml-smi/platforms/tpu/BUILD.bazel index 4f3f2503..a65377b3 100644 --- a/bin/zml-smi/platforms/tpu/BUILD.bazel +++ b/bin/zml-smi/platforms/tpu/BUILD.bazel @@ -1,11 +1,10 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "tpuinfo", srcs = ["tpuinfo.cc"], hdrs = ["tpuinfo.h"], - target_compatible_with = ["@platforms//os:linux"], deps = [ "@cloud_accelerator_diagnostics//:tpu_metric_service_cc_grpc", "@grpc//:grpc++_unsecure", @@ -15,16 +14,17 @@ cc_library( zig_library( name = "tpu", import_name = "zml-smi/platforms/tpu", - main = "Tpu.zig", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["//bin/zml-smi:__subpackages__"], + main = "metrics.zig", + srcs = [ + "process.zig", + "tpuinfo.zig", + ], + visibility = ["//visibility:public"], deps = [ ":tpuinfo", - "//bin/zml-smi/smi", + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:sysfs", ], ) - -zig_test( - name = "test", - deps = [":tpu"], -) diff --git a/bin/zml-smi/platforms/tpu/Tpu.zig b/bin/zml-smi/platforms/tpu/Tpu.zig deleted file mode 100644 index 536db90c..00000000 --- a/bin/zml-smi/platforms/tpu/Tpu.zig +++ /dev/null @@ -1,162 +0,0 @@ -//! Google TPUs, through a PCI scan and the runtime metrics service of libtpu. -const std = @import("std"); -const c = @import("c"); -const smi = @import("zml-smi"); - -const Tpu = @This(); - -platform: smi.Platform, -io: std.Io, -/// The service numbers the devices chip after chip. -devices_per_chip: u32, - -const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes }; - -/// Each query waits at most 1 s, so a stalled service delays a poll by up to 3 s. -const address = "localhost:8431"; -const max_devices: usize = c.TPUINFO_MAX_DEVICES; -const google = 0x1ae0; - -const Chip = struct { device: u16, subsystem: ?u16 = null, name: []const u8, devices_per_chip: u32 }; - -/// From tpu_info/device.py in cloud-accelerator-diagnostics (c94f463). -const chips = [_]Chip{ - .{ .device = 0x0027, .subsystem = 0x004e, .name = "TPU v2", .devices_per_chip = 2 }, - .{ .device = 0x0027, .subsystem = 0x004f, .name = "TPU v3", .devices_per_chip = 2 }, - .{ .device = 0x005e, .name = "TPU v4", .devices_per_chip = 1 }, - .{ .device = 0x0063, .name = "TPU v5e", .devices_per_chip = 1 }, - .{ .device = 0x0062, .name = "TPU v5p", .devices_per_chip = 1 }, - .{ .device = 0x006f, .name = "TPU v6e", .devices_per_chip = 1 }, - .{ .device = 0x0076, .name = "TPU7x", .devices_per_chip = 2 }, -}; - -pub fn detect(io: std.Io) bool { - return smi.sysfs.exists(io, "/dev/accel0") or smi.sysfs.exists(io, "/dev/vfio/0"); -} - -pub fn init(arena: std.mem.Allocator, _: std.mem.Allocator, io: std.Io) !Tpu { - var scan: smi.pci.Devices = try .open(io, google); - defer scan.close(); - var found: ?Chip = null; - var count: u32 = 0; - while (scan.next()) |entry| { - defer entry.dir.close(); - found = chipOf(entry.id, entry.dir.int(u16, "subsystem_device") catch null) orelse continue; - count += 1; - } - const chip = found orelse return error.TpuUnavailable; - - const devices = try arena.alloc(smi.Device, count * chip.devices_per_chip); - @memset(devices, .{ .target = .tpu, .name = chip.name }); - return .{ - .platform = .{ .devices = devices, .vtable = &vtable }, - .io = io, - .devices_per_chip = chip.devices_per_chip, - }; -} - -fn chipOf(device: u16, subsystem: ?u16) ?Chip { - for (chips) |chip| { - if (chip.device == device and (chip.subsystem == null or chip.subsystem == subsystem)) return chip; - } - return null; -} - -fn poll(_: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { - const used = query(c_longlong, "tpu.runtime.hbm.memory.usage.bytes"); - const total = query(c_longlong, "tpu.runtime.hbm.memory.total.bytes"); - const duty = query(f64, "tpu.runtime.tensorcore.dutycycle.percent"); - for (samples[0..@min(samples.len, max_devices)], 0..) |*sample, id| { - sample.tpu = .{ .util_percent = duty[id], .mem_used_bytes = used[id], .mem_total_bytes = total[id] }; - } -} - -/// One metric of every device, by device id. A failed query leaves them all null. -fn query(comptime T: type, metric: [:0]const u8) [max_devices]?u64 { - var ids: [max_devices]c_longlong = undefined; - var values: [max_devices]T = undefined; - const count = switch (T) { - c_longlong => c.tpu_query_int(address, metric, &ids, &values, max_devices), - f64 => c.tpu_query_double(address, metric, &ids, &values, max_devices), - else => @compileError("a metric is an int or a double"), - }; - const n = std.math.cast(usize, count) orelse 0; - return byId(T, ids[0..n], values[0..n]); -} - -fn byId(comptime T: type, ids: []const c_longlong, values: []const T) [max_devices]?u64 { - var out: [max_devices]?u64 = @splat(null); - for (ids, values) |id, value| { - const index = std.math.cast(usize, id) orelse continue; - if (index >= max_devices) continue; - out[index] = switch (T) { - c_longlong => std.math.cast(u64, value), - f64 => if (std.math.isFinite(value)) std.math.lossyCast(u64, @round(value)) else null, - else => @compileError("a metric is an int or a double"), - }; - } - return out; -} - -fn processes(platform: *smi.Platform, _: std.mem.Allocator, samples: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { - const self: *Tpu = @alignCast(@fieldParentPtr("platform", platform)); - var holders: smi.proc.FdHolders = .init(self.io, &.{ "/dev/accel", "/dev/vfio/" }); - defer holders.deinit(); - while (holders.next()) |holder| try addHolder(out, samples, self.devices_per_chip, holder.pid, holder.rest); -} - -/// Lists a process that holds the node of a chip on every device of that chip, once. -fn addHolder(out: *smi.ProcessList, samples: []const smi.Sample, devices_per_chip: u32, pid: u32, node: []const u8) !void { - const chip = std.fmt.parseInt(u16, node, 10) catch return; - for (0..devices_per_chip) |core| { - const device = @as(usize, chip) * devices_per_chip + core; - if (device >= samples.len) return; - for (out.rows.items) |row| { - if (row.pid == pid and row.device_idx == device) break; - } else { - const tpu = samples[device].tpu; - try out.append(.{ - .pid = pid, - .device_idx = @intCast(device), - .dev_util_percent = if (tpu.util_percent) |percent| std.math.cast(u16, percent) else null, - .dev_mem_kib = if (tpu.mem_used_bytes) |bytes| bytes / 1024 else null, - }); - } - } -} - -test chipOf { - try std.testing.expectEqualStrings("TPU v3", chipOf(0x0027, 0x004f).?.name); - try std.testing.expectEqualStrings("TPU v5e", chipOf(0x0063, 0x0001).?.name); - try std.testing.expect(chipOf(0x0027, null) == null); - try std.testing.expect(chipOf(0x0001, null) == null); -} - -test byId { - const used = byId(c_longlong, &.{ 1, 0, -1, 64 }, &.{ 2048, -5, 7, 7 }); - try std.testing.expectEqual(2048, used[1]); - try std.testing.expectEqual(null, used[0]); - const duty = byId(f64, &.{ 0, 1 }, &.{ 12.5, std.math.nan(f64) }); - try std.testing.expectEqual(13, duty[0]); - try std.testing.expectEqual(null, duty[1]); -} - -test addHolder { - var list: smi.ProcessList = .init(std.testing.allocator); - defer list.deinit(); - const samples = [_]smi.Sample{ .{ .tpu = .{} }, .{ .tpu = .{} }, .{ .tpu = .{ .util_percent = 40, .mem_used_bytes = 8192 } }, .{ .tpu = .{} } }; - - // Chip 1 of a two-device chip is devices 2 and 3, listed once however many descriptors. - try addHolder(&list, &samples, 2, 7, "1"); - try addHolder(&list, &samples, 2, 7, "1"); - // The vfio container and a chip beyond the devices make no row. - try addHolder(&list, &samples, 2, 7, "vfio"); - try addHolder(&list, &samples, 2, 7, "2"); - - try std.testing.expectEqual(2, list.rows.items.len); - try std.testing.expectEqual(2, list.rows.items[0].device_idx); - try std.testing.expectEqual(40, list.rows.items[0].dev_util_percent); - try std.testing.expectEqual(8, list.rows.items[0].dev_mem_kib); - try std.testing.expectEqual(3, list.rows.items[1].device_idx); - try std.testing.expectEqual(null, list.rows.items[1].dev_mem_kib); -} diff --git a/bin/zml-smi/platforms/tpu/metrics.zig b/bin/zml-smi/platforms/tpu/metrics.zig new file mode 100644 index 00000000..9410b3f8 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/metrics.zig @@ -0,0 +1,168 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const tpuinfo = @import("tpuinfo.zig"); +const device_info = @import("zml-smi/info").device_info; +const DeviceInfo = device_info.DeviceInfo; +const TpuInfo = device_info.TpuInfo; +const Collector = @import("zml-smi/collector").Collector; +const tpu_process = @import("process.zig"); + +const address = "localhost:8431"; + +pub fn start(collector: *Collector) !void { + const chip = scanPciChips(collector.arena, collector.io) orelse return error.TpuUnavailable; + const device_count = chip.chip_count * chip.devices_per_chip; + var tpu_infos: std.ArrayList(*DeviceInfo) = .empty; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + for (0..device_count) |_| { + const chip_name = try collector.arena.dupe(u8, chip.name); + const initial: TpuInfo = .{ .name = chip_name }; + const info = try collector.addDevice(.{ .tpu = .{ .values = .{ initial, initial } } }); + try tpu_infos.append(collector.arena, info); + } + + try collector.spawnPoll(pollAllDevices, .{tpu_infos.items}); + + if (tpu_infos.items.len > 0) { + const processes = try collector.createProcessList(); + try tpu_process.init(collector, chip.devices_per_chip, tpu_infos.items, processes, dev_offset); + } +} + +const MetricKind = enum { int, double }; + +fn pollAllDevices(devs: []*DeviceInfo) void { + var mem_used: [tpuinfo.max_devices]?u64 = undefined; + var mem_total: [tpuinfo.max_devices]?u64 = undefined; + var util: [tpuinfo.max_devices]?u64 = undefined; + + for (devs, 0..) |info, i| { + const front = info.tpu.front(); + mem_used[i] = front.mem_used_bytes; + mem_total[i] = front.mem_total_bytes; + util[i] = front.util_percent; + } + + queryRaw("tpu.runtime.hbm.memory.usage.bytes", .int, mem_used[0..devs.len]); + queryRaw("tpu.runtime.hbm.memory.total.bytes", .int, mem_total[0..devs.len]); + queryRaw("tpu.runtime.tensorcore.dutycycle.percent", .double, util[0..devs.len]); + + for (devs, 0..) |info, i| { + const back = info.tpu.back(); + back.* = info.tpu.front().*; + back.mem_used_bytes = mem_used[i]; + back.mem_total_bytes = mem_total[i]; + back.util_percent = util[i]; + info.tpu.swap(); + } +} + +fn queryRaw(metric_name: [:0]const u8, comptime kind: MetricKind, out: []?u64) void { + var ids: [tpuinfo.max_devices]c_longlong = undefined; + switch (kind) { + .int => { + var vals: [tpuinfo.max_devices]c_longlong = undefined; + const n = tpuinfo.queryInt(address, metric_name, &ids, &vals) catch { + @memset(out, null); + return; + }; + + for (ids[0..n], vals[0..n]) |id, val| { + if (id >= 0 and id < out.len) out[@intCast(id)] = @intCast(val); + } + }, + .double => { + var vals: [tpuinfo.max_devices]f64 = undefined; + const n = tpuinfo.queryDouble(address, metric_name, &ids, &vals) catch { + @memset(out, null); + return; + }; + + for (ids[0..n], vals[0..n]) |id, val| { + if (id >= 0 and id < out.len and std.math.isFinite(val)) + out[@intCast(id)] = @intFromFloat(@round(val)); + } + }, + } +} + +// -- Discover available chips +// We need to scan /sys/bus/pci/devices for Google TPU chips. +// From: https://github.com/AI-Hypercomputer/cloud-accelerator-diagnostics/blob/c94f463dd3bf99eaecf4f5af46f87cc2a5ff1b22/tpu_info/tpu_info/device.py#L43-L92 + +const ChipInfo = struct { + name: []const u8, + devices_per_chip: u32, + chip_count: u32, +}; + +const google_pci_vendor_id = "0x1ae0"; +const pci_base = "/sys/bus/pci/devices"; + +const ChipEntry = struct { + device_id: []const u8, + subsystem_id: ?[]const u8 = null, + name: []const u8, + devices_per_chip: u32, +}; + +const chip_table = [_]ChipEntry{ + .{ .device_id = "0x0027", .subsystem_id = "0x004e", .name = "TPU v2", .devices_per_chip = 2 }, + .{ .device_id = "0x0027", .subsystem_id = "0x004f", .name = "TPU v3", .devices_per_chip = 2 }, + .{ .device_id = "0x005e", .name = "TPU v4", .devices_per_chip = 1 }, + .{ .device_id = "0x0063", .name = "TPU v5e", .devices_per_chip = 1 }, + .{ .device_id = "0x0062", .name = "TPU v5p", .devices_per_chip = 1 }, + .{ .device_id = "0x006f", .name = "TPU v6e", .devices_per_chip = 1 }, + .{ .device_id = "0x0076", .name = "TPU7x", .devices_per_chip = 2 }, +}; + +fn scanPciChips(allocator: std.mem.Allocator, io: std.Io) ?ChipInfo { + var pci_dir = std.Io.Dir.openDir(.cwd(), io, pci_base, .{ .iterate = true }) catch return null; + defer pci_dir.close(io); + + var it = pci_dir.iterate(); + var chip_count: u32 = 0; + var result: ChipInfo = undefined; + + while (it.next(io) catch null) |entry| { + const vendor = readSysfs(allocator, io, entry.name, "vendor") catch continue; + if (!std.mem.eql(u8, vendor, google_pci_vendor_id)) { + continue; + } + + const device_raw = readSysfs(allocator, io, entry.name, "device") catch continue; + const subsystem_raw: ?[]const u8 = readSysfs(allocator, io, entry.name, "subsystem_device") catch null; + + for (chip_table) |chip| { + const dev_match = std.mem.eql(u8, device_raw, chip.device_id); + const sub_match = if (chip.subsystem_id) |expected_sub| + if (subsystem_raw) |actual_sub| std.mem.eql(u8, actual_sub, expected_sub) else false + else + true; + + if (dev_match and sub_match) { + result = .{ + .name = chip.name, + .devices_per_chip = chip.devices_per_chip, + .chip_count = undefined, + }; + + chip_count += 1; + break; + } + } + } + + if (chip_count > 0) { + result.chip_count = chip_count; + return result; + } + + return null; +} + +fn readSysfs(allocator: std.mem.Allocator, io: std.Io, slot: []const u8, file: []const u8) ![]const u8 { + const path = try std.fmt.allocPrint(allocator, pci_base ++ "/{s}/{s}", .{ slot, file }); + return sysfs.readString(allocator, io, path); +} diff --git a/bin/zml-smi/platforms/tpu/process.zig b/bin/zml-smi/platforms/tpu/process.zig new file mode 100644 index 00000000..cca8f6a1 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/process.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u16) !void { + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, devices_per_chip, device_infos, dev_offset }); +} + +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u16) void { + scan(allocator, io, list.back(), devices_per_chip, device_infos, dev_offset); + list.swap(); +} + +fn scan(allocator: std.mem.Allocator, io: std.Io, back: *std.ArrayList(pi.ProcessInfo), devices_per_chip: u32, infos: []*DeviceInfo, dev_offset: u16) void { + back.clearRetainingCapacity(); + + var proc_dir = std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }) catch return; + defer proc_dir.close(io); + + var proc_it = proc_dir.iterate(); + while (proc_it.next(io) catch null) |proc_entry| { + const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; + + var fd_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const fd_sub = std.fmt.bufPrint(&fd_path_buf, "{d}/fd", .{pid}) catch continue; + var fd_dir = proc_dir.openDir(io, fd_sub, .{ .iterate = true }) catch continue; + defer fd_dir.close(io); + + var fd_it = fd_dir.iterate(); + while (fd_it.next(io) catch null) |fd_entry| { + var link_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const len = fd_dir.readLink(io, fd_entry.name, &link_buf) catch continue; + const target = link_buf[0..len]; + + if (parseChipIndex(target)) |chip_idx| { + const base: u16 = @intCast(chip_idx * devices_per_chip); + for (0..devices_per_chip) |d| { + const local_idx: u16 = @intCast(base + d); + var info: pi.ProcessInfo = .{ .pid = pid, .device_idx = local_idx + dev_offset }; + + if (local_idx < infos.len) { + const tpu = infos[local_idx].tpu.front().*; + + if (tpu.mem_used_bytes) |mem| { + info.dev_mem_kib = @intCast(mem / 1024); + } + + if (tpu.util_percent) |util| { + info.dev_util_percent = @intCast(util); + } + } + + back.append(allocator, info) catch return; + } + } + } + } +} + +fn parseChipIndex(target: []const u8) ?u32 { + if (std.mem.startsWith(u8, target, "/dev/accel")) { + return std.fmt.parseInt(u32, target["/dev/accel".len..], 10) catch null; + } + + if (std.mem.startsWith(u8, target, "/dev/vfio/")) { + return std.fmt.parseInt(u32, target["/dev/vfio/".len..], 10) catch null; + } + + return null; +} diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.zig b/bin/zml-smi/platforms/tpu/tpuinfo.zig new file mode 100644 index 00000000..1baf62df --- /dev/null +++ b/bin/zml-smi/platforms/tpu/tpuinfo.zig @@ -0,0 +1,37 @@ +const std = @import("std"); +const c = @import("c"); + +pub const max_devices: usize = c.TPUINFO_MAX_DEVICES; + +pub const Error = error{ + TpuUnavailable, + QueryFailed, +}; + +pub fn queryInt( + address: [:0]const u8, + metric_name: [:0]const u8, + device_ids: []c_longlong, + values: []c_longlong, +) Error!u32 { + const n = c.tpu_query_int(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); + if (n < 0) { + return error.QueryFailed; + } + + return @intCast(n); +} + +pub fn queryDouble( + address: [:0]const u8, + metric_name: [:0]const u8, + device_ids: []c_longlong, + values: []f64, +) Error!u32 { + const n = c.tpu_query_double(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); + if (n < 0) { + return error.QueryFailed; + } + + return @intCast(n); +} diff --git a/bin/zml-smi/prometheus/BUILD.bazel b/bin/zml-smi/prometheus/BUILD.bazel new file mode 100644 index 00000000..c00a26b5 --- /dev/null +++ b/bin/zml-smi/prometheus/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "prometheus", + import_name = "zml-smi/prometheus", + main = "prometheus.zig", + srcs = [ + "server.zig", + "exposition.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + ], +) diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig new file mode 100644 index 00000000..54d48cb6 --- /dev/null +++ b/bin/zml-smi/prometheus/exposition.zig @@ -0,0 +1,159 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const HostInfo = smi_info.host_info.HostInfo; + +const device_metrics = .{ + // Shared across all device types + .{ .field = "util_percent", .metric = "zml_device_utilization_percent", .help = "Device utilization percentage" }, + .{ .field = "mem_used_bytes", .metric = "zml_device_memory_used_bytes", .help = "Device memory used in bytes" }, + .{ .field = "mem_total_bytes", .metric = "zml_device_memory_total_bytes", .help = "Device memory total in bytes" }, + // GPU (CUDA/ROCm/oneAPI) + .{ .field = "encoder_util_percent", .metric = "zml_device_encoder_utilization_percent", .help = "Encoder utilization percentage" }, + .{ .field = "decoder_util_percent", .metric = "zml_device_decoder_utilization_percent", .help = "Decoder utilization percentage" }, + .{ .field = "power_mw", .metric = "zml_device_power_watts", .help = "Power draw in watts", .fmt = fmtMilliwattsAsWatts }, + .{ .field = "power_limit_mw", .metric = "zml_device_power_limit_watts", .help = "Power limit in watts", .fmt = fmtMilliwattsAsWatts }, + .{ .field = "temperature", .metric = "zml_device_temperature_celsius", .help = "Device temperature in celsius" }, + .{ .field = "fan_speed_percent", .metric = "zml_device_fan_speed_percent", .help = "Fan speed percentage" }, + .{ .field = "clock_graphics_mhz", .metric = "zml_device_clock_graphics_mhz", .help = "Graphics clock frequency in MHz" }, + .{ .field = "clock_sm_mhz", .metric = "zml_device_clock_sm_mhz", .help = "SM clock frequency in MHz" }, + .{ .field = "clock_mem_mhz", .metric = "zml_device_clock_memory_mhz", .help = "Memory clock frequency in MHz" }, + .{ .field = "pcie_tx_kbps", .metric = "zml_device_pcie_tx_bytes_per_second", .help = "PCIe TX throughput in bytes per second", .fmt = fmtScaled(128) }, + .{ .field = "pcie_rx_kbps", .metric = "zml_device_pcie_rx_bytes_per_second", .help = "PCIe RX throughput in bytes per second", .fmt = fmtScaled(128) }, + // Neuron + .{ .field = "nc_tensors", .metric = "zml_device_neuron_tensors_bytes", .help = "Neuron core tensor memory in bytes" }, + .{ .field = "nc_constants", .metric = "zml_device_neuron_constants_bytes", .help = "Neuron core constants memory in bytes" }, + .{ .field = "nc_model_code", .metric = "zml_device_neuron_model_code_bytes", .help = "Neuron core model code memory in bytes" }, + .{ .field = "nc_shared_scratchpad", .metric = "zml_device_neuron_shared_scratchpad_bytes", .help = "Neuron core shared scratchpad memory in bytes" }, + .{ .field = "nc_nonshared_scratchpad", .metric = "zml_device_neuron_nonshared_scratchpad_bytes", .help = "Neuron core non-shared scratchpad memory in bytes" }, + .{ .field = "nc_runtime", .metric = "zml_device_neuron_runtime_bytes", .help = "Neuron core runtime memory in bytes" }, + .{ .field = "nc_driver", .metric = "zml_device_neuron_driver_bytes", .help = "Neuron core driver memory in bytes" }, + .{ .field = "nc_dma_rings", .metric = "zml_device_neuron_dma_rings_bytes", .help = "Neuron core DMA rings memory in bytes" }, + .{ .field = "nc_collectives", .metric = "zml_device_neuron_collectives_bytes", .help = "Neuron core collectives memory in bytes" }, + .{ .field = "nc_notifications", .metric = "zml_device_neuron_notifications_bytes", .help = "Neuron core notifications memory in bytes" }, + .{ .field = "nc_uncategorized", .metric = "zml_device_neuron_uncategorized_bytes", .help = "Neuron core uncategorized memory in bytes" }, +}; + +const host_metrics = .{ + .{ .field = "cpu_cores", .metric = "zml_host_cpu_cores", .help = "Number of CPU cores" }, + .{ .field = "mem_total_kib", .metric = "zml_host_memory_total_bytes", .help = "Total host memory in bytes", .fmt = fmtScaled(1024) }, + .{ .field = "mem_available_kib", .metric = "zml_host_memory_available_bytes", .help = "Available host memory in bytes", .fmt = fmtScaled(1024) }, + .{ .field = "load_1", .metric = "zml_host_load_1", .help = "1-minute load average" }, + .{ .field = "load_5", .metric = "zml_host_load_5", .help = "5-minute load average" }, + .{ .field = "load_15", .metric = "zml_host_load_15", .help = "15-minute load average" }, + .{ .field = "uptime_seconds", .metric = "zml_host_uptime_seconds", .help = "System uptime in seconds" }, +}; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, host: *const HostInfo) !void { + try writer.writeAll("# HELP zml_device_info Device metadata\n"); + try writer.writeAll("# TYPE zml_device_info gauge\n"); + for (devices, 0..) |dev, i| { + inline for (@typeInfo(DeviceInfo).@"union".field_names) |type_name| { + const tag = @field(smi_info.device_info.Target, type_name); + + switch (dev.*) { + tag => |*db| { + const val = db.front().*; + + try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, type_name }); + + inline for (.{ + .{ "name", "name" }, + .{ "driver_version", "driver" }, + .{ "cuda_driver_version", "cuda_version" }, + }) |label| { + if (@hasField(@TypeOf(val), label[0])) { + if (@field(val, label[0])) |v| { + try writer.print(",{s}=\"", .{label[1]}); + try std.zig.stringEscape(v, writer); + try writer.writeAll("\""); + } + } + } + + try writer.writeAll("} 1\n"); + }, + else => {}, + } + } + } + try writer.writeAll("\n"); + + try writeAllDeviceMetrics(writer, device_metrics, devices); + try writeHostMetrics(writer, host.front().*); +} + +fn writeAllDeviceMetrics( + writer: *std.Io.Writer, + comptime metrics: anytype, + devices: []const *DeviceInfo, +) !void { + inline for (metrics) |m| { + var header_written = false; + + for (devices, 0..) |dev, i| { + switch (dev.*) { + inline else => |*db| { + const val = db.front().*; + if (@hasField(@TypeOf(val), m.field)) { + if (@field(val, m.field)) |raw| { + if (!header_written) { + try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n", .{ m.metric, m.help, m.metric }); + header_written = true; + } + + const device_type = @tagName(std.meta.activeTag(dev.*)); + try writer.print("{s}{{device=\"{d}\",type=\"{s}\",name=\"", .{ m.metric, i, device_type }); + try std.zig.stringEscape(val.name orelse "", writer); + try writer.writeAll("\"} "); + try fmtValue(m, writer, raw); + try writer.writeAll("\n"); + } + } + }, + } + } + + if (header_written) { + try writer.writeAll("\n"); + } + } +} + +fn writeHostMetrics(writer: *std.Io.Writer, val: anytype) !void { + inline for (host_metrics) |m| { + if (@field(val, m.field)) |raw| { + try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n{s} ", .{ m.metric, m.help, m.metric, m.metric }); + try fmtValue(m, writer, raw); + try writer.writeAll("\n\n"); + } + } +} + +fn fmtValue(comptime m: anytype, writer: *std.Io.Writer, raw: anytype) !void { + if (@hasField(@TypeOf(m), "fmt")) { + try m.fmt(writer, raw); + } else { + try writeValue(writer, raw); + } +} + +fn writeValue(writer: *std.Io.Writer, val: anytype) std.Io.Writer.Error!void { + switch (@typeInfo(@TypeOf(val))) { + .float, .comptime_float => try writer.print("{d:.2}", .{val}), + .pointer => try writer.writeAll(val), + else => try writer.print("{d}", .{val}), + } +} + +fn fmtMilliwattsAsWatts(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { + try writer.print("{d}.{d:0>3}", .{ val / 1000, val % 1000 }); +} + +fn fmtScaled(comptime scale: u64) fn (*std.Io.Writer, u64) std.Io.Writer.Error!void { + return struct { + fn f(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { + try writer.print("{d}", .{val * scale}); + } + }.f; +} diff --git a/bin/zml-smi/prometheus/prometheus.zig b/bin/zml-smi/prometheus/prometheus.zig new file mode 100644 index 00000000..cfa2fc28 --- /dev/null +++ b/bin/zml-smi/prometheus/prometheus.zig @@ -0,0 +1,2 @@ +pub const Server = @import("server.zig").Server; +pub const exposition = @import("exposition.zig"); diff --git a/bin/zml-smi/prometheus/server.zig b/bin/zml-smi/prometheus/server.zig new file mode 100644 index 00000000..9517ecac --- /dev/null +++ b/bin/zml-smi/prometheus/server.zig @@ -0,0 +1,87 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const HostInfo = smi_info.host_info.HostInfo; +const exposition = @import("exposition.zig"); + +pub const Server = struct { + io: std.Io, + devices: []const *DeviceInfo, + host: *const HostInfo, + tcp: std.Io.net.Server, + connection_group: std.Io.Group = .init, + + pub fn init(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host: *const HostInfo) !Server { + const address = try std.Io.net.IpAddress.parseLiteral(listen); + var tcp = try address.listen(io, .{ .reuse_address = true }); + errdefer tcp.deinit(io); + + return .{ + .io = io, + .devices = devices, + .host = host, + .tcp = tcp, + }; + } + + pub fn run(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host_info: *const HostInfo) void { + var self = init(io, listen, devices, host_info) catch |err| { + std.log.err("prometheus server failed to start: {s}", .{@errorName(err)}); + return; + }; + defer self.deinit(); + self.serve(); + } + + pub fn deinit(self: *Server) void { + self.connection_group.await(self.io) catch {}; + self.tcp.deinit(self.io); + } + + pub fn serve(self: *Server) void { + while (true) { + const stream = self.tcp.accept(self.io) catch break; + self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; + } + } + + fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { + defer stream.close(self.io); + self.handleConn(stream) catch {}; + } + + fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { + var read_buf: [8192]u8 = undefined; + var reader = stream.reader(self.io, &read_buf); + + var write_buf: [4096]u8 = undefined; + var writer = stream.writer(self.io, &write_buf); + + var http_server: std.http.Server = .init(&reader.interface, &writer.interface); + + while (true) { + var request = http_server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => return err, + }; + + if (request.head.method != .GET) { + try request.respond("Method Not Allowed\n", .{ .status = .method_not_allowed }); + continue; + } + + var body_buf: [4096]u8 = undefined; + var body_writer = try request.respondStreaming(&body_buf, .{ + .respond_options = .{ + .status = .ok, + .extra_headers = &.{ + .{ .name = "content-type", .value = "text/plain" }, + }, + }, + }); + + try exposition.write(&body_writer.writer, self.devices, self.host); + try body_writer.end(); + } + } +}; diff --git a/bin/zml-smi/smi/BUILD.bazel b/bin/zml-smi/smi/BUILD.bazel deleted file mode 100644 index 00614864..00000000 --- a/bin/zml-smi/smi/BUILD.bazel +++ /dev/null @@ -1,37 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "smi", - srcs = [ - "api.zig", - "collector.zig", - "counter.zig", - "csv.zig", - "double_buffer.zig", - "dynlib.zig", - "fdinfo.zig", - "host/linux.zig", - "host/macos.zig", - "hwmon.zig", - "json.zig", - "model.zig", - "pci.zig", - "proc.zig", - "prometheus.zig", - "remote.zig", - "sandbox.zig", - "sysfs.zig", - "units.zig", - ], - import_name = "zml-smi", - main = "smi.zig", - visibility = ["//bin/zml-smi:__subpackages__"], - deps = ["@rules_zig//zig/runfiles"], -) - -zig_test( - name = "test", - # The sandbox test looks the directory up through the runfiles. - data = ["//bin/zml-smi:sandbox"], - deps = [":smi"], -) diff --git a/bin/zml-smi/smi/api.zig b/bin/zml-smi/smi/api.zig deleted file mode 100644 index 16cf37e9..00000000 --- a/bin/zml-smi/smi/api.zig +++ /dev/null @@ -1,65 +0,0 @@ -const std = @import("std"); -const smi = @import("smi.zig"); - -const log = std.log.scoped(.@"zml-smi/api"); - -pub const Format = enum { - /// The API: what --json prints, which --remotes reads. - json, - prometheus, -}; - -/// Answers every request on `address` with the latest values, until canceled. -pub fn serve(gpa: std.mem.Allocator, io: std.Io, address: std.Io.net.IpAddress, collector: *smi.Collector, format: Format) void { - var server = address.listen(io, .{ .reuse_address = true }) catch |err| { - log.err("{t} server cannot listen on {f}: {t}", .{ format, address, err }); - return; - }; - defer server.deinit(io); - - var connections: std.Io.Group = .init; - defer connections.cancel(io); - while (true) { - const stream = server.accept(io) catch return; - connections.concurrent(io, answer, .{ gpa, io, stream, collector, format }) catch { - stream.close(io); - return; - }; - } -} - -fn answer(gpa: std.mem.Allocator, io: std.Io, stream: std.Io.net.Stream, collector: *smi.Collector, format: Format) void { - defer stream.close(io); - var read_buffer: [8192]u8 = undefined; - var write_buffer: [4096]u8 = undefined; - var reader = stream.reader(io, &read_buffer); - var writer = stream.writer(io, &write_buffer); - var http: std.http.Server = .init(&reader.interface, &writer.interface); - - // The body is written to memory first, so a slow client never holds a lock. - var body: std.Io.Writer.Allocating = .init(gpa); - defer body.deinit(); - - while (true) { - var request = http.receiveHead() catch return; - if (format == .prometheus and request.head.method != .GET) { - request.respond("Method Not Allowed\n", .{ .status = .method_not_allowed }) catch return; - continue; - } - - body.clearRetainingCapacity(); - const written = switch (format) { - .json => smi.json.write(&body.writer, collector), - .prometheus => smi.prometheus.write(&body.writer, collector), - }; - written catch return; - - request.respond(body.written(), .{ .extra_headers = switch (format) { - .json => &.{ - .{ .name = "content-type", .value = "application/json" }, - .{ .name = "access-control-allow-origin", .value = "*" }, - }, - .prometheus => &.{.{ .name = "content-type", .value = "text/plain" }}, - } }) catch return; - } -} diff --git a/bin/zml-smi/smi/collector.zig b/bin/zml-smi/smi/collector.zig deleted file mode 100644 index 450913d0..00000000 --- a/bin/zml-smi/smi/collector.zig +++ /dev/null @@ -1,291 +0,0 @@ -const std = @import("std"); -const smi = @import("smi.zig"); - -/// Owns everything a platform does not: tasks, global device indices, snapshots -/// and the merged process list. -pub const Collector = struct { - /// Every device of every platform and remote host, in the order they were added. - devices: std.ArrayList(smi.Device) = .empty, - /// Every process list merged into one, with host details. Written by the host step. - processes: smi.DoubleBuffer(smi.ProcessList), - host: smi.DoubleBuffer(smi.HostData), - /// What the host reader read once. Each host step starts from it, so a value a platform did not measure is null. - host_once: smi.HostData, - /// A source derives values from two polls, so a one-shot print runs two rounds. - two_readings: bool = false, - - sources: std.ArrayList(*Source) = .empty, - host_reader: smi.Host, - host_scratch: std.heap.ArenaAllocator, - arena: std.mem.Allocator, - gpa: std.mem.Allocator, - io: std.Io, - interval: std.Io.Duration, - tasks: std.Io.Group = .init, - stopping: std.atomic.Value(bool) = .init(false), - - pub const Options = struct { poll_interval_ms: u16 = 500 }; - - const Source = struct { - platform: *smi.Platform, - /// The global index of its first device. - first: u16, - sides: smi.DoubleBuffer(Side), - scratch: std.heap.ArenaAllocator, - }; - - const Side = struct { - samples: []smi.Sample, - processes: smi.ProcessList, - /// What the platform measured of the host. - host: smi.HostData = .{}, - }; - - pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, options: Options) Collector { - var host: smi.HostData = .{}; - const host_reader: smi.Host = .init(arena, gpa, io, &host); - return .{ - .processes = .{ .values = .{ .init(gpa), .init(gpa) } }, - .host = .{ .values = .{ host, host } }, - .host_once = host, - .host_reader = host_reader, - .host_scratch = .init(gpa), - .arena = arena, - .gpa = gpa, - .io = io, - .interval = .fromMilliseconds(options.poll_interval_ms), - }; - } - - /// Stops the tasks, then calls `deinit` on the platforms that have one. - pub fn deinit(self: *Collector) void { - self.stopping.store(true, .release); - self.tasks.cancel(self.io); - for (self.sources.items) |source| { - if (source.platform.vtable.deinit) |deinitPlatform| deinitPlatform(source.platform); - for (&source.sides.values) |*side| side.processes.deinit(); - source.scratch.deinit(); - } - for (&self.processes.values) |*list| list.deinit(); - self.host_reader.deinit(); - self.host_scratch.deinit(); - } - - /// Registers an opened platform or remote host. Both sides of its samples start empty. - pub fn add(self: *Collector, platform: *smi.Platform) !void { - const count = platform.devices.len; - const source = try self.arena.create(Source); - source.* = .{ - .platform = platform, - .first = std.math.cast(u16, self.devices.items.len) orelse return error.TooManyDevices, - .sides = .{ .values = .{ - .{ .samples = try self.arena.alloc(smi.Sample, count), .processes = .init(self.gpa) }, - .{ .samples = try self.arena.alloc(smi.Sample, count), .processes = .init(self.gpa) }, - } }, - .scratch = .init(self.gpa), - }; - for (&source.sides.values) |side| { - for (side.samples, platform.devices) |*sample_, device| sample_.* = .init(device.target); - } - try self.devices.appendSlice(self.arena, platform.devices); - try self.sources.append(self.arena, source); - self.two_readings = self.two_readings or platform.two_readings; - } - - /// One round: every source once, then the host step. - pub fn round(self: *Collector) void { - for (self.sources.items) |source| self.poll(source); - self.hostStep(); - } - - /// Runs one round, then starts one task per source and the host task. No `add` may follow. - pub fn start(self: *Collector) !void { - const begin: std.Io.Timestamp = .now(self.io, .awake); - self.round(); - for (self.sources.items) |source| try self.tasks.concurrent(self.io, run, .{ self, source, begin }); - try self.tasks.concurrent(self.io, run, .{ self, null, begin }); - } - - /// The latest sample of a device, by value. - pub fn sample(self: *Collector, device: smi.Device.Index) smi.Sample { - const index = @backingInt(device); - for (self.sources.items) |source| { - const position = index -% source.first; - if (index < source.first or position >= source.platform.devices.len) continue; - defer source.sides.unlock(self.io); - return source.sides.lock(self.io).samples[position]; - } - unreachable; - } - - fn run(self: *Collector, source: ?*Source, first_round: std.Io.Timestamp) std.Io.Cancelable!void { - var begin = first_round; - while (true) { - const elapsed = begin.untilNow(self.io, .awake); - if (elapsed.nanoseconds < self.interval.nanoseconds) { - try self.io.sleep(.fromNanoseconds(self.interval.nanoseconds - elapsed.nanoseconds), .awake); - } - begin = .now(self.io, .awake); - if (source) |s| self.poll(s) else self.hostStep(); - if (self.stopping.load(.acquire)) return; - } - } - - fn poll(self: *Collector, source: *Source) void { - const platform = source.platform; - const back = source.sides.back(); - const scratch = source.scratch.allocator(); - defer _ = source.scratch.reset(.retain_capacity); - - for (back.samples, platform.devices) |*sample_, device| sample_.* = .init(device.target); - back.processes.clear(); - - platform.vtable.poll(platform, scratch, back.samples); - for (back.samples, platform.devices) |*sample_, device| { - if (std.meta.activeTag(sample_.*) != device.target) sample_.* = .init(device.target); - } - - if (platform.vtable.processes) |processes| { - if (processes(platform, scratch, back.samples, &back.processes)) { - // A row that names a device the source does not have is dropped. - const rows = &back.processes.rows; - var kept: usize = 0; - for (rows.items) |row| { - if (row.device_idx >= platform.devices.len) continue; - rows.items[kept] = row; - rows.items[kept].device_idx += source.first; - kept += 1; - } - rows.shrinkRetainingCapacity(kept); - } else |_| { - back.processes.clear(); - } - } - if (platform.vtable.host) |measureHost| { - back.host = .{}; - measureHost(platform, &back.host); - } - source.sides.swap(self.io); - } - - /// Merges the rows of every source, adds the host details, and reads the host. - fn hostStep(self: *Collector) void { - const merged = self.processes.back(); - merged.clear(); - for (self.sources.items) |source| { - const side = source.sides.lock(self.io); - defer source.sides.unlock(self.io); - for (side.processes.rows.items) |row| { - var copy = row; - copy.username = merged.dupe(row.username) catch ""; - copy.comm = merged.dupe(row.comm) catch ""; - merged.append(copy) catch break; - } - } - self.host_reader.enrich(merged.strings.allocator(), self.host_scratch.allocator(), merged.rows.items); - _ = self.host_scratch.reset(.retain_capacity); - - const host = self.host.back(); - host.* = self.host_once; - self.host_reader.sample(host); - for (self.sources.items) |source| { - if (source.platform.vtable.host == null) continue; - const measured = source.sides.lock(self.io).host; - source.sides.unlock(self.io); - inline for (@typeInfo(smi.HostData).@"struct".field_names) |field| { - if (@field(measured, field)) |value| @field(host, field) = value; - } - } - - self.processes.swap(self.io); - self.host.swap(self.io); - } -}; - -const Fake = struct { - platform: smi.Platform, - polls: u64 = 0, - - const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .host = host }; - - fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { - const self: *Fake = @alignCast(@fieldParentPtr("platform", platform)); - self.polls += 1; - samples[0].cuda.util_percent = self.polls; - // A tag that is not the device's target. - if (samples.len > 1) samples[1] = .{ .tpu = .{ .util_percent = 1 } }; - } - - fn processes(platform: *smi.Platform, _: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { - const self: *Fake = @alignCast(@fieldParentPtr("platform", platform)); - if (self.polls % 2 == 0) return error.Unavailable; - // A device the platform does not have. - try out.append(.{ .pid = 7, .device_idx = @intCast(self.platform.devices.len) }); - try out.append(.{ .pid = 42, .device_idx = @intCast(self.platform.devices.len - 1), .remote = true, .username = try out.dupe("alice") }); - } - - /// Measures the CPU power on odd polls only. - fn host(platform: *smi.Platform, data: *smi.HostData) void { - const self: *Fake = @alignCast(@fieldParentPtr("platform", platform)); - if (self.polls % 2 == 1) data.cpu_power_mw = self.polls; - } -}; - -test "collector rounds" { - const io = std.testing.io; - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - - var collector: Collector = .init(arena.allocator(), std.testing.allocator, io, .{}); - defer collector.deinit(); - - var gpus: Fake = .{ .platform = .{ .devices = &.{ .{ .target = .cuda }, .{ .target = .cuda } }, .vtable = &Fake.vtable } }; - var other: Fake = .{ .platform = .{ .devices = &.{.{ .target = .cuda }}, .two_readings = true, .vtable = &Fake.vtable } }; - try collector.add(&gpus.platform); - try std.testing.expect(!collector.two_readings); - try collector.add(&other.platform); - try std.testing.expect(collector.two_readings); - try std.testing.expectEqual(3, collector.devices.items.len); - - // Before any round, samples are empty. - try std.testing.expectEqual(null, collector.sample(@fromBackingInt(0)).cuda.util_percent); - - collector.round(); - try std.testing.expectEqual(1, collector.sample(@fromBackingInt(0)).cuda.util_percent); - // The wrong tag was replaced with an empty sample of the device's target. - try std.testing.expectEqual(null, collector.sample(@fromBackingInt(1)).cuda.util_percent); - try std.testing.expectEqual(1, collector.sample(@fromBackingInt(2)).cuda.util_percent); - { - const merged = collector.processes.lock(io); - defer collector.processes.unlock(io); - // A row naming a missing device is dropped. The others get the global index - // of their device, and keep their strings. - try std.testing.expectEqual(2, merged.rows.items.len); - try std.testing.expectEqual(1, merged.rows.items[0].device_idx); - try std.testing.expectEqual(2, merged.rows.items[1].device_idx); - try std.testing.expectEqualStrings("alice", merged.rows.items[1].username); - } - try std.testing.expectEqual(1, collector.host.read(io).cpu_power_mw); - - // `processes` fails: no rows this round, not the previous ones. - collector.round(); - try std.testing.expectEqual(2, collector.sample(@fromBackingInt(0)).cuda.util_percent); - // Host values a platform did not measure this round are null, not the previous ones. - try std.testing.expectEqual(null, collector.host.read(io).cpu_power_mw); - const merged = collector.processes.lock(io); - defer collector.processes.unlock(io); - try std.testing.expectEqual(0, merged.rows.items.len); -} - -test "collector tasks stop" { - const io = std.testing.io; - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - - var collector: Collector = .init(arena.allocator(), std.testing.allocator, io, .{ .poll_interval_ms = 1 }); - var fake: Fake = .{ .platform = .{ .devices = &.{.{ .target = .cuda }}, .vtable = &Fake.vtable } }; - try collector.add(&fake.platform); - try collector.start(); - while (collector.sample(@fromBackingInt(0)).cuda.util_percent.? < 3) try io.sleep(.fromMilliseconds(1), .awake); - collector.deinit(); -} diff --git a/bin/zml-smi/smi/counter.zig b/bin/zml-smi/smi/counter.zig deleted file mode 100644 index cd0996f3..00000000 --- a/bin/zml-smi/smi/counter.zig +++ /dev/null @@ -1,139 +0,0 @@ -const std = @import("std"); - -/// A running total (energy, busy time, CPU ticks) read at two polls. -pub const Counter = struct { - last: ?Reading = null, - - pub const Reading = struct { total: u64, at: std.Io.Timestamp }; - - /// Reads the monotonic clock, so the time is that of the read. - /// Null on the first reading, if the total went backwards, or if no time passed. - pub fn advance(self: *Counter, io: std.Io, total: u64) ?Growth { - return self.step(.{ .total = total, .at = .now(io, .awake) }); - } - - fn step(self: *Counter, reading: Reading) ?Growth { - const last = self.last; - self.last = reading; - const previous = last orelse return null; - if (reading.total < previous.total or reading.at.nanoseconds <= previous.at.nanoseconds) return null; - return .{ .amount = reading.total - previous.total, .elapsed = previous.at.durationTo(reading.at) }; - } -}; - -pub const Growth = struct { - amount: u64, - elapsed: std.Io.Duration, - - /// Growth per second: microjoules become microwatts. A result that does not fit is the largest u64. - pub fn perSecond(self: Growth) u64 { - const per_second = @as(u128, self.amount) * std.time.ns_per_s / @as(u128, @intCast(self.elapsed.nanoseconds)); - return std.math.cast(u64, per_second) orelse std.math.maxInt(u64); - } - - /// For a total of busy nanoseconds: its share of the elapsed time, at most 100. - pub fn percentOfTime(self: Growth) u8 { - return percent(self.amount, @intCast(self.elapsed.nanoseconds)); - } - - /// Its share of another growth, at most 100: busy cycles over total cycles. 0 when the other did not grow. - pub fn percentOf(self: Growth, whole: Growth) u8 { - return percent(self.amount, whole.amount); - } - - /// Adds a growth to `sum`, which starts null: the busy time of several processes over one poll. - pub fn add(sum: *?Growth, growth: Growth) void { - if (sum.*) |*total| total.amount +|= growth.amount else sum.* = growth; - } -}; - -fn percent(part: u128, whole: u128) u8 { - if (whole == 0) return 0; - return @intCast(@min(part * 100 / whole, 100)); -} - -/// One counter per key, for totals kept per process. -pub fn Counters(comptime Key: type) type { - return struct { - const Self = @This(); - - map: std.AutoHashMapUnmanaged(Key, Entry) = .empty, - gpa: std.mem.Allocator, - - const Entry = struct { counter: Counter = .{}, seen: bool = false }; - - pub fn init(gpa: std.mem.Allocator) Self { - return .{ .gpa = gpa }; - } - - pub fn deinit(self: *Self) void { - self.map.deinit(self.gpa); - } - - /// Advances the counter of `key` and marks the key as seen. - pub fn advance(self: *Self, key: Key, io: std.Io, total: u64) error{OutOfMemory}!?Growth { - const entry = try self.map.getOrPutValue(self.gpa, key, .{}); - entry.value_ptr.seen = true; - return entry.value_ptr.counter.advance(io, total); - } - - /// Forgets the keys not advanced since the last sweep. Called once per poll. - pub fn sweep(self: *Self) void { - var it = self.map.iterator(); - while (it.next()) |entry| { - if (entry.value_ptr.seen) { - entry.value_ptr.seen = false; - } else { - self.map.removeByPtr(entry.key_ptr); - } - } - } - }; -} - -fn at(ms: i64) std.Io.Timestamp { - return .fromNanoseconds(@as(i96, ms) * std.time.ns_per_ms); -} - -test Counter { - var counter: Counter = .{}; - try std.testing.expectEqual(null, counter.step(.{ .total = 100, .at = at(0) })); - - const growth = counter.step(.{ .total = 600, .at = at(500) }).?; - try std.testing.expectEqual(500, growth.amount); - try std.testing.expectEqual(500, growth.elapsed.toMilliseconds()); - - // The total went backwards, then no time passed. - try std.testing.expectEqual(null, counter.step(.{ .total = 10, .at = at(1000) })); - try std.testing.expectEqual(null, counter.step(.{ .total = 20, .at = at(1000) })); -} - -test Growth { - const half_second: std.Io.Duration = .fromMilliseconds(500); - // 10 mJ in 500 ms is 20 mW. - try std.testing.expectEqual(20_000, (Growth{ .amount = 10_000, .elapsed = half_second }).perSecond()); - try std.testing.expectEqual(std.math.maxInt(u64), (Growth{ .amount = std.math.maxInt(u64), .elapsed = .fromNanoseconds(1) }).perSecond()); - - try std.testing.expectEqual(50, (Growth{ .amount = 250_000_000, .elapsed = half_second }).percentOfTime()); - try std.testing.expectEqual(100, (Growth{ .amount = 2_000_000_000, .elapsed = half_second }).percentOfTime()); - - const cycles: Growth = .{ .amount = 30, .elapsed = half_second }; - try std.testing.expectEqual(30, cycles.percentOf(.{ .amount = 100, .elapsed = half_second })); - try std.testing.expectEqual(0, cycles.percentOf(.{ .amount = 0, .elapsed = half_second })); -} - -test Counters { - const io = std.testing.io; - var counters: Counters(u32) = .init(std.testing.allocator); - defer counters.deinit(); - - try std.testing.expectEqual(null, try counters.advance(1, io, 10)); - try std.testing.expectEqual(null, try counters.advance(2, io, 10)); - counters.sweep(); - - _ = try counters.advance(1, io, 20); - counters.sweep(); - // Key 2 was not advanced since the last sweep, so it starts over. - try std.testing.expect(counters.map.contains(1)); - try std.testing.expect(!counters.map.contains(2)); -} diff --git a/bin/zml-smi/smi/csv.zig b/bin/zml-smi/smi/csv.zig deleted file mode 100644 index f757362f..00000000 --- a/bin/zml-smi/smi/csv.zig +++ /dev/null @@ -1,66 +0,0 @@ -const std = @import("std"); -const smi = @import("smi.zig"); - -/// One table per target: index, type, remote, the strings of the device, then the fields of its sample. -/// A device column that is empty for the whole table is left out. -pub fn write(writer: *std.Io.Writer, collector: *smi.Collector) !void { - inline for (@typeInfo(smi.Target).@"enum".field_names) |name| { - try writeTable(writer, collector, @field(smi.Target, name)); - } -} - -fn writeTable(writer: *std.Io.Writer, collector: *smi.Collector, comptime target: smi.Target) !void { - const fields = @typeInfo(@FieldType(smi.Sample, @tagName(target))).@"struct".field_names; - - var rows: usize = 0; - var names = false; - var drivers = false; - var key_buffer: [16][]const u8 = undefined; - var keys: std.ArrayList([]const u8) = .initBuffer(&key_buffer); - for (collector.devices.items) |device| { - if (device.target != target) continue; - rows += 1; - names = names or device.name != null; - drivers = drivers or device.driver_version != null; - for (device.details) |detail| { - if (!contains(keys.items, detail.key)) keys.appendBounded(detail.key) catch {}; - } - } - if (rows == 0) return; - - try writer.writeAll("index,type,remote"); - if (names) try writer.writeAll(",name"); - if (drivers) try writer.writeAll(",driver_version"); - for (keys.items) |key| try writer.print(",{s}", .{key}); - inline for (fields) |field| try writer.writeAll("," ++ field); - try writer.writeAll("\n"); - - for (collector.devices.items, 0..) |device, i| { - if (device.target != target) continue; - try writer.print("{d},{t},{any}", .{ i, target, device.remote }); - if (names) try writer.print(",{s}", .{device.name orelse ""}); - if (drivers) try writer.print(",{s}", .{device.driver_version orelse ""}); - for (keys.items) |key| try writer.print(",{s}", .{detailValue(device, key) orelse ""}); - - const sample = @field(collector.sample(@fromBackingInt(@intCast(i))), @tagName(target)); - inline for (fields) |field| { - try writer.writeAll(","); - if (@field(sample, field)) |value| try writer.print("{d}", .{value}); - } - try writer.writeAll("\n"); - } -} - -fn contains(keys: []const []const u8, key: []const u8) bool { - for (keys) |other| { - if (std.mem.eql(u8, other, key)) return true; - } - return false; -} - -fn detailValue(device: smi.Device, key: []const u8) ?[]const u8 { - for (device.details) |item| { - if (std.mem.eql(u8, item.key, key)) return item.value; - } - return null; -} diff --git a/bin/zml-smi/smi/double_buffer.zig b/bin/zml-smi/smi/double_buffer.zig deleted file mode 100644 index c8e3d5cc..00000000 --- a/bin/zml-smi/smi/double_buffer.zig +++ /dev/null @@ -1,59 +0,0 @@ -const std = @import("std"); - -/// Two copies of a value: one writer fills the back while readers copy the front. -/// The writer holds the lock only to swap; a reader holds it while it copies. -pub fn DoubleBuffer(comptime T: type) type { - return struct { - const Self = @This(); - - values: [2]T, - front_index: u1 = 0, - mutex: std.Io.Mutex = .init, - - /// The side the writer fills. Only the writer calls it. - pub fn back(self: *Self) *T { - return &self.values[~self.front_index]; - } - - /// The side readers see. Only the writer reads it without the lock. - pub fn front(self: *Self) *T { - return &self.values[self.front_index]; - } - - /// Publishes the back. - pub fn swap(self: *Self, io: std.Io) void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - self.front_index = ~self.front_index; - } - - /// Locks the front until `unlock`. - pub fn lock(self: *Self, io: std.Io) *const T { - self.mutex.lockUncancelable(io); - return &self.values[self.front_index]; - } - - pub fn unlock(self: *Self, io: std.Io) void { - self.mutex.unlock(io); - } - - /// A copy of the front. - pub fn read(self: *Self, io: std.Io) T { - defer self.unlock(io); - return self.lock(io).*; - } - }; -} - -test "swap publishes the back" { - const io = std.testing.io; - var buffer: DoubleBuffer(u32) = .{ .values = .{ 1, 2 } }; - - buffer.back().* = 3; - try std.testing.expectEqual(1, buffer.lock(io).*); - buffer.unlock(io); - - buffer.swap(io); - try std.testing.expectEqual(3, buffer.lock(io).*); - buffer.unlock(io); -} diff --git a/bin/zml-smi/smi/dynlib.zig b/bin/zml-smi/smi/dynlib.zig deleted file mode 100644 index e24e131a..00000000 --- a/bin/zml-smi/smi/dynlib.zig +++ /dev/null @@ -1,108 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const sandbox = @import("sandbox.zig"); - -const log = std.log.scoped(.@"zml-smi/dynlib"); - -const flags: std.c.RTLD = .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }; - -/// Opens the first system library that loads and resolves every field of `Api` by its name. -/// A field is a `Checked`, a function pointer, or a pointer to a data symbol. -pub fn open(comptime Api: type, names: []const [:0]const u8) !Api { - for (names) |name| { - const handle = std.c.dlopen(name, flags) orelse continue; - return resolve(Api, handle, name); - } - log.warn("{s}", .{lastError()}); - return error.LibraryNotFound; -} - -/// Opens a library shipped with zml-smi, from its sandbox directory. -pub fn openBundled(comptime Api: type, io: std.Io, name: []const u8) !Api { - var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const directory = sandbox.path(io, &buffer) orelse { - log.warn("{s}: no sandbox directory", .{name}); - return error.LibraryNotFound; - }; - var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrintSentinel(&path_buffer, "{s}/lib/{s}", .{ directory, name }, 0); - const handle = std.c.dlopen(path, flags) orelse { - log.warn("{s}", .{lastError()}); - return error.LibraryNotFound; - }; - return resolve(Api, handle, name); -} - -fn lastError() []const u8 { - return std.mem.span(std.c.dlerror() orelse return "no library to open"); -} - -fn resolve(comptime Api: type, handle: *anyopaque, library: []const u8) !Api { - var api: Api = undefined; - const info = @typeInfo(Api).@"struct"; - inline for (info.field_names, info.field_types) |name, Field| { - const symbol = std.c.dlsym(handle, name) orelse { - log.warn("{s} has no symbol {s}", .{ library, name }); - return error.MissingSymbol; - }; - @field(api, name) = if (@typeInfo(Field) == .@"struct") .{ .function = @ptrCast(@alignCast(symbol)) } else @ptrCast(@alignCast(symbol)); - } - return api; -} - -/// A C function that reports failure through its return value, turned into an error by `check`. -pub fn Checked(comptime function: anytype, comptime check: anytype) type { - const Function = @TypeOf(function); - const Error = @typeInfo(@typeInfo(@TypeOf(check)).@"fn".return_type.?).error_union.error_set; - - return struct { - function: *const Function, - - const params = @typeInfo(Function).@"fn".param_types; - const Result = @typeInfo(params[params.len - 1].?).pointer.child; - const Inputs = inputs: { - var types: [params.len - 1]type = undefined; - for (&types, params[0 .. params.len - 1]) |*T, param| T.* = param.?; - const final = types; - break :inputs @Tuple(&final); - }; - - pub fn call(self: @This(), args: std.meta.ArgsTuple(Function)) Error!void { - return check(@call(.auto, self.function, args)); - } - - /// For a function whose only result is its last parameter: returns that result. - pub fn get(self: @This(), args: Inputs) Error!Result { - var result: Result = std.mem.zeroes(Result); - try check(@call(.auto, self.function, args ++ .{&result})); - return result; - } - }; -} - -const Libc = struct { - getpid: *const @TypeOf(std.c.getpid), - clock_gettime: Checked(std.c.clock_gettime, checkLibc), - close: Checked(std.c.close, checkLibc), - - fn checkLibc(status: c_int) error{Failed}!void { - if (status != 0) return error.Failed; - } -}; - -const libc_names: []const [:0]const u8 = switch (builtin.os.tag) { - .macos => &.{"/usr/lib/libSystem.B.dylib"}, - else => &.{ "libc.so.6", "libc.so" }, -}; - -test open { - const libc = try open(Libc, libc_names); - try std.testing.expectEqual(std.c.getpid(), libc.getpid()); - try std.testing.expect((try libc.clock_gettime.get(.{.MONOTONIC})).sec > 0); - try std.testing.expectError(error.Failed, libc.close.call(.{-1})); - try std.testing.expectError(error.LibraryNotFound, open(Libc, &.{"libmissing.so"})); -} - -test openBundled { - try std.testing.expectError(error.LibraryNotFound, openBundled(Libc, std.testing.io, "libmissing.so")); -} diff --git a/bin/zml-smi/smi/fdinfo.zig b/bin/zml-smi/smi/fdinfo.zig deleted file mode 100644 index be79558d..00000000 --- a/bin/zml-smi/smi/fdinfo.zig +++ /dev/null @@ -1,166 +0,0 @@ -const std = @import("std"); -const sysfs = @import("sysfs.zig"); - -/// One /proc//fdinfo/ file of a DRM client. Its strings point into the text it was parsed from. -pub const Fdinfo = struct { - /// The PCI address of the device: "0000:03:00.0". - pdev: ?[]const u8 = null, - client_id: ?u64 = null, - region_buffer: [8]Region = undefined, - region_count: u8 = 0, - engine_buffer: [16]Engine = undefined, - engine_count: u8 = 0, - - /// Memory of one region, in bytes. - pub const Region = struct { - name: []const u8 = "", - total: ?u64 = null, - resident: ?u64 = null, - }; - - /// The busy time of one engine class: nanoseconds, or for xe, busy cycles out of total cycles. - pub const Engine = struct { - class: []const u8 = "", - busy_ns: ?u64 = null, - cycles: ?u64 = null, - total_cycles: ?u64 = null, - }; - - pub fn regions(self: *const Fdinfo) []const Region { - return self.region_buffer[0..self.region_count]; - } - - pub fn engines(self: *const Fdinfo) []const Engine { - return self.engine_buffer[0..self.engine_count]; - } - - fn region(self: *Fdinfo, name: []const u8) ?*Region { - return slot(Region, &self.region_buffer, &self.region_count, "name", name); - } - - fn engine(self: *Fdinfo, class: []const u8) ?*Engine { - return slot(Engine, &self.engine_buffer, &self.engine_count, "class", class); - } -}; - -/// Reads one fdinfo file into `buffer`, which the strings of the result point into. -pub fn read(dir: sysfs.Dir, sub_path: []const u8, buffer: []u8) !Fdinfo { - return parse(try dir.read(sub_path, buffer)); -} - -pub fn parse(text: []const u8) Fdinfo { - var info: Fdinfo = .{}; - var lines = std.mem.tokenizeScalar(u8, text, '\n'); - while (lines.next()) |line| { - const colon = std.mem.indexOfScalar(u8, line, ':') orelse continue; - const key = line[0..colon]; - const value = std.mem.trim(u8, line[colon + 1 ..], " \t"); - - if (std.mem.eql(u8, key, "drm-pdev") or std.mem.eql(u8, key, "pdev")) { - info.pdev = value; - } else if (std.mem.eql(u8, key, "drm-client-id")) { - info.client_id = sysfs.firstNumber(value); - } else if (std.mem.startsWith(u8, key, "drm-engine-capacity-")) { - continue; - } else if (std.mem.cutPrefix(u8, key, "drm-engine-")) |class| { - if (info.engine(class)) |engine| engine.busy_ns = sysfs.firstNumber(value); - } else if (std.mem.cutPrefix(u8, key, "drm-total-cycles-")) |class| { - if (info.engine(class)) |engine| engine.total_cycles = sysfs.firstNumber(value); - } else if (std.mem.cutPrefix(u8, key, "drm-cycles-")) |class| { - if (info.engine(class)) |engine| engine.cycles = sysfs.firstNumber(value); - } else if (std.mem.cutPrefix(u8, key, "drm-total-")) |name| { - if (info.region(name)) |region| region.total = bytes(value); - } else if (std.mem.cutPrefix(u8, key, "drm-resident-")) |name| { - if (info.region(name)) |region| region.resident = bytes(value); - } else if (std.mem.cutPrefix(u8, key, "drm-memory-")) |name| { - // The older amdgpu name of resident memory. - if (info.region(name)) |region| region.resident = bytes(value); - } else if (std.mem.endsWith(u8, key, " mem")) { - // The oldest amdgpu one: "vram mem". - if (info.region(key[0 .. key.len - " mem".len])) |region| region.resident = bytes(value); - } - } - return info; -} - -/// The entry called `name`, added when missing. Null past the capacity. -fn slot(comptime T: type, buffer: []T, count: *u8, comptime key: []const u8, name: []const u8) ?*T { - for (buffer[0..count.*]) |*item| { - if (std.mem.eql(u8, @field(item, key), name)) return item; - } - if (count.* == buffer.len) return null; - const item = &buffer[count.*]; - count.* += 1; - item.* = .{}; - @field(item, key) = name; - return item; -} - -/// A size such as "4096 KiB", in bytes. A size without a unit is in bytes. -fn bytes(value: []const u8) ?u64 { - var words = std.mem.tokenizeAny(u8, value, " \t"); - const size = std.fmt.parseInt(u64, words.next() orelse return null, 10) catch return null; - const unit = words.next() orelse return size; - const shift: u6 = if (std.ascii.eqlIgnoreCase(unit, "KiB") or std.ascii.eqlIgnoreCase(unit, "kB")) - 10 - else if (std.ascii.eqlIgnoreCase(unit, "MiB")) - 20 - else if (std.ascii.eqlIgnoreCase(unit, "GiB")) - 30 - else - 0; - return std.math.mul(u64, size, @as(u64, 1) << shift) catch null; -} - -test "i915 engines and regions" { - const info = parse( - \\pos: 0 - \\drm-driver: i915 - \\drm-pdev: 0000:03:00.0 - \\drm-client-id: 42 - \\drm-total-system0: 2048 KiB - \\drm-resident-local0: 3 MiB - \\drm-engine-render: 100 ns - \\drm-engine-capacity-video: 2 - \\drm-engine-video-decode: 50 ns - ); - try std.testing.expectEqualStrings("0000:03:00.0", info.pdev.?); - try std.testing.expectEqual(42, info.client_id); - try std.testing.expectEqual(2, info.regions().len); - try std.testing.expectEqualStrings("system0", info.regions()[0].name); - try std.testing.expectEqual(2048 * 1024, info.regions()[0].total); - try std.testing.expectEqual(3 * 1024 * 1024, info.regions()[1].resident); - try std.testing.expectEqual(2, info.engines().len); - try std.testing.expectEqualStrings("render", info.engines()[0].class); - try std.testing.expectEqual(100, info.engines()[0].busy_ns); - try std.testing.expectEqualStrings("video-decode", info.engines()[1].class); -} - -test "xe cycles" { - const info = parse( - \\drm-pdev: 0000:03:00.0 - \\drm-cycles-rcs: 10 - \\drm-total-cycles-rcs: 100 - \\drm-cycles-ccs: 30 - \\drm-total-cycles-ccs: 100 - \\drm-total-vram0: 4096 KiB - \\drm-resident-vram0: 3072 KiB - ); - try std.testing.expectEqual(2, info.engines().len); - try std.testing.expectEqual(10, info.engines()[0].cycles); - try std.testing.expectEqual(100, info.engines()[0].total_cycles); - try std.testing.expectEqual(null, info.engines()[0].busy_ns); - try std.testing.expectEqual(4096 * 1024, info.regions()[0].total); - try std.testing.expectEqual(3072 * 1024, info.regions()[0].resident); -} - -test "amdgpu keys" { - const old = parse("pdev:\t0000:0c:00.0\nvram mem:\t1024 kB\n"); - try std.testing.expectEqualStrings("0000:0c:00.0", old.pdev.?); - try std.testing.expectEqualStrings("vram", old.regions()[0].name); - try std.testing.expectEqual(1024 * 1024, old.regions()[0].resident); - - const current = parse("drm-pdev:\t0000:0c:00.0\ndrm-memory-vram:\t2048 KiB\ndrm-engine-gfx:\t5 ns\n"); - try std.testing.expectEqual(2048 * 1024, current.regions()[0].resident); - try std.testing.expectEqual(5, current.engines()[0].busy_ns); -} diff --git a/bin/zml-smi/smi/host/linux.zig b/bin/zml-smi/smi/host/linux.zig deleted file mode 100644 index 19f7ec78..00000000 --- a/bin/zml-smi/smi/host/linux.zig +++ /dev/null @@ -1,140 +0,0 @@ -//! The host, from /proc and /sys. -const std = @import("std"); -const smi = @import("../smi.zig"); - -pub const Host = struct { - io: std.Io, - proc: ?smi.sysfs.Dir, - /// CPU ticks of every CPU together. - total: smi.Counter = .{}, - /// CPU ticks of each process. - ticks: smi.Counters(u32), - - /// Reads what does not change: hostname, kernel, CPU, core count and total memory. - pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, data: *smi.HostData) Host { - const proc = smi.sysfs.Dir.open(io, "/proc") catch null; - var buffer: [256]u8 = undefined; - if (proc) |dir| { - data.hostname = dupe(arena, dir.read("sys/kernel/hostname", &buffer) catch null); - data.kernel = dupe(arena, dir.read("sys/kernel/osrelease", &buffer) catch null); - data.cpu_name = dupe(arena, dir.field("cpuinfo", "model name", &buffer) catch null); - data.mem_total_kib = smi.sysfs.firstNumber(dir.field("meminfo", "MemTotal", &buffer) catch ""); - } - if (smi.sysfs.Dir.open(io, "/sys/devices/system/cpu")) |cpu| { - defer cpu.close(); - data.cpu_cores = cores(cpu.read("present", &buffer) catch ""); - } else |_| {} - return .{ .io = io, .proc = proc, .ticks = .init(gpa) }; - } - - pub fn deinit(self: *Host) void { - self.ticks.deinit(); - if (self.proc) |proc| proc.close(); - } - - /// Overwrites what changes: available memory, load and uptime. A read that fails gives null. - pub fn sample(self: *Host, data: *smi.HostData) void { - data.mem_available_kib = null; - data.load_1, data.load_5, data.load_15 = .{ null, null, null }; - data.uptime_seconds = null; - const proc = self.proc orelse return; - - var buffer: [256]u8 = undefined; - data.mem_available_kib = smi.sysfs.firstNumber(proc.field("meminfo", "MemAvailable", &buffer) catch ""); - if (proc.read("loadavg", &buffer)) |loadavg| { - var words = std.mem.tokenizeScalar(u8, loadavg, ' '); - inline for (.{ &data.load_1, &data.load_5, &data.load_15 }) |load| { - load.* = std.fmt.parseFloat(f32, words.next() orelse "") catch null; - } - } else |_| {} - if (proc.read("uptime", &buffer)) |uptime| { - data.uptime_seconds = std.fmt.parseInt(u64, std.mem.sliceTo(uptime, '.'), 10) catch null; - } else |_| {} - } - - /// Adds user, command, CPU and RSS to the local rows. Each process is read once, - /// however many rows it has. `strings` holds the strings of the rows. - pub fn enrich(self: *Host, strings: std.mem.Allocator, scratch: std.mem.Allocator, rows: []smi.ProcessInfo) void { - const proc = self.proc orelse return; - defer self.ticks.sweep(); - - var buffer: [512]u8 = undefined; - const total = if (proc.head("stat", &buffer)) |stat| self.total.advance(self.io, totalTicks(stat)) else |_| null; - - var first_row: std.AutoHashMapUnmanaged(u32, usize) = .empty; - for (rows, 0..) |*row, i| { - // A remote host already filled its rows. - if (row.remote) continue; - const entry = first_row.getOrPut(scratch, row.pid) catch continue; - if (entry.found_existing) { - const done = rows[entry.value_ptr.*]; - row.uid, row.username, row.comm, row.cpu_percent, row.rss_kib = .{ done.uid, done.username, done.comm, done.cpu_percent, done.rss_kib }; - continue; - } - entry.value_ptr.* = i; - self.read(proc, strings, row, total); - } - } - - fn read(self: *Host, proc: smi.sysfs.Dir, strings: std.mem.Allocator, row: *smi.ProcessInfo, total: ?smi.Growth) void { - var path: [32]u8 = undefined; - var buffer: [4096]u8 = undefined; - - if (proc.read(processPath(&path, row.pid, "status"), &buffer)) |status| { - var lines = std.mem.tokenizeScalar(u8, status, '\n'); - while (lines.next()) |line| { - if (smi.sysfs.fieldValue(line, "Uid")) |uid| row.uid = @intCast(smi.sysfs.firstNumber(uid) orelse 0); - if (smi.sysfs.fieldValue(line, "VmRSS")) |rss| row.rss_kib = smi.sysfs.firstNumber(rss) orelse 0; - } - if (std.c.getpwuid(row.uid)) |user| { - if (user.name) |name| row.username = strings.dupe(u8, std.mem.span(name)) catch ""; - } - } else |_| {} - - if (proc.head(processPath(&path, row.pid, "cmdline"), &buffer)) |cmdline| { - const command = buffer[0..cmdline.len]; - std.mem.replaceScalar(u8, command, 0, ' '); - row.comm = strings.dupe(u8, std.mem.trimEnd(u8, command, " ")) catch ""; - } else |_| {} - - if (proc.read(processPath(&path, row.pid, "stat"), &buffer)) |stat| { - const ticks = processTicks(stat) orelse return; - const growth = (self.ticks.advance(row.pid, self.io, ticks) catch return) orelse return; - const whole = total orelse return; - if (whole.amount > 0) row.cpu_percent = @intCast(@min(growth.amount *| 1000 / whole.amount, std.math.maxInt(u16))); - } else |_| {} - } -}; - -fn processPath(buffer: []u8, pid: u32, file: []const u8) []const u8 { - return std.fmt.bufPrint(buffer, "{d}/{s}", .{ pid, file }) catch ""; -} - -fn dupe(arena: std.mem.Allocator, string: ?[]const u8) ?[]const u8 { - return arena.dupe(u8, string orelse return null) catch null; -} - -/// The CPUs present, "0-47": the highest index, plus one. -fn cores(present: []const u8) ?u64 { - const dash = std.mem.indexOfScalar(u8, present, '-') orelse return if (present.len > 0) 1 else null; - return (std.fmt.parseInt(u64, present[dash + 1 ..], 10) catch return null) + 1; -} - -/// The sum of the first line of /proc/stat: "cpu user nice system idle ...". -fn totalTicks(stat: []const u8) u64 { - var words = std.mem.tokenizeAny(u8, std.mem.sliceTo(stat, '\n'), " \t"); - _ = words.next(); - var total: u64 = 0; - while (words.next()) |word| total +|= std.fmt.parseInt(u64, word, 10) catch 0; - return total; -} - -/// utime plus stime of /proc//stat, the 12th and 13th fields after the command. -fn processTicks(stat: []const u8) ?u64 { - const end = std.mem.lastIndexOfScalar(u8, stat, ')') orelse return null; - var words = std.mem.tokenizeScalar(u8, stat[end + 1 ..], ' '); - for (0..11) |_| _ = words.next(); - const utime = std.fmt.parseInt(u64, words.next() orelse return null, 10) catch return null; - const stime = std.fmt.parseInt(u64, words.next() orelse return null, 10) catch return null; - return utime + stime; -} diff --git a/bin/zml-smi/smi/host/macos.zig b/bin/zml-smi/smi/host/macos.zig deleted file mode 100644 index 8fcd1199..00000000 --- a/bin/zml-smi/smi/host/macos.zig +++ /dev/null @@ -1,227 +0,0 @@ -//! The host, from sysctl, the Mach host statistics and libproc. -const std = @import("std"); -const smi = @import("../smi.zig"); - -const c = std.c; - -pub const Host = struct { - io: std.Io, - /// CPU nanoseconds of each process. - cpu_time: smi.Counters(u32), - cpus: u64, - /// Converts the CPU time libproc reports, in Mach ticks, to nanoseconds. - timebase: c.mach_timebase_info_data, - - /// Reads what does not change: hostname, kernel, CPU, core count and total memory. - pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, data: *smi.HostData) Host { - data.hostname = sysctlString(arena, "kern.hostname"); - data.kernel = sysctlString(arena, "kern.osrelease"); - data.cpu_name = sysctlString(arena, "machdep.cpu.brand_string"); - data.cpu_cores = if (sysctl(u32, "hw.logicalcpu")) |cores| cores else null; - data.mem_total_kib = if (sysctl(u64, "hw.memsize")) |bytes| bytes / 1024 else null; - - var timebase: c.mach_timebase_info_data = .{ .numer = 1, .denom = 1 }; - if (c.mach_timebase_info(&timebase) != KERN_SUCCESS or timebase.denom == 0) timebase = .{ .numer = 1, .denom = 1 }; - return .{ .io = io, .cpu_time = .init(gpa), .cpus = @max(1, data.cpu_cores orelse 1), .timebase = timebase }; - } - - pub fn deinit(self: *Host) void { - self.cpu_time.deinit(); - } - - /// Overwrites what changes: available memory, load and uptime. A read that fails gives null. - pub fn sample(_: *Host, data: *smi.HostData) void { - data.mem_available_kib = memAvailable(); - - var load: [3]f64 = undefined; - const loaded = getloadavg(&load, 3) == 3; - inline for (.{ &data.load_1, &data.load_5, &data.load_15 }, 0..) |field, i| { - field.* = if (loaded) @floatCast(load[i]) else null; - } - - data.uptime_seconds = null; - var now: c.timeval = undefined; - if (sysctl(c.timeval, "kern.boottime")) |boot| { - if (c.gettimeofday(&now, null) == 0) data.uptime_seconds = @intCast(now.sec - boot.sec); - } - } - - /// Adds user, command, CPU and RSS to the local rows. Each process is read once, - /// however many rows it has. `strings` holds the strings of the rows. - /// The CPU and RSS of another user's process read 0, unless zml-smi runs as root. - pub fn enrich(self: *Host, strings: std.mem.Allocator, scratch: std.mem.Allocator, rows: []smi.ProcessInfo) void { - defer self.cpu_time.sweep(); - - var first_row: std.AutoHashMapUnmanaged(u32, usize) = .empty; - for (rows, 0..) |*row, i| { - // A remote host already filled its rows. - if (row.remote) continue; - const entry = first_row.getOrPut(scratch, row.pid) catch continue; - if (entry.found_existing) { - const done = rows[entry.value_ptr.*]; - row.uid, row.username, row.comm, row.cpu_percent, row.rss_kib = .{ done.uid, done.username, done.comm, done.cpu_percent, done.rss_kib }; - continue; - } - entry.value_ptr.* = i; - self.read(strings, row); - } - } - - fn read(self: *Host, strings: std.mem.Allocator, row: *smi.ProcessInfo) void { - const pid = std.math.cast(c_int, row.pid) orelse return; - - var short: ProcBsdShortInfo = undefined; - if (proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 0, &short, @sizeOf(ProcBsdShortInfo)) == @sizeOf(ProcBsdShortInfo)) { - row.uid = short.uid; - if (c.getpwuid(row.uid)) |user| { - if (user.name) |name| row.username = strings.dupe(u8, std.mem.span(name)) catch ""; - } - } - - var path: [PROC_PIDPATHINFO_MAXSIZE]u8 = undefined; - const length = proc_pidpath(pid, &path, path.len); - if (length > 0) row.comm = strings.dupe(u8, path[0..@intCast(length)]) catch ""; - - var task: ProcTaskInfo = undefined; - if (proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &task, @sizeOf(ProcTaskInfo)) != @sizeOf(ProcTaskInfo)) return; - row.rss_kib = task.resident_size / 1024; - const ticks = @as(u128, task.total_user) + task.total_system; - const nanoseconds = std.math.lossyCast(u64, ticks * self.timebase.numer / self.timebase.denom); - const growth = (self.cpu_time.advance(row.pid, self.io, nanoseconds) catch return) orelse return; - // Tenths of a percent of every CPU, as on Linux. - const whole = @as(u128, @intCast(growth.elapsed.nanoseconds)) * self.cpus; - row.cpu_percent = @intCast(@min(@as(u128, growth.amount) * 1000 / whole, std.math.maxInt(u16))); - } -}; - -const PROC_PIDTASKINFO: c_int = 4; -const PROC_PIDT_SHORTBSDINFO: c_int = 13; -const PROC_PIDPATHINFO_MAXSIZE = 4096; - -/// struct proc_bsdshortinfo, which libproc gives for every process. -const ProcBsdShortInfo = extern struct { - pid: u32, - ppid: u32, - pgid: u32, - status: u32, - comm: [16]u8, - flags: u32, - uid: c.uid_t, - gid: c.gid_t, - ruid: c.uid_t, - rgid: c.gid_t, - svuid: c.uid_t, - svgid: c.gid_t, - rfu: u32, -}; - -/// struct proc_taskinfo, which libproc gives for the user's own processes, or any as root. -const ProcTaskInfo = extern struct { - virtual_size: u64, - resident_size: u64, - total_user: u64, - total_system: u64, - threads_user: u64, - threads_system: u64, - policy: i32, - faults: i32, - pageins: i32, - cow_faults: i32, - messages_sent: i32, - messages_received: i32, - syscalls_mach: i32, - syscalls_unix: i32, - csw: i32, - threadnum: i32, - numrunning: i32, - priority: i32, -}; - -extern "c" fn proc_pidinfo(pid: c_int, flavor: c_int, arg: u64, buffer: *anyopaque, size: c_int) c_int; -extern "c" fn proc_pidpath(pid: c_int, buffer: [*]u8, size: u32) c_int; - -const vm_statistics64 = extern struct { - free_count: c_uint, - active_count: c_uint, - inactive_count: c_uint, - wire_count: c_uint, - zero_fill_count: u64, - reactivations: u64, - pageins: u64, - pageouts: u64, - faults: u64, - cow_faults: u64, - lookups: u64, - hits: u64, - purges: u64, - purgeable_count: c_uint, - speculative_count: c_uint, - decompressions: u64, - compressions: u64, - swapins: u64, - swapouts: u64, - compressor_page_count: c_uint, - throttled_count: c_uint, - external_page_count: c_uint, - internal_page_count: c_uint, - total_uncompressed_pages_in_compressor: u64, -}; - -const HOST_VM_INFO64: c_int = 4; -const KERN_SUCCESS: c_int = 0; - -extern "c" fn host_statistics64(host: c.mach_port_t, flavor: c_int, info: *anyopaque, count: *c_uint) c.kern_return_t; -extern "c" fn getloadavg(loadavg: *[3]f64, nelem: c_int) c_int; - -/// Free, inactive and purgeable pages. -fn memAvailable() ?u64 { - var stats: vm_statistics64 = undefined; - var count: c_uint = @sizeOf(vm_statistics64) / @sizeOf(c_int); - if (host_statistics64(c.mach_host_self(), HOST_VM_INFO64, &stats, &count) != KERN_SUCCESS) return null; - - var page_size: u64 = 0; - if (c._host_page_size(c.mach_host_self(), &page_size) != KERN_SUCCESS) return null; - - const pages = @as(u64, stats.free_count) + stats.inactive_count + stats.purgeable_count; - return pages * page_size / 1024; -} - -fn sysctl(comptime T: type, name: [*:0]const u8) ?T { - var value: T = undefined; - var len: usize = @sizeOf(T); - if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) return null; - return value; -} - -fn sysctlString(arena: std.mem.Allocator, name: [*:0]const u8) ?[]const u8 { - var buffer: [256]u8 = undefined; - var len: usize = buffer.len; - if (c.sysctlbyname(name, &buffer, &len, null, 0) != 0) return null; - return arena.dupe(u8, std.mem.sliceTo(buffer[0..len], 0)) catch null; -} - -test "enrich" { - const io = std.testing.io; - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - var data: smi.HostData = .{}; - var host: Host = .init(arena.allocator(), std.testing.allocator, io, &data); - defer host.deinit(); - - // This process on two devices, launchd, and a row a remote host filled. - const self: u32 = @intCast(c.getpid()); - var rows = [_]smi.ProcessInfo{ .{ .pid = self }, .{ .pid = 1 }, .{ .pid = self, .device_idx = 1 }, .{ .pid = 1, .remote = true } }; - host.enrich(arena.allocator(), arena.allocator(), &rows); - try std.testing.expectEqual(c.getuid(), rows[0].uid); - try std.testing.expect(rows[0].rss_kib > 0); - try std.testing.expectEqualStrings(rows[0].comm, rows[2].comm); - try std.testing.expectEqualStrings("root", rows[1].username); - try std.testing.expectEqualStrings("/sbin/launchd", rows[1].comm); - try std.testing.expectEqualStrings("", rows[3].comm); - - // CPU share needs a second reading. - const start: std.Io.Timestamp = .now(io, .awake); - while (start.untilNow(io, .awake).toMilliseconds() < 20) {} - host.enrich(arena.allocator(), arena.allocator(), &rows); - try std.testing.expect(rows[0].cpu_percent > 0); -} diff --git a/bin/zml-smi/smi/hwmon.zig b/bin/zml-smi/smi/hwmon.zig deleted file mode 100644 index 46da85e1..00000000 --- a/bin/zml-smi/smi/hwmon.zig +++ /dev/null @@ -1,90 +0,0 @@ -const std = @import("std"); -const sysfs = @import("sysfs.zig"); - -/// The hwmon directory of a device: the one called `name`, or the first when `name` is null. -pub fn find(device: sysfs.Dir, name: ?[]const u8) !sysfs.Dir { - const hwmon = try device.openDir("hwmon"); - defer hwmon.close(); - - var it = hwmon.iterate(); - while (try it.next(device.io)) |entry| { - if (!std.mem.startsWith(u8, entry.name, "hwmon")) continue; - const dir = hwmon.openDir(entry.name) catch continue; - const wanted = name orelse return dir; - var buffer: [64]u8 = undefined; - if (std.mem.eql(u8, dir.read("name", &buffer) catch "", wanted)) return dir; - dir.close(); - } - return error.NotFound; -} - -/// A file that may hold a value: `temp2_input`, or with a label, the `temp_input` -/// whose `temp_label` is the label. -pub const Candidate = struct { - prefix: []const u8, - label: ?[]const u8 = null, - suffix: []const u8 = "input", -}; - -/// The name of the first candidate that reads as a number, copied into `arena`. -pub fn resolve(arena: std.mem.Allocator, dir: sysfs.Dir, candidates: []const Candidate) ?[]const u8 { - var buffer: [64]u8 = undefined; - for (candidates) |candidate| { - const name = fileName(dir, candidate, &buffer) orelse continue; - _ = dir.int(u64, name) catch continue; - return arena.dupe(u8, name) catch null; - } - return null; -} - -fn fileName(dir: sysfs.Dir, candidate: Candidate, buffer: []u8) ?[]const u8 { - const label = candidate.label orelse - return std.fmt.bufPrint(buffer, "{s}_{s}", .{ candidate.prefix, candidate.suffix }) catch null; - - for (1..16) |index| { - var label_name: [64]u8 = undefined; - var text: [64]u8 = undefined; - const path = std.fmt.bufPrint(&label_name, "{s}{d}_label", .{ candidate.prefix, index }) catch return null; - if (!std.ascii.eqlIgnoreCase(dir.read(path, &text) catch continue, label)) continue; - return std.fmt.bufPrint(buffer, "{s}{d}_{s}", .{ candidate.prefix, index, candidate.suffix }) catch null; - } - return null; -} - -/// A fan PWM duty, 0 to 255, as a percentage. -pub fn pwmPercent(pwm: u64) u64 { - return (@as(u64, @min(pwm, 255)) * 100 + 127) / 255; -} - -test resolve { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); - defer tmp.cleanup(); - try tmp.dir.createDirPath(io, "hwmon/hwmon2"); - try tmp.dir.createDirPath(io, "hwmon/hwmon3"); - try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon2/name", .data = "nvme\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon3/name", .data = "xe\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon3/temp2_label", .data = "PKG\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon3/temp2_input", .data = "58000\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon3/power1_max", .data = "230000000\n" }); - - const device: sysfs.Dir = .{ .handle = tmp.dir, .io = io }; - try std.testing.expectError(error.NotFound, find(device, "i915")); - const dir = try find(device, "xe"); - defer dir.close(); - - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - const temperature = resolve(arena.allocator(), dir, &.{ .{ .prefix = "temp", .label = "pkg" }, .{ .prefix = "temp1" } }); - try std.testing.expectEqualStrings("temp2_input", temperature.?); - const power_limit = resolve(arena.allocator(), dir, &.{ .{ .prefix = "power1", .suffix = "cap" }, .{ .prefix = "power1", .suffix = "max" } }); - try std.testing.expectEqualStrings("power1_max", power_limit.?); - try std.testing.expectEqual(null, resolve(arena.allocator(), dir, &.{.{ .prefix = "fan1" }})); -} - -test pwmPercent { - try std.testing.expectEqual(0, pwmPercent(0)); - try std.testing.expectEqual(50, pwmPercent(128)); - try std.testing.expectEqual(100, pwmPercent(255)); - try std.testing.expectEqual(100, pwmPercent(300)); -} diff --git a/bin/zml-smi/smi/json.zig b/bin/zml-smi/smi/json.zig deleted file mode 100644 index 43d53d72..00000000 --- a/bin/zml-smi/smi/json.zig +++ /dev/null @@ -1,21 +0,0 @@ -const std = @import("std"); -const smi = @import("smi.zig"); - -/// Every device with its latest sample, then the merged processes. -pub fn write(writer: *std.Io.Writer, collector: *smi.Collector) !void { - var json: std.json.Stringify = .{ .writer = writer }; - try json.beginObject(); - try json.objectField("devices"); - try json.beginArray(); - for (collector.devices.items, 0..) |device, i| { - try json.write(smi.Row{ .device = device, .sample = collector.sample(@fromBackingInt(@intCast(i))) }); - } - try json.endArray(); - - try json.objectField("processes"); - const processes = collector.processes.lock(collector.io); - defer collector.processes.unlock(collector.io); - try json.write(processes.rows.items); - try json.endObject(); - try writer.writeAll("\n"); -} diff --git a/bin/zml-smi/smi/model.zig b/bin/zml-smi/smi/model.zig deleted file mode 100644 index caba0d73..00000000 --- a/bin/zml-smi/smi/model.zig +++ /dev/null @@ -1,225 +0,0 @@ -const std = @import("std"); - -pub const Target = enum { cuda, rocm, oneapi, neuron, tpu, furiosa, metal }; - -/// What does not change while we run. Its strings live as long as the program. -pub const Device = struct { - target: Target, - /// It comes from another zml-smi, over --remotes. - remote: bool = false, - name: ?[]const u8 = null, - driver_version: ?[]const u8 = null, - /// The strings only one platform has, such as the CUDA version. - details: []const Detail = &.{}, - - pub const Detail = struct { - /// Its name in JSON, CSV and Prometheus labels: "cuda_version". - key: []const u8, - /// Its name in the TUI: "CUDA". - label: []const u8, - value: []const u8, - /// Shown beside the driver version in the card title. - in_title: bool = false, - }; - - /// Devices are referred to by index, never by pointer. - pub const Index = enum(u16) { _ }; -}; - -/// What is read every poll. Numbers only. Its tag is the device's target. -pub const Sample = union(Target) { - cuda: Gpu, - rocm: Gpu, - oneapi: Gpu, - neuron: Neuron, - tpu: Tpu, - furiosa: Furiosa, - metal: Metal, - - /// A sample of the given target with every field null. - pub fn init(target: Target) Sample { - return switch (target) { - inline else => |tag| @unionInit(Sample, @tagName(tag), .{}), - }; - } -}; - -/// A device and its latest sample: what JSON writes and what a remote host sends. -pub const Row = struct { device: Device, sample: Sample }; - -/// Every GPU target. CSV columns and JSON keys follow the field order. -pub const Gpu = struct { - util_percent: ?u64 = null, - encoder_util_percent: ?u64 = null, - decoder_util_percent: ?u64 = null, - power_mw: ?u64 = null, - power_limit_mw: ?u64 = null, - /// Degrees Celsius. - temperature: ?u64 = null, - fan_speed_percent: ?u64 = null, - clock_graphics_mhz: ?u64 = null, - clock_sm_mhz: ?u64 = null, - clock_soc_mhz: ?u64 = null, - clock_mem_mhz: ?u64 = null, - clock_graphics_max_mhz: ?u64 = null, - clock_mem_max_mhz: ?u64 = null, - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, - /// Bits. - mem_bus_width: ?u64 = null, - pcie_tx_kbps: ?u64 = null, - pcie_rx_kbps: ?u64 = null, - /// The line rate of the negotiated link across all its lanes. - pcie_bandwidth_mbps: ?u64 = null, - pcie_link_gen: ?u64 = null, - pcie_link_width: ?u64 = null, -}; - -/// One Neuron core. -pub const Neuron = struct { - util_percent: ?u64 = null, - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, - // Device memory of the core by category, in bytes. - nc_tensors: ?u64 = null, - nc_constants: ?u64 = null, - nc_model_code: ?u64 = null, - nc_shared_scratchpad: ?u64 = null, - nc_nonshared_scratchpad: ?u64 = null, - nc_runtime: ?u64 = null, - nc_driver: ?u64 = null, - nc_dma_rings: ?u64 = null, - nc_collectives: ?u64 = null, - nc_notifications: ?u64 = null, - nc_uncategorized: ?u64 = null, -}; - -pub const Tpu = struct { - util_percent: ?u64 = null, - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, -}; - -/// One FuriosaAI RNGD chip. -pub const Furiosa = struct { - util_percent: ?u64 = null, - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, - power_mw: ?u64 = null, - power_limit_mw: ?u64 = null, - /// Degrees Celsius, at the hottest point of the SoC. - temperature: ?u64 = null, - /// Degrees Celsius. - ambient_temperature: ?u64 = null, - /// A `Governor`. - governor: ?u64 = null, - /// A `Liveness`. - liveness: ?u64 = null, - /// A bitmask whose bits are `ThrottleReason`s. - throttle_reasons: ?u64 = null, - - pub const Governor = enum(u64) { performance, powersave }; - pub const Liveness = enum(u64) { unknown, alive }; - pub const ThrottleReason = enum(u6) { idle, thermal_slowdown, app_power_cap, app_clock_cap, hw_clock_cap, hw_bus_limit, hw_power_cap, gov_clock_cap, other }; -}; - -/// An Apple GPU. Its memory is the host's. -pub const Metal = struct { - util_percent: ?u64 = null, - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, - power_mw: ?u64 = null, - /// Degrees Celsius, of the hottest GPU sensor. - temperature: ?u64 = null, - /// The fastest fan. - fan_speed_percent: ?u64 = null, - clock_graphics_mhz: ?u64 = null, - clock_graphics_max_mhz: ?u64 = null, - /// The Neural Engine. - ane_power_mw: ?u64 = null, -}; - -pub const ProcessInfo = struct { - pid: u32 = 0, - /// The global index of the device, once the collector has the row. - device_idx: u16 = 0, - dev_util_percent: ?u16 = null, - dev_mem_kib: ?u64 = null, - uid: u32 = 0, - username: []const u8 = "", - comm: []const u8 = "", - /// Tenths of a percent of every CPU. - cpu_percent: u16 = 0, - rss_kib: u64 = 0, - remote: bool = false, -}; - -pub const HostData = struct { - hostname: ?[]const u8 = null, - kernel: ?[]const u8 = null, - cpu_name: ?[]const u8 = null, - cpu_cores: ?u64 = null, - mem_total_kib: ?u64 = null, - mem_available_kib: ?u64 = null, - load_1: ?f32 = null, - load_5: ?f32 = null, - load_15: ?f32 = null, - uptime_seconds: ?u64 = null, - /// Measured by a platform, through `Platform.VTable.host`. - cpu_power_mw: ?u64 = null, - system_power_mw: ?u64 = null, -}; - -/// A source of devices: a local platform or a remote host. -/// It is a field of its implementation, which finds itself with @fieldParentPtr. -pub const Platform = struct { - /// The devices found when the platform was opened. The list never changes afterwards. - devices: []const Device, - /// Values are derived from two polls, so a one-shot print polls twice. - two_readings: bool = false, - vtable: *const VTable, - - pub const VTable = struct { - /// Called every poll, with one empty sample per device, in the order of `devices`. - poll: *const fn (platform: *Platform, scratch: std.mem.Allocator, samples: []Sample) void, - /// Called right after `poll`, with the samples it just filled. A row's device is its position in `devices`. - processes: ?*const fn (platform: *Platform, scratch: std.mem.Allocator, samples: []const Sample, out: *ProcessList) ProcessList.Error!void = null, - deinit: ?*const fn (platform: *Platform) void = null, - /// For a platform that also measures the host. Called right after `processes`, in the same task, - /// with host data whose fields start null: it fills those it read. - host: ?*const fn (platform: *Platform, data: *HostData) void = null, - }; -}; - -/// The process rows of one source for one poll, and the strings they point into. -pub const ProcessList = struct { - rows: std.ArrayList(ProcessInfo) = .empty, - /// Owns every string a row holds. - strings: std.heap.ArenaAllocator, - gpa: std.mem.Allocator, - - pub const Error = error{ OutOfMemory, Unavailable }; - - pub fn init(gpa: std.mem.Allocator) ProcessList { - return .{ .strings = .init(gpa), .gpa = gpa }; - } - - pub fn deinit(list: *ProcessList) void { - list.rows.deinit(list.gpa); - list.strings.deinit(); - } - - pub fn clear(list: *ProcessList) void { - list.rows.clearRetainingCapacity(); - _ = list.strings.reset(.retain_capacity); - } - - pub fn append(list: *ProcessList, row: ProcessInfo) error{OutOfMemory}!void { - try list.rows.append(list.gpa, row); - } - - /// Copies a string into the list, for a row to hold. - pub fn dupe(list: *ProcessList, string: []const u8) error{OutOfMemory}![]const u8 { - return list.strings.allocator().dupe(u8, string); - } -}; diff --git a/bin/zml-smi/smi/pci.zig b/bin/zml-smi/smi/pci.zig deleted file mode 100644 index 104eb4d5..00000000 --- a/bin/zml-smi/smi/pci.zig +++ /dev/null @@ -1,164 +0,0 @@ -const std = @import("std"); -const sysfs = @import("sysfs.zig"); - -/// The PCI devices of one vendor, from /sys/bus/pci/devices. -pub const Devices = struct { - root: sysfs.Dir, - it: std.Io.Dir.Iterator, - vendor: u16, - - pub const Entry = struct { - /// The device id. - id: u16, - /// The caller closes it. - dir: sysfs.Dir, - }; - - pub fn open(io: std.Io, vendor: u16) !Devices { - const root = try sysfs.Dir.open(io, "/sys/bus/pci/devices"); - return .{ .root = root, .it = root.iterate(), .vendor = vendor }; - } - - pub fn close(self: *Devices) void { - self.root.close(); - } - - pub fn next(self: *Devices) ?Entry { - while (self.it.next(self.root.io) catch null) |entry| { - const dir = self.root.openDir(entry.name) catch continue; - if ((dir.int(u16, "vendor") catch 0) == self.vendor) { - if (dir.int(u16, "device")) |id| return .{ .id = id, .dir = dir } else |_| {} - } - dir.close(); - } - return null; - } -}; - -pub const Link = struct { - gen: ?u64 = null, - width: ?u64 = null, - - /// The line rate across every lane, in megabits per second. - pub fn mbps(self: Link) ?u64 { - return (laneMbps(self.gen orelse return null) orelse return null) * (self.width orelse return null); - } -}; - -/// The link a device negotiated, or the widest it supports. -pub fn link(dir: sysfs.Dir, comptime which: enum { current, max }) Link { - var buffer: [32]u8 = undefined; - const speed = dir.read(@tagName(which) ++ "_link_speed", &buffer) catch null; - return .{ - .gen = generation(speed orelse ""), - .width = dir.int(u64, @tagName(which) ++ "_link_width") catch null, - }; -} - -/// The device, or the bridge above it that carries its link: the one with the widest link, -/// climbing only while the vendor is the device's own. -pub fn upstream(device: sysfs.Dir) !sysfs.Dir { - const max_depth = 8; - const vendor = try device.int(u16, "vendor"); - var path: [3 * max_depth]u8 = undefined; - var best_depth: usize = 0; - var best_mbps = link(device, .max).mbps() orelse 0; - - for (1..max_depth + 1) |depth| { - const dir = device.openDir(parents(&path, depth)) catch break; - defer dir.close(); - if ((dir.int(u16, "vendor") catch break) != vendor) break; - const mbps = link(dir, .max).mbps() orelse 0; - if (mbps > best_mbps) { - best_depth = depth; - best_mbps = mbps; - } - } - return device.openDir(parents(&path, best_depth)); -} - -fn parents(buffer: []u8, depth: usize) []const u8 { - if (depth == 0) return "."; - for (0..depth) |i| @memcpy(buffer[3 * i ..][0..3], "../"); - return buffer[0 .. 3 * depth - 1]; -} - -/// The generation of a link speed such as "32.0 GT/s PCIe". -pub fn generation(speed: []const u8) ?u64 { - const transfers = std.fmt.parseFloat(f32, std.mem.sliceTo(speed, ' ')) catch return null; - if (!(transfers > 0 and transfers < 100)) return null; - return switch (@as(u32, @intFromFloat(transfers * 10))) { - 25 => 1, - 50 => 2, - 80 => 3, - 160 => 4, - 320 => 5, - 640 => 6, - else => null, - }; -} - -/// The line rate of one lane, in megabits per second. -pub fn laneMbps(gen: u64) ?u64 { - return switch (gen) { - 1 => 2_500, - 2 => 5_000, - 3 => 8_000, - 4 => 16_000, - 5 => 32_000, - 6 => 64_000, - else => null, - }; -} - -test generation { - try std.testing.expectEqual(1, generation("2.5 GT/s PCIe")); - try std.testing.expectEqual(3, generation("8 GT/s")); - try std.testing.expectEqual(4, generation("16.0 GT/s PCIe")); - try std.testing.expectEqual(null, generation("Unknown")); -} - -test Link { - try std.testing.expectEqual(10_000, (Link{ .gen = 1, .width = 4 }).mbps()); - try std.testing.expectEqual(512_000, (Link{ .gen = 5, .width = 16 }).mbps()); - try std.testing.expectEqual(null, (Link{ .gen = 5 }).mbps()); -} - -test upstream { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); - defer tmp.cleanup(); - try tmp.dir.createDirPath(io, "root/bridge/gpu"); - try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/vendor", .data = "0x8086\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/max_link_speed", .data = "32.0 GT/s PCIe\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/max_link_width", .data = "16\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/gpu/vendor", .data = "0x8086\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/gpu/max_link_speed", .data = "2.5 GT/s PCIe\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/gpu/max_link_width", .data = "1\n" }); - - const root: sysfs.Dir = .{ .handle = tmp.dir, .io = io }; - const gpu = try root.openDir("root/bridge/gpu"); - defer gpu.close(); - const dir = try upstream(gpu); - defer dir.close(); - try std.testing.expectEqual(512_000, link(dir, .max).mbps()); -} - -test Devices { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); - defer tmp.cleanup(); - try tmp.dir.createDirPath(io, "0000:03:00.0"); - try tmp.dir.createDirPath(io, "0000:04:00.0"); - try tmp.dir.writeFile(io, .{ .sub_path = "0000:03:00.0/vendor", .data = "0x8086\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "0000:03:00.0/device", .data = "0xe223\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "0000:04:00.0/vendor", .data = "0x10de\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "0000:04:00.0/device", .data = "0x2b85\n" }); - - const root: sysfs.Dir = .{ .handle = tmp.dir, .io = io }; - var devices: Devices = .{ .root = root, .it = root.iterate(), .vendor = 0x8086 }; - const entry = devices.next().?; - defer entry.dir.close(); - try std.testing.expectEqual(0xe223, entry.id); - try std.testing.expectEqual(null, devices.next()); -} diff --git a/bin/zml-smi/smi/proc.zig b/bin/zml-smi/smi/proc.zig deleted file mode 100644 index 07042dd8..00000000 --- a/bin/zml-smi/smi/proc.zig +++ /dev/null @@ -1,94 +0,0 @@ -const std = @import("std"); - -/// The open files of every process whose path starts with one of the prefixes. -/// One walk of /proc, whatever the number of prefixes. Skips zml-smi itself, -/// which may hold device nodes through a library it opened. -pub const FdHolders = struct { - io: std.Io, - prefixes: []const []const u8, - proc: ?std.Io.Dir, - pids: std.Io.Dir.Iterator = undefined, - fds: ?std.Io.Dir = null, - fd_names: std.Io.Dir.Iterator = undefined, - pid: u32 = 0, - own_pid: u32, - link: [std.Io.Dir.max_path_bytes]u8 = undefined, - - pub const Holder = struct { - pid: u32, - /// The descriptor, for /proc//fdinfo/. - fd: u32, - /// The path after the prefix, valid until the next call: "0" for /dev/accel0. - rest: []const u8, - }; - - pub fn init(io: std.Io, prefixes: []const []const u8) FdHolders { - return initAt(io, std.Io.Dir.cwd().openDir(io, "/proc", .{ .iterate = true }) catch null, prefixes); - } - - fn initAt(io: std.Io, proc: ?std.Io.Dir, prefixes: []const []const u8) FdHolders { - var self: FdHolders = .{ .io = io, .prefixes = prefixes, .proc = proc, .own_pid = @intCast(std.c.getpid()) }; - if (proc) |dir| self.pids = dir.iterate(); - return self; - } - - pub fn next(self: *FdHolders) ?Holder { - const proc = self.proc orelse return null; - while (true) { - if (self.fds) |fds| { - while (self.fd_names.next(self.io) catch null) |entry| { - const fd = std.fmt.parseInt(u32, entry.name, 10) catch continue; - const len = fds.readLink(self.io, entry.name, &self.link) catch continue; - const target = self.link[0..len]; - for (self.prefixes) |prefix| { - if (std.mem.startsWith(u8, target, prefix)) return .{ .pid = self.pid, .fd = fd, .rest = target[prefix.len..] }; - } - } - fds.close(self.io); - self.fds = null; - } - - const entry = (self.pids.next(self.io) catch null) orelse return null; - const pid = std.fmt.parseInt(u32, entry.name, 10) catch continue; - if (pid == self.own_pid) continue; - var path: [16]u8 = undefined; - const fd_path = std.fmt.bufPrint(&path, "{d}/fd", .{pid}) catch continue; - const fds = proc.openDir(self.io, fd_path, .{ .iterate = true }) catch continue; - self.fds = fds; - self.fd_names = fds.iterate(); - self.pid = pid; - } - } - - pub fn deinit(self: *FdHolders) void { - if (self.fds) |fds| fds.close(self.io); - if (self.proc) |proc| proc.close(self.io); - } -}; - -test FdHolders { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); - defer tmp.cleanup(); - try tmp.dir.createDirPath(io, "100/fd"); - try tmp.dir.createDirPath(io, "200/fd"); - try tmp.dir.symLink(io, "/dev/accel3", "100/fd/5", .{}); - try tmp.dir.symLink(io, "/dev/null", "100/fd/0", .{}); - try tmp.dir.symLink(io, "/dev/vfio/1", "200/fd/9", .{}); - - var holders: FdHolders = .initAt(io, try tmp.dir.openDir(io, ".", .{ .iterate = true }), &.{ "/dev/accel", "/dev/vfio/" }); - defer holders.deinit(); - - var count: usize = 0; - while (holders.next()) |holder| : (count += 1) { - switch (holder.pid) { - 100 => { - try std.testing.expectEqual(5, holder.fd); - try std.testing.expectEqualStrings("3", holder.rest); - }, - 200 => try std.testing.expectEqualStrings("1", holder.rest), - else => return error.TestUnexpectedResult, - } - } - try std.testing.expectEqual(2, count); -} diff --git a/bin/zml-smi/smi/prometheus.zig b/bin/zml-smi/smi/prometheus.zig deleted file mode 100644 index 03f9d62c..00000000 --- a/bin/zml-smi/smi/prometheus.zig +++ /dev/null @@ -1,209 +0,0 @@ -const std = @import("std"); -const smi = @import("smi.zig"); - -/// A metric family. `decimals` moves the decimal point and `factor` multiplies, so conversions are exact. -const Metric = struct { name: []const u8, help: []const u8, decimals: u8 = 0, factor: u64 = 1 }; - -/// One entry per field of `T`. A missing or misspelled field does not compile. -fn Metrics(comptime T: type) type { - return std.enums.EnumFieldStruct(std.meta.FieldEnum(T), ?Metric, null); -} - -const utilization: Metric = .{ .name = "zml_device_utilization_percent", .help = "Device utilization percentage" }; -const memory_used: Metric = .{ .name = "zml_device_memory_used_bytes", .help = "Device memory used in bytes" }; -const memory_total: Metric = .{ .name = "zml_device_memory_total_bytes", .help = "Device memory total in bytes" }; - -const gpu: Metrics(smi.Gpu) = .{ - .util_percent = utilization, - .encoder_util_percent = .{ .name = "zml_device_encoder_utilization_percent", .help = "Encoder utilization percentage" }, - .decoder_util_percent = .{ .name = "zml_device_decoder_utilization_percent", .help = "Decoder utilization percentage" }, - // Milliwatts, written as watts. - .power_mw = .{ .name = "zml_device_power_watts", .help = "Power draw in watts", .decimals = 3 }, - .power_limit_mw = .{ .name = "zml_device_power_limit_watts", .help = "Power limit in watts", .decimals = 3 }, - .temperature = .{ .name = "zml_device_temperature_celsius", .help = "Device temperature in celsius" }, - .fan_speed_percent = .{ .name = "zml_device_fan_speed_percent", .help = "Fan speed percentage" }, - .clock_graphics_mhz = .{ .name = "zml_device_clock_graphics_mhz", .help = "Graphics clock frequency in MHz" }, - .clock_sm_mhz = .{ .name = "zml_device_clock_sm_mhz", .help = "SM clock frequency in MHz" }, - .clock_soc_mhz = null, - .clock_mem_mhz = .{ .name = "zml_device_clock_memory_mhz", .help = "Memory clock frequency in MHz" }, - .clock_graphics_max_mhz = null, - .clock_mem_max_mhz = null, - .mem_used_bytes = memory_used, - .mem_total_bytes = memory_total, - .mem_bus_width = null, - // Kilobytes per second, written as bytes per second. - .pcie_tx_kbps = .{ .name = "zml_device_pcie_tx_bytes_per_second", .help = "PCIe TX throughput in bytes per second", .factor = 1000 }, - .pcie_rx_kbps = .{ .name = "zml_device_pcie_rx_bytes_per_second", .help = "PCIe RX throughput in bytes per second", .factor = 1000 }, - .pcie_bandwidth_mbps = null, - .pcie_link_gen = null, - .pcie_link_width = null, -}; - -const neuron: Metrics(smi.Neuron) = .{ - .util_percent = utilization, - .mem_used_bytes = memory_used, - .mem_total_bytes = memory_total, - .nc_tensors = .{ .name = "zml_device_neuron_tensors_bytes", .help = "Neuron core tensor memory in bytes" }, - .nc_constants = .{ .name = "zml_device_neuron_constants_bytes", .help = "Neuron core constants memory in bytes" }, - .nc_model_code = .{ .name = "zml_device_neuron_model_code_bytes", .help = "Neuron core model code memory in bytes" }, - .nc_shared_scratchpad = .{ .name = "zml_device_neuron_shared_scratchpad_bytes", .help = "Neuron core shared scratchpad memory in bytes" }, - .nc_nonshared_scratchpad = .{ .name = "zml_device_neuron_nonshared_scratchpad_bytes", .help = "Neuron core non-shared scratchpad memory in bytes" }, - .nc_runtime = .{ .name = "zml_device_neuron_runtime_bytes", .help = "Neuron core runtime memory in bytes" }, - .nc_driver = .{ .name = "zml_device_neuron_driver_bytes", .help = "Neuron core driver memory in bytes" }, - .nc_dma_rings = .{ .name = "zml_device_neuron_dma_rings_bytes", .help = "Neuron core DMA rings memory in bytes" }, - .nc_collectives = .{ .name = "zml_device_neuron_collectives_bytes", .help = "Neuron core collectives memory in bytes" }, - .nc_notifications = .{ .name = "zml_device_neuron_notifications_bytes", .help = "Neuron core notifications memory in bytes" }, - .nc_uncategorized = .{ .name = "zml_device_neuron_uncategorized_bytes", .help = "Neuron core uncategorized memory in bytes" }, -}; - -const tpu: Metrics(smi.Tpu) = .{ - .util_percent = utilization, - .mem_used_bytes = memory_used, - .mem_total_bytes = memory_total, -}; - -const furiosa: Metrics(smi.Furiosa) = .{ - .util_percent = utilization, - .mem_used_bytes = memory_used, - .mem_total_bytes = memory_total, - .power_mw = gpu.power_mw, - .power_limit_mw = gpu.power_limit_mw, - .temperature = gpu.temperature, - .ambient_temperature = .{ .name = "zml_device_ambient_temperature_celsius", .help = "Ambient temperature in celsius" }, - .governor = .{ .name = "zml_device_governor", .help = "Governor profile: 0 performance, 1 powersave" }, - .liveness = .{ .name = "zml_device_liveness", .help = "Liveness: 1 alive, 0 unknown" }, - .throttle_reasons = .{ .name = "zml_device_throttle_reasons", .help = "Active throttle reasons as a bitmask" }, -}; - -const metal: Metrics(smi.Metal) = .{ - .util_percent = utilization, - .mem_used_bytes = memory_used, - .mem_total_bytes = memory_total, - .power_mw = gpu.power_mw, - .temperature = gpu.temperature, - .fan_speed_percent = gpu.fan_speed_percent, - .clock_graphics_mhz = gpu.clock_graphics_mhz, - .clock_graphics_max_mhz = null, - .ane_power_mw = .{ .name = "zml_device_ane_power_watts", .help = "Neural Engine power draw in watts", .decimals = 3 }, -}; - -const host: Metrics(smi.HostData) = .{ - .hostname = null, - .kernel = null, - .cpu_name = null, - .cpu_cores = .{ .name = "zml_host_cpu_cores", .help = "Number of CPU cores" }, - .mem_total_kib = .{ .name = "zml_host_memory_total_bytes", .help = "Total host memory in bytes", .factor = 1024 }, - .mem_available_kib = .{ .name = "zml_host_memory_available_bytes", .help = "Available host memory in bytes", .factor = 1024 }, - .load_1 = .{ .name = "zml_host_load_1", .help = "1-minute load average", .decimals = 2 }, - .load_5 = .{ .name = "zml_host_load_5", .help = "5-minute load average", .decimals = 2 }, - .load_15 = .{ .name = "zml_host_load_15", .help = "15-minute load average", .decimals = 2 }, - .uptime_seconds = .{ .name = "zml_host_uptime_seconds", .help = "System uptime in seconds" }, - .cpu_power_mw = .{ .name = "zml_host_cpu_power_watts", .help = "CPU power draw in watts", .decimals = 3 }, - .system_power_mw = .{ .name = "zml_host_system_power_watts", .help = "System power draw in watts", .decimals = 3 }, -}; - -fn metricsOf(comptime T: type) Metrics(T) { - return switch (T) { - smi.Gpu => gpu, - smi.Neuron => neuron, - smi.Tpu => tpu, - smi.Furiosa => furiosa, - smi.Metal => metal, - else => @compileError("no metrics for " ++ @typeName(T)), - }; -} - -/// The distinct families of every struct, in the order the structs give them. -/// TPU first keeps the order of the families: utilization and memory, then GPU, then Neuron. -const families: []const Metric = families: { - @setEvalBranchQuota(10_000); - var list: []const Metric = &.{}; - for (.{ smi.Tpu, smi.Gpu, smi.Neuron, smi.Furiosa, smi.Metal }) |T| { - const metrics = metricsOf(T); - next: for (@typeInfo(T).@"struct".field_names) |field| { - const metric = @field(metrics, field) orelse continue; - for (list) |seen| { - if (!std.mem.eql(u8, seen.name, metric.name)) continue; - if (!std.mem.eql(u8, seen.help, metric.help) or seen.decimals != metric.decimals or seen.factor != metric.factor) { - @compileError(metric.name ++ " has two definitions"); - } - continue :next; - } - list = list ++ .{metric}; - } - } - break :families list; -}; - -/// The field of `T` written as `family`, or null. -fn fieldOf(comptime T: type, comptime family: Metric) ?[]const u8 { - // Every call made while `write` is analyzed counts against one quota. - @setEvalBranchQuota(100_000); - const metrics = metricsOf(T); - for (@typeInfo(T).@"struct".field_names) |field| { - const metric = @field(metrics, field) orelse continue; - if (std.mem.eql(u8, metric.name, family.name)) return field; - } - return null; -} - -pub fn write(writer: *std.Io.Writer, collector: *smi.Collector) !void { - const devices = collector.devices.items; - - try writer.writeAll("# HELP zml_device_info Device metadata\n# TYPE zml_device_info gauge\n"); - for (devices, 0..) |device, i| { - try writer.print("zml_device_info{{device=\"{d}\",type=\"{t}\"", .{ i, device.target }); - if (device.name) |name| try label(writer, "name", name); - if (device.driver_version) |driver| try label(writer, "driver", driver); - for (device.details) |detail| try label(writer, detail.key, detail.value); - try writer.writeAll("} 1\n"); - } - try writer.writeAll("\n"); - - inline for (families) |family| { - var written = false; - for (devices, 0..) |device, i| { - const value: u64 = switch (collector.sample(@fromBackingInt(@intCast(i)))) { - inline else => |fields| if (comptime fieldOf(@TypeOf(fields), family)) |field| @field(fields, field) else null, - } orelse continue; - if (!written) try header(writer, family); - written = true; - try writer.print("{s}{{device=\"{d}\",type=\"{t}\",name=\"", .{ family.name, i, device.target }); - try std.zig.stringEscape(device.name orelse "", writer); - try writer.writeAll("\"} "); - try number(writer, family, value); - try writer.writeAll("\n"); - } - if (written) try writer.writeAll("\n"); - } - - const data = collector.host.read(collector.io); - inline for (@typeInfo(smi.HostData).@"struct".field_names) |field| { - if (comptime @field(host, field)) |family| { - if (@field(data, field)) |value| { - try header(writer, family); - try writer.print("{s} ", .{family.name}); - try number(writer, family, value); - try writer.writeAll("\n\n"); - } - } - } -} - -fn header(writer: *std.Io.Writer, comptime family: Metric) !void { - try writer.writeAll("# HELP " ++ family.name ++ " " ++ family.help ++ "\n# TYPE " ++ family.name ++ " gauge\n"); -} - -fn label(writer: *std.Io.Writer, name: []const u8, value: []const u8) !void { - try writer.print(",{s}=\"", .{name}); - try std.zig.stringEscape(value, writer); - try writer.writeAll("\""); -} - -fn number(writer: *std.Io.Writer, comptime family: Metric, value: anytype) !void { - if (@TypeOf(value) == f32) return writer.print(std.fmt.comptimePrint("{{d:.{d}}}", .{family.decimals}), .{value}); - const scaled = value *| family.factor; - if (family.decimals == 0) return writer.print("{d}", .{scaled}); - const one = comptime std.math.pow(u64, 10, family.decimals); - try writer.print(std.fmt.comptimePrint("{{d}}.{{d:0>{d}}}", .{family.decimals}), .{ scaled / one, scaled % one }); -} diff --git a/bin/zml-smi/smi/remote.zig b/bin/zml-smi/smi/remote.zig deleted file mode 100644 index 78320dd9..00000000 --- a/bin/zml-smi/smi/remote.zig +++ /dev/null @@ -1,147 +0,0 @@ -const std = @import("std"); -const smi = @import("smi.zig"); - -const log = std.log.scoped(.@"zml-smi/remote"); - -/// The devices of another zml-smi, read from its API. -pub const Remote = struct { - platform: smi.Platform, - io: std.Io, - /// "http://host:9090/metrics". - url: []const u8, - /// For each row of the peer, the position of its device here, or null when this build does not know its target. - positions: []const ?u16, - timeout: std.Io.Duration, - /// What the poll of this round fetched, in scratch, for `processes`. - fetched: ?std.json.Value = null, - warned: bool = false, - - const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes }; - - /// Fetches the devices once. `interval` bounds every later fetch, to at least a second. - pub fn open(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, host: []const u8, interval: std.Io.Duration) !Remote { - if (!std.mem.startsWith(u8, host, "http://") and !std.mem.startsWith(u8, host, "https://")) { - log.err("{s}: a remote host starts with http:// or https://", .{host}); - return error.InvalidUrl; - } - const url = try std.fmt.allocPrint(arena, "{s}/metrics", .{host}); - - var scratch: std.heap.ArenaAllocator = .init(gpa); - defer scratch.deinit(); - const rows = items(try fetch(scratch.allocator(), io, url, .fromSeconds(3)), "devices"); - - var devices: std.ArrayList(smi.Device) = try .initCapacity(arena, rows.len); - const positions = try arena.alloc(?u16, rows.len); - for (rows, positions) |item, *position| { - const row = std.json.parseFromValueLeaky(smi.Row, scratch.allocator(), item, .{ .ignore_unknown_fields = true }) catch { - position.* = null; - continue; - }; - position.* = @intCast(devices.items.len); - devices.appendAssumeCapacity(try copy(arena, row.device)); - } - - return .{ - .platform = .{ .devices = devices.items, .vtable = &vtable }, - .io = io, - .url = url, - .positions = positions, - .timeout = if (interval.nanoseconds < std.time.ns_per_s) .fromSeconds(1) else interval, - }; - } - - fn poll(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []smi.Sample) void { - const self: *Remote = @alignCast(@fieldParentPtr("platform", platform)); - self.fetched = null; - const root = fetch(scratch, self.io, self.url, self.timeout) catch return; - - const rows = items(root, "devices"); - if (rows.len != self.positions.len) return self.changed(samples); - for (rows, self.positions) |item, maybe_position| { - const position = maybe_position orelse continue; - const row = std.json.parseFromValueLeaky(smi.Row, scratch, item, .{ .ignore_unknown_fields = true }) catch return self.changed(samples); - const device = self.platform.devices[position]; - if (row.device.target != device.target or !std.mem.eql(u8, row.device.name orelse "", device.name orelse "")) { - return self.changed(samples); - } - samples[position] = row.sample; - } - self.fetched = root; - } - - /// The peer's devices are not the ones it had when it was opened: show nothing. - fn changed(self: *Remote, samples: []smi.Sample) void { - for (samples, self.platform.devices) |*sample, device| sample.* = .init(device.target); - if (!self.warned) log.warn("{s}: the devices changed since zml-smi started", .{self.url}); - self.warned = true; - } - - fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { - const self: *Remote = @alignCast(@fieldParentPtr("platform", platform)); - const root = self.fetched orelse return error.Unavailable; - for (items(root, "processes")) |item| { - var row = std.json.parseFromValueLeaky(smi.ProcessInfo, scratch, item, .{ .ignore_unknown_fields = true }) catch continue; - if (row.device_idx >= self.positions.len) continue; - row.device_idx = self.positions[row.device_idx] orelse continue; - row.remote = true; - row.username = try out.dupe(row.username); - row.comm = try out.dupe(row.comm); - try out.append(row); - } - } -}; - -fn copy(arena: std.mem.Allocator, device: smi.Device) !smi.Device { - const details = try arena.alloc(smi.Device.Detail, device.details.len); - for (details, device.details) |*detail, source| { - detail.* = .{ - .key = try arena.dupe(u8, source.key), - .label = try arena.dupe(u8, source.label), - .value = try arena.dupe(u8, source.value), - .in_title = source.in_title, - }; - } - return .{ - .target = device.target, - .remote = true, - .name = if (device.name) |name| try arena.dupe(u8, name) else null, - .driver_version = if (device.driver_version) |version| try arena.dupe(u8, version) else null, - .details = details, - }; -} - -fn items(root: std.json.Value, key: []const u8) []const std.json.Value { - if (root != .object) return &.{}; - const value = root.object.get(key) orelse return &.{}; - return if (value == .array) value.array.items else &.{}; -} - -/// GETs `url` and parses its body into `scratch`, giving up after `timeout`. -/// std.http has no timeout, so the fetch races a timer and the loser is canceled. -fn fetch(scratch: std.mem.Allocator, io: std.Io, url: []const u8, timeout: std.Io.Duration) !std.json.Value { - const Race = union(enum) { - body: @typeInfo(@TypeOf(get)).@"fn".return_type.?, - timer: std.Io.Cancelable!void, - }; - var results: [2]Race = undefined; - var race: std.Io.Select(Race) = .init(io, &results); - defer race.cancelDiscard(); - - try race.concurrent(.body, get, .{ scratch, io, url }); - try race.concurrent(.timer, std.Io.sleep, .{ io, timeout, .awake }); - const body = switch (try race.await()) { - .body => |body| try body, - .timer => return error.Timeout, - }; - return std.json.parseFromSliceLeaky(std.json.Value, scratch, body, .{}); -} - -fn get(scratch: std.mem.Allocator, io: std.Io, url: []const u8) ![]const u8 { - var client: std.http.Client = .{ .allocator = scratch, .io = io }; - defer client.deinit(); - - var body: std.Io.Writer.Allocating = .init(scratch); - const result = try client.fetch(.{ .location = .{ .url = url }, .response_writer = &body.writer }); - if (result.status != .ok) return error.HttpError; - return body.written(); -} diff --git a/bin/zml-smi/smi/sandbox.zig b/bin/zml-smi/smi/sandbox.zig deleted file mode 100644 index 9f796cdb..00000000 --- a/bin/zml-smi/smi/sandbox.zig +++ /dev/null @@ -1,30 +0,0 @@ -const std = @import("std"); -const bazel_runfiles = @import("runfiles"); -const bazel_builtin = @import("bazel_builtin"); - -/// The directory of the libraries shipped with zml-smi, found through the runfiles of the binary. -pub fn path(io: std.Io, buffer: *[std.Io.Dir.max_path_bytes]u8) ?[]const u8 { - var exe: [std.Io.Dir.max_path_bytes]u8 = undefined; - const exe_len = std.process.executablePath(io, &exe) catch return null; - - var runfiles = (bazel_runfiles.Runfiles.create(.{ - .allocator = std.heap.c_allocator, - .io = io, - .argv0 = exe[0..exe_len], - .directory = env("RUNFILES_DIR"), - .manifest = env("RUNFILES_MANIFEST_FILE"), - }) catch return null) orelse return null; - defer runfiles.deinit(std.heap.c_allocator); - - return runfiles.withSourceRepo(bazel_builtin.current_repository).rlocation("zml/bin/zml-smi/sandbox", buffer) catch null; -} - -fn env(name: [*:0]const u8) ?[]const u8 { - return if (std.c.getenv(name)) |value| std.mem.span(value) else null; -} - -test path { - const io = std.testing.io; - var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - try std.Io.Dir.cwd().access(io, path(io, &buffer) orelse return error.NoSandbox, .{}); -} diff --git a/bin/zml-smi/smi/smi.zig b/bin/zml-smi/smi/smi.zig deleted file mode 100644 index 858859ab..00000000 --- a/bin/zml-smi/smi/smi.zig +++ /dev/null @@ -1,58 +0,0 @@ -//! The core of zml-smi: the model, the collector, the blocks platforms are made of, and the outputs. -const builtin = @import("builtin"); - -const model = @import("model.zig"); -pub const Target = model.Target; -pub const Device = model.Device; -pub const Sample = model.Sample; -pub const Row = model.Row; -pub const Gpu = model.Gpu; -pub const Neuron = model.Neuron; -pub const Tpu = model.Tpu; -pub const Furiosa = model.Furiosa; -pub const Metal = model.Metal; -pub const ProcessInfo = model.ProcessInfo; -pub const HostData = model.HostData; -pub const Platform = model.Platform; -pub const ProcessList = model.ProcessList; - -pub const Collector = @import("collector.zig").Collector; -pub const Remote = @import("remote.zig").Remote; -pub const Host = switch (builtin.os.tag) { - .linux => @import("host/linux.zig").Host, - .macos => @import("host/macos.zig").Host, - else => @compileError("zml-smi runs on Linux and macOS"), -}; - -pub const sysfs = @import("sysfs.zig"); -pub const hwmon = @import("hwmon.zig"); -pub const pci = @import("pci.zig"); -pub const fdinfo = @import("fdinfo.zig"); -pub const proc = @import("proc.zig"); -pub const dynlib = @import("dynlib.zig"); -pub const units = @import("units.zig"); -const counter = @import("counter.zig"); -pub const Counter = counter.Counter; -pub const Counters = counter.Counters; -pub const Growth = counter.Growth; -pub const DoubleBuffer = @import("double_buffer.zig").DoubleBuffer; - -pub const csv = @import("csv.zig"); -pub const json = @import("json.zig"); -pub const prometheus = @import("prometheus.zig"); -pub const api = @import("api.zig"); - -test { - _ = @import("collector.zig"); - _ = counter; - _ = @import("double_buffer.zig"); - _ = dynlib; - _ = fdinfo; - _ = hwmon; - _ = pci; - _ = proc; - _ = @import("sandbox.zig"); - _ = sysfs; - _ = units; - _ = Host; -} diff --git a/bin/zml-smi/smi/sysfs.zig b/bin/zml-smi/smi/sysfs.zig deleted file mode 100644 index 44c0bf75..00000000 --- a/bin/zml-smi/smi/sysfs.zig +++ /dev/null @@ -1,148 +0,0 @@ -const std = @import("std"); -const units = @import("units.zig"); - -/// A directory of /sys, /proc or /dev, kept open for the life of a device. -pub const Dir = struct { - handle: std.Io.Dir, - io: std.Io, - - pub fn open(io: std.Io, path: []const u8) !Dir { - return .{ .handle = try std.Io.Dir.cwd().openDir(io, path, .{ .iterate = true }), .io = io }; - } - - pub fn openDir(self: Dir, sub_path: []const u8) !Dir { - return .{ .handle = try self.handle.openDir(self.io, sub_path, .{ .iterate = true }), .io = self.io }; - } - - pub fn close(self: Dir) void { - self.handle.close(self.io); - } - - /// Trimmed content of a file. The slice points into `buffer`. - /// A file that does not fit is error.StreamTooLong, never a silent cut. - pub fn read(self: Dir, sub_path: []const u8, buffer: []u8) ![]const u8 { - const file = try self.handle.openFile(self.io, sub_path, .{}); - defer file.close(self.io); - - const len = try readAll(file, self.io, buffer); - var extra: [1]u8 = undefined; - if (len == buffer.len and try readAll(file, self.io, &extra) != 0) return error.StreamTooLong; - return std.mem.trim(u8, buffer[0..len], &std.ascii.whitespace); - } - - /// The first bytes of a file, cut at `buffer.len`: for /proc//cmdline. - pub fn head(self: Dir, sub_path: []const u8, buffer: []u8) ![]const u8 { - const file = try self.handle.openFile(self.io, sub_path, .{}); - defer file.close(self.io); - - return buffer[0..try readAll(file, self.io, buffer)]; - } - - /// A number, in decimal or in hex with a 0x prefix. - pub fn int(self: Dir, comptime T: type, sub_path: []const u8) !T { - var buffer: [64]u8 = undefined; - return parseInt(T, try self.read(sub_path, &buffer)); - } - - /// `int`, converted between unit prefixes. - pub fn scaled(self: Dir, sub_path: []const u8, from: units.Prefix, to: units.Prefix) !u64 { - return units.scale(try self.int(u64, sub_path), from, to); - } - - /// Value of `key` in a "key: value" file such as /proc/meminfo. - /// Reads line by line, with `buffer` as the line. - pub fn field(self: Dir, sub_path: []const u8, key: []const u8, buffer: []u8) ![]const u8 { - const file = try self.handle.openFile(self.io, sub_path, .{}); - defer file.close(self.io); - - var reader = file.readerStreaming(self.io, buffer); - while (true) { - const line = reader.interface.takeDelimiter('\n') catch |err| switch (err) { - // A line longer than the buffer is not the one we look for. - error.StreamTooLong => { - _ = reader.interface.discardDelimiterInclusive('\n') catch return error.NotFound; - continue; - }, - error.ReadFailed => return reader.err.?, - } orelse return error.NotFound; - if (fieldValue(line, key)) |value| return value; - } - } - - pub fn iterate(self: Dir) std.Io.Dir.Iterator { - return self.handle.iterate(); - } - - pub fn readLink(self: Dir, sub_path: []const u8, buffer: []u8) ![]const u8 { - return buffer[0..try self.handle.readLink(self.io, sub_path, buffer)]; - } -}; - -/// Whether a path exists: the usual body of `detect`. -pub fn exists(io: std.Io, path: []const u8) bool { - std.Io.Dir.cwd().access(io, path, .{}) catch return false; - return true; -} - -/// A number, in decimal or in hex with a 0x prefix. -pub fn parseInt(comptime T: type, text: []const u8) !T { - if (std.mem.startsWith(u8, text, "0x")) return std.fmt.parseInt(T, text[2..], 16); - return std.fmt.parseInt(T, text, 10); -} - -/// The first number of a value: "100 ns" is 100. -pub fn firstNumber(text: []const u8) ?u64 { - var words = std.mem.tokenizeAny(u8, text, " \t"); - return std.fmt.parseInt(u64, words.next() orelse return null, 10) catch null; -} - -/// The value of `key` on a "key: value" or "key=value" line. -pub fn fieldValue(line: []const u8, key: []const u8) ?[]const u8 { - if (!std.mem.startsWith(u8, line, key)) return null; - const rest = line[key.len..]; - if (rest.len > 0 and std.mem.indexOfScalar(u8, ":= \t", rest[0]) == null) return null; - return std.mem.trim(u8, rest, ":= \t"); -} - -fn readAll(file: std.Io.File, io: std.Io, buffer: []u8) !usize { - var len: usize = 0; - while (len < buffer.len) { - len += file.readStreaming(io, &.{buffer[len..]}) catch |err| switch (err) { - error.EndOfStream => break, - else => |e| return e, - }; - } - return len; -} - -test Dir { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); - defer tmp.cleanup(); - try tmp.dir.writeFile(io, .{ .sub_path = "vendor", .data = "0x8086\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "temp1_input", .data = "58500\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "meminfo", .data = "MemTotal: 65137624 kB\nMemTotalX: 1 kB\nFlags: fpu vme de pse tsc msr pae mce cx8\nMemAvailable: 33697096 kB\n" }); - try tmp.dir.writeFile(io, .{ .sub_path = "cmdline", .data = "python\x00-m\x00llmd\x00" }); - - const dir: Dir = .{ .handle = tmp.dir, .io = io }; - var buffer: [16]u8 = undefined; - - try std.testing.expectEqualStrings("0x8086", try dir.read("vendor", &buffer)); - try std.testing.expectEqual(0x8086, try dir.int(u16, "vendor")); - try std.testing.expectEqual(59, try dir.scaled("temp1_input", .milli, .one)); - try std.testing.expectError(error.StreamTooLong, dir.read("vendor", buffer[0..4])); - try std.testing.expectEqualStrings("python\x00-m", try dir.head("cmdline", buffer[0..9])); - - var line: [64]u8 = undefined; - try std.testing.expectEqualStrings("33697096 kB", try dir.field("meminfo", "MemAvailable", &line)); - try std.testing.expectEqualStrings("65137624 kB", try dir.field("meminfo", "MemTotal", &line)); - try std.testing.expectError(error.NotFound, dir.field("meminfo", "Mem", &line)); - // A line longer than the buffer is skipped, not cut. - try std.testing.expectEqualStrings("33697096 kB", try dir.field("meminfo", "MemAvailable", line[0..30])); -} - -test fieldValue { - try std.testing.expectEqualStrings("0000:03:00.0", fieldValue("PCI_SLOT_NAME=0000:03:00.0", "PCI_SLOT_NAME").?); - try std.testing.expectEqualStrings("AMD Ryzen", fieldValue("model name\t: AMD Ryzen", "model name").?); - try std.testing.expectEqual(null, fieldValue("model names: x", "model name")); -} diff --git a/bin/zml-smi/smi/units.zig b/bin/zml-smi/smi/units.zig deleted file mode 100644 index ca2b01fc..00000000 --- a/bin/zml-smi/smi/units.zig +++ /dev/null @@ -1,30 +0,0 @@ -const std = @import("std"); - -/// Decimal prefixes, a factor of 1000 apart. -pub const Prefix = enum(u3) { nano, micro, milli, one, kilo, mega, giga }; - -/// Converts between prefixes of one unit, rounding to nearest: `scale(1500, .micro, .milli)` is 2. -/// A result that does not fit is the largest u64. -pub fn scale(value: u64, from: Prefix, to: Prefix) u64 { - const from_step: u8 = @backingInt(from); - const to_step: u8 = @backingInt(to); - if (from_step >= to_step) { - return std.math.mul(u64, value, factor(from_step - to_step)) catch std.math.maxInt(u64); - } - const divisor = factor(to_step - from_step); - const rounded_up: u64 = @intFromBool(value % divisor * 2 >= divisor); - return value / divisor + rounded_up; -} - -fn factor(steps: u8) u64 { - return std.math.powi(u64, 1000, steps) catch unreachable; -} - -test scale { - try std.testing.expectEqual(2, scale(1500, .micro, .milli)); - try std.testing.expectEqual(1, scale(1499, .micro, .milli)); - try std.testing.expectEqual(58, scale(58_000, .milli, .one)); - try std.testing.expectEqual(230_000, scale(230_000_000, .micro, .milli)); - try std.testing.expectEqual(3_000_000, scale(3, .one, .micro)); - try std.testing.expectEqual(std.math.maxInt(u64), scale(std.math.maxInt(u64) / 10, .giga, .nano)); -} diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index 47c513e7..29f4824c 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -13,9 +13,7 @@ zig_library( "lib/utils.zig", "pages/detail.zig", "pages/detail/detail_common.zig", - "pages/detail/furiosa_detail.zig", "pages/detail/gpu_detail.zig", - "pages/detail/metal_detail.zig", "pages/detail/neuron_detail.zig", "pages/detail/tpu_detail.zig", "pages/overview.zig", @@ -45,10 +43,14 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//bazel/stamp:zig", - "//bin/zml-smi/smi", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", "//zml:logo", "@libvaxis//:vaxis", - ], + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "//conditions:default": ["//bin/zml-smi/platforms/linux"], + }), ) zig_test( diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index 79a4e62c..bc6171b7 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -1,27 +1,74 @@ const std = @import("std"); -const smi = @import("zml-smi"); +const builtin = @import("builtin"); +const smi_info = @import("zml-smi/info"); +const di = smi_info.device_info; +pub const DeviceInfo = di.DeviceInfo; +pub const GpuInfo = di.GpuInfo; +pub const NeuronInfo = di.NeuronInfo; +pub const TpuInfo = di.TpuInfo; +pub const Target = di.Target; +pub const Targets = smi_info.Targets; +const hi = smi_info.host_info; +pub const HostInfo = hi.HostInfo; +pub const HostData = hi.HostData; +pub const pi = smi_info.process_info; +pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +pub const ProcessEnricher = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos").process.ProcessEnricher +else + @import("zml-smi/platforms/linux").process.ProcessEnricher; -/// The samples of one metric, oldest first. -pub const History = std.ArrayList(u64); +pub const history_len: usize = 500; + +pub fn RingBuffer(comptime T: type, comptime capacity: usize) type { + return struct { + const Self = @This(); + + buf: [capacity]T = @splat(0), + head: usize = 0, + len: usize = 0, + + pub fn push(self: *Self, value: T) void { + self.buf[self.head] = value; + self.head = (self.head + 1) % capacity; + if (self.len < capacity) { + self.len += 1; + } + } + + pub fn sliceLast(self: *const Self, arena: std.mem.Allocator, n: usize) std.mem.Allocator.Error![]const T { + const count = @min(n, self.len); + if (count == 0) { + return &.{}; + } + const out = try arena.alloc(T, count); + const start = (self.head + capacity - count) % capacity; + for (0..count) |i| { + out[i] = self.buf[(start + i) % capacity]; + } + return out; + } + }; +} pub const HistoryBuffers = struct { - util: []History = &.{}, - mem_util: []History = &.{}, - temp: []History = &.{}, - power: []History = &.{}, + util: []RingBuffer(u64, history_len) = &.{}, + mem_util: []RingBuffer(u64, history_len) = &.{}, + temp: []RingBuffer(u64, history_len) = &.{}, + power: []RingBuffer(u64, history_len) = &.{}, pub fn init(allocator: std.mem.Allocator, count: usize) !HistoryBuffers { const h: HistoryBuffers = .{ - .util = try allocator.alloc(History, count), - .mem_util = try allocator.alloc(History, count), - .temp = try allocator.alloc(History, count), - .power = try allocator.alloc(History, count), + .util = try allocator.alloc(RingBuffer(u64, history_len), count), + .mem_util = try allocator.alloc(RingBuffer(u64, history_len), count), + .temp = try allocator.alloc(RingBuffer(u64, history_len), count), + .power = try allocator.alloc(RingBuffer(u64, history_len), count), }; - @memset(h.util, .empty); - @memset(h.mem_util, .empty); - @memset(h.temp, .empty); - @memset(h.power, .empty); + @memset(h.util, .{}); + @memset(h.mem_util, .{}); + @memset(h.temp, .{}); + @memset(h.power, .{}); return h; } @@ -35,19 +82,24 @@ pub const HistoryBuffers = struct { }; pub const SystemState = struct { - collector: *smi.Collector, + devices: []*DeviceInfo, + host: *HostInfo, history: HistoryBuffers = .{}, - /// Charts plot one sample per braille dot, two per cell: the history - /// keeps enough of them to fill the widest terminal seen. - history_len: usize = 0, + targets: Targets, + process_lists: []*ProcessDoubleBuffer, + enricher: *ProcessEnricher, gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, tui_refresh_rate: u16, pub const Config = struct { - collector: *smi.Collector, + devices: []*DeviceInfo, + host: *HostInfo, + targets: Targets, tui_refresh_rate: u16, + process_lists: []*ProcessDoubleBuffer, + enricher: *ProcessEnricher, gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, @@ -55,9 +107,13 @@ pub const SystemState = struct { pub fn init(cfg: Config) !SystemState { return .{ - .collector = cfg.collector, - .history = try HistoryBuffers.init(cfg.arena, cfg.collector.devices.items.len), + .devices = cfg.devices, + .host = cfg.host, + .history = try HistoryBuffers.init(cfg.arena, cfg.devices.len), + .targets = cfg.targets, .tui_refresh_rate = cfg.tui_refresh_rate, + .process_lists = cfg.process_lists, + .enricher = cfg.enricher, .gpa = cfg.gpa, .arena = cfg.arena, .io = cfg.io, @@ -69,63 +125,31 @@ pub const SystemState = struct { } pub fn deviceCount(self: *const SystemState) usize { - return self.collector.devices.items.len; - } - - pub fn device(self: *const SystemState, id: u16) smi.Device { - return self.collector.devices.items[id]; - } - - pub fn sample(self: *const SystemState, id: u16) smi.Sample { - return self.collector.sample(@fromBackingInt(id)); - } - - pub fn host(self: *const SystemState) smi.HostData { - return self.collector.host.read(self.io); + return self.devices.len; } pub fn recordHistory(self: *SystemState) void { for (0..self.deviceCount()) |i| { - switch (self.sample(@intCast(i))) { - .cuda, .rocm, .oneapi => |gpu| { - self.recordUsage(i, gpu.util_percent, gpu.mem_used_bytes, gpu.mem_total_bytes); - self.record(&self.history.temp[i], gpu.temperature orelse 0); - self.record(&self.history.power[i], gpu.power_mw orelse 0); + switch (self.devices[i].*) { + .cuda, .rocm, .oneapi => |*sv| { + const gpu = sv.front().*; + self.history.util[i].push(gpu.util_percent orelse 0); + const used = gpu.mem_used_bytes orelse 0; + const total = gpu.mem_total_bytes orelse 0; + self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); + self.history.temp[i].push(gpu.temperature orelse 0); + self.history.power[i].push(gpu.power_mw orelse 0); }, - .neuron => |core| { - self.recordUsage(i, core.util_percent, core.mem_used_bytes, core.mem_total_bytes); - self.record(&self.history.temp[i], 0); - self.record(&self.history.power[i], 0); - }, - .tpu => |chip| { - self.recordUsage(i, chip.util_percent, chip.mem_used_bytes, chip.mem_total_bytes); - self.record(&self.history.temp[i], 0); - self.record(&self.history.power[i], 0); - }, - .furiosa => |chip| { - self.recordUsage(i, chip.util_percent, chip.mem_used_bytes, chip.mem_total_bytes); - self.record(&self.history.temp[i], chip.temperature orelse 0); - self.record(&self.history.power[i], chip.power_mw orelse 0); - }, - .metal => |gpu| { - self.recordUsage(i, gpu.util_percent, gpu.mem_used_bytes, gpu.mem_total_bytes); - self.record(&self.history.temp[i], gpu.temperature orelse 0); - self.record(&self.history.power[i], gpu.power_mw orelse 0); + inline else => |*sv| { + const info = sv.front().*; + self.history.util[i].push(info.util_percent orelse 0); + const used = info.mem_used_bytes orelse 0; + const total = info.mem_total_bytes orelse 0; + self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); + self.history.temp[i].push(0); + self.history.power[i].push(0); }, } } } - - fn recordUsage(self: *SystemState, i: usize, util: ?u64, used: ?u64, total: ?u64) void { - self.record(&self.history.util[i], util orelse 0); - const total_bytes = total orelse 0; - self.record(&self.history.mem_util[i], if (total_bytes > 0) (used orelse 0) * 100 / total_bytes else 0); - } - - fn record(self: *SystemState, history: *History, value: u64) void { - if (history.items.len > 0 and history.items.len >= self.history_len) { - _ = history.orderedRemove(0); - } - history.append(self.arena, value) catch {}; - } }; diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig index f04be14d..1aea4778 100644 --- a/bin/zml-smi/tui/image_cache.zig +++ b/bin/zml-smi/tui/image_cache.zig @@ -1,35 +1,37 @@ const std = @import("std"); const vaxis = @import("vaxis"); -const smi = @import("zml-smi"); const ImageCache = @This(); pub var global: ImageCache = .{}; -/// A target without a logo shows none. -const logos: std.EnumMap(smi.Target, []const u8) = .init(.{ - .cuda = @embedFile("assets/nvidia.png"), - .rocm = @embedFile("assets/amd.png"), - .oneapi = @embedFile("assets/intel.png"), - .neuron = @embedFile("assets/neuron.png"), - .tpu = @embedFile("assets/tpu.png"), -}); +map: std.StringHashMapUnmanaged(vaxis.Image) = .empty, -zml: ?vaxis.Image = null, -targets: std.EnumMap(smi.Target, vaxis.Image) = .{}, +pub fn deinit(self: *ImageCache, allocator: std.mem.Allocator) void { + self.map.deinit(allocator); +} -pub fn deinit(self: *ImageCache) void { - self.* = .{}; +pub fn load( + self: *ImageCache, + vx: *vaxis.Vaxis, + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + key: []const u8, + data: []const u8, +) void { + const image = vx.loadImage(allocator, writer, .{ .mem = data }) catch return; + self.map.put(allocator, key, image) catch return; } pub fn loadAll(self: *ImageCache, vx: *vaxis.Vaxis, allocator: std.mem.Allocator, writer: *std.Io.Writer) void { - self.zml = vx.loadImage(allocator, writer, .{ .mem = @embedFile("assets/logo.png") }) catch null; - for (std.enums.values(smi.Target)) |target| { - const png = logos.get(target) orelse continue; - self.targets.put(target, vx.loadImage(allocator, writer, .{ .mem = png }) catch continue); - } + self.load(vx, allocator, writer, "logo", @embedFile("assets/logo.png")); + self.load(vx, allocator, writer, "gpu_cuda", @embedFile("assets/nvidia.png")); + self.load(vx, allocator, writer, "gpu_rocm", @embedFile("assets/amd.png")); + self.load(vx, allocator, writer, "gpu_oneapi", @embedFile("assets/intel.png")); + self.load(vx, allocator, writer, "gpu_neuron", @embedFile("assets/neuron.png")); + self.load(vx, allocator, writer, "gpu_tpu", @embedFile("assets/tpu.png")); } -pub fn get(self: *const ImageCache, target: smi.Target) ?vaxis.Image { - return self.targets.get(target); +pub fn get(self: *const ImageCache, key: []const u8) ?vaxis.Image { + return self.map.get(key); } diff --git a/bin/zml-smi/tui/lib/image.zig b/bin/zml-smi/tui/lib/image.zig index 18b6bf90..a365ac4d 100644 --- a/bin/zml-smi/tui/lib/image.zig +++ b/bin/zml-smi/tui/lib/image.zig @@ -24,7 +24,13 @@ pub fn draw(self: *const Image, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surf }; } - const cols = ui.imageCellSize(self.image, target_rows, ctx.cell_size).cols; + // Compute cell columns from image aspect ratio and terminal cell pixel size + const cell_h: u32 = if (ctx.cell_size.height > 0) ctx.cell_size.height else 20; + const cell_w: u32 = if (ctx.cell_size.width > 0) ctx.cell_size.width else 10; + const height_px = @as(u32, target_rows) * cell_h; + const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(self.image.height)); + const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(self.image.width)) * scale_f); + const cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); const size: vxfw.Size = .{ .width = cols, .height = target_rows }; var surf = try vxfw.Surface.init(ctx.arena, ui.widget(self), size); @@ -73,22 +79,6 @@ test Image { try std.testing.expectEqual(@as(?u16, 20), size.cols); } -test "Image rounds its width to the nearest column" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - vxfw.DrawContext.init(.unicode); - - // One 17px row scales a square icon to 17px: closer to 2 columns of 8px than to 3. - const img: Image = .{ .image = .{ .id = 1, .width = 64, .height = 64 }, .rows = 1 }; - const surface = try img.draw(.{ - .arena = arena.allocator(), - .min = .{}, - .max = .{ .width = 40, .height = 10 }, - .cell_size = .{ .width = 8, .height = 17 }, - }); - try std.testing.expectEqual(@as(u16, 2), surface.size.width); -} - test "refAllDecls" { std.testing.refAllDecls(@This()); } diff --git a/bin/zml-smi/tui/lib/ui.zig b/bin/zml-smi/tui/lib/ui.zig index 4a533903..6008d1e0 100644 --- a/bin/zml-smi/tui/lib/ui.zig +++ b/bin/zml-smi/tui/lib/ui.zig @@ -76,14 +76,12 @@ pub const ImageCellSize = struct { /// Compute the cell dimensions for an image scaled to a target row count, /// maintaining aspect ratio based on the terminal's cell pixel size. -/// The terminal stretches the image over exactly these cells, so the width -/// is rounded to the nearest column: rounding up would widen the image. pub fn imageCellSize(img: vaxis.Image, target_rows: u16, cell_size: vxfw.Size) ImageCellSize { const cell_h: u32 = if (cell_size.height > 0) cell_size.height else 20; const cell_w: u32 = if (cell_size.width > 0) cell_size.width else 10; const height_px = @as(u32, target_rows) * cell_h; const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(img.height)); const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(img.width)) * scale_f); - const img_cols: u16 = @intCast(@max(1, (width_px + cell_w / 2) / cell_w)); + const img_cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); return .{ .cols = img_cols, .rows = target_rows }; } diff --git a/bin/zml-smi/tui/lib/utils.zig b/bin/zml-smi/tui/lib/utils.zig index 4f9a05c8..993703c9 100644 --- a/bin/zml-smi/tui/lib/utils.zig +++ b/bin/zml-smi/tui/lib/utils.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const smi = @import("zml-smi"); pub fn bytesToMb(val: ?u64) u64 { return if (val) |v| v / (1024 * 1024) else 0; @@ -29,17 +28,3 @@ pub fn normalizeRange(arena: std.mem.Allocator, raw: []const u64, min_val: u64, } return result; } - -/// " (driver, CUDA 13.4)": the driver version and the details, all of them or those shown in titles. -pub fn versions(arena: std.mem.Allocator, device: smi.Device, comptime details: enum { in_title, all }) std.mem.Allocator.Error![]const u8 { - var text: std.Io.Writer.Allocating = .init(arena); - const writer = &text.writer; - if (device.driver_version) |driver| writer.writeAll(driver) catch return error.OutOfMemory; - for (device.details) |detail| { - if (details == .in_title and !detail.in_title) continue; - if (text.written().len > 0) writer.writeAll(", ") catch return error.OutOfMemory; - writer.print("{s} {s}", .{ detail.label, detail.value }) catch return error.OutOfMemory; - } - if (text.written().len == 0) return ""; - return std.fmt.allocPrint(arena, " ({s})", .{text.written()}); -} diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig index 03b6a615..86d4b7a3 100644 --- a/bin/zml-smi/tui/pages/detail.zig +++ b/bin/zml-smi/tui/pages/detail.zig @@ -7,8 +7,6 @@ const ProcessTable = @import("../widgets/process_table.zig"); const gpu_detail = @import("detail/gpu_detail.zig"); const neuron_detail = @import("detail/neuron_detail.zig"); const tpu_detail = @import("detail/tpu_detail.zig"); -const furiosa_detail = @import("detail/furiosa_detail.zig"); -const metal_detail = @import("detail/metal_detail.zig"); const Detail = @This(); @@ -19,14 +17,12 @@ process_table: *ProcessTable = undefined, pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const w = ctx.max.width orelse 80; const id = self.device_id; - const device = self.state.device(id); + const dev = self.state.devices[id]; const wgt = ui.widget(self); - return switch (self.state.sample(id)) { - .cuda, .rocm, .oneapi => |gpu| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, device, gpu, wgt), - .neuron => |core| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, device, core, wgt), - .tpu => |chip| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, device, chip, wgt), - .furiosa => |chip| try furiosa_detail.draw(self.state, self.process_table, ctx, w, id, device, chip, wgt), - .metal => |gpu| try metal_detail.draw(self.state, self.process_table, ctx, w, id, device, gpu, wgt), + return switch (dev.*) { + .cuda, .rocm, .oneapi => |*sv| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + .neuron => |*sv| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + .tpu => |*sv| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), }; } diff --git a/bin/zml-smi/tui/pages/detail/detail_common.zig b/bin/zml-smi/tui/pages/detail/detail_common.zig index 52a68fda..93e8addb 100644 --- a/bin/zml-smi/tui/pages/detail/detail_common.zig +++ b/bin/zml-smi/tui/pages/detail/detail_common.zig @@ -1,7 +1,6 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; -const smi = @import("zml-smi"); const data = @import("../../data.zig"); const theme = @import("../../theme.zig"); const utils = @import("../../lib/utils.zig"); @@ -10,10 +9,9 @@ const compose = @import("../../lib/compose.zig"); const Chart = @import("../../widgets/chart.zig"); const ColumnLayout = @import("../../widgets/column_layout.zig"); -/// "Device 0: NVIDIA GeForce RTX 5090 (615.71.09, CUDA 13.4)", with every detail. -pub fn headerText(arena: std.mem.Allocator, id: u16, device: smi.Device) std.mem.Allocator.Error!vxfw.Text { +pub fn headerText(arena: std.mem.Allocator, id: u16, name: []const u8) std.mem.Allocator.Error!vxfw.Text { return .{ - .text = try std.fmt.allocPrint(arena, "Device {d}: {s}{s}", .{ id, device.name orelse "Unknown", try utils.versions(arena, device, .all) }), + .text = try std.fmt.allocPrint(arena, "Device {d}: {s}", .{ id, name }), .style = theme.header_style, .softwrap = false, .overflow = .clip, @@ -37,7 +35,7 @@ pub fn historyChart( ) std.mem.Allocator.Error!Chart { return .{ .title = opts.title, - .data = try utils.normalizeRange(ctx.arena, history[id].items, opts.y_min, opts.y_max), + .data = try utils.normalizeRange(ctx.arena, try history[id].sliceLast(ctx.arena, data.history_len), opts.y_min, opts.y_max), .value_label = opts.value_label, .info_line = opts.info_line, .y_min = opts.y_min, diff --git a/bin/zml-smi/tui/pages/detail/furiosa_detail.zig b/bin/zml-smi/tui/pages/detail/furiosa_detail.zig deleted file mode 100644 index 877a4f0d..00000000 --- a/bin/zml-smi/tui/pages/detail/furiosa_detail.zig +++ /dev/null @@ -1,113 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const smi = @import("zml-smi"); -const data = @import("../../data.zig"); -const utils = @import("../../lib/utils.zig"); -const ui = @import("../../lib/ui.zig"); -const Chart = @import("../../widgets/chart.zig"); -const ColumnLayout = @import("../../widgets/column_layout.zig"); -const MetricCard = @import("../../widgets/metric_card.zig"); -const ProcessTable = @import("../../widgets/process_table.zig"); -const common = @import("detail_common.zig"); - -pub fn draw( - state: *const data.SystemState, - process_table: *ProcessTable, - ctx: vxfw.DrawContext, - w: u16, - id: u16, - device: smi.Device, - chip: smi.Furiosa, - parent_widget: vxfw.Widget, -) std.mem.Allocator.Error!vxfw.Surface { - const header = try common.headerText(ctx.arena, id, device); - - const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ - .title = "NPU Utilization", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{chip.util_percent orelse 0}), - }); - - const mem_used = chip.mem_used_bytes orelse 0; - const mem_total = chip.mem_total_bytes orelse 0; - const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ - .title = "Memory", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), - .info_line = try std.fmt.allocPrint(ctx.arena, "DRAM {d} / {d} MB", .{ utils.bytesToMb(chip.mem_used_bytes), utils.bytesToMb(chip.mem_total_bytes) }), - }); - - const temp_chart = try common.historyChart(ctx, state, id, state.history.temp, .{ - .title = "Temperature", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C", .{chip.temperature orelse 0}), - .info_line = try std.fmt.allocPrint(ctx.arena, "Ambient {d}\u{00b0}C", .{chip.ambient_temperature orelse 0}), - .y_min = 20, - .y_max = 100, - .y_unit = "\u{00b0}C", - }); - - const power_limit_w = (chip.power_limit_mw orelse 0) / 1000; - const power_raw = state.history.power[id].items; - const power_watts = try ctx.arena.alloc(u64, power_raw.len); - for (power_raw, power_watts) |mw, *watts| watts.* = mw / 1000; - const y_max = if (power_limit_w > 0) power_limit_w else 1; - const power_chart: Chart = .{ - .title = "Power", - .data = try utils.normalizeRange(ctx.arena, power_watts, 0, y_max), - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}W", .{(chip.power_mw orelse 0) / 1000}), - .info_line = if (power_limit_w > 0) try std.fmt.allocPrint(ctx.arena, "Limit {d}W", .{power_limit_w}) else "N/A", - .y_min = 0, - .y_max = @intCast(@min(y_max, std.math.maxInt(u32))), - .y_unit = "W", - .chart_height = 8, - .tui_refresh_rate = state.tui_refresh_rate, - }; - - const charts_flow: ColumnLayout = .{ - .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart), ui.widget(&temp_chart), ui.widget(&power_chart) }, - .min_child_width = 60, - .gap = 1, - }; - - const state_card: MetricCard = .{ - .title = "State", - .lines = &.{ - .{ .label = "Liveness ", .value = try code(ctx.arena, smi.Furiosa.Liveness, chip.liveness) }, - .{ .label = "Governor ", .value = try code(ctx.arena, smi.Furiosa.Governor, chip.governor) }, - .{ .label = "Throttle ", .value = try flags(ctx.arena, smi.Furiosa.ThrottleReason, chip.throttle_reasons) }, - }, - }; - - return common.pageFrame(ctx, .{ - .children = &.{ - header.widget(), - ui.widget(&charts_flow), - ui.widget(&state_card), - ui.widget(process_table), - }, - .gap = 1, - }, w, parent_widget); -} - -/// The name of a code, or its number when it has none. -fn code(arena: std.mem.Allocator, comptime E: type, value: ?u64) std.mem.Allocator.Error![]const u8 { - const number = value orelse return " N/A"; - if (std.enums.fromInt(E, number)) |known| return std.fmt.allocPrint(arena, " {t}", .{known}); - return std.fmt.allocPrint(arena, " {d}", .{number}); -} - -/// The names of the bits set in a bitmask, or their positions when they have none. -fn flags(arena: std.mem.Allocator, comptime E: type, value: ?u64) std.mem.Allocator.Error![]const u8 { - const mask = value orelse return " N/A"; - if (mask == 0) return " none"; - var text: std.Io.Writer.Allocating = .init(arena); - for (0..64) |bit| { - if (mask >> @intCast(bit) & 1 == 0) continue; - const separator = if (text.written().len == 0) " " else ", "; - if (std.enums.fromInt(E, bit)) |flag| { - text.writer.print("{s}{t}", .{ separator, flag }) catch return error.OutOfMemory; - } else { - text.writer.print("{s}bit {d}", .{ separator, bit }) catch return error.OutOfMemory; - } - } - return text.written(); -} diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig index 7cd21e42..7884d6f8 100644 --- a/bin/zml-smi/tui/pages/detail/gpu_detail.zig +++ b/bin/zml-smi/tui/pages/detail/gpu_detail.zig @@ -1,7 +1,6 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; -const smi = @import("zml-smi"); const data = @import("../../data.zig"); const utils = @import("../../lib/utils.zig"); const ui = @import("../../lib/ui.zig"); @@ -17,12 +16,23 @@ pub fn draw( ctx: vxfw.DrawContext, w: u16, id: u16, - device: smi.Device, - gpu: smi.Gpu, + gpu: data.GpuInfo, parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { // ── Header ────────────────────────────────────────────── - const header = try common.headerText(ctx.arena, id, device); + const name_suffix: []const u8 = blk: { + const drv = gpu.driver_version orelse ""; + const cuda = gpu.cuda_driver_version orelse ""; + break :blk if (drv.len > 0 and cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) + else if (drv.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) + else if (cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) + else + @as([]const u8, ""); + }; + const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ gpu.name orelse "Unknown", name_suffix })); // ── GPU Utilization + Memory charts ───────────────────── const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ @@ -60,7 +70,7 @@ pub fn draw( const power_limit_mw = gpu.power_limit_mw orelse 0; const power_limit_w = power_limit_mw / 1000; - const power_raw = state.history.power[id].items; + const power_raw = try state.history.power[id].sliceLast(ctx.arena, data.history_len); const power_watts = try ctx.arena.alloc(u64, power_raw.len); for (power_raw, 0..) |p, j| { power_watts[j] = p / 1000; diff --git a/bin/zml-smi/tui/pages/detail/metal_detail.zig b/bin/zml-smi/tui/pages/detail/metal_detail.zig deleted file mode 100644 index 5ccb1cb0..00000000 --- a/bin/zml-smi/tui/pages/detail/metal_detail.zig +++ /dev/null @@ -1,103 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const smi = @import("zml-smi"); -const data = @import("../../data.zig"); -const utils = @import("../../lib/utils.zig"); -const ui = @import("../../lib/ui.zig"); -const Chart = @import("../../widgets/chart.zig"); -const ColumnLayout = @import("../../widgets/column_layout.zig"); -const MetricCard = @import("../../widgets/metric_card.zig"); -const ProcessTable = @import("../../widgets/process_table.zig"); -const common = @import("detail_common.zig"); - -pub fn draw( - state: *const data.SystemState, - process_table: *ProcessTable, - ctx: vxfw.DrawContext, - w: u16, - id: u16, - device: smi.Device, - gpu: smi.Metal, - parent_widget: vxfw.Widget, -) std.mem.Allocator.Error!vxfw.Surface { - const header = try common.headerText(ctx.arena, id, device); - - const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ - .title = "GPU Utilization", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{gpu.util_percent orelse 0}), - }); - const mem_total = gpu.mem_total_bytes orelse 0; - const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ - .title = "Memory", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) (gpu.mem_used_bytes orelse 0) * 100 / mem_total else 0}), - .info_line = try std.fmt.allocPrint(ctx.arena, "In use {d} / {d} MB unified", .{ - utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), - }), - }); - const temp_chart = try common.historyChart(ctx, state, id, state.history.temp, .{ - .title = "Temperature", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C", .{gpu.temperature orelse 0}), - .info_line = if (gpu.fan_speed_percent) |fan| try std.fmt.allocPrint(ctx.arena, "Fan {d}%", .{fan}) else "Fan N/A", - .y_min = 20, - .y_max = 100, - .y_unit = "\u{00b0}C", - }); - - // No power limit: the scale follows the highest power in the history, in whole watts. - const power_mw = state.history.power[id].items; - const highest = if (power_mw.len > 0) std.mem.max(u64, power_mw) else 0; - const y_max: u32 = std.math.lossyCast(u32, @max(2, (highest + 999) / 1000)); - const power_chart: Chart = .{ - .title = "Power", - .data = try utils.normalizeRange(ctx.arena, power_mw, 0, @as(u64, y_max) * 1000), - .value_label = try watts(ctx.arena, gpu.power_mw), - .info_line = try std.fmt.allocPrint(ctx.arena, "ANE {s}", .{try watts(ctx.arena, gpu.ane_power_mw)}), - .y_min = 0, - .y_max = y_max, - .y_unit = "W", - .chart_height = 8, - .tui_refresh_rate = state.tui_refresh_rate, - }; - - const charts_flow: ColumnLayout = .{ - .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart), ui.widget(&temp_chart), ui.widget(&power_chart) }, - .min_child_width = 60, - .gap = 1, - }; - - const clocks_card: MetricCard = .{ .title = "Clocks", .lines = &.{ - .{ .label = "Graphics ", .value = if (gpu.clock_graphics_mhz == null and gpu.clock_graphics_max_mhz == null) - " N/A" - else - try std.fmt.allocPrint(ctx.arena, " {d} / {d} MHz", .{ gpu.clock_graphics_mhz orelse 0, gpu.clock_graphics_max_mhz orelse 0 }) }, - } }; - // The CPU and the whole machine, which the GPU's platform measures. - const host = state.host(); - const power_card: MetricCard = .{ .title = "Power", .lines = &.{ - .{ .label = "GPU ", .value = try std.fmt.allocPrint(ctx.arena, " {s}", .{try watts(ctx.arena, gpu.power_mw)}) }, - .{ .label = "ANE ", .value = try std.fmt.allocPrint(ctx.arena, " {s}", .{try watts(ctx.arena, gpu.ane_power_mw)}) }, - .{ .label = "CPU ", .value = try std.fmt.allocPrint(ctx.arena, " {s}", .{try watts(ctx.arena, host.cpu_power_mw)}) }, - .{ .label = "System ", .value = try std.fmt.allocPrint(ctx.arena, " {s}", .{try watts(ctx.arena, host.system_power_mw)}) }, - } }; - const cards_flow: ColumnLayout = .{ - .children = &.{ ui.widget(&clocks_card), ui.widget(&power_card) }, - .min_child_width = 30, - }; - - return common.pageFrame(ctx, .{ - .children = &.{ - header.widget(), - ui.widget(&charts_flow), - ui.widget(&cards_flow), - ui.widget(process_table), - }, - .gap = 1, - }, w, parent_widget); -} - -/// "12.3W", or "N/A". -fn watts(arena: std.mem.Allocator, milliwatts: ?u64) std.mem.Allocator.Error![]const u8 { - const mw = milliwatts orelse return "N/A"; - return std.fmt.allocPrint(arena, "{d}.{d}W", .{ mw / 1000, mw % 1000 / 100 }); -} diff --git a/bin/zml-smi/tui/pages/detail/neuron_detail.zig b/bin/zml-smi/tui/pages/detail/neuron_detail.zig index 594d5fff..e0dc3bcd 100644 --- a/bin/zml-smi/tui/pages/detail/neuron_detail.zig +++ b/bin/zml-smi/tui/pages/detail/neuron_detail.zig @@ -1,7 +1,6 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; -const smi = @import("zml-smi"); const data = @import("../../data.zig"); const utils = @import("../../lib/utils.zig"); const ui = @import("../../lib/ui.zig"); @@ -16,12 +15,15 @@ pub fn draw( ctx: vxfw.DrawContext, w: u16, id: u16, - device: smi.Device, - nc: smi.Neuron, + nc: data.NeuronInfo, parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { // ── Header ────────────────────────────────────────────── - const header = try common.headerText(ctx.arena, id, device); + const name_suffix: []const u8 = if (nc.driver_version) |v| + try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) + else + ""; + const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ nc.name orelse "Unknown", name_suffix })); // ── Utilization + Memory charts ───────────────────────── const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ diff --git a/bin/zml-smi/tui/pages/detail/tpu_detail.zig b/bin/zml-smi/tui/pages/detail/tpu_detail.zig index 953a894e..ca6d3b66 100644 --- a/bin/zml-smi/tui/pages/detail/tpu_detail.zig +++ b/bin/zml-smi/tui/pages/detail/tpu_detail.zig @@ -1,7 +1,6 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; -const smi = @import("zml-smi"); const data = @import("../../data.zig"); const utils = @import("../../lib/utils.zig"); const ui = @import("../../lib/ui.zig"); @@ -15,12 +14,11 @@ pub fn draw( ctx: vxfw.DrawContext, w: u16, id: u16, - device: smi.Device, - tp: smi.Tpu, + tp: data.TpuInfo, parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { // ── Header ────────────────────────────────────────────── - const header = try common.headerText(ctx.arena, id, device); + const header = try common.headerText(ctx.arena, id, tp.name orelse "Unknown"); // ── Utilization + Memory charts ───────────────────────── const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ diff --git a/bin/zml-smi/tui/pages/overview.zig b/bin/zml-smi/tui/pages/overview.zig index 2407f3d6..e91253bd 100644 --- a/bin/zml-smi/tui/pages/overview.zig +++ b/bin/zml-smi/tui/pages/overview.zig @@ -48,7 +48,7 @@ pub fn deinit(self: *Overview, allocator: std.mem.Allocator) void { } fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { - const logo: Logo = .{ .image = image_cache.global.zml }; + const logo: Logo = .{ .image = image_cache.global.get("logo") }; const info_lines: InfoLines = .{ .state = self.state }; const children = [2]vxfw.Widget{ @@ -60,7 +60,7 @@ fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16 } fn drawWideBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { - const logo: Logo = .{ .image = image_cache.global.zml }; + const logo: Logo = .{ .image = image_cache.global.get("logo") }; const info_lines: InfoLines = .{ .state = self.state }; const logo_box_w = Logo.logo_width + 6; // +4 centering + 2 absorbed page margin diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig index d1e33507..238534cd 100644 --- a/bin/zml-smi/tui/print.zig +++ b/bin/zml-smi/tui/print.zig @@ -48,7 +48,7 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Envir } image_cache.global.loadAll(&vx, allocator, tty.writer()); - defer image_cache.global.deinit(); + defer image_cache.global.deinit(allocator); vxfw.DrawContext.init(vx.screen.width_method); diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig index 9fe0e336..74b3796b 100644 --- a/bin/zml-smi/tui/top.zig +++ b/bin/zml-smi/tui/top.zig @@ -51,7 +51,7 @@ const Model = struct { fn deinit(self: *Model) void { self.process_table.deinit(self.allocator); self.overview.deinit(self.allocator); - image_cache.global.deinit(); + image_cache.global.deinit(self.allocator); } pub fn handleEvent(self: *Model, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { @@ -111,7 +111,6 @@ const Model = struct { pub fn draw(self: *Model, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const screen = ctx.max.size(); const content_w = @max(screen.width, min_width); - self.state.history_len = @max(self.state.history_len, 2 * @as(usize, content_w)); if (self.viewing_device != self.prev_viewing_device) { self.scroll.reset(); @@ -119,7 +118,7 @@ const Model = struct { self.prev_viewing_device = self.viewing_device; } - self.process_table.prepare(self.state, self.viewing_device, ctx.arena); + self.process_table.prepare(self.state, self.viewing_device); var content_surf: vxfw.Surface = undefined; diff --git a/bin/zml-smi/tui/tui.zig b/bin/zml-smi/tui/tui.zig index 0dce23b5..015e3ce2 100644 --- a/bin/zml-smi/tui/tui.zig +++ b/bin/zml-smi/tui/tui.zig @@ -1,7 +1,3 @@ pub const data = @import("data.zig"); pub const print = @import("print.zig"); pub const top = @import("top.zig"); - -test { - _ = @import("lib/image.zig"); -} diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig index 5db21cb4..c7f7ac38 100644 --- a/bin/zml-smi/tui/widgets/device_card.zig +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -1,7 +1,6 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; -const smi = @import("zml-smi"); const data = @import("../data.zig"); const theme = @import("../theme.zig"); const utils = @import("../lib/utils.zig"); @@ -40,71 +39,67 @@ pub fn handleEvent(self: *DeviceCard, ctx: *vxfw.EventContext, event: vxfw.Event } } -/// What a card shows of a device: utilization and memory, with a suffix after utilization. -const Usage = struct { - label: []const u8, - util_label: []const u8, - util_percent: ?u64, - mem_used: ?u64, - mem_total: ?u64, - suffix: []const u8 = "", -}; - pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const i = self.device_id; - const device = self.state.device(i); - const usage: Usage = switch (self.state.sample(i)) { - .cuda, .rocm, .oneapi => |gpu| .{ - .label = "GPU", - .util_label = "GPU", - .util_percent = gpu.util_percent, - .mem_used = gpu.mem_used_bytes, - .mem_total = gpu.mem_total_bytes, - .suffix = try gpuSuffix(ctx.arena, gpu), - }, - .neuron => |core| .{ - .label = "NC", - .util_label = "Core", - .util_percent = core.util_percent, - .mem_used = core.mem_used_bytes, - .mem_total = core.mem_total_bytes, - }, - .tpu => |chip| .{ - .label = "TPU", - .util_label = "Duty", - .util_percent = chip.util_percent, - .mem_used = chip.mem_used_bytes, - .mem_total = chip.mem_total_bytes, - }, - .furiosa => |chip| .{ - .label = "NPU", - .util_label = "NPU", - .util_percent = chip.util_percent, - .mem_used = chip.mem_used_bytes, - .mem_total = chip.mem_total_bytes, - .suffix = try gpuSuffix(ctx.arena, .{ .temperature = chip.temperature, .power_mw = chip.power_mw, .power_limit_mw = chip.power_limit_mw }), - }, - .metal => |gpu| .{ - .label = "GPU", - .util_label = "GPU", - .util_percent = gpu.util_percent, - .mem_used = gpu.mem_used_bytes, - .mem_total = gpu.mem_total_bytes, - // An Apple GPU idles below a watt. - .suffix = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C | {d}.{d} W", .{ gpu.temperature orelse 0, (gpu.power_mw orelse 0) / 1000, (gpu.power_mw orelse 0) % 1000 / 100 }), + const dev = self.state.devices[i]; + const target: data.Target = std.meta.activeTag(dev.*); + const cf = switch (dev.*) { + inline else => |*sv| blk: { + const f = sv.front().*; + break :blk .{ + .name = f.name orelse "Unknown", + .util_percent = @as(u8, @intCast(@min(f.util_percent orelse 0, 100))), + .mem_used = f.mem_used_bytes orelse 0, + .mem_total = f.mem_total_bytes orelse 0, + }; }, }; - const util_percent: u8 = @intCast(@min(usage.util_percent orelse 0, 100)); - const mem_used = usage.mem_used orelse 0; - const mem_total = usage.mem_total orelse 0; - const mem_pct: u8 = if (mem_total > 0) @intCast(@min(mem_used * 100 / mem_total, 100)) else 0; + const mem_pct: u8 = if (cf.mem_total > 0) @intCast(@min(cf.mem_used * 100 / cf.mem_total, 100)) else 0; - const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}{s}", .{ - usage.label, i, device.name orelse "Unknown", try utils.versions(ctx.arena, device, .in_title), - }); + const device_label, const util_label = switch (target) { + .cuda, .rocm, .oneapi => .{ "GPU", "GPU" }, + .neuron => .{ "NC", "Core" }, + .tpu => .{ "TPU", "Duty" }, + }; + const driver_ver: []const u8 = switch (dev.*) { + .cuda => |*sv| blk: { + const gpu = sv.front(); + const drv = gpu.driver_version orelse ""; + const cuda = gpu.cuda_driver_version orelse ""; + break :blk if (drv.len > 0 and cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) + else if (drv.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) + else if (cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) + else + @as([]const u8, ""); + }, + .rocm, .oneapi => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", + .neuron => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", + else => "", + }; + const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}{s}", .{ device_label, i, cf.name, driver_ver }); - const mem_used_mb = utils.bytesToMb(mem_used); - const mem_total_mb = utils.bytesToMb(mem_total); + const dev_suffix = switch (dev.*) { + .cuda, .rocm, .oneapi => |*sv| blk: { + const gpu = sv.front().*; + const power = (gpu.power_mw orelse 0) / 1000; + const power_limit = (gpu.power_limit_mw orelse 0) / 1000; + + const power_limit_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power_limit}); + const power_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power}); + + const pad = try utils.repeatStr(ctx.arena, " ", @intCast(power_limit_str.len -| power_str.len)); + break :blk try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C | {s}{s} W", .{ + gpu.temperature orelse 0, pad, power_str, + }); + }, + else => "", + }; + + const mem_used_mb = utils.bytesToMb(cf.mem_used); + const mem_total_mb = utils.bytesToMb(cf.mem_total); const mem_total_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_total_mb}); const mem_used_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_used_mb}); @@ -115,21 +110,29 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx var card: MetricCard = .{ .title = card_title, - .title_image = image_cache.global.get(device.target), - .title_right = if (device.remote) "remote" else null, + .title_image = image_cache.global.get(switch (dev.*) { + .cuda => "gpu_cuda", + .rocm => "gpu_rocm", + .oneapi => "gpu_oneapi", + .neuron => "gpu_neuron", + .tpu => "gpu_tpu", + }), + .title_right = if (self.state.devices[i].isRemote()) "remote" else null, .cell_size = ctx.cell_size, .highlighted = self.highlighted, }; - const util_chart_data = try utils.normalizeRange(ctx.arena, self.state.history.util[i].items, 0, 100); - const mem_chart_data = try utils.normalizeRange(ctx.arena, self.state.history.mem_util[i].items, 0, 100); + const util_hist = try self.state.history.util[i].sliceLast(ctx.arena, data.history_len); + const mem_hist = try self.state.history.mem_util[i].sliceLast(ctx.arena, data.history_len); + const util_chart_data = try utils.normalizeRange(ctx.arena, util_hist, 0, 100); + const mem_chart_data = try utils.normalizeRange(ctx.arena, mem_hist, 0, 100); const chart_entries = [2]MetricCard.ChartEntry{ - .{ .label = usage.util_label, .value = util_percent, .suffix = usage.suffix, .data = util_chart_data }, + .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix, .data = util_chart_data }, .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix, .data = mem_chart_data }, }; const gauge_entries = [2]MetricCard.GaugeEntry{ - .{ .label = usage.util_label, .value = util_percent, .suffix = usage.suffix }, + .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix }, .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix }, }; @@ -143,11 +146,3 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx card_surf.widget = ui.widget(self); return card_surf; } - -/// "41°C | 10 W", the power padded to the width of the limit. -fn gpuSuffix(arena: std.mem.Allocator, gpu: smi.Gpu) std.mem.Allocator.Error![]const u8 { - const power_str = try std.fmt.allocPrint(arena, "{d}", .{(gpu.power_mw orelse 0) / 1000}); - const power_limit_str = try std.fmt.allocPrint(arena, "{d}", .{(gpu.power_limit_mw orelse 0) / 1000}); - const pad = try utils.repeatStr(arena, " ", @intCast(power_limit_str.len -| power_str.len)); - return std.fmt.allocPrint(arena, "{d}\u{00b0}C | {s}{s} W", .{ gpu.temperature orelse 0, pad, power_str }); -} diff --git a/bin/zml-smi/tui/widgets/info_lines.zig b/bin/zml-smi/tui/widgets/info_lines.zig index 00ba8f5a..fcf1ae0d 100644 --- a/bin/zml-smi/tui/widgets/info_lines.zig +++ b/bin/zml-smi/tui/widgets/info_lines.zig @@ -34,7 +34,7 @@ const Entry = struct { pub fn draw(self: *const InfoLines, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const w = ctx.max.width orelse 80; - const host = self.state.host(); + const host = self.state.host.front().*; const hostname_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.hostname orelse "Unknown"}); const kernel_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.kernel orelse "Unknown"}); diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig index 9ac7e09a..58b20d51 100644 --- a/bin/zml-smi/tui/widgets/process_table.zig +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -7,7 +7,8 @@ const ui = @import("../lib/ui.zig"); const compose = @import("../lib/compose.zig"); const TitledBorder = @import("titled_border.zig"); const ColumnLayout = @import("column_layout.zig"); -const ProcessInfo = @import("zml-smi").ProcessInfo; +const pi = @import("zml-smi/info").process_info; +const ProcessInfo = pi.ProcessInfo; const RichText = vxfw.RichText; const Segment = vaxis.Cell.Segment; @@ -23,39 +24,41 @@ scroll_bars: vxfw.ScrollBars = .{ }, .draw_horizontal_scrollbar = false, }, -processes: std.ArrayList(ProcessInfo) = .empty, +merged: std.ArrayList(ProcessInfo) = .empty, pub fn deinit(self: *ProcessTable, allocator: std.mem.Allocator) void { - self.processes.deinit(allocator); + self.merged.deinit(allocator); } -/// Copies the rows to draw, and their strings into `frame`, the arena of the frame. -pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16, frame: std.mem.Allocator) void { +pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16) void { self.scroll_bars.scroll_view.children.builder.userdata = self; - self.processes.clearRetainingCapacity(); + self.merged.clearRetainingCapacity(); - const merged = state.collector.processes.lock(state.io); - defer state.collector.processes.unlock(state.io); - for (merged.rows.items) |entry| { + for (state.process_lists) |pl| { + const pl_items = pl.front().items; if (device_id) |did| { - if (entry.device_idx != did) { - continue; + for (pl_items) |entry| { + if (entry.device_idx != did) { + continue; + } + self.merged.append(state.gpa, entry) catch break; } + } else { + self.merged.appendSlice(state.gpa, pl_items) catch break; } - var row = entry; - row.username = frame.dupe(u8, entry.username) catch ""; - row.comm = frame.dupe(u8, entry.comm) catch ""; - self.processes.append(state.gpa, row) catch break; } + // Enrich with host data (uid, username, cpu%, rss, cmdline) + state.enricher.enrich(state.io, self.merged.items); + // Sort by dev_mem descending - std.sort.insertion(ProcessInfo, self.processes.items, {}, struct { + std.sort.insertion(ProcessInfo, self.merged.items, {}, struct { fn cmp(_: void, a: ProcessInfo, b: ProcessInfo) bool { return (a.dev_mem_kib orelse 0) > (b.dev_mem_kib orelse 0); } }.cmp); - const count: u32 = @intCast(self.processes.items.len); + const count: u32 = @intCast(self.merged.items.len); self.scroll_bars.estimated_content_height = count; self.scroll_bars.scroll_view.item_count = count; } @@ -67,7 +70,7 @@ pub fn resetScroll(self: *ProcessTable) void { pub fn draw(self: *ProcessTable, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const w = ctx.max.width orelse 80; - const row_count = self.processes.items.len; + const row_count = self.merged.items.len; var header_rt: RichText = .{ .text = &header_segs, .softwrap = false, .overflow = .clip }; const empty_rt: RichText = .{ @@ -101,10 +104,10 @@ pub fn draw(self: *ProcessTable, ctx: vxfw.DrawContext) std.mem.Allocator.Error! /// Builds only data rows (no header) for the scroll view. fn buildRow(ptr: *const anyopaque, idx: usize, _: usize) ?vxfw.Widget { const self: *ProcessTable = @ptrCast(@alignCast(@constCast(ptr))); - if (idx >= self.processes.items.len) { + if (idx >= self.merged.items.len) { return null; } - return .{ .userdata = @ptrCast(&self.processes.items[idx]), .drawFn = drawProcessRow }; + return .{ .userdata = @ptrCast(&self.merged.items[idx]), .drawFn = drawProcessRow }; } // Column format strings — shared between header (comptime) and row (runtime). diff --git a/bin/zml-smi/utils/BUILD.bazel b/bin/zml-smi/utils/BUILD.bazel new file mode 100644 index 00000000..a154556c --- /dev/null +++ b/bin/zml-smi/utils/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "double_buffer", + import_name = "zml-smi/double_buffer", + main = "double_buffer.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "dynlib", + import_name = "zml-smi/dynlib", + main = "dynlib.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "sandbox", + import_name = "zml-smi/sandbox", + main = "sandbox.zig", + visibility = ["//visibility:public"], + deps = ["@rules_zig//zig/runfiles"], +) + +zig_library( + name = "sysfs", + import_name = "zml-smi/sysfs", + main = "sysfs.zig", + visibility = ["//visibility:public"], +) diff --git a/bin/zml-smi/utils/double_buffer.zig b/bin/zml-smi/utils/double_buffer.zig new file mode 100644 index 00000000..add3037b --- /dev/null +++ b/bin/zml-smi/utils/double_buffer.zig @@ -0,0 +1,32 @@ +const std = @import("std"); + +pub fn DoubleBuffer(comptime T: type) type { + return struct { + const Self = @This(); + pub const Value = T; + + values: [2]T, + current: std.atomic.Value(u8) = std.atomic.Value(u8).init(0), + + pub fn front(self: *const Self) *const T { + return &self.values[self.current.load(.acquire)]; + } + + pub fn back(self: *Self) *T { + return &self.values[1 - self.current.load(.acquire)]; + } + + pub fn swap(self: *Self) void { + self.current.store(1 - self.current.load(.acquire), .release); + } + + pub fn jsonStringify(self: *const Self, jw: *std.json.Stringify) !void { + try jw.write(self.front().*); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { + const val = try std.json.innerParseFromValue(T, allocator, source, options); + return .{ .values = .{ val, val } }; + } + }; +} diff --git a/bin/zml-smi/utils/dynlib.zig b/bin/zml-smi/utils/dynlib.zig new file mode 100644 index 00000000..4a478107 --- /dev/null +++ b/bin/zml-smi/utils/dynlib.zig @@ -0,0 +1,10 @@ +const std = @import("std"); + +pub fn lookupStruct(dynlib: *std.DynLib, comptime VTable: type) !VTable { + var result: VTable = undefined; + inline for (comptime std.meta.fieldNames(VTable), comptime std.meta.fieldTypes(VTable)) |field_name, Field| { + @field(result, field_name) = dynlib.lookup(Field, field_name) orelse return error.SymbolResolutionFailed; + } + + return result; +} diff --git a/bin/zml-smi/utils/sandbox.zig b/bin/zml-smi/utils/sandbox.zig new file mode 100644 index 00000000..d4e268bd --- /dev/null +++ b/bin/zml-smi/utils/sandbox.zig @@ -0,0 +1,44 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const bazel_runfiles = @import("runfiles"); +const bazel_builtin = @import("bazel_builtin"); + +var runfiles_global: bazel_runfiles.Runfiles = undefined; + +const init_array_section = switch (builtin.object_format) { + .macho => "__DATA,__mod_init_func", + .elf => ".init_array", + else => "", +}; + +export const _ align(@alignOf(usize)) linksection(init_array_section) = &struct { + fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = argv; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + const io = threaded.io(); + + var exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const exe_size = std.process.executablePath(io, &exe_path_buf) catch + std.debug.panic("Unable to get executable path", .{}); + + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = io, + .argv0 = exe_path_buf[0..exe_size], + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; + } +}.call; + +pub fn path(buf: *[std.Io.Dir.max_path_bytes]u8) ?[]const u8 { + const with_repo = runfiles_global.withSourceRepo(bazel_builtin.current_repository); + + return with_repo.rlocation("zml/bin/zml-smi/sandbox", buf) catch null; +} diff --git a/bin/zml-smi/utils/sysfs.zig b/bin/zml-smi/utils/sysfs.zig new file mode 100644 index 00000000..636883b2 --- /dev/null +++ b/bin/zml-smi/utils/sysfs.zig @@ -0,0 +1,70 @@ +const std = @import("std"); + +pub fn readInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { + const data = try readFirstLine(allocator, io, path); + return std.fmt.parseInt(u64, data, 10); +} + +pub fn readString(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { + return readFirstLine(allocator, io, path); +} + +pub fn readFieldInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) !u64 { + return nthTokenInt(try findField(allocator, io, path, key), 0); +} + +pub fn nthTokenInt(data: []const u8, n: usize) u64 { + var i: usize = 0; + + var iter = std.mem.tokenizeAny(u8, data, " \t"); + while (iter.next()) |tok| : (i += 1) { + if (i == n) { + return std.fmt.parseInt(u64, tok, 10) catch 0; + } + } + + return 0; +} + +pub fn readFieldString(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { + const value = try findField(allocator, io, path, key); + return std.mem.trimEnd(u8, value, &std.ascii.whitespace); +} + +fn findField(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + + var line_writer: std.Io.Writer.Allocating = .init(allocator); + errdefer line_writer.deinit(); + + while (true) { + line_writer.clearRetainingCapacity(); + _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch return error.NotFound; + reader.interface.toss(1); + + const line = line_writer.written(); + if (std.mem.startsWith(u8, line, key)) { + const owned = line_writer.toOwnedSlice() catch return error.NotFound; + return std.mem.trimStart(u8, owned[key.len..], " :\t"); + } + } +} + +pub fn readFirstLine(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]u8 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + var writer: std.Io.Writer.Allocating = .init(allocator); + errdefer writer.deinit(); + _ = reader.interface.streamDelimiter(&writer.writer, '\n') catch |err| switch (err) { + error.EndOfStream => 0, + else => |e| return e, + }; + return writer.toOwnedSlice(); +} diff --git a/dsl/fly/README.md b/dsl/fly/README.md index b886988b..687c99dc 100644 --- a/dsl/fly/README.md +++ b/dsl/fly/README.md @@ -81,7 +81,7 @@ const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ ``` Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` -and `zml/kernel.zig`. +and `zml/kernel/fly/test.zig`. Model kernels live beside their consumers: diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md index ec1ae61f..b01b7189 100644 --- a/dsl/tcl/README.md +++ b/dsl/tcl/README.md @@ -104,8 +104,7 @@ def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16 across the partitions; other kernels stay per chip. Its arguments and results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis `C`, sized to the chip count, across chips (each partition holds its share), - the default `.broadcast` keeps a copy per chip. Call it inside - `zml.ops.manualComputation` so it sees each partition's shards. + the default `.broadcast` keeps a copy per chip. * **Convolutions** pad and slide the input in its read: a slide replaces its axis by `window_axis, frame_axis`, then `contract` with the filter. On its own (without a slide) the padding is dropped. diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 7e519bf0..34a78bfe 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -42,7 +42,7 @@ const DemoModel = struct { const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); - return zml.ops.manualComputation2( + return zml.ops.manualComputation( (struct { y: zml.Tensor, gate: zml.Tensor, @@ -70,7 +70,7 @@ fn usage() void { \\Simple sharding demo: \\ - explicit partitioning on data/model axes \\ - tensor print - \\ - manualComputation2 with multiple inputs and one output + \\ - manualComputation with multiple inputs and one output \\ \\Mesh modes: \\ - auto: physical_mesh=auto, inferred cpu device_count=2 diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md index b66b7abe..9673870f 100644 --- a/mlir/dialects/fly/REGENERATING.md +++ b/mlir/dialects/fly/REGENERATING.md @@ -71,7 +71,7 @@ dialect today; when one appears, extend the generator and this file. `test.zig`. - Callers that no longer compile are the point of generating: an operation whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`dsl/fly/builder.zig`, `zml/kernel.zig`, `zml/kernels/*/fly/`, + the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, and downstream llmd's `llmd/kernels/*/fly/`). - Emit the IR of the fly kernels before and after (for instance llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 623c4f0d..afb26864 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611121233.371.1.9ff7ddf1e19f/zml-furiosa-linux-amd64.tar.zst", - sha256 = "d732b56832b1f009cbe4d808c70c6f8d1dc3857a83eb06804fc53b39bef9dd6b", + url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", + sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", ) return mctx.extension_metadata( diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index ebbd9e3c..ee33fd58 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -50,7 +50,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); - try setEnv("FURIOSA_VISIBLE_DEVICES", "0", 0); const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); log.info("Loading Furiosa plugin: {s}", .{library}); diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 6016e389..a3c968d3 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" ONEAPI_VERSION = "2026.1" @@ -248,6 +248,7 @@ filegroup( name = "mkl_core_runtime", srcs = [ "{ONEAPI_MKL_LIB}/libmkl_core.so.3", + "{ONEAPI_MKL_LIB}/libmkl_def.so.3", "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index a3e69ad4..140ce102 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -216,7 +216,7 @@ "name": "libigc2", "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -227,7 +227,7 @@ "name": "libigdfcl2", "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -249,7 +249,7 @@ "name": "libze1", "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" ], "version": "1.32.0-1~24.04~ppa1" }, diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 272bc02c..5a829ac5 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -15,9 +15,11 @@ sources: - channel: noble main url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z - # IGC 2.41.15 is newer than the graphics snapshot above. + # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics + # packages are pinned below to keep their versions from it. Use snapshots, + # not the live PPA: the PPA deletes superseded packages from its pool. - channel: noble main - url: https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z archs: - "amd64" @@ -43,9 +45,9 @@ packages: - "libnl-genl-3-200" - "libnl-route-3-200" - "libnuma1" - - "libigdgmm12" + - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" - "libigc2 (= 2.41.15-1~24.04)" - "libigdfcl2 (= 2.41.15-1~24.04)" - - "libze-intel-gpu1" + - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" - "libze1" - "zlib1g" diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index 4c1c50ed..6a0fdfae 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -482,7 +482,7 @@ fn flashAttention( } /// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in -/// `zml/kernel.zig` opens the builder, calls this `run`, and finalizes the +/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the /// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ diff --git a/stdx/flags.zig b/stdx/flags.zig index 52a9e6a4..d0c4352d 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -591,25 +591,19 @@ fn fields_to_comma_list(comptime E: type) []const u8 { } } +/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. pub fn flag_name(comptime field_name: []const u8) []const u8 { - // TODO(Zig): Cleanup when this is fixed after Zig 0.11. - // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. - return comptime blk: { - assert(!std.mem.eql(u8, field_name, "positional")); - - var result: []const u8 = "--"; - var index = 0; - while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { - result = result ++ field_name[index..][0..i] ++ "-"; - index = index + i + 1; - } - result = result ++ field_name[index..]; - break :blk result; - }; + comptime assert(!std.mem.eql(u8, field_name, "positional")); + comptime var name = field_name[0..field_name.len].*; + comptime std.mem.replaceScalar(u8, &name, '_', '-'); + const dashed = name; + return "--" ++ dashed; } test flag_name { - try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); + try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); + try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); + try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); } fn flag_name_positional(comptime field_name: []const u8) []const u8 { diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index fff3b8a8..f52ecdd5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -87,11 +87,10 @@ zig_library( "exe.zig", "floats.zig", "io.zig", - "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", + "kernels/moe/api.zig", "kernels/moe/moe.zig", - "kernels/moe/options.zig", "kernels/moe/tests.zig", "kernels/moe/weights.zig", "kernels/paged_attention/options.zig", @@ -103,7 +102,6 @@ zig_library( "nn.zig", "ops.zig", "ops/manual_computation.zig", - "ops/manual_computation2.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", @@ -115,6 +113,7 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", + "//zml/kernel", "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 0ac2a95c..edd9b4ca 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -97,7 +97,6 @@ pub const Options = @import("Options.zig"); pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; - const InputSpecs = @FieldType(zml.ops.ManualPartitioning(Attention.forward), "in_specs"); in_specs: struct { q: Logical, @@ -105,7 +104,7 @@ pub const Partitioning = struct { v: Logical, token_index: Logical = .replicated, /// Omit to replicate backend metadata tensors. - metadata: ?@FieldType(InputSpecs, "metadata") = null, + metadata: ?zml.ops.PartitionSpecs(Metadata) = null, }, out_specs: Logical, @@ -135,20 +134,13 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T /// `attention`, causal or not depending on `opts`. pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; - return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, .{ + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ .in_specs = .{ .q = partitioning.in_specs.q, .k = partitioning.in_specs.k, .v = partitioning.in_specs.v, .token_index = partitioning.in_specs.token_index, - .metadata = partitioning.in_specs.metadata orelse switch (metadata) { - .stablehlo => .{ .stablehlo = {} }, - .flash_attention => |metadata_| .{ .flash_attention = switch (metadata_) { - .cute => .{ .cute = {} }, - .nki => .{ .nki = {} }, - .metal => .{ .metal = .{ .num_tokens = .replicated } }, - } }, - }, + .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), }, .out_specs = partitioning.out_specs, }); diff --git a/zml/kernel.zig b/zml/kernel.zig deleted file mode 100644 index d85b6a56..00000000 --- a/zml/kernel.zig +++ /dev/null @@ -1,1627 +0,0 @@ -const std = @import("std"); - -const cuda_tile_builder = @import("dsl/cuda_tile/builder"); -const cute_builder = @import("dsl/cute/builder"); -const fly_builder = @import("dsl/fly/builder"); -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); -const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const tcl_builder = @import("dsl/tcl/builder"); -const triton_builder = @import("dsl/triton/builder"); - -const Compiler = @import("Compiler.zig"); -const DataType = @import("dtype.zig").DataType; -const mlirx = @import("mlirx.zig"); -const ops = @import("ops.zig"); -const platform_ = @import("platform.zig"); -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub const triton = struct { - pub const Builder = triton_builder.Builder; - pub const Value = triton_builder.Value; - pub const DType = triton_builder.DType; - pub const FinishError = triton_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(triton_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - debug: bool = false, - /// Bytes for Triton's implicit global scratch argument, across all CTAs. - global_scratch_memory_size: i32 = 0, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try triton_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ttir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ttir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const call_opts: ops.TritonOps = .{ - .debug = opts.debug, - .name = name, - .ir = ttir, - .grid = opts.grid, - .num_stages = opts.num_stages, - .num_warps = opts.num_warps, - .output_operand_aliases = aliases.constSlice(), - .is_tma_allowed = opts.global_scratch_memory_size > 0, - .global_scratch_memory_size = opts.global_scratch_memory_size, - }; - - const triton_results = if (opts.global_scratch_memory_size > 0) blk: { - // XLA maps the final custom-call result to Triton's implicit - // scratch pointer; it must not appear in the TTIR signature. - const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; - const res = ops.triton(inputs_arr, extended, call_opts); - break :blk res[0..spec.outputs.len].*; - } else ops.triton(inputs_arr, outputs_arr, call_opts); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; - return results; - } - }; - } -}; - -pub const mosaic_tpu = struct { - const Builder = mosaic_tpu_builder.Builder; - const DType = mosaic_tpu_builder.DType; - const FinishError = mosaic_tpu_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(mosaic_tpu_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - dynamic_grid_bounds: []const Tensor = &.{}, - }; - - pub const CallOpts = struct { - cfg: ConfigT, - extras: CallExtras = .{}, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finishOpts(&.{}, .{ .canonicalize = true }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const backend_config = buildBackendConfig(cur.allocator, ir, .{ - .vmem_limit_bytes = opts.extras.vmem_limit_bytes, - .disable_bounds_checks = opts.extras.disable_bounds_checks, - .disable_semaphore_checks = opts.extras.disable_semaphore_checks, - .has_communication = opts.extras.has_communication, - }) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); - defer cur.allocator.free(backend_config); - - var values: [spec.inputs.len]*const mlir.Value = undefined; - inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); - - var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (&res_types, spec.outputs) |*r, fname| { - r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); - } - - const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); - - const op = callTpuCustomCall(.{ - .inputs = &values, - .result_types = &res_types, - .backend_config = backend_config, - .aliases = aliases.constSlice(), - .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, - .additional_attributes = opts.extras.additional_attributes, - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - @field(results, fname) = Tensor._result(out, op.result(i)); - } - return results; - } - }; - } - - const CustomCallConfig = struct { - body: []const u8, - has_communication: ?bool = null, - serialization_format: i64 = 1, - needs_layout_passes: bool = true, - shape_invariant_numerics: bool = false, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - }; - - const ScopedMemoryConfig = struct { - memory_space: i64 = 1, - offset: i64 = 0, - size: i64, - }; - - const BackendConfig = struct { - custom_call_config: CustomCallConfig, - scoped_memory_configs: ?[]const ScopedMemoryConfig = null, - }; - - const BackendConfigOpts = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - }; - - fn buildBackendConfig( - allocator: std.mem.Allocator, - ir: [:0]const u8, - opts: BackendConfigOpts, - ) ![]u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - const module = try mlir.Module.parse(ctx, ir); - defer module.deinit(); - - mosaic_tpu_dialect.registerMosaicSerdePass(); - const prev_allow_unregistered = ctx.allowUnregisteredDialects(); - ctx.setAllowUnregisteredDialects(true); - defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); - - const pm = mlir.PassManager.init(ctx); - defer pm.deinit(); - try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); - try pm.runOnOp(module.operation()); - - const Encoder = std.base64.standard.Encoder; - const b64 = b64: { - var bytecode: std.Io.Writer.Allocating = .init(allocator); - defer bytecode.deinit(); - try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); - const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); - break :b64 Encoder.encode(buf, bytecode.written()); - }; - defer allocator.free(b64); - - const config = BackendConfig{ - .custom_call_config = .{ - .body = b64, - .has_communication = opts.has_communication, - .disable_bounds_checks = opts.disable_bounds_checks, - .disable_semaphore_checks = opts.disable_semaphore_checks, - }, - .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| - &.{.{ .size = vmem }} - else - null, - }; - - var json: std.Io.Writer.Allocating = .init(allocator); - errdefer json.deinit(); - - try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - return try json.toOwnedSlice(); - } - - const TpuCustomCallArgs = struct { - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - backend_config: []const u8, - aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, - dynamic_grid_bounds: []const Tensor, - additional_attributes: []const mlir.NamedAttribute, - }; - - fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = Compiler.current(); - - var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; - for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); - all_inputs.appendSliceAssumeCapacity(args.inputs); - - return stablehlo.custom_call( - cur.mlir_ctx, - all_inputs.constSlice(), - args.result_types, - .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = args.backend_config }, - .has_side_effect = false, - .api_version = .original, - .additional_attributes = args.additional_attributes, - .output_operand_aliases = args.aliases, - }, - cur.location, - ).appendTo(cur.currentScope().block); - } -}; - -pub const cuda_tile = struct { - pub const Builder = cuda_tile_builder.Builder; - pub const Value = cuda_tile_builder.Value; - pub const DType = cuda_tile_builder.DType; - pub const FinishError = cuda_tile_builder.FinishError; - pub const EntryHint = cuda_tile_builder.EntryHint; - pub const Arch = cuda_tile_builder.Arch; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cuda_tile_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i4, .u4 => .i4, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - .f4e2m1 => .f4e2m1fn, - else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// No `num_warps`/`num_stages`: warp and CTA tuning is - /// `optimization_hints` inside the IR (`Opts.hints`). - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Outputs XLA zeroes before the launch, by position in - /// `spec.outputs`, ascending. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(&.{}); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const cute = struct { - pub const AlgebraExpr = cute_builder.AlgebraExpr; - pub const Builder = cute_builder.Builder; - pub const ArgSpec = cute_builder.ArgSpec; - pub const Value = cute_builder.Value; - pub const View = cute_builder.View; - pub const Atom = cute_builder.Atom; - pub const DType = cute_builder.DType; - pub const AlgebraToken = cute_builder.AlgebraToken; - pub const LayoutSpec = cute_builder.LayoutSpec; - pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; - pub const FinishError = cute_builder.FinishError; - pub const TmaConfig = cute_builder.TmaConfig; - pub const TmaFormat = cute_builder.TmaFormat; - pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; - pub const CtaGroup = cute_builder.CtaGroup; - pub const ctaCount = cute_builder.ctaCount; - pub const MBarrierScope = cute_builder.MBarrierScope; - pub const OperandMajorMode = cute_builder.OperandMajorMode; - pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; - pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; - pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; - pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; - pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; - pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cute_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4e2m1fn, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared - /// memory is what the kernel declares. - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - block: [3]i32, - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - // XLA passes one pointer per input then per output; it does - // not check the count against the kernel. - const expected = spec.inputs.len + spec.outputs.len; - if (b.args.len != expected) { - std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); - } - - return b.finish(block); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| - std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .block = opts.block, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } - - fn ProgramSpec(comptime Config: type) type { - return struct { - /// Public host entry selected by XLA. The callback may give the - /// nested CUDA kernel a different (private) symbol. - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A complete CuTe program: one or more `cuda.kernel` operations inside - /// a `gpu.module`, followed by a public `func.func` which constructs the - /// launch-time CuTe objects and calls `cuda.launch_ex`. - /// - /// This is the source-level equivalent of invoking a Python `@cute.jit` - /// object whose `__call__` builds TMA descriptors and launches an - /// `@cute.kernel`. XLA executes the generated host function on its CUDA - /// stream, so the custom call deliberately carries no external grid or - /// block dimensions. - pub fn Program( - comptime ConfigT: type, - comptime spec: ProgramSpec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.openProgram(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - return b.finishProgram(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .zeroed_outputs = opts.zeroed_outputs, - .scalars = opts.scalars, - .symmetric_workspace = opts.symmetric_workspace, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const tcl = struct { - pub const Builder = tcl_builder.Builder; - pub const TensorOperation = tcl_builder.TensorOperation; - pub const Value = tcl_builder.Value; - pub const Axis = tcl_builder.Axis; - pub const Expr = tcl_builder.Expr; - pub const Dram = tcl_builder.Dram; - pub const Context = tcl_builder.Context; - pub const DType = tcl_builder.DType; - pub const VeOp = tcl_builder.VeOp; - pub const ReduceMode = tcl_builder.ReduceMode; - pub const Predicate = tcl_builder.Predicate; - pub const Tactic = tcl_builder.Tactic; - pub const FinishError = tcl_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(tcl_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .bool, - .i4 => .i4, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4_e2, - .f8e4m3fn => .f8_e4, - .f8e5m2 => .f8_e5, - else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A TCL function compiled by the Furiosa plugin as one native unit: - /// `declareArgs` declares the inputs in order, `ret` the outputs. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - return build(allocator, cfg, null, null, null); - } - - /// `updated`, when given, receives the input each output updates - /// in place (`Builder.update`), or null. - fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try Builder.open(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - - const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); - if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { - std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); - } - - if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); - if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); - if (updated) |u| for (u, results) |*dst, r| { - dst.* = b.updatedArgument(r); - }; - - return b.finish(); - } - - fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { - var same = from(s.dtype()) == t.dtype; - if (t.shards > 1) { - // A partition holds one chip's share of a chip-split DRAM tensor. - var n: i64 = 1; - for (t.axes) |a| n *= a.size; - same = same and n == @as(i64, @intCast(s.count())) * t.shards; - } else { - same = same and s.rank() == t.axes.len; - if (same) for (t.axes, 0..) |a, i| { - same = same and a.size == s.dim(i); - }; - } - - if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - var input_shapes: [spec.inputs.len]Shape = undefined; - inline for (spec.inputs, 0..) |fname, i| { - inputs_arr[i] = @field(inputs, fname); - input_shapes[i] = inputs_arr[i].shape(); - } - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var updated: [spec.outputs.len]?usize = undefined; - const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| - std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ - .output_index = @intCast(output), - .operand_index = @intCast(i), - }); - const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - @field(results, fname) = tensor_results[i]; - } - - return results; - } - }; - } -}; - -pub const metal = struct { - pub const Builder = metal_builder.Builder; - pub const Value = metal_builder.Value; - pub const Type = metal_builder.Type; - pub const Struct = metal_builder.Struct; - pub const ArgSpec = metal_builder.ArgSpec; - pub const LoopOpts = metal_builder.LoopOpts; - pub const FinishError = metal_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(metal_builder.registerDialects); - } - - pub fn from(dt: DataType) Type { - return switch (dt) { - .bool => .bool, - .i8 => .char, - .i16 => .short, - .i32 => .int, - .i64 => .long, - .u8 => .uchar, - .u16 => .ushort, - .u32 => .uint, - .u64 => .ulong, - .f16 => .half, - .bf16 => .bfloat, - .f32 => .float, - else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), - }; - } - - /// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host - /// `extern struct` laid out like a `constant S &` argument. - pub const Constant = struct { - buffer: [256]u8 = undefined, - len: usize = 0, - - pub fn of(value: anytype) Constant { - const raw = std.mem.asBytes(&value); - var c: Constant = .{ .len = raw.len }; - @memcpy(c.buffer[0..raw.len], raw); - return c; - } - - pub fn bytes(self: *const Constant) []const u8 { - return self.buffer[0..self.len]; - } - }; - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - /// The arguments bound to input tensors. - inputs: []const [:0]const u8, - /// The arguments bound to output tensors; one named like an - /// input updates it in place. - outputs: []const [:0]const u8, - /// The `constant` arguments given by value at the call. - constants: []const [:0]const u8 = &.{}, - run: *const fn (*Builder, Config) FinishError!void, - fast_math: bool = true, - }; - } - - /// A Metal kernel: `run` declares its arguments with `declareArgs`, by - /// name, and builds its body. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const Constants = StructOf(spec.constants, Constant); - - pub const CallOpts = struct { - cfg: ConfigT, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - constants: Constants = std.mem.zeroes(Constants), - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - return build(allocator, cfg, &arguments); - } - - fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); - defer b.deinit(); - try spec.run(&b, cfg); - const ir = try b.finish(); - errdefer allocator.free(ir); - try arguments.appendSlice(allocator, b.arguments.items); - return ir; - } - - fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { - for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; - return null; - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const allocator = cur.allocator; - - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - const ir = build(allocator, opts.cfg, &arguments) catch |err| - std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); - var constants: [spec.constants.len][]const u8 = undefined; - inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); - - var bound: [spec.outputs.len]bool = @splat(false); - var bound_inputs: [spec.inputs.len]bool = @splat(false); - var bound_constants: [spec.constants.len]bool = @splat(false); - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - var bindings: std.ArrayList(i32) = .empty; - defer bindings.deinit(allocator); - for (arguments.items) |arg| { - if (arg.kind == .threadgroup) { - // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. - if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); - continue; - } - std.debug.assert(arg.index == bindings.items.len); - const input = indexOf(spec.inputs, arg.name); - const output = indexOf(spec.outputs, arg.name); - if (input) |i| bound_inputs[i] = true; - if (output) |o| bound[o] = true; - const binding: i32 = if (input) |i| blk: { - if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); - break :blk @intCast(i); - } else if (output) |o| - @intCast(spec.inputs.len + o) - else if (indexOf(spec.constants, arg.name)) |k| blk: { - if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); - bound_constants[k] = true; - break :blk -1 - @as(i32, @intCast(k)); - } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); - bindings.append(allocator, binding) catch @panic("OOM"); - } - for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); - for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); - for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); - - const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .threadgroup = opts.threadgroup, - .threadgroup_memory = opts.threadgroup_memory, - .arguments = bindings.items, - .constants = &constants, - .output_operand_aliases = aliases.constSlice(), - }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const fly = struct { - /// Threads per wavefront: 64 on CDNA, 32 on RDNA. - pub fn waveSize(p: *const platform_.Platform) i32 { - const cc = platform_.rocm.computeCapability(p) orelse return 64; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => 64, - .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, - }; - } - - /// The plugin sizes a block in wavefronts, so a thread count has to divide - /// evenly: a kernel's layouts are built for an exact number of threads and - /// spare ones would partition out of range. - fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { - const wave = waveSize(p); - if (@rem(threads, wave) != 0) { - std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); - } - return @divExact(threads, wave); - } - - pub const Builder = fly_builder.Builder; - pub const Value = fly_builder.Value; - pub const DType = fly_builder.DType; - pub const FinishError = fly_builder.FinishError; - pub const TiledCopy = fly_builder.TiledCopy; - pub const TiledMma = fly_builder.TiledMma; - pub const Arch = fly_builder.Arch; - pub const MmaFlavor = fly_builder.MmaFlavor; - pub const fly_rocdl = fly_builder.fly_rocdl; - - /// The matrix atom this device provides, or null when it has none: WMMA - /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. - pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { - const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, - .rdna3, .rdna3_5 => .gfx11_wmma, - .rdna4 => .gfx120x_wmma, - .rdna2 => null, - }; - } - pub const layout = fly_builder.layout; - pub const Layout = fly_builder.Layout; - pub const Tile = fly_builder.Tile; - pub const IntTuple = fly_builder.IntTuple; - pub const L = fly_builder.L; - pub const rowMajor = fly_builder.rowMajor; - pub const colMajor = fly_builder.colMajor; - pub const ordered = fly_builder.ordered; - pub const tile = fly_builder.tile; - pub const it = fly_builder.it; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(fly_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - .f8e5m2fnuz => .f8e5m2fnuz, - else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const arg_names = spec.inputs ++ spec.outputs; - /// By name, as `!fly.memref` views. - pub const Args = StructOf(arg_names, Value); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Threads per block. Converted to the wavefronts the plugin - /// wants using the device's wave size, which is 64 on CDNA and - /// 32 on RDNA, so a kernel states the thread count its layouts - /// were built for and runs on both. - threads: i32, - /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. - waves_per_eu: i32 = 0, - /// Required when the body uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Inputs first, then outputs; see `zeroedOutput`. A no-op - /// inside command buffers. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - /// The arguments `run` was declared with. - pub fn args(b: *Builder) Args { - var out: Args = undefined; - inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); - return out; - } - - /// `zeroed_args` index of an output. - pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); - } - - /// `shapes` is inputs, then outputs. - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { - std.debug.assert(shapes.len == arg_names.len); - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try fly_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - var specs: [arg_names.len]Builder.ArgSpec = undefined; - inline for (arg_names, 0..) |fname, i| { - const shape = shapes[i]; - specs[i] = .{ - .name = fname, - .dtype = from(shape.dtype()), - .dims = if (shape.rank() > 0) shape.dims() else null, - }; - } - try b.declareArgsLow(&specs); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var shapes: [arg_names.len]Shape = undefined; - inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); - inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; - - const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| - std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .num_warps = warpsFor(cur.platform, opts.threads), - .waves_per_eu = opts.waves_per_eu, - .shared_mem_bytes = opts.shared_mem_bytes, - .zeroed_args = opts.zeroed_args, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -const FlyVectorAdd = struct { - const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), - }; - - const K = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); - } -}; - -test "fly kernel emits a module XLA can parse" { - const shapes = [_]Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; - const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); -} - -/// C = A @ B^T with one MFMA-tiled block. -const FlyTiledMma = struct { - const Cfg = struct { flavor: fly.MmaFlavor }; - const K = fly.Kernel(Cfg, .{ - .name = "tiled_mma", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); - } -}; - -test "fly kernels run on rocm" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward`, upload both inputs, run, and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); - defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); - defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); - defer bb.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // vector add: elementwise over a ragged (100, 1000), predicated at the edge. - { - const M, const N = .{ 100, 1000 }; - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, - .threads = 8 * 16, - }).c; - } - }; - - const ha = try allocator.alloc(f32, M * N); - defer allocator.free(ha); - const hb = try allocator.alloc(f32, M * N); - defer allocator.free(hb); - for (ha, hb, 0..) |*va, *vb, i| { - va.* = @floatFromInt(i % 977); - vb.* = @floatFromInt(1000 + (i % 313)); - } - - var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); - } - - // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix - // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, - // whose verifier rejects f32 operands, so the operand type and K come from - // the flavour rather than being written into the test. - if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { - inline else => |f| { - const Elem = switch (comptime f.operandDType()) { - .f32 => f32, - .f16 => f16, - else => @compileError("unhandled MMA operand type"), - }; - const M, const N = .{ 64, 64 }; - const Kd = comptime f.blockK(); - const operand_dtype: DataType = switch (comptime f.operandDType()) { - .f32 => .f32, - .f16 => .f16, - else => unreachable, - }; - - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = Shape.init(.{ M, N }, .f32) }, .{ - .cfg = .{ .flavor = f }, - .grid = .{ 1, 1, 1 }, - .threads = comptime f.blockThreads(), - }).c; - } - }; - - var ha: [M * Kd]Elem = undefined; - var hb: [N * Kd]Elem = undefined; - for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; - for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; - - var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); - defer host.free(allocator); - - // f16 operands accumulate in f32, so the error is set by the input - // rounding rather than the sum. - const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; - const out = host.items(f32); - for (0..M) |m| for (0..N) |n| { - var acc: f32 = 0; - for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); - try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); - }; - }, - }; -} - -/// `furiosa.tcl.examples.softmax` over f32. -const TclSoftmax = struct { - const Cfg = struct { b: i64, s: i64, d: i64 }; - const K = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"x"}, - .outputs = &.{"y"}, - .run = run, - }); - - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.x, .{}); - const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); - b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); - } -}; - -test "tcl kernels run on furiosa" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; - - const ir = try TclSoftmax.K.emit(allocator, cfg); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); - - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - - const Mod = struct { - pub fn forward(x: Tensor) Tensor { - return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; - } - }; - const x: Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); - defer exe.deinit(); - - var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); - defer x_buffer.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - - const out = host.items(f32); - var row: usize = 0; - while (row < cfg.b * cfg.s) : (row += 1) { - const in = input[row * cfg.d ..][0..cfg.d]; - var max: f32 = in[0]; - for (in) |v| max = @max(max, v); - var sum: f32 = 0; - for (in) |v| sum += @exp(v - max); - for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); - } -} - -test "cute kernel emits the module the CuTe compiler takes" { - const Cfg = struct { n: i64 }; - const AddOne = cute.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - }); - const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); - var guard = b.openIf(i.lt(cfg.n)); - { - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.close(); - } - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); -} - -test "cuda_tile kernel emits a module XLA can parse" { - const Cfg = struct { n: i64 }; - const AddOne = cuda_tile.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); -} - -/// exp over a tensor split across two chips in one launch, returning the -/// split result and its all_gather. -const TclSplitExp = struct { - const Cfg = struct {}; - const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); - fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - } - fn body(ctx: struct { x: Tensor }, shapes: [2]Shape) [2]Tensor { - const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); - return .{ r.y, r.all }; - } -}; - -// Needs FURIOSA_VISIBLE_DEVICES=0,1. -test "tcl multi-chip kernels run on two furiosa chips" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const tp = zml.testing.tpMesh(); - const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); - const Mod = struct { - pub fn forward(x: Tensor) [2]Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); - return .{ r[0], r[1] }; - } - }; - const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{tp} }); - defer exe.deinit(); - var hx: [2 * 1024]f32 = undefined; - for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var res: [2]zml.Buffer = undefined; - runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); - defer for (&res) |*r| r.deinit(); - for (res) |r| { - var host = try r.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); - } -} - -/// `out[i] = a[i] * scale + b[i]` over `n` floats. -const MetalAxpy = struct { - const Cfg = struct {}; - const K = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -test "metal kernels run on metal" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - - const ir = try MetalAxpy.K.emit(allocator, .{}); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); - - const platform = zml.testing.env(); - if (platform.target != .metal) return error.SkipZigTest; - - const n = 1000; - const Mod = struct { - pub fn forward(x: Tensor, y: Tensor) Tensor { - return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ (n + 255) / 256, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, - }).out; - } - }; - const x: Tensor = .init(.{n}, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, Tensor.init(.{n}, .f32) }, platform, .{}); - defer exe.deinit(); - - var hx: [n]f32 = undefined; - var hy: [n]f32 = undefined; - for (&hx, &hy, 0..) |*a, *b, i| { - a.* = @floatFromInt(i); - b.* = 1; - } - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); - defer by.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); -} diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel new file mode 100644 index 00000000..61ab44f8 --- /dev/null +++ b/zml/kernel/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "kernel", + srcs = [ + "kernel.zig", + "//zml/kernel/cuda_tile", + "//zml/kernel/cute", + "//zml/kernel/fly", + "//zml/kernel/metal", + "//zml/kernel/mosaic_tpu", + "//zml/kernel/tcl", + "//zml/kernel/triton", + ], +) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..89b7977e --- /dev/null +++ b/zml/kernel/cuda_tile/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cuda_tile", + srcs = [ + "cuda_tile.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..3f6d8734 --- /dev/null +++ b/zml/kernel/cuda_tile/cuda_tile.zig @@ -0,0 +1,163 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +pub const Builder = cuda_tile_builder.Builder; +pub const Value = cuda_tile_builder.Value; +pub const DType = cuda_tile_builder.DType; +pub const FinishError = cuda_tile_builder.FinishError; +pub const EntryHint = cuda_tile_builder.EntryHint; +pub const Arch = cuda_tile_builder.Arch; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig new file mode 100644 index 00000000..c929b0b9 --- /dev/null +++ b/zml/kernel/cuda_tile/test.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +const cuda_tile = @import("cuda_tile.zig"); + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel new file mode 100644 index 00000000..7a613072 --- /dev/null +++ b/zml/kernel/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig new file mode 100644 index 00000000..2e8c919f --- /dev/null +++ b/zml/kernel/cute/cute.zig @@ -0,0 +1,264 @@ +const std = @import("std"); + +const cute_builder = @import("dsl/cute/builder"); +pub const AlgebraExpr = cute_builder.AlgebraExpr; +pub const Builder = cute_builder.Builder; +pub const ArgSpec = cute_builder.ArgSpec; +pub const Value = cute_builder.Value; +pub const View = cute_builder.View; +pub const Atom = cute_builder.Atom; +pub const DType = cute_builder.DType; +pub const AlgebraToken = cute_builder.AlgebraToken; +pub const LayoutSpec = cute_builder.LayoutSpec; +pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; +pub const FinishError = cute_builder.FinishError; +pub const TmaConfig = cute_builder.TmaConfig; +pub const TmaFormat = cute_builder.TmaFormat; +pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; +pub const CtaGroup = cute_builder.CtaGroup; +pub const ctaCount = cute_builder.ctaCount; +pub const MBarrierScope = cute_builder.MBarrierScope; +pub const OperandMajorMode = cute_builder.OperandMajorMode; +pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; +pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; +pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; +pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; +pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; +pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} + +fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A complete CuTe program: one or more `cuda.kernel` operations inside +/// a `gpu.module`, followed by a public `func.func` which constructs the +/// launch-time CuTe objects and calls `cuda.launch_ex`. +/// +/// This is the source-level equivalent of invoking a Python `@cute.jit` +/// object whose `__call__` builds TMA descriptors and launches an +/// `@cute.kernel`. XLA executes the generated host function on its CUDA +/// stream, so the custom call deliberately carries no external grid or +/// block dimensions. +pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig new file mode 100644 index 00000000..f4d457a3 --- /dev/null +++ b/zml/kernel/cute/test.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const cute = @import("cute.zig"); + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel new file mode 100644 index 00000000..57203521 --- /dev/null +++ b/zml/kernel/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "test.zig", + ], +) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig new file mode 100644 index 00000000..b7d6c943 --- /dev/null +++ b/zml/kernel/fly/fly.zig @@ -0,0 +1,248 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); +pub const Builder = fly_builder.Builder; +pub const Value = fly_builder.Value; +pub const DType = fly_builder.DType; +pub const FinishError = fly_builder.FinishError; +pub const TiledCopy = fly_builder.TiledCopy; +pub const TiledMma = fly_builder.TiledMma; +pub const Arch = fly_builder.Arch; +pub const MmaFlavor = fly_builder.MmaFlavor; +pub const fly_rocdl = fly_builder.fly_rocdl; +pub const layout = fly_builder.layout; +pub const Layout = fly_builder.Layout; +pub const Tile = fly_builder.Tile; +pub const IntTuple = fly_builder.IntTuple; +pub const L = fly_builder.L; +pub const rowMajor = fly_builder.rowMajor; +pub const colMajor = fly_builder.colMajor; +pub const ordered = fly_builder.ordered; +pub const tile = fly_builder.tile; +pub const it = fly_builder.it; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +/// Threads per wavefront: 64 on CDNA, 32 on RDNA. +pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; +} + +/// The plugin sizes a block in wavefronts, so a thread count has to divide +/// evenly: a kernel's layouts are built for an exact number of threads and +/// spare ones would partition out of range. +fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); +} + +/// The matrix atom this device provides, or null when it has none: WMMA +/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. +pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig new file mode 100644 index 00000000..ae788d96 --- /dev/null +++ b/zml/kernel/fly/test.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); + +const fly = @import("fly.zig"); +const zml = @import("../../zml.zig"); + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +test "fly kernels run on rocm" { + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig new file mode 100644 index 00000000..204c6204 --- /dev/null +++ b/zml/kernel/kernel.zig @@ -0,0 +1,12 @@ +pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); +pub const cute = @import("cute/cute.zig"); +pub const fly = @import("fly/fly.zig"); +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const tcl = @import("tcl/tcl.zig"); +pub const triton = @import("triton/triton.zig"); + +test { + const std = @import("std"); + std.testing.refAllDecls(@This()); +} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel new file mode 100644 index 00000000..4007beb4 --- /dev/null +++ b/zml/kernel/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + "test.zig", + ], +) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig new file mode 100644 index 00000000..9626a1de --- /dev/null +++ b/zml/kernel/metal/metal.zig @@ -0,0 +1,237 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +pub const Builder = metal_builder.Builder; +pub const Value = metal_builder.Value; +pub const Type = metal_builder.Type; +pub const Struct = metal_builder.Struct; +pub const ArgSpec = metal_builder.ArgSpec; +pub const LoopOpts = metal_builder.LoopOpts; +pub const FinishError = metal_builder.FinishError; +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); +} + +pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; +} + +/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host +/// `extern struct` laid out like a `constant S &` argument. +pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } +}; + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; +} + +/// A Metal kernel: `run` declares its arguments with `declareArgs`, by +/// name, and builds its body. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig new file mode 100644 index 00000000..0a342af0 --- /dev/null +++ b/zml/kernel/metal/test.zig @@ -0,0 +1,71 @@ +const std = @import("std"); + +const metal = @import("metal.zig"); +const zml = @import("../../zml.zig"); + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: zml.Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ba3b6b66 --- /dev/null +++ b/zml/kernel/mosaic_tpu/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = ["mosaic_tpu.zig"], +) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..a09e8368 --- /dev/null +++ b/zml/kernel/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,284 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const Builder = mosaic_tpu_builder.Builder; +const DType = mosaic_tpu_builder.DType; +const FinishError = mosaic_tpu_builder.FinishError; +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; +} + +const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, +}; + +const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, +}; + +const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, +}; + +const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, +}; + +fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, +) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); +} + +const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, +}; + +fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); +} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel new file mode 100644 index 00000000..bc7a0aa2 --- /dev/null +++ b/zml/kernel/tcl/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "tcl", + srcs = [ + "tcl.zig", + "test.zig", + ], +) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig new file mode 100644 index 00000000..023e862f --- /dev/null +++ b/zml/kernel/tcl/tcl.zig @@ -0,0 +1,206 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +pub const Builder = tcl_builder.Builder; +pub const TensorOperation = tcl_builder.TensorOperation; +pub const Value = tcl_builder.Value; +pub const Axis = tcl_builder.Axis; +pub const Expr = tcl_builder.Expr; +pub const Dram = tcl_builder.Dram; +pub const Context = tcl_builder.Context; +pub const DType = tcl_builder.DType; +pub const VeOp = tcl_builder.VeOp; +pub const ReduceMode = tcl_builder.ReduceMode; +pub const Predicate = tcl_builder.Predicate; +pub const Tactic = tcl_builder.Tactic; +pub const FinishError = tcl_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A TCL function compiled by the Furiosa plugin as one native unit: +/// `declareArgs` declares the inputs in order, `ret` the outputs. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; +} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig new file mode 100644 index 00000000..b9ae1875 --- /dev/null +++ b/zml/kernel/tcl/test.zig @@ -0,0 +1,123 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const tcl = @import("tcl.zig"); + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: zml.Tensor) zml.Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const tp = zml.testing.tpMesh(); + const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); + return .{ r[0], r[1] }; + } + }; + const x: zml.Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel new file mode 100644 index 00000000..1e992352 --- /dev/null +++ b/zml/kernel/triton/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "triton", + srcs = ["triton.zig"], +) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig new file mode 100644 index 00000000..6a91a03f --- /dev/null +++ b/zml/kernel/triton/triton.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const triton_builder = @import("dsl/triton/builder"); +pub const Builder = triton_builder.Builder; +pub const Value = triton_builder.Value; +pub const DType = triton_builder.DType; +pub const FinishError = triton_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const ops = @import("../../ops.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; +} diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig index 4faa1e82..de4b68da 100644 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -4,9 +4,8 @@ const std = @import("std"); -const kernel = @import("../../../kernel.zig"); +const metal = @import("../../../kernel/kernel.zig").metal; -const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/kernels/moe/options.zig b/zml/kernels/moe/api.zig similarity index 75% rename from zml/kernels/moe/options.zig rename to zml/kernels/moe/api.zig index 7f8d26de..b66593c1 100644 --- a/zml/kernels/moe/options.zig +++ b/zml/kernels/moe/api.zig @@ -130,3 +130,35 @@ pub const ExpertsLayout = struct { mxfp4_interleaved8, }; }; + +/// Routing for a local token batch and a contiguous range of experts. +/// IDs retain the model's global expert numbering, while the projection weights +/// contain only the local experts. Kernels compute those experts contributions and +/// `forwardMoe` sums them along the expert axis. +pub const LocalRouting = struct { + /// Global expert IDs for local tokens, with shape {.t, .topk} + ids: Tensor, + weights: Tensor, + global_num_experts: u32, + local_num_experts: u32, + /// Scalar i32 containing the first local expert's global ID. + /// Null when all experts are local and global IDs already index the weights. + expert_start: ?Tensor = null, + + /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. + /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. + pub fn localIds(self: LocalRouting) Tensor { + const start = self.expert_start orelse return self.ids; + const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(self.ids.sub(start), .scalar(-1, .i32)); + } + + /// Returns a lookup table from every global expert ID to its local weight + /// index or -1 if nonlocal. Returns null when every expert is local. + pub fn expertMap(self: LocalRouting) ?Tensor { + const start = self.expert_start orelse return null; + const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(ids.sub(start), .scalar(-1, .i32)); + } +}; diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index bd482093..8bc03c9c 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,8 +3,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const Options = @import("../options.zig").Options; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Options = @import("../api.zig").Options; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -41,7 +42,7 @@ pub fn validateOptions(opts: Options) void { /// interleave by 8 and its scale atoms are MN-major packed i32 words. pub fn fusedExperts( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index f651813f..ff6edec2 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -4,16 +4,17 @@ //! rows in route order, and the top-k reduction. MXFP4 applies routing weights //! before the down projection, with the reference model's BF16 rounding. const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); +const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; +const triton_kernels = @import("../../triton/moe.zig"); pub const mxfp4 = @import("mxfp4.zig"); pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); -const RoutingWeightPlacement = @import("../../options.zig").RoutingWeightPlacement; -const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { /// `[tokens, hidden]` BF16. diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig index d1a0e1b5..92317c7c 100644 --- a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -5,10 +5,12 @@ const platforms = @import("platforms"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const GlobalOptions = @import("../options.zig").Options; -const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const LocalRouting = @import("../api.zig").LocalRouting; const unpackedWeight = @import("../weights.zig").unpackedWeight; const contract = @import("contract.zig"); +const Attributes = contract.Attributes; const runtime = @import("runtime.zig"); const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); @@ -50,8 +52,6 @@ const Inputs = contract.Inputs(zml.Tensor); const Outputs = contract.Outputs(zml.Shape); -const Attributes = contract.Attributes; - fn currentRunners() !*runtime.Runners { const platform = zml.Compiler.current().platform; const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; @@ -311,7 +311,7 @@ pub fn fusedExpertsNvfp4( pub fn fusedExperts( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index d041a5ab..ba3d3c00 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,11 +6,12 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const ExpertsLayout = @import("../options.zig").ExpertsLayout; -const ProjectionLayout = @import("../options.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; -const Activation = @import("../options.zig").Activation; -const Options = @import("../options.zig").Options; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const fly = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -21,7 +22,7 @@ test { pub fn fusedExperts( hidden_states: Tensor, - routing_: zml.moe.LocalRouting, + routing_: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options, @@ -135,7 +136,7 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { +pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index 988a4f51..f835c889 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -4,8 +4,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../options.zig").Activation; -const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); @@ -68,7 +69,7 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso pub fn fusedExperts( hidden_states: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 2a0aa8d7..eb697f36 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,17 +4,18 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; +pub const Activation = @import("api.zig").Activation; +pub const ActivationKind = @import("api.zig").ActivationKind; pub const cute = @import("cute/cute.zig"); -pub const ExpertsLayout = @import("options.zig").ExpertsLayout; +pub const ExpertsLayout = @import("api.zig").ExpertsLayout; pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); pub const fly = @import("fly/fly.zig"); +pub const LocalRouting = @import("api.zig").LocalRouting; pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const Options = @import("options.zig").Options; -pub const Activation = @import("options.zig").Activation; -pub const ActivationKind = @import("options.zig").ActivationKind; -pub const ProjectionLayout = @import("options.zig").ProjectionLayout; -pub const RoutingWeightPlacement = @import("options.zig").RoutingWeightPlacement; +pub const Options = @import("api.zig").Options; +pub const ProjectionLayout = @import("api.zig").ProjectionLayout; +pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); @@ -138,38 +139,6 @@ pub const Backend = enum { } }; -/// Routing for a local token batch and a contiguous range of experts. -/// IDs retain the model's global expert numbering, while the projection weights -/// contain only the local experts. Kernels compute those experts contributions and -/// `forwardMoe` sums them along the expert axis. -pub const LocalRouting = struct { - /// Global expert IDs for local tokens, with shape {.t, .topk} - ids: zml.Tensor, - weights: zml.Tensor, - global_num_experts: u32, - local_num_experts: u32, - /// Scalar i32 containing the first local expert's global ID. - /// Null when all experts are local and global IDs already index the weights. - expert_start: ?zml.Tensor = null, - - /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. - /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. - pub fn localIds(self: LocalRouting) zml.Tensor { - const start = self.expert_start orelse return self.ids; - const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(self.ids.sub(start), .scalar(-1, .i32)); - } - - /// Returns a lookup table from every global expert ID to its local weight - /// index or -1 if nonlocal. Returns null when every expert is local. - pub fn expertMap(self: LocalRouting) ?zml.Tensor { - const start = self.expert_start orelse return null; - const ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(ids.sub(start), .scalar(-1, .i32)); - } -}; - /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); @@ -297,7 +266,7 @@ pub fn forwardMoe( .global_experts = @intCast(gate_up.weight.dim(.expert)), }; const Logical = zml.Sharding.Partitioning.Logical; - var projection_specs: [2]zml.meta.MapRestrict(zml.Tensor, Logical).map(zml.nn.Linear) = undefined; + var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { zml.meta.mapAlloc(struct { fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { @@ -316,7 +285,7 @@ pub fn forwardMoe( .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, }, linear, specs) catch @panic("OOM"); } - return zml.ops.manualComputation2(Call.forward, args, output_shape, mesh, .{ + return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ .in_specs = .{ .input = partitioning.in_specs.input, .ids = partitioning.in_specs.ids, diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index 4b33b585..bb60594b 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,8 +5,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../options.zig").Activation; -const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -214,7 +215,7 @@ pub fn callGmmEp( pub fn fusedExperts( hidden_states: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index 7f84835b..ba723234 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -7,9 +7,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const GlobalOptions = @import("../options.zig").Options; -const Activation = @import("../options.zig").Activation; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; pub fn activate(x: Tensor, activation: Activation) Tensor { return switch (activation) { @@ -191,7 +192,7 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { +pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel index 4057eea3..44b56e08 100644 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -2,5 +2,8 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "tcl", - srcs = ["tcl.zig"], + srcs = [ + "moe.zig", + "tcl.zig", + ], ) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig new file mode 100644 index 00000000..23fb2274 --- /dev/null +++ b/zml/kernels/moe/tcl/moe.zig @@ -0,0 +1,457 @@ +//! TCL kernel schedules for Furiosa RNGD fused experts. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); + +const zml = @import("../../../zml.zig"); +const tcl = zml.kernel.tcl; +const ActivationKind = @import("../api.zig").ActivationKind; + +pub const Format = enum { bf16, fp8_block128 }; + +/// Dispatch in `tcl.zig` uses the first matching schedule. +/// +/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected +/// expert on the whole batch to reuse its weights, with zero routing weight +/// for unrouted pairs. +/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather +/// weights for each route to avoid the overhead of grouping routes. +/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to +/// reuse weights within each block, then gather results back by route. +pub const Schedule = enum { per_route, blockwise, small_batch }; + +pub const Cfg = struct { + num_tokens: i64, + top_k: i64, + local_num_experts: i64, + hidden_size: i64, + expert_intermediate_size: i64, + weight_scale_dtype: tcl.DType = .bf16, + activation: ActivationKind = .swiglu, +}; + +const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; +const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; + +pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { + if (schedule == .small_batch and format != .fp8_block128) { + @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); + } + return tcl.Kernel(Cfg, .{ + .name = "moe_" ++ @tagName(schedule) ++ switch (format) { + .bf16 => "", + .fp8_block128 => "_fp8", + }, + .inputs = switch (format) { + .bf16 => base_inputs, + .fp8_block128 => base_inputs ++ scale_inputs, + }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, schedule); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, format); + switch (schedule) { + .per_route => try perRoute(b, a, in, cfg.activation, format), + .blockwise => try blockwise(b, a, in, cfg, format), + .small_batch => try fp8SmallBatch(b, a, in, cfg), + } + } + }.run, + }); +} + +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.num_tokens), + .K = b.axis("K", cfg.top_k), + .E = b.axis("E", cfg.local_num_experts), + .D = b.axis("D", cfg.hidden_size), + .I = b.axis("I", cfg.expert_intermediate_size), + .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } +}; + +fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { + const weight_dtype: tcl.DType = switch (format) { + .bf16 => .bf16, + .fp8_block128 => .f8_e4, + }; + const spec = .{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, + }; + if (format == .bf16) { + const t = try b.declareArgs(spec); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + } + const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); + const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); + const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); + const t = try b.declareArgs(.{ + .x = spec.x, + .ids = spec.ids, + .w = spec.w, + .gate_up = spec.gate_up, + .down = spec.down, + .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, + .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; +} + +const Args = struct { + x: builder.Tensor, + ids: builder.Tensor, + w: builder.Tensor, + gate_up: builder.Tensor, + down: builder.Tensor, + gate_up_scale: ?builder.Tensor = null, + down_scale: ?builder.Tensor = null, +}; + +const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; + +fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { + if (format == .bf16) return .{}; + const IG = b.axis("IG", @divExact(a.I.size, 128)); + const DG = b.axis("DG", @divExact(a.D.size, 128)); + const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); + var gu_axes: [8]tcl.Axis = undefined; + var dn_axes: [8]tcl.Axis = undefined; + @memcpy(gu_axes[0..batch.len], batch); + @memcpy(dn_axes[0..batch.len], batch); + @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); + @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); + return .{ + .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), + .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), + }; +} + +fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { + var n: usize = 0; + for (axes) |a| { + if (std.mem.eql(u8, a.name, axis.name)) { + storage[n] = group; + n += 1; + storage[n] = lane; + } else storage[n] = a; + n += 1; + } + return storage[0..n]; +} + +/// Block scales multiply each K-block partial before K-block reduction. +/// Only the requested experts are gathered; their bank remains FP8 in HBM. +fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { + if (format == .bf16) { + const op = b.tensorOperation(.{}); + return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); + } + const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); + const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); + const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); + const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); + var x_axes: [16]tcl.Axis = undefined; + var w_axes: [16]tcl.Axis = undefined; + var w2_axes: [16]tcl.Axis = undefined; + var out_axes: [16]tcl.Axis = undefined; + var partial_axes: [16]tcl.Axis = undefined; + const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); + const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); + const out = blockedAxes(to, output, OG, OL, &out_axes); + @memcpy(partial_axes[0..out.len], out); + partial_axes[out.len] = CG; + + // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. + var op = b.tensorOperation(.{}); + const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); + op = b.tensorOperation(.{}); + // Native F32 reciprocal accuracy is qualified separately (within one ULP). + const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); + const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); + op = b.tensorOperation(.{}); + const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); + return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); +} + +/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` +/// (gate first), times `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // The builder can infer the TCL `Interleaving` tactic for two inputs with + // matching axes, such as `gate` and `up`. That tactic allows exactly one + // binary instruction, so force `.elementwise` for this activation chain. + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + const activated = if (format == .fp8_block128) blk: { + if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); + + // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal + // BF16 SiLU write, even through as_dram/as_logical. Make its RNE + // rounding observable as integer bits before multiplying by up. + // Remove once the native reproducer preserves the BF16 boundary. + const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const v = op.fetch(bits, .{}); + const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); + const silu = try op.commit(rounded, .{ .dtype = .bf16 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + break :blk op.fetch(silu, .{}); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), + }; + var v = activated.mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); +} + +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// Use IO-bound tactics for per-route and FP8 small-batch schedules and +/// compute-bound tactics for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { + const config = comptime blk: { + const common = .{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "None", + }; + const tactics = switch (schedule) { + .blockwise => .{ + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + }, + .per_route, .small_batch => .{ + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }, + }; + // The transaction-simulation setting is only emitted for IO-bound schedules. + const common_info = @typeInfo(@TypeOf(common)).@"struct"; + const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; + const Config = @Struct( + .auto, + null, + common_info.field_names ++ tactics_info.field_names, + common_info.field_types ++ tactics_info.field_types, + common_info.field_attrs ++ tactics_info.field_attrs, + ); + var merged: Config = undefined; + for (common_info.field_names) |field| @field(merged, field) = @field(common, field); + for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); + break :blk merged; + }; + b.compilerConfig(config); +} + +/// Gather the selected experts for every token. +fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); + const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); + if (format == .fp8_block128) { + const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); + } else { + // The BF16 recipe weights activations and contracts experts together. + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } +} + +/// Rows per expert block: the next power of two of the routes per expert. +fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); +} + +const blocks_per_chunk = 32; + +/// Group routes into expert-major blocks, then run both projections in a +/// device loop over chunks of blocks. +fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); + const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); + const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); +} + +/// Number of experts processed per device-loop iteration, based on the +/// batch's route count (`cfg.num_tokens * cfg.top_k`). +fn expertsPerChunk(cfg: Cfg) i64 { + const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); + return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); +} + +/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. +/// Load each selected expert once and run it for every token. Routing weights +/// scale the F32 down-projection results before summing the experts and steps. +fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { + const chunk = expertsPerChunk(cfg); + const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; + const C = b.axis("C", chunk); + const Go = b.axis("Go", steps); + const J = b.axis("J", steps * chunk); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Each token's routing weight per expert, and the used experts. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); + op = b.tensorOperation(.{}); + const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); + // A used expert's rank among them, -1 for the others. + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); + + // Slot j holds the used expert of rank j and its tokens' weights; + // slots past the last hold expert 0 with no weight. + const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); + op = b.tensorOperation(.{}); + const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); + // A reduction's condition is read plainly, so broadcast it first. + op = b.tensorOperation(.{}); + const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); + op = b.tensorOperation(.{}); + const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); + const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); + const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); + { + const ids = try b.indexRead(step_experts, loop.iv); + const weights = try b.indexRead(step_gate, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); + const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); + const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); + const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); + op = b.tensorOperation(.{}); + const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); +} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index e7d7a01b..f05727ce 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -1,55 +1,88 @@ //! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. -//! -//! Few tokens (at most `experts / top_k`): activation stationary, each token -//! gathers the weights of its experts and the down projection contracts over -//! the experts and the intermediate axis at once. -//! -//! More tokens: blockwise. The kernel routes on the device: routes are placed -//! in expert-major blocks of `BB` rows (one-hot, cumulative sums), then a -//! device loop over the used chunks of `BGi` blocks gathers each block's -//! expert weights and runs both projections; outputs are gathered back per -//! route and weighted. -//! -//! FP8 experts split across devices (expert parallel): each device runs its own -//! experts for the routes that reach them and an all-reduce sums the devices' -//! outputs. Up to `expert_stationary_tokens` tokens, expert stationary: a -//! device loop over the used experts, a few at a time, reads each once and -//! runs it for every token, scaled by the token's routing weight (zero for -//! the tokens it does not serve); more tokens take blockwise. const std = @import("std"); -const builder = @import("dsl/tcl/builder"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const Options = @import("../options.zig").Options; +const kernels = @import("moe.zig"); +const api = @import("../api.zig"); -/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` -/// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. -/// SwiGLU, routing weights applied after the down projection. -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { +/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` +/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is +/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with +/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. +pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { const topk_ids = routing.localIds(); - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); + stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); + const format = expertFormat(gate_up_linear, down_linear); + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + const gate_up = gate_up_linear.weight; const down = down_linear.weight; - const quantized = gate_up_linear.quantization != null or down_linear.quantization != null; - const scales: ?Scales = if (quantized) .{ .gate_up = fp8Scales(gate_up_linear), .down = fp8Scales(down_linear) } else null; - stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32 and (quantized or (gate_up.dtype() == .bf16 and down.dtype() == .bf16)), "TCL MoE takes BF16 activations, BF16/block-128 FP8 experts and i32 ids", .{}); + const cfg: kernels.Cfg = .{ + .num_tokens = input.dim(.t), + .top_k = topk_ids.dim(.topk), + .local_num_experts = gate_up.dim(.expert), + .hidden_size = input.dim(.d), + .expert_intermediate_size = down.dim(.dout), + .weight_scale_dtype = switch (format) { + .bf16 => .bf16, + .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), + }, + .activation = std.meta.activeTag(opts.activation), + }; + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); + const weights = routing.weights.convert(.f32); - const local_weights = if (scales != null) topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)) else weights; - return runExperts(input, topk_ids, local_weights, gate_up, down, scales); + const local_weights = switch (format) { + .bf16 => weights, + .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), + }; + return switch (format) { + .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), + .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ + .x = input, + .ids = topk_ids, + .w = local_weights, + .gate_up = gate_up, + .down = down, + .gate_up_scale = s.gate_up, + .down_scale = s.down, + }), + }; } const Scales = struct { gate_up: Tensor, down: Tensor }; +const Format = union(kernels.Format) { + bf16: void, + fp8_block128: Scales, +}; + +fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { + const scheme = gate_up.quantizationScheme(); + stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); + return switch (scheme orelse { + stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); + return .bf16; + }) { + .fp8_block128 => blk: { + const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; + stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); + break :blk .{ .fp8_block128 = scales }; + }, + else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), + }; +} + fn fp8Scales(linear: zml.nn.Linear) Tensor { - const q = linear.quantization orelse std.debug.panic("TCL MoE requires both FP8 projections", .{}); - stdx.debug.assert(q.scheme == .fp8_block128 and linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); + const q = linear.quantization.?; + stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); const w = linear.weight; const scale = q.scales; @@ -57,471 +90,76 @@ fn fp8Scales(linear: zml.nn.Linear) Tensor { return scale; } -/// Tokens up to which a device runs its experts stationary. -pub const expert_stationary_tokens = 64; +// Maximum total input tokens per MoE call before padding for the FP8 +// small-batch schedule. It loads each used expert's weights once and reuses +// them across the batch. +// Larger batches fall back to per-route or blockwise dispatch. +const fp8_small_batch_max_tokens = 64; -fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales) Tensor { - var cfg: Cfg = .{ - .t = input.dim(.t), - .k = topk_ids.dim(.topk), - .e = gate_up.dim(.expert), - .d = input.dim(.d), - .i = down.dim(.dout), - }; - stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); - if (scales) |s| { - cfg.scale_dtype = scaleDtype(s); - if (cfg.t <= expert_stationary_tokens) { - // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two - // panics its adder tree ("Fail to cover valid range", 5 tokens padded - // to 8): pad with unrouted tokens. - var padded = cfg; - padded.t = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.t)) catch unreachable); - const pad = padded.t - cfg.t; - const x = input.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.d }); - const ids = topk_ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); - const weights = topk_weights.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); - const y = ExpertStationaryFp8.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }, .{ .y = x.shape() }, .{ .cfg = padded }).y; - return y.slice(0, .{ .end = cfg.t }).reshape(input.shape()); - } +fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { + return run(format, .small_batch, cfg, inputs); } + if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { + return run(format, .per_route, cfg, inputs); + } + return run(format, .blockwise, cfg, inputs); +} - const x = input.reshape(.{ cfg.t, cfg.d }); - const ids = topk_ids.reshape(.{ cfg.t, cfg.k }); - const weights = topk_weights.reshape(.{ cfg.t, cfg.k }); - const out_shape = x.shape(); - if (scales) |s| { - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); - const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = valid.select(ids, Tensor.scalar(0, .i32)), .w = valid.select(weights, Tensor.scalar(0, .f32)), .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; - const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationaryFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y - else - BlockwiseFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y; - return y.reshape(input.shape()); - } - const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y +fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + const call_cfg, const prepared = if (schedule == .small_batch) + prepareFp8SmallBatch(cfg, inputs) else - Blockwise.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; - return y.reshape(input.shape()); + .{ cfg, prepareRoutedInputs(cfg, inputs) }; + + const shape = prepared.x.shape(); + var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; + + if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); + return y.reshape(inputs.x.shape()); } -fn scaleDtype(scales: Scales) tcl.DType { - stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); - return if (scales.gate_up.dtype() == .bf16) .bf16 else .f32; +fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { + var prepared = inputs; + prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); + const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); + const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); + // These kernels gather every route, and blockwise also scatters each route + // into an expert slot. Give nonlocal routes a valid slot and zero weight. + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); + prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); + prepared.w = valid.select(w, Tensor.scalar(0, .f32)); + return prepared; } -pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64, scale_dtype: tcl.DType = .bf16 }; - -const Axes = struct { - T: tcl.Axis, - K: tcl.Axis, - E: tcl.Axis, - D: tcl.Axis, - I: tcl.Axis, - I2: tcl.Axis, - S2: tcl.Axis, - S1: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.t), - .K = b.axis("K", cfg.k), - .E = b.axis("E", cfg.e), - .D = b.axis("D", cfg.d), - .I = b.axis("I", cfg.i), - .I2 = b.axis("I2", 2 * cfg.i), - .S2 = b.axis("S2", 2), - .S1 = b.axis("S1", 1), - }; - } -}; - -fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime fp8: bool) tcl.FinishError!Args { - const spec = .{ - .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, - .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, - .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.D, a.I } }, - }; - if (!fp8) { - const t = try b.declareArgs(spec); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; - } - const IG = b.axis("IG", @divExact(cfg.i, 128)); - const DG = b.axis("DG", @divExact(cfg.d, 128)); - const I2G = b.axis("I2G", 2 * @divExact(cfg.i, 128)); - const t = try b.declareArgs(.{ - .x = spec.x, - .ids = spec.ids, - .w = spec.w, - .gate_up = spec.gate_up, - .down = spec.down, - .gate_up_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, I2G, DG } }, - .down_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, DG, IG } }, - }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; +fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { + // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two + // panics its adder tree ("Fail to cover valid range", 5 tokens padded + // to 8): pad with unrouted tokens. + var padded = cfg; + padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); + const pad = padded.num_tokens - cfg.num_tokens; + var prepared = inputs; + prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); + prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + return .{ padded, prepared }; } -const Args = struct { - x: builder.Tensor, - ids: builder.Tensor, - w: builder.Tensor, - gate_up: builder.Tensor, - down: builder.Tensor, - gate_up_scale: ?builder.Tensor = null, - down_scale: ?builder.Tensor = null, -}; - -const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; - -fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis) tcl.FinishError!GatheredScales { - const gu = in.gate_up_scale orelse return .{}; - const IG = b.axis("IG", @divExact(a.I.size, 128)); - const DG = b.axis("DG", @divExact(a.D.size, 128)); - const gate_up = try b.reshape(gu, &.{ a.E, a.S2, IG, DG }, .{}); - var gu_axes: [8]tcl.Axis = undefined; - var dn_axes: [8]tcl.Axis = undefined; - @memcpy(gu_axes[0..batch.len], batch); - @memcpy(dn_axes[0..batch.len], batch); - @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); - @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); - return .{ - .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), - .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), - }; -} - -fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { - var n: usize = 0; - for (axes) |a| { - if (std.mem.eql(u8, a.name, axis.name)) { - storage[n] = group; - n += 1; - storage[n] = lane; - } else storage[n] = a; - n += 1; - } - return storage[0..n]; -} - -/// Block scales multiply each K-block partial before K-block reduction. -/// Only the requested experts are gathered; their bank remains FP8 in HBM. -fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType) tcl.FinishError!builder.Tensor { - if (scale == null) { - const op = b.tensorOperation(.{}); - return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); - } - const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); - const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); - const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); - const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); - var x_axes: [16]tcl.Axis = undefined; - var w_axes: [16]tcl.Axis = undefined; - var w2_axes: [16]tcl.Axis = undefined; - var out_axes: [16]tcl.Axis = undefined; - var partial_axes: [16]tcl.Axis = undefined; - const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); - const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); - const out = blockedAxes(to, output, OG, OL, &out_axes); - @memcpy(partial_axes[0..out.len], out); - partial_axes[out.len] = CG; - - // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. - var op = b.tensorOperation(.{}); - const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); - op = b.tensorOperation(.{}); - // Native F32 reciprocal accuracy is qualified separately (within one ULP). - const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); - const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); - op = b.tensorOperation(.{}); - const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); - return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); -} - -/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` -/// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, comptime fp8: bool) tcl.FinishError!builder.Tensor { - var sliced: [4]tcl.Axis = undefined; - var flat: [3]tcl.Axis = undefined; - @memcpy(sliced[0..batch.len], batch); - @memcpy(flat[0..batch.len], batch); - sliced[batch.len] = a.S1; - sliced[batch.len + 1] = a.I; - flat[batch.len] = a.I; - const s = sliced[0 .. batch.len + 2]; - const f = flat[0 .. batch.len + 1]; - const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); - const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // Interleaving requires one binary instruction; SwiGLU chains several. - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const g = op.fetch(gate, .{}); - const activated = if (fp8) blk: { - // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal - // BF16 SiLU write, even through as_dram/as_logical. Make its RNE - // rounding observable as integer bits before multiplying by up. - // Remove once the native reproducer preserves the BF16 boundary. - const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const v = op.fetch(bits, .{}); - const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); - const silu = try op.commit(rounded, .{ .dtype = .bf16 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - break :blk op.fetch(silu, .{}); - } else g.sigmoid().mulf(g); - var v = activated.mulf(op.fetch(up, .{})); - if (weights) |w| v = v.mulf(op.fetch(w, .{})); - return op.commit(v, .{ .dtype = .bf16 }); -} - -/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): -/// IO-bound tactics for activation stationary, compute-bound for blockwise. -/// Keep the compiler's default lowering mode and omit the adaptive-einsum and -/// half-VRF settings removed in TCC 2026.5. -fn configure(b: *tcl.Builder, blockwise: bool) void { - if (blockwise) b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, - .dma_preference = 1.2, - .propagate_sparse_axis_from_op = "None", - }) else b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - .propagate_sparse_axis_from_op = "None", - }); -} - -pub const ActivationStationary = activationStationaryKernel(false); -pub const ActivationStationaryFp8 = activationStationaryKernel(true); - -fn activationStationaryKernel(comptime fp8: bool) type { - return tcl.Kernel(Cfg, .{ - .name = if (fp8) "moe_activation_stationary_fp8" else "moe_activation_stationary", - .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, fp8); - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - - const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }); - const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, fp8); - if (fp8) { - const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32); - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); - } else { - // The BF16 recipe weights activations and contracts experts together. - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } - } - }.run, - }); -} - -/// Rows per expert block: the next power of two of the routes per expert. -pub fn blockRows(cfg: Cfg) i64 { - const per_expert = std.math.divCeil(i64, cfg.t * cfg.k, cfg.e) catch unreachable; - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); -} - -const blocks_per_chunk = 32; - -pub const Blockwise = blockwiseKernel(false); -pub const BlockwiseFp8 = blockwiseKernel(true); - -fn blockwiseKernel(comptime fp8: bool) type { - return tcl.Kernel(Cfg, .{ - .name = if (fp8) "moe_blockwise_fp8" else "moe_blockwise", - .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, true); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, fp8); - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. - const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); - op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); - op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); - - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(used, .{try b.scratchpad(if (fp8) .f32 else .bf16, &.{ Go, BGi, BB, D })}); - { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{BGi}); - const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ BGi, BB }, null, fp8); - const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (fp8) .f32 else .bf16); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); - } - }.run, - }); -} - -/// Experts per `ExpertStationary` loop step: more as the routes, so the -/// experts they use, grow. -pub fn stationaryChunk(cfg: Cfg) i64 { - const routes: u64 = @intCast(std.math.divCeil(i64, cfg.t * cfg.k, 16) catch unreachable); - return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); -} - -/// The experts that `ids` use (-1 for none), `stationaryChunk` at a time in a -/// device loop: each is read once and runs for every token. Routing weights -/// scale the F32 down-projection results before summing the experts and steps. -pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ - .name = "moe_expert_stationary_fp8", - .inputs = &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, true); - const chunk = stationaryChunk(cfg); - const steps = std.math.divCeil(i64, @min(cfg.e, cfg.t * cfg.k), chunk) catch unreachable; - const C = b.axis("C", chunk); - const Go = b.axis("Go", steps); - const J = b.axis("J", steps * chunk); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Each token's routing weight per expert, and the used experts. - const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); - op = b.tensorOperation(.{}); - const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); - // A used expert's rank among them, -1 for the others. - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - - // Slot j holds the used expert of rank j and its tokens' weights; - // slots past the last hold expert 0 with no weight. - const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); - op = b.tensorOperation(.{}); - const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); - // A reduction's condition is read plainly, so broadcast it first. - op = b.tensorOperation(.{}); - const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); - op = b.tensorOperation(.{}); - const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); - const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); - const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); - { - const ids = try b.indexRead(step_experts, loop.iv); - const weights = try b.indexRead(step_gate, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{C}); - const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16); - const act = try gated(b, a, h, &.{ C, T }, null, true); - const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32); - op = b.tensorOperation(.{}); - const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); - } - }.run, -}); - test "tcl fused experts run on furiosa" { + try testFusedExperts(.{ .swiglu = .{} }); + try testFusedExperts(.geglu_tanh); +} + +fn testFusedExperts(activation: zml.moe.Activation) !void { const platform = zml.testing.env(); if (platform.target != .furiosa) return error.SkipZigTest; const allocator = std.testing.allocator; const io = std.testing.io; const bf16 = zml.floats.BFloat16; - // 2 * 4 <= 8 experts is activation stationary, 16 tokens is blockwise. + // With two routes per token and eight experts, four tokens use per-route + // dispatch and sixteen use blockwise dispatch. for ([_]i64{ 4, 16 }) |t| { const k = 2; const e = 8; @@ -533,15 +171,15 @@ test "tcl fused experts run on furiosa" { const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ - .activation = .{ .swiglu = .{} }, + .activation = act, .quantize_input = false, .routing_weight_placement = .after_down, }); } }; - var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, .{}); + var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); @@ -593,7 +231,12 @@ test "tcl fused experts run on furiosa" { g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); } - act[n] = g / (1 + @exp(-g)) * u; + const gate = switch (activation) { + .swiglu => g / (1 + @exp(-g)), + .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), + else => unreachable, + }; + act[n] = gate * u; } for (0..d) |m| { var y: f32 = 0; diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index f78cd1ef..c74406c3 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,10 +5,10 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const ExpertsLayout = @import("options.zig").ExpertsLayout; -const GlobalOptions = @import("options.zig").Options; -const Activation = @import("options.zig").Activation; -const Placement = @import("options.zig").RoutingWeightPlacement; +const Activation = @import("api.zig").Activation; +const ExpertsLayout = @import("api.zig").ExpertsLayout; +const GlobalOptions = @import("api.zig").Options; +const Placement = @import("api.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); const Case = struct { diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 0776a590..6e4006fa 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,7 +12,7 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index ed6c9f5f..8ba799a5 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -7,11 +7,12 @@ const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; const toDType = zml.kernel.triton.from; -const ExpertsLayout = @import("../options.zig").ExpertsLayout; -const ProjectionLayout = @import("../options.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; -const Activation = @import("../options.zig").Activation; -const Options = @import("../options.zig").Options; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const kernels = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/triton"); @@ -38,7 +39,7 @@ test { pub fn fusedExperts( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, @@ -58,7 +59,7 @@ pub fn fusedExperts( pub fn fusedExpertsBase( hidden_states: Tensor, - routing_: zml.moe.LocalRouting, + routing_: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, layout: ProjectionLayout, @@ -900,7 +901,7 @@ fn validateInputs( /// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. pub fn fusedExpertsMxfp4( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 3e05fed8..587af69f 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -125,7 +125,7 @@ fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation2(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 337432e7..04f4c1ac 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,9 +3,8 @@ const std = @import("std"); -const kernel = @import("../../../kernel.zig"); +const metal = @import("../../../kernel/kernel.zig").metal; -const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index aaa42b20..4a97a38e 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -207,7 +207,7 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z _ = k; _ = v; const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; - return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, partitioning); + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); } const Attention = struct { @@ -435,12 +435,9 @@ test pagedAttention { const parameters: Parameters = .init(.fromBackend(backend_options_args)); const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - const ParameterSpecs = zml.meta.MapRestrict(zml.Tensor, zml.Sharding.Partitioning.Logical).map(Parameters); const partitioning: Partitioning = .{ .in_specs = .{ - .parameters = switch (parameters) { - inline else => |_, tag| @unionInit(ParameterSpecs, @tagName(tag), .{ .block_table = .replicated, .seq_lens = .replicated, .query_start_len = .replicated }), - }, + .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), .q = head_spec, .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, .opts = .{ .mask = .{ .standard = {} }, .sink = null }, diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index 881593aa..d04b684e 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -67,22 +67,9 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); - if (compiler.meshes[0].numDevices() > 1) return zml.ops.manualComputation(Shard.body, .{ .parameters = parameters, .q = q, .k_cache = k_cache, .v_cache = v_cache, .opts = opts }, q.shape()); return attend(parameters, q, k_cache, v_cache, opts); } - const Shard = struct { - parameters: Parameters, - q: Tensor, - k_cache: Tensor, - v_cache: Tensor, - opts: AttentionOptions, - - fn body(self: Shard, _: zml.Shape) Tensor { - return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); - } - }; - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); diff --git a/zml/nn.zig b/zml/nn.zig index a2b582ee..eceb316b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,11 +2,13 @@ const std = @import("std"); const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; @@ -320,13 +322,23 @@ fn onesGrid(shape: Shape, dt: DataType) Shape { } fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { - return ops.customCall( - "zml$scaled_dot_unmatched", - .{ in[0], in[1], in[2], in[3] }, - out_shape, - {}, - .{ .has_side_effect = false }, - ); + // Keep the unmatched-composite marker in the graph for backend lowering. + const ctx = zml.Compiler.current(); + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, + &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, + .{ + .call_target_name = "zml$scaled_dot_unmatched", + .has_side_effect = false, + .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); + } + return Tensor._result(out_shape, op.result(0)); } fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { diff --git a/zml/ops.zig b/zml/ops.zig index a9ae0f8d..f8fdba7e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -18,9 +18,10 @@ const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const manual_computation = @import("ops/manual_computation.zig"); pub const manualComputation = manual_computation.manualComputation; -const manual_computation2 = @import("ops/manual_computation2.zig"); -pub const manualComputation2 = manual_computation2.manualComputation2; -pub const ManualPartitioning = manual_computation2.ManualPartitioning; +pub const ManualPartitioning = manual_computation.ManualPartitioning; +pub const PartitionSpecs = manual_computation.PartitionSpecs; +pub const partitionSpecs = manual_computation.partitionSpecs; +pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; @@ -30,7 +31,6 @@ const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; test { _ = manual_computation; - _ = manual_computation2; } const log = std.log.scoped(.@"zml/Compiler"); @@ -89,7 +89,7 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ /// ``` /// /// Each expert shard now has the complete output for its data shard. To expose -/// this result through `manualComputation2`, an output spec such as +/// this result through `manualComputation`, an output spec such as /// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the /// result replicated along `.experts`. pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { @@ -2760,11 +2760,6 @@ pub fn CustomCall( comptime params: struct { /// Name of the custom call, must be unique. name: [:0]const u8, - /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies - /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, - /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output - /// sharding, this setting will avoid the shuffling of the buffer's data. - sharding_aware: bool, /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. has_side_effect: bool, /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. @@ -2798,23 +2793,7 @@ pub fn CustomCall( .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), .compute_on_host = params.compute_on_host, }; - if (params.sharding_aware) { - return shardingAwareTypedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } else { - return typedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } + return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); } fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { @@ -2842,44 +2821,6 @@ pub fn CustomCallOutputOperandAliases(I: type, O: type) type { ); } -pub fn shardingAwareTypedCustomCall( - comptime target_name: [:0]const u8, - comptime opts: CustomCallOptions, - input: anytype, - output: anytype, - attributes: anytype, -) ShapeToTensor(@TypeOf(output)) { - const Input = @TypeOf(input); - const Output = @TypeOf(output); - const Attributes = @TypeOf(attributes); - - var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - output_shapes[i] = @field(output, field_name); - } - - const Handler = struct { - input: Input, - attributes: Attributes, - - fn body(self: @This(), sharded_output_shapes: []const Shape) []const Tensor { - return typedCustomCall(target_name, opts, self.input, sharded_output_shapes, self.attributes); - } - }; - const output_tensors = manualComputation( - Handler.body, - .{ .input = input, .attributes = attributes }, - @as([]const Shape, &output_shapes), - ); - - // Convert the slice back to a struct - var out: ShapeToTensor(Output) = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - @field(out, field_name) = output_tensors[i]; - } - return out; -} - pub fn typedCustomCall( target_name: [:0]const u8, opts: CustomCallOptions, @@ -2894,7 +2835,7 @@ pub fn typedCustomCall( const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); - stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); + stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { @@ -2904,7 +2845,6 @@ pub fn typedCustomCall( } break :b &input_tensors; }, - // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -2920,7 +2860,6 @@ pub fn typedCustomCall( } break :b &output_shapes; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -2985,7 +2924,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + if (ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -2997,7 +2936,6 @@ pub fn typedCustomCall( } return out; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -3050,7 +2988,7 @@ test customCall { const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3061,9 +2999,17 @@ test customCall { const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); - const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + const Local = struct { + fn body(x: Tensor, output_shape: Shape) Tensor { + return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + }); + } + }; + const output = manualComputation(Local.body, input, shape, tp, .{ + .in_specs = .parse(.{ ._0 = .model }), + .out_specs = .parse(.{ ._0 = .model }), }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index 45400206..a0ec0037 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); @@ -7,44 +6,126 @@ const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("../Compiler.zig"); -const DataType = @import("../dtype.zig").DataType; +const meta = @import("../meta.zig"); const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; -const zml = struct { - // Nested to disambiguate from stdx.meta - const meta = @import("../meta.zig"); - const testing = @import("../testing.zig"); -}; - const log = std.log.scoped(.@"zml/Compiler"); -/// ZML operator describes computation across several devices, -/// but `manualComputation` allows to partition the devices in small group, -/// split the `Tensor` on those partitions, -/// and describes sub-computation that work inside a partition. +/// Partitioning declarations for the tensors in `T`. +/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. +/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes +/// `struct { x: Logical, bias: ?Logical }`. /// -/// See: -/// https://github.com/openxla/shardy/blob/main/docs/sdy_dialect.md#sdymanual_computation-sdymanualcomputationop +/// Initialize individual fields to declare different shardings, or use +/// `partitionSpecs` to apply one spec to every tensor in a value. +pub fn PartitionSpecs(comptime T: type) type { + return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); +} + +/// Declares the same sharding for a group of tensors at a `manualComputation` +/// boundary. /// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R` where: -/// - `I` is user defined and contains inputs tensors and any metadata required by the computation -/// - `O` contains the partionned output shapes, types must be `void`, `Shape` or `[]const Shape` for 0, 1 or many outputs -/// - `R` must be `void`, `Tensor, or `[]const Tensor` for 0, 1 or many outputs -/// * `inputs` must be using the same mesh (using .withPartitioning), or be fully replicated -/// * `outputs` contains the global output shapes, -/// and `manualComputation` will pass to `body_fn` the shapes of each local output shard. +/// ```zig +/// .in_specs = .{ +/// .query = .parse(.{ .h = .model }), +/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), +/// .metadata = partitionSpecs(metadata, .replicated), +/// } +/// ``` /// -/// This depart from the rest of ZML where output shapes are generally inferred, -/// but in this case since `body_fn` explains how to compute parts of the result, -/// it's not clear how to put them back together, -/// hence why the user is asked to explicitly provided the expected output shape. +/// Requires an active compiler and uses its arena for allocations. Use +/// `partitionSpecsAlloc` outside graph construction. Use named struct fields +/// when mixing tensors with non-tensor metadata. +pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { + return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); +} + +/// Builds the same declarations as `partitionSpecs` using an explicit allocator. +pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { + var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; + try meta.mapAlloc(struct { + fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { + return spec_; + } + }.assign, allocator, spec, tensors, &specs); + return specs; +} + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = PartitionSpecs(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), ) stdx.meta.FnReturn(body_fn) { const Inputs = stdx.meta.FnParam(body_fn, 0); const OutputShapes = stdx.meta.FnParam(body_fn, 1); @@ -55,13 +136,42 @@ pub fn manualComputation( defer arena_state.deinit(); const arena = arena_state.allocator(); - var owned_inputs: Inputs = inputs; - const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - defer arena.free(flat_input_tensors); + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); - var owned_output_shapes: OutputShapes = output_shapes; - const flat_output_shapes: []*Shape = zml.meta.collectMutPtrs(Shape, arena, &owned_output_shapes) catch @panic("OOM"); - defer arena.free(flat_output_shapes); + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } const WrappedInputs = struct { inputs: *const Inputs, @@ -73,15 +183,17 @@ pub fn manualComputation( outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); } }; var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &owned_output_shapes }; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + _ = manualComputationInternal( compiler, arena, + mesh, @typeName(@TypeOf(body_fn)), &wrapped_inputs, flat_input_tensors, @@ -91,6 +203,7 @@ pub fn manualComputation( ) catch |err| switch (err) { error.OutOfMemory => @panic("OOM"), }; + return outputs_struct; } @@ -101,27 +214,21 @@ const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyo noinline fn manualComputationInternal( compiler: *Compiler, arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, functor_name: []const u8, input_struct: *anyopaque, input_tensors: []*Tensor, output_struct: *anyopaque, - output_shapes: []*Shape, + global_output_shapes: []const Shape, local_fn: LocalComputationFn, ) error{OutOfMemory}![]*Tensor { const global_scope = compiler.currentScope(); const mlir_ctx = compiler.mlir_ctx; - const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { - .mesh => |m| m, - .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {d}: {f} and input {d}: {f}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), - }; - const manual_axes = collectManualAxes(mesh, input_tensors); - log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); + log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); const global_input_shapes = try arena.alloc(Shape, input_tensors.len); for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - const global_output_shapes = try arena.alloc(Shape, output_shapes.len); - for (global_output_shapes, output_shapes) |*shape, shape_ptr| shape.* = shape_ptr.*; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -135,11 +242,17 @@ noinline fn manualComputationInternal( const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } input_val.* = global_scope.value(tensor_ptr.*); } - const manual_block = mlir.Block.init(input_types, input_locs); - errdefer manual_block.deinit(); + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); const local_scope = compiler.pushBlock(manual_block); defer local_scope.pop(); @@ -151,17 +264,14 @@ noinline fn manualComputationInternal( }; } - for (output_shapes, global_output_shapes) |shape_ptr, global| { - shape_ptr.* = Sharding.shardedShape(global) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - compiler.manual_computation_depth += 1; defer compiler.manual_computation_depth -= 1; const local_outputs = try local_fn(arena, input_struct, output_struct); stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, output_shapes) |i, local_tensor, local_shape| { - stdx.debug.assert(local_shape.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, local_shape.*, local_tensor._shape }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); } const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); @@ -218,95 +328,11 @@ noinline fn manualComputationInternal( // Returned tensors describe the operation's global output shapes and sharding. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape = output_shape; + output_tensor_ptr._shape._sharding = output_shape._sharding; } return local_outputs; } -const ChosenMesh = union(enum) { - mesh: *const Sharding.Mesh, - incompatible: [2]u32, -}; - -fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ChosenMesh { - var mesh: ?*const Sharding.Mesh = null; - var mesh_source_id: u32 = @intCast(inputs.len); - for (0.., inputs) |i, input| { - if (input.shape()._sharding.mesh) |shape_mesh| { - if (mesh) |m| { - if (m != shape_mesh) - return .{ .incompatible = .{ mesh_source_id, @intCast(i) } }; - } else { - mesh = shape_mesh; - mesh_source_id = @intCast(i); - } - } - } - return .{ .mesh = mesh orelse fallback }; -} - -test chooseCommonMesh { - const fallback: Sharding.Mesh = .initTest(.fallback, .{.all}); - const first: Sharding.Mesh = .initTest(.first, .{ .data, .model }); - const second: Sharding.Mesh = .initTest(.second, .{ .experts, .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); - const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); - - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); -} - -fn collectManualAxes(mesh: *const Sharding.Mesh, inputs: []const *const Tensor) stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) { - var used: [Sharding.MAX_MESH_RANK]bool = @splat(false); - for (inputs) |input| { - const sharding = input.shape()._sharding; - if (sharding.mesh == null) continue; - std.debug.assert(sharding.mesh.? == mesh); - for (0..input.rank()) |dim| { - if (sharding.partition.get(dim).meshAxis()) |axis| used[axis] = true; - } - } - - var axes: stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) = .empty; - for (mesh.logical.axes.constSlice(), 0..) |axis, i| { - if (used[i]) axes.appendAssumeCapacity(axis); - } - return axes; -} - -test collectManualAxes { - const mesh: Sharding.Mesh = .initTest(.manual, .{ .data, .model, .unused }); - var data = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .data }); - var model = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const open = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .open }); - - const axes = collectManualAxes(&mesh, &.{ &model, &replicated, &data, &model, &open }); - data = replicated; - model = replicated; - try std.testing.expectEqual(@as(usize, 2), axes.len); - try std.testing.expectEqualStrings("data", std.mem.span(axes.get(0))); - try std.testing.expectEqualStrings("model", std.mem.span(axes.get(1))); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{ &replicated, &open }).len); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{}).len); - - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const expected = shardy.ManualAxesAttr.get(ctx, &.{ - .init(ctx, "data"), - .init(ctx, "model"), - }).attribute(); - try std.testing.expect(shardy_.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); -} - const gspmd = struct { fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); @@ -349,54 +375,172 @@ const gspmd = struct { } }; -const shardy_ = struct { - pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { - var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; - for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); - return shardy.ManualAxesAttr.get(mlir_ctx, attrs.constSlice()).attribute(); - } -}; - -test "manualComputation passes local output shapes to the callback" { +test manualComputation { + const zml = @import("../zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const mesh = zml.testing.tpMesh(); - const partitions = mesh.numPartitionsForLogicalAxis(.model); - if (partitions == 1) return error.SkipZigTest; - var compiler: Compiler = .init(allocator, io, platform, .{ .meshes = &.{mesh} }); - defer compiler.deinit(); - compiler.activate(); - defer compiler.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - defer block.deinit(); - const scope = compiler.pushBlock(block); - defer scope.pop(); - - const global_shape = Shape.init(.{ .b = 4, .h = 3 * partitions, .hd = 128 }, .bf16) - .withPartitioning(mesh, .{ .h = .model }); - const local_shape = Shape.init(.{ .b = 4, .h = 3, .hd = 128 }, .bf16); - const input = Tensor.zeroes(global_shape).withPartitioning(mesh, .{ .h = .model }); - var received_shape: ?Shape = null; - const Context = struct { - input: Tensor, - received_shape: *?Shape, + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, fn body(self: @This(), output_shape: Shape) Tensor { - self.received_shape.* = output_shape; - return self.input; + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); } }; - const output = manualComputation( - Context.body, - .{ .input = input, .received_shape = &received_shape }, - global_shape, - ); - try std.testing.expect(received_shape != null); - try std.testing.expect(received_shape.?.eql(local_shape)); - try std.testing.expect(output.shape().eql(global_shape)); - try std.testing.expectEqualDeep(global_shape._sharding, output.shape()._sharding); + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} + +test "partitionSpecs preserves optional tensors and active cache variants" { + const zml = @import("../zml.zig"); + const Logical = Sharding.Partitioning.Logical; + const Inputs = struct { + layers: []const Tensor, + bias: ?Tensor, + cache: union(enum) { disabled, values: [2]Tensor }, + scale: f32, + }; + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); + var inputs: Inputs = .{ + .layers = &.{ tensor, tensor }, + .bias = null, + .cache = .{ .values = .{ tensor, tensor } }, + .scale = 2, + }; + const heads: Logical = .parse(.{ .h = .model }); + const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); + const expected: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{ heads, heads }, + .bias = null, + .cache = .{ .values = .{ heads, heads } }, + }; + try std.testing.expectEqualDeep(expected, specs); + + inputs.layers = &.{}; + inputs.bias = tensor; + inputs.cache = .disabled; + const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); + const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{}, + .bias = .replicated, + .cache = .disabled, + }; + try std.testing.expectEqualDeep(expected_replicated, replicated); } diff --git a/zml/ops/manual_computation2.zig b/zml/ops/manual_computation2.zig deleted file mode 100644 index 2d3f0aec..00000000 --- a/zml/ops/manual_computation2.zig +++ /dev/null @@ -1,465 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const shardy = @import("mlir/dialects/shardy"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../Compiler.zig"); -const meta = @import("../meta.zig"); -const mlirx = @import("../mlirx.zig"); -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const Tensor = @import("../tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/Compiler"); - -/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. -/// Returns the type of the `partitioning` argument to `manualComputation`. -/// -/// * `in_specs` specifies the sharding of each tensor in the body's first argument. -/// It follows the argument's structure, omitting non-tensor metadata. -/// * `out_specs` specifies the sharding of each shape in the body's second argument. -/// It follows the output shapes' structure and can be omitted for a `void` argument. -/// -/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as -/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across -/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` -/// replicates the entire tensor. The specs must not contain open dimensions. -/// Slice lengths, optional presence and active union tags must match the arguments. -/// -/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single -/// output `Shape`, the partitioning can be declared as: -/// -/// ```zig -/// const partitioning: ManualPartitioning(body) = .{ -/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, -/// .out_specs = .parse(.{ .hidden = .model }), -/// }; -/// ``` -pub fn ManualPartitioning(comptime body_fn: anytype) type { - const Logical = Sharding.Partitioning.Logical; - - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - - const InputSpecs = meta.MapRestrict(Tensor, Logical).map(Inputs); - const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); - - if (OutputSpecs == void) return struct { - in_specs: InputSpecs, - out_specs: void = {}, - }; - - return struct { - in_specs: InputSpecs, - out_specs: OutputSpecs, - }; -} - -/// Describes a computation on local shards of tensors distributed across `mesh`. -/// ZML operations normally describe work on global tensors. The partitioner -/// distributes the inputs according to `in_specs`, and `body_fn` describes the work -/// performed on each shard. The results are exposed to the surrounding graph as -/// global tensors with the sharding declared by `out_specs`. -/// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: -/// - `I` contains the local input tensors and any metadata needed by the body. -/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. -/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes -/// in traversal order. -/// * `inputs` contains the global tensors and metadata. The partitioner reshards -/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. -/// * `output_shapes` contains the global output shapes. The body receives their -/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. -/// * `mesh` is the logical mesh whose axis names are referenced by the specs. -/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. -/// Each partitioned dimension must be divisible by the size of its mesh axis. -/// -/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives -/// the body a dimension of size 8. A replicated dimension keeps its global size. -/// The caller supplies the global output shapes because the body only describes -/// local results: their shapes alone do not determine the global tensor shapes. -pub fn manualComputation2( - comptime body_fn: anytype, - inputs: stdx.meta.FnParam(body_fn, 0), - output_shapes: stdx.meta.FnParam(body_fn, 1), - mesh: *const Sharding.Mesh, - partitioning: ManualPartitioning(body_fn), -) stdx.meta.FnReturn(body_fn) { - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - const Outputs = stdx.meta.FnReturn(body_fn); - - const compiler = Compiler.current(); - var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // The body may return containers borrowed from its inputs or output shapes. - const body_arena = compiler.arena.allocator(); - - var owned_inputs: Inputs = undefined; - meta.mapAlloc((struct { - fn copy(_: void, tensor: Tensor) Tensor { - return tensor; - } - }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); - - const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); - stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); - - for (flat_input_tensors, in_specs) |tensor, spec| { - tensor._shape = tensor.shape().withPartitioning(mesh, spec); - stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); - } - - var local_output_shapes: OutputShapes = undefined; - meta.mapAlloc((struct { - fn copy(_: void, shape: Shape) Shape { - return shape; - } - }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); - - const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); - const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); - stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); - - const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); - for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { - global.* = local.withPartitioning(mesh, spec); - stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); - local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - - const WrappedInputs = struct { - inputs: *const Inputs, - output_shapes: *const OutputShapes, - - fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { - const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); - const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); - - outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - - return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); - } - }; - - var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; - - _ = manualComputationInternal( - compiler, - arena, - mesh, - @typeName(@TypeOf(body_fn)), - &wrapped_inputs, - flat_input_tensors, - @ptrCast(&outputs_struct), - flat_output_shapes, - WrappedInputs.wrap, - ) catch |err| switch (err) { - error.OutOfMemory => @panic("OOM"), - }; - - return outputs_struct; -} - -const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; - -/// Reified version of manualComputation2. -/// This one is compiled exactly once, user code just need to compile the `collectPtrs` -noinline fn manualComputationInternal( - compiler: *Compiler, - arena: std.mem.Allocator, - mesh: *const Sharding.Mesh, - functor_name: []const u8, - input_struct: *anyopaque, - input_tensors: []*Tensor, - output_struct: *anyopaque, - global_output_shapes: []const Shape, - local_fn: LocalComputationFn, -) error{OutOfMemory}![]*Tensor { - const global_scope = compiler.currentScope(); - const mlir_ctx = compiler.mlir_ctx; - - log.debug("manualComputation2({s}) will use mesh {s}", .{ functor_name, mesh.name }); - - const global_input_shapes = try arena.alloc(Shape, input_tensors.len); - for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - - const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); - for (input_types, input_tensors) |*input_ty, tensor_ptr| { - const global = tensor_ptr._shape; - const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); - log.debug("\t- {f} -> {f}", .{ global, local }); - input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); - } - const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); - @memset(input_locs, compiler.unknown_location); - - const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); - for (input_values, input_tensors) |*input_val, tensor_ptr| { - if (compiler.partitioner == .gspmd) { - tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); - } - input_val.* = global_scope.value(tensor_ptr.*); - } - - const manual_block = switch (compiler.partitioner) { - .shardy => mlir.Block.init(input_types, input_locs), - .gspmd => global_scope.block, - }; - errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); - - const local_scope = compiler.pushBlock(manual_block); - defer local_scope.pop(); - for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { - const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above - tensor_ptr.* = switch (compiler.partitioner) { - .shardy => Tensor._result(local, manual_block.argument(i)), - .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), - }; - } - - compiler.manual_computation_depth += 1; - defer compiler.manual_computation_depth -= 1; - - const local_outputs = try local_fn(arena, input_struct, output_struct); - stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation2 body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { - const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); - stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation2 body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); - } - - const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |output_shape, i| { - global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); - } - - const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); - for (local_output_values, local_outputs) |*output_val, tensor_ptr| { - output_val.* = local_scope.value(tensor_ptr.*); - } - - const op: *const mlir.Operation = switch (compiler.partitioner) { - .shardy => op: { - // Shardy creates a sdy.manual_computation block. - _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); - - const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); - for (out_mesh_attrs, global_output_shapes) |*attr, shape| { - attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); - } - - break :op shardy.manual_computation( - mlir_ctx, - input_values, - global_output_types, - manual_block, - try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), - .get(mlir_ctx, out_mesh_attrs), - try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), - compiler.location, - ).appendTo(global_scope.block); - }, - .gspmd => op: { - // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier - // Shortcut to avoid empty optimization barrier - if (global_output_shapes.len == 0) return &.{}; - - const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); - const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |global_shape, i| { - global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); - } - - break :op stablehlo.optimizationBarrier( - mlir_ctx, - global_values, - global_types, - compiler.location, - ).appendTo(global_scope.block); - }, - }; - - // Returned tensors describe the operation's global output shapes and sharding. - for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { - output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape._sharding = output_shape._sharding; - } - return local_outputs; -} - -const gspmd = struct { - fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { - const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); - const full_to_shard = stablehlo.custom_call( - mlir_ctx, - &.{scope.value(local_tensor)}, - &.{local_type}, - .{ - .call_target_name = "SPMDFullToShardShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), - }, - }, - location, - ).appendTo(scope.block); - return Tensor._result(local_shape, full_to_shard.result(0)); - } - - fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { - const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); - const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); - - const shard_to_full = stablehlo.custom_call( - compiler.mlir_ctx, - &.{local_value}, - &.{global_type}, - .{ - .call_target_name = "SPMDShardToFullShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), - }, - }, - compiler.location, - ).appendTo(scope.block); - return .{ shard_to_full.result(0), global_type }; - } -}; - -test manualComputation2 { - const zml = @import("../zml.zig"); - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const mesh = zml.testing.tpMesh(); - - const LocalSum = struct { - x: Tensor, - bias: Tensor, - scale: i32, - - fn body(self: @This(), output_shape: Shape) Tensor { - return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); - } - - fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { - return manualComputation2( - body, - .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, - x.shape(), - mesh_, - .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, - ); - } - }; - - // Each shard sums two elements and broadcasts the result to its local output shape. - const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); - const values = try allocator.alloc(i32, shape.count()); - defer allocator.free(values); - - for (values, 0..) |*value, i| value.* = @intCast(i + 1); - - var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - const actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - - for (actual.constItems(i32), 0..) |value, i| { - const pair = values[i / 2 * 2 ..][0..2]; - try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); - } - - try std.testing.expect(output.shape().eqlWithTags(shape)); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } -} - -test "manualComputation2 composes across logical and physical meshes" { - const zml = @import("../zml.zig"); - if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - - const Topologies = enum { flat, torus }; - - const Model = struct { - fn increment(x: Tensor, _: Shape) Tensor { - return x.addConstant(1); - } - - fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { - const partitioning: ManualPartitioning(increment) = .{ - .in_specs = .parse(.{ .x = .model }), - .out_specs = .parse(.{ .x = .model }), - }; - - const first = manualComputation2(increment, x, x.shape(), mesh, partitioning); - const converted = first.convert(.f32); - return manualComputation2(increment, converted, converted.shape(), mesh, partitioning); - } - }; - - const shape: Shape = .init(.{ .x = 16 }, .i32); - - var values: [16]i32 = undefined; - var expected: [16]f32 = undefined; - for (&values, &expected, 0..) |*value, *result, i| { - value.* = @intCast(i); - result.* = @floatFromInt(i + 2); - } - - for ([_]Topologies{ .flat, .torus }) |topology| { - const platform = try zml.Platform.init(allocator, io, .cpu, .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, - }); - defer platform.deinit(allocator, io); - - // Fold three physical axes into one logical axis, or split one into three. - const logical: Sharding.LogicalMesh = if (topology == .torus) - try .init(.{.model}, &.{.balanced}, &.{8}) - else - try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); - - const mesh = try platform.registerMesh("manual", logical); - - var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); - try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } - } -} diff --git a/zml/platform.zig b/zml/platform.zig index 6c6081aa..54fb90e4 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1015,12 +1015,15 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; + const buffers = call_frame.args.buffers(); + if (buffers.len <= 0) return null; + const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = call_frame.args.buffers()[0]; + const buffer = buffers[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -1044,9 +1047,13 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); + defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); + + const event = pjrt_buffer.readyEvent(pjrt_api); + defer event.deinit(pjrt_api); + try event.awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); diff --git a/zml/tensor.zig b/zml/tensor.zig index a7782844..ee084835 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -282,7 +282,13 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { - .cpu, .neuron, .metal => return self, + .cpu, .neuron, .metal => { + // A single memory needs no copy, but the result must still be a distinct + // tensor: `reuseBuffer` donations are keyed by tensor id. + var res = _result(self._shape, self.value()); + res._shape._sharding = self._shape._sharding; + return res; + }, .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, } @@ -4813,7 +4819,7 @@ pub const Tensor = struct { defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation2((struct { + ops.manualComputation((struct { input: Tensor, name: []const u8, diff --git a/zml/zml.zig b/zml/zml.zig index 35c2f9df..dafc3a0f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -26,7 +26,7 @@ pub const Exe = exe.Exe; pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); -pub const kernel = @import("kernel.zig"); +pub const kernel = @import("kernel/kernel.zig"); pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized;