From 6975d34bcedbf13ceb7f2820afb12f3a6dc908f6 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 10 Nov 2026 14:34:26 +0100 Subject: [PATCH 01/19] bin/zml-smi: derive AMD per-process utilization from engine time amdsmi reports the time a process spent on the GFX and ENC engines as a running total in ns, which was cast to u16 as if it were a percentage. Utilization is now its growth between two polls over the elapsed time. One-shot runs sample once beforehand, as neuron does. --- bin/zml-smi/platforms/amdsmi/BUILD.bazel | 7 +++- bin/zml-smi/platforms/amdsmi/metrics.zig | 4 ++ bin/zml-smi/platforms/amdsmi/process.zig | 50 ++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel index dac6b935..4fde01d2 100644 --- a/bin/zml-smi/platforms/amdsmi/BUILD.bazel +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "amdsmi", @@ -20,3 +20,8 @@ zig_library( "@amdrocm-amdsmi//:amdsmi", ], ) + +zig_test( + name = "test", + deps = [":amdsmi"], +) diff --git a/bin/zml-smi/platforms/amdsmi/metrics.zig b/bin/zml-smi/platforms/amdsmi/metrics.zig index 8aa0487c..b443230c 100644 --- a/bin/zml-smi/platforms/amdsmi/metrics.zig +++ b/bin/zml-smi/platforms/amdsmi/metrics.zig @@ -149,3 +149,7 @@ const metrics = .{ .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, }; + +test { + _ = process; +} diff --git a/bin/zml-smi/platforms/amdsmi/process.zig b/bin/zml-smi/platforms/amdsmi/process.zig index 8d1fdd01..df687eb0 100644 --- a/bin/zml-smi/platforms/amdsmi/process.zig +++ b/bin/zml-smi/platforms/amdsmi/process.zig @@ -7,6 +7,22 @@ const Collector = @import("zml-smi/collector").Collector; const bdf_len = "0000:00:00.0".len; +/// amdsmi reports the time a process spent on the engines as a running total, +/// so utilization is how much it grew between two polls. +const Sample = struct { + engine_ns: u64, + timestamp_ns: u64, + + fn utilSince(self: Sample, previous: Sample) ?u16 { + if (self.timestamp_ns <= previous.timestamp_ns or self.engine_ns < previous.engine_ns) return null; + return @intCast(@min(100, (self.engine_ns - previous.engine_ns) * 100 / (self.timestamp_ns - previous.timestamp_ns))); + } +}; + +/// Last sample of each process on each device. +const Samples = std.AutoHashMapUnmanaged(Key, Sample); +const Key = struct { pid: u32, device: u32 }; + pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16) !void { const device_count = amdsmi.deviceCount(); const pci_slots = try collector.arena.alloc(?[bdf_len]u8, device_count); @@ -23,13 +39,25 @@ pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const Am }); } - try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, amdsmi, dev_offset, device_count, pci_slots }); + const samples = try collector.arena.create([2]Samples); + samples.* = .{ .empty, .empty }; + + const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.gpa, collector.arena, collector.io, list, amdsmi, dev_offset, device_count, pci_slots, samples }; + // A one-shot run polls once: sample beforehand so that poll has a utilization to report. + if (collector.poll_only) { + @call(.auto, pollOnce, args); + collector.io.sleep(.fromMilliseconds(collector.worker.poll_interval_ms), .awake) catch {}; + } + + try collector.spawnPoll(pollOnce, args); } -fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16, device_count: u32, pci_slots: []?[bdf_len]u8) void { +fn pollOnce(allocator: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16, device_count: u32, pci_slots: []?[bdf_len]u8, samples: *[2]Samples) void { const back = list.back(); + const previous, const current = .{ &samples[0], &samples[1] }; back.clearRetainingCapacity(); + current.clearRetainingCapacity(); for (0..device_count) |dev_idx| { const handle = amdsmi.handleByIndex(@intCast(dev_idx)) catch continue; @@ -58,19 +86,25 @@ fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer allocator.free(procs); } } + const timestamp_ns: u64 = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); const pci_slot = if (dev_idx < pci_slots.len) &(pci_slots[dev_idx] orelse continue) else continue; for (procs) |proc| { + const key: Key = .{ .pid = proc.pid, .device = @intCast(dev_idx) }; + const sample: Sample = .{ .engine_ns = proc.engine_usage.gfx +| proc.engine_usage.enc, .timestamp_ns = timestamp_ns }; + current.put(arena, key, sample) catch {}; + back.append(allocator, .{ .pid = proc.pid, .device_idx = @intCast(dev_idx + dev_offset), - .dev_util_percent = @intCast(proc.engine_usage.gfx + proc.engine_usage.enc), + .dev_util_percent = if (previous.get(key)) |p| sample.utilSince(p) else null, .dev_mem_kib = readProcessGpuVram(io, proc.pid, pci_slot), }) catch break; } } + std.mem.swap(Samples, previous, current); list.swap(); } @@ -137,3 +171,13 @@ fn formatBdf(bdf_id: u64) [bdf_len]u8 { return buf; } + +test "utilSince is the engine time growth over the elapsed time" { + const previous: Sample = .{ .engine_ns = 100, .timestamp_ns = 1000 }; + + try std.testing.expectEqual(50, (Sample{ .engine_ns = 600, .timestamp_ns = 2000 }).utilSince(previous)); + // gfx and enc can both be busy + try std.testing.expectEqual(100, (Sample{ .engine_ns = 2100, .timestamp_ns = 2000 }).utilSince(previous)); + // the pid went to another process + try std.testing.expectEqual(null, (Sample{ .engine_ns = 10, .timestamp_ns = 2000 }).utilSince(previous)); +} From 86aed617718c085f2518d20a646a881ea2fc8b67 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 10 Nov 2026 14:34:26 +0100 Subject: [PATCH 02/19] bin/zml-smi: enrich processes once per poll Processes were enriched by whoever read them: on every API request, from every connection at once, and on every TUI draw. That raced on the enricher and made CPU usage a delta over an arbitrary window. A poll of the collector now merges and enriches them once per interval, and the API, the TUI and --json read its result. --- bin/zml-smi/api/BUILD.bazel | 5 +- bin/zml-smi/api/server.zig | 46 +++++--------- bin/zml-smi/collector/collector.zig | 4 ++ bin/zml-smi/main.zig | 20 ++---- bin/zml-smi/platforms/linux/BUILD.bazel | 1 + bin/zml-smi/platforms/linux/process.zig | 74 ++++++++++++----------- bin/zml-smi/platforms/macos/BUILD.bazel | 1 + bin/zml-smi/platforms/macos/process.zig | 30 +++++++-- bin/zml-smi/tui/BUILD.bazel | 5 +- bin/zml-smi/tui/data.zig | 14 +---- bin/zml-smi/tui/widgets/process_table.zig | 32 ++++------ 11 files changed, 107 insertions(+), 125 deletions(-) diff --git a/bin/zml-smi/api/BUILD.bazel b/bin/zml-smi/api/BUILD.bazel index 1a09f597..7f1f7a2d 100644 --- a/bin/zml-smi/api/BUILD.bazel +++ b/bin/zml-smi/api/BUILD.bazel @@ -14,8 +14,5 @@ zig_library( "//bin/zml-smi/info", "//bin/zml-smi/json", "//bin/zml-smi/utils:double_buffer", - ] + select({ - "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], - "//conditions:default": ["//bin/zml-smi/platforms/linux"], - }), + ], ) diff --git a/bin/zml-smi/api/server.zig b/bin/zml-smi/api/server.zig index e05aa7fe..517d0cd9 100644 --- a/bin/zml-smi/api/server.zig +++ b/bin/zml-smi/api/server.zig @@ -1,25 +1,19 @@ const std = @import("std"); -const builtin = @import("builtin"); const smi_info = @import("zml-smi/info"); const DeviceInfo = smi_info.device_info.DeviceInfo; const ProcessInfo = smi_info.process_info.ProcessInfo; const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(ProcessInfo)); const json = @import("zml-smi/json"); -const ProcessEnricher = if (builtin.os.tag == .macos) - @import("zml-smi/platforms/macos").process.ProcessEnricher -else - @import("zml-smi/platforms/linux").process.ProcessEnricher; pub const Server = struct { allocator: std.mem.Allocator, io: std.Io, devices: []const *DeviceInfo, - process_lists: []const *const ProcessDoubleBuffer, - enricher: *ProcessEnricher, + processes: *const ProcessDoubleBuffer, tcp: std.Io.net.Server, connection_group: std.Io.Group = .init, - pub fn init(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) !Server { + pub fn init(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, processes: *const ProcessDoubleBuffer) !Server { const address: std.Io.net.IpAddress = .{ .ip4 = .{ .bytes = .{ 0, 0, 0, 0 }, .port = port, @@ -31,14 +25,13 @@ pub const Server = struct { .allocator = allocator, .io = io, .devices = devices, - .process_lists = process_lists, - .enricher = enricher, + .processes = processes, .tcp = tcp, }; } - pub fn run(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) void { - var self = init(allocator, io, port, devices, process_lists, enricher) catch |err| { + pub fn run(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, processes: *const ProcessDoubleBuffer) void { + var self = init(allocator, io, port, devices, processes) catch |err| { std.log.err("api server failed to start: {s}", .{@errorName(err)}); return; }; @@ -72,8 +65,8 @@ pub const Server = struct { var http_server: std.http.Server = .init(&reader.interface, &writer.interface); - var procs: std.ArrayList(ProcessInfo) = .empty; - defer procs.deinit(self.allocator); + var body: std.Io.Writer.Allocating = .init(self.allocator); + defer body.deinit(); while (true) { var request = http_server.receiveHead() catch |err| switch (err) { @@ -81,25 +74,18 @@ pub const Server = struct { else => return err, }; - procs.clearRetainingCapacity(); - for (self.process_lists) |pl| { - procs.appendSlice(self.allocator, pl.front().items) catch {}; - } - self.enricher.enrich(self.io, procs.items); + // The processes are only valid until they are polled again, so the body + // is written out before it is sent: a slow client cannot outlive them. + body.clearRetainingCapacity(); + try json.write(&body.writer, self.devices, self.processes.front().items); - var body_buf: [4096]u8 = undefined; - var body_writer = try request.respondStreaming(&body_buf, .{ - .respond_options = .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - .{ .name = "access-control-allow-origin", .value = "*" }, - }, + try request.respond(body.written(), .{ + .status = .ok, + .extra_headers = &.{ + .{ .name = "content-type", .value = "application/json" }, + .{ .name = "access-control-allow-origin", .value = "*" }, }, }); - - try json.write(&body_writer.writer, self.devices, procs.items); - try body_writer.end(); } } }; diff --git a/bin/zml-smi/collector/collector.zig b/bin/zml-smi/collector/collector.zig index d98b204d..3453e942 100644 --- a/bin/zml-smi/collector/collector.zig +++ b/bin/zml-smi/collector/collector.zig @@ -8,6 +8,8 @@ const Worker = @import("worker.zig").Worker; pub const Collector = struct { device_infos: std.ArrayList(*DeviceInfo) = .empty, process_lists: std.ArrayList(*ProcessDoubleBuffer) = .empty, + /// Every process list merged into one, written by the process poll of the host. + processes: ProcessDoubleBuffer = .{ .values = .{ .empty, .empty } }, poll_arenas: std.ArrayList(*std.heap.ArenaAllocator) = .empty, arena: std.mem.Allocator, gpa: std.mem.Allocator, @@ -84,6 +86,8 @@ pub const Collector = struct { list.values[1].deinit(self.gpa); } self.process_lists.deinit(self.arena); + self.processes.values[0].deinit(self.gpa); + self.processes.values[1].deinit(self.gpa); for (self.poll_arenas.items) |poll_arena| { poll_arena.deinit(); diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index f066dd34..921f07b0 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -16,7 +16,6 @@ const platform = if (builtin.os.tag == .macos) @import("zml-smi/platforms/macos") else @import("zml-smi/platforms/linux"); -const ProcessEnricher = platform.process.ProcessEnricher; const host = platform.metrics; const c = @import("c"); const csv = @import("zml-smi/csv"); @@ -94,9 +93,6 @@ pub fn main(init: std.process.Init) !void { var host_info: HostInfo = .{ .values = .{ .{}, .{} } }; try host.init(&collector, &host_info); - var enricher: ProcessEnricher = try .init(gpa, io); - defer enricher.deinit(); - if (args.api) { std.log.info("serving api metrics on port {d}", .{args.api_port}); } @@ -120,13 +116,14 @@ pub fn main(init: std.process.Init) !void { try api.addRemotes(&collector, hosts); } + try platform.process.start(&collector); + var state = try data.SystemState.init(.{ .devices = collector.device_infos.items, .host = &host_info, .targets = targets, .tui_refresh_rate = args.tui_refresh_rate, - .process_lists = collector.process_lists.items, - .enricher = &enricher, + .processes = &collector.processes, .gpa = gpa, .arena = arena, .io = io, @@ -138,7 +135,7 @@ pub fn main(init: std.process.Init) !void { } if (args.api) { - try api_group.concurrent(io, api.Server.run, .{ gpa, io, args.api_port, collector.device_infos.items, collector.process_lists.items, &enricher }); + try api_group.concurrent(io, api.Server.run, .{ gpa, io, args.api_port, collector.device_infos.items, &collector.processes }); } if (args.prometheus_listen != null or args.api) { @@ -150,17 +147,10 @@ pub fn main(init: std.process.Init) !void { try csv.write(&csv_writer.interface, collector.device_infos.items); try csv_writer.flush(); } else if (args.json) { - var procs: std.ArrayList(smi_info.process_info.ProcessInfo) = .empty; - defer procs.deinit(gpa); - for (collector.process_lists.items) |pl| { - try procs.appendSlice(gpa, pl.front().items); - } - enricher.enrich(io, procs.items); - var json_buf: [4096]u8 = undefined; var json_writer = std.Io.File.stdout().writer(io, &json_buf); - try json.write(&json_writer.interface, collector.device_infos.items, procs.items); + try json.write(&json_writer.interface, collector.device_infos.items, collector.processes.front().items); try json_writer.flush(); } else if (args.top) { try tui.run(gpa, io, init.environ_map, &state); diff --git a/bin/zml-smi/platforms/linux/BUILD.bazel b/bin/zml-smi/platforms/linux/BUILD.bazel index 57bb96d7..38cbdab8 100644 --- a/bin/zml-smi/platforms/linux/BUILD.bazel +++ b/bin/zml-smi/platforms/linux/BUILD.bazel @@ -12,6 +12,7 @@ zig_library( deps = [ "//bin/zml-smi/collector", "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", "//bin/zml-smi/utils:sysfs", ], ) diff --git a/bin/zml-smi/platforms/linux/process.zig b/bin/zml-smi/platforms/linux/process.zig index c57d1725..fcbd6d7d 100644 --- a/bin/zml-smi/platforms/linux/process.zig +++ b/bin/zml-smi/platforms/linux/process.zig @@ -1,49 +1,54 @@ const std = @import("std"); const sysfs = @import("zml-smi/sysfs"); const pi = @import("zml-smi/info").process_info; +const Collector = @import("zml-smi/collector").Collector; const ProcessInfo = pi.ProcessInfo; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(ProcessInfo)); -pub const ProcessEnricher = struct { - gpa: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - prev_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}, +/// Merges every process list into `collector.processes`, completed with what +/// the host knows about each local process. +pub fn start(collector: *Collector) !void { + const enricher = try collector.arena.create(ProcessEnricher); + enricher.* = .{ .arenas = .{ try collector.createPollArena(), try collector.createPollArena() } }; + try collector.spawnPoll(pollOnce, .{ enricher, collector.gpa, collector.io, collector.process_lists.items, &collector.processes }); +} + +fn pollOnce(enricher: *ProcessEnricher, gpa: std.mem.Allocator, io: std.Io, lists: []const *const ProcessDoubleBuffer, processes: *ProcessDoubleBuffer) void { + // Each side of the double buffer owns its strings, in the arena of the same index. + const arena = enricher.arenas[1 - processes.current.load(.acquire)]; + _ = arena.reset(.retain_capacity); + + const back = processes.back(); + back.clearRetainingCapacity(); + for (lists) |list| { + back.appendSlice(gpa, list.front().items) catch break; + } + + enricher.enrich(arena.allocator(), io, back.items); + processes.swap(); +} + +const ProcessEnricher = struct { + arenas: [2]*std.heap.ArenaAllocator, + /// Lives in the arena of the previous poll, which is reset one poll later. + prev_ticks: std.AutoHashMapUnmanaged(u32, u64) = .empty, prev_total_ticks: u64 = 0, - pub fn init(gpa: std.mem.Allocator, io: std.Io) !ProcessEnricher { - var arena = std.heap.ArenaAllocator.init(gpa); - errdefer arena.deinit(); - - const prev_total_ticks = readTotalCpuTicks(arena.allocator(), io); - var enricher: ProcessEnricher = .{ - .gpa = gpa, - .arena = arena, - .prev_total_ticks = prev_total_ticks, - .prev_ticks = .{}, - }; - - try enricher.prev_ticks.ensureTotalCapacity(gpa, 64); - - return enricher; - } - - pub fn deinit(self: *ProcessEnricher) void { - self.prev_ticks.deinit(self.gpa); - self.arena.deinit(); - } - - pub fn enrich(self: *ProcessEnricher, io: std.Io, procs: []ProcessInfo) void { - _ = self.arena.reset(.retain_capacity); - const alloc = self.arena.allocator(); - + fn enrich(self: *ProcessEnricher, alloc: std.mem.Allocator, io: std.Io, procs: []ProcessInfo) void { const curr_total = readTotalCpuTicks(alloc, io); const delta_total = curr_total -| self.prev_total_ticks; - var curr_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}; - curr_ticks.ensureTotalCapacity(self.gpa, @intCast(procs.len)) catch return; + var curr_ticks: std.AutoHashMapUnmanaged(u32, u64) = .empty; + curr_ticks.ensureTotalCapacity(alloc, @intCast(procs.len)) catch {}; for (procs) |*info| { - if (info.remote) continue; + if (info.remote) { + // Already enriched by its host: only the strings are taken over. + info.username = alloc.dupe(u8, info.username) catch ""; + info.comm = alloc.dupe(u8, info.comm) catch ""; + continue; + } // Read Uid, VmRSS from /proc/{pid}/status const status_path = std.fmt.allocPrint(alloc, "/proc/{d}/status", .{info.pid}) catch continue; const uid: u32 = @intCast(sysfs.readFieldInt(alloc, io, status_path, "Uid") catch 0); @@ -64,10 +69,9 @@ pub const ProcessEnricher = struct { } } - curr_ticks.getOrPutAssumeCapacity(info.pid).value_ptr.* = ticks; + curr_ticks.put(alloc, info.pid, ticks) catch continue; } - self.prev_ticks.deinit(self.gpa); self.prev_ticks = curr_ticks; self.prev_total_ticks = curr_total; } diff --git a/bin/zml-smi/platforms/macos/BUILD.bazel b/bin/zml-smi/platforms/macos/BUILD.bazel index a92e298e..57d4709c 100644 --- a/bin/zml-smi/platforms/macos/BUILD.bazel +++ b/bin/zml-smi/platforms/macos/BUILD.bazel @@ -12,5 +12,6 @@ zig_library( deps = [ "//bin/zml-smi/collector", "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", ], ) diff --git a/bin/zml-smi/platforms/macos/process.zig b/bin/zml-smi/platforms/macos/process.zig index a48e9831..fd9c2fc4 100644 --- a/bin/zml-smi/platforms/macos/process.zig +++ b/bin/zml-smi/platforms/macos/process.zig @@ -1,12 +1,30 @@ const std = @import("std"); const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; -pub const ProcessEnricher = struct { - pub fn init(_: std.mem.Allocator, _: std.Io) !ProcessEnricher { - return .{}; +/// Merges every process list into `collector.processes`. No backend runs on macOS: +/// the processes all come from remotes, already enriched by their host. +pub fn start(collector: *Collector) !void { + const arenas: [2]*std.heap.ArenaAllocator = .{ try collector.createPollArena(), try collector.createPollArena() }; + try collector.spawnPoll(pollOnce, .{ arenas, collector.gpa, collector.process_lists.items, &collector.processes }); +} + +fn pollOnce(arenas: [2]*std.heap.ArenaAllocator, gpa: std.mem.Allocator, lists: []const *const ProcessDoubleBuffer, processes: *ProcessDoubleBuffer) void { + // Each side of the double buffer owns its strings, in the arena of the same index. + const arena = arenas[1 - processes.current.load(.acquire)]; + _ = arena.reset(.retain_capacity); + + const back = processes.back(); + back.clearRetainingCapacity(); + for (lists) |list| { + back.appendSlice(gpa, list.front().items) catch break; } - pub fn deinit(_: *ProcessEnricher) void {} + for (back.items) |*info| { + info.username = arena.allocator().dupe(u8, info.username) catch ""; + info.comm = arena.allocator().dupe(u8, info.comm) catch ""; + } - pub fn enrich(_: *ProcessEnricher, _: std.Io, _: []pi.ProcessInfo) void {} -}; + processes.swap(); +} diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index 29f4824c..e4a41e3c 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -47,10 +47,7 @@ zig_library( "//bin/zml-smi/utils:double_buffer", "//zml:logo", "@libvaxis//:vaxis", - ] + select({ - "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], - "//conditions:default": ["//bin/zml-smi/platforms/linux"], - }), + ], ) zig_test( diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index bc6171b7..d25382ee 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const smi_info = @import("zml-smi/info"); const di = smi_info.device_info; pub const DeviceInfo = di.DeviceInfo; @@ -13,10 +12,6 @@ pub const HostInfo = hi.HostInfo; pub const HostData = hi.HostData; pub const pi = smi_info.process_info; pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -pub const ProcessEnricher = if (builtin.os.tag == .macos) - @import("zml-smi/platforms/macos").process.ProcessEnricher -else - @import("zml-smi/platforms/linux").process.ProcessEnricher; pub const history_len: usize = 500; @@ -86,8 +81,7 @@ pub const SystemState = struct { host: *HostInfo, history: HistoryBuffers = .{}, targets: Targets, - process_lists: []*ProcessDoubleBuffer, - enricher: *ProcessEnricher, + processes: *const ProcessDoubleBuffer, gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, @@ -98,8 +92,7 @@ pub const SystemState = struct { host: *HostInfo, targets: Targets, tui_refresh_rate: u16, - process_lists: []*ProcessDoubleBuffer, - enricher: *ProcessEnricher, + processes: *const ProcessDoubleBuffer, gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, @@ -112,8 +105,7 @@ pub const SystemState = struct { .history = try HistoryBuffers.init(cfg.arena, cfg.devices.len), .targets = cfg.targets, .tui_refresh_rate = cfg.tui_refresh_rate, - .process_lists = cfg.process_lists, - .enricher = cfg.enricher, + .processes = cfg.processes, .gpa = cfg.gpa, .arena = cfg.arena, .io = cfg.io, diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig index 58b20d51..af740f3d 100644 --- a/bin/zml-smi/tui/widgets/process_table.zig +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -24,41 +24,33 @@ scroll_bars: vxfw.ScrollBars = .{ }, .draw_horizontal_scrollbar = false, }, -merged: std.ArrayList(ProcessInfo) = .empty, +processes: std.ArrayList(ProcessInfo) = .empty, pub fn deinit(self: *ProcessTable, allocator: std.mem.Allocator) void { - self.merged.deinit(allocator); + self.processes.deinit(allocator); } pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16) void { self.scroll_bars.scroll_view.children.builder.userdata = self; - self.merged.clearRetainingCapacity(); + self.processes.clearRetainingCapacity(); - for (state.process_lists) |pl| { - const pl_items = pl.front().items; + for (state.processes.front().items) |entry| { if (device_id) |did| { - for (pl_items) |entry| { - if (entry.device_idx != did) { - continue; - } - self.merged.append(state.gpa, entry) catch break; + if (entry.device_idx != did) { + continue; } - } else { - self.merged.appendSlice(state.gpa, pl_items) catch break; } + self.processes.append(state.gpa, entry) catch break; } - // Enrich with host data (uid, username, cpu%, rss, cmdline) - state.enricher.enrich(state.io, self.merged.items); - // Sort by dev_mem descending - std.sort.insertion(ProcessInfo, self.merged.items, {}, struct { + std.sort.insertion(ProcessInfo, self.processes.items, {}, struct { fn cmp(_: void, a: ProcessInfo, b: ProcessInfo) bool { return (a.dev_mem_kib orelse 0) > (b.dev_mem_kib orelse 0); } }.cmp); - const count: u32 = @intCast(self.merged.items.len); + const count: u32 = @intCast(self.processes.items.len); self.scroll_bars.estimated_content_height = count; self.scroll_bars.scroll_view.item_count = count; } @@ -70,7 +62,7 @@ pub fn resetScroll(self: *ProcessTable) void { pub fn draw(self: *ProcessTable, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const w = ctx.max.width orelse 80; - const row_count = self.merged.items.len; + const row_count = self.processes.items.len; var header_rt: RichText = .{ .text = &header_segs, .softwrap = false, .overflow = .clip }; const empty_rt: RichText = .{ @@ -104,10 +96,10 @@ pub fn draw(self: *ProcessTable, ctx: vxfw.DrawContext) std.mem.Allocator.Error! /// Builds only data rows (no header) for the scroll view. fn buildRow(ptr: *const anyopaque, idx: usize, _: usize) ?vxfw.Widget { const self: *ProcessTable = @ptrCast(@alignCast(@constCast(ptr))); - if (idx >= self.merged.items.len) { + if (idx >= self.processes.items.len) { return null; } - return .{ .userdata = @ptrCast(&self.merged.items[idx]), .drawFn = drawProcessRow }; + return .{ .userdata = @ptrCast(&self.processes.items[idx]), .drawFn = drawProcessRow }; } // Column format strings — shared between header (comptime) and row (runtime). From 906f00e0c4bd4749235661290ad61313570e114f Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 10 Nov 2026 14:34:26 +0100 Subject: [PATCH 03/19] bin/zml-smi: keep enough history to fill charts at any terminal width Charts plot one sample per braille dot, but the history was capped at 500 samples: a chart wider than 250 cells never reached its left edge. The history now holds two samples per column of the widest terminal seen. --- bin/zml-smi/tui/data.zig | 84 +++++++------------ .../tui/pages/detail/detail_common.zig | 2 +- bin/zml-smi/tui/pages/detail/gpu_detail.zig | 2 +- bin/zml-smi/tui/top.zig | 1 + bin/zml-smi/tui/widgets/device_card.zig | 6 +- 5 files changed, 37 insertions(+), 58 deletions(-) diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index d25382ee..553b1023 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -13,57 +13,27 @@ pub const HostData = hi.HostData; pub const pi = smi_info.process_info; pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -pub const history_len: usize = 500; - -pub fn RingBuffer(comptime T: type, comptime capacity: usize) type { - return struct { - const Self = @This(); - - buf: [capacity]T = @splat(0), - head: usize = 0, - len: usize = 0, - - pub fn push(self: *Self, value: T) void { - self.buf[self.head] = value; - self.head = (self.head + 1) % capacity; - if (self.len < capacity) { - self.len += 1; - } - } - - pub fn sliceLast(self: *const Self, arena: std.mem.Allocator, n: usize) std.mem.Allocator.Error![]const T { - const count = @min(n, self.len); - if (count == 0) { - return &.{}; - } - const out = try arena.alloc(T, count); - const start = (self.head + capacity - count) % capacity; - for (0..count) |i| { - out[i] = self.buf[(start + i) % capacity]; - } - return out; - } - }; -} +/// The samples of one metric, oldest first. +pub const History = std.ArrayList(u64); pub const HistoryBuffers = struct { - util: []RingBuffer(u64, history_len) = &.{}, - mem_util: []RingBuffer(u64, history_len) = &.{}, - temp: []RingBuffer(u64, history_len) = &.{}, - power: []RingBuffer(u64, history_len) = &.{}, + util: []History = &.{}, + mem_util: []History = &.{}, + temp: []History = &.{}, + power: []History = &.{}, pub fn init(allocator: std.mem.Allocator, count: usize) !HistoryBuffers { const h: HistoryBuffers = .{ - .util = try allocator.alloc(RingBuffer(u64, history_len), count), - .mem_util = try allocator.alloc(RingBuffer(u64, history_len), count), - .temp = try allocator.alloc(RingBuffer(u64, history_len), count), - .power = try allocator.alloc(RingBuffer(u64, history_len), count), + .util = try allocator.alloc(History, count), + .mem_util = try allocator.alloc(History, count), + .temp = try allocator.alloc(History, count), + .power = try allocator.alloc(History, count), }; - @memset(h.util, .{}); - @memset(h.mem_util, .{}); - @memset(h.temp, .{}); - @memset(h.power, .{}); + @memset(h.util, .empty); + @memset(h.mem_util, .empty); + @memset(h.temp, .empty); + @memset(h.power, .empty); return h; } @@ -80,6 +50,9 @@ pub const SystemState = struct { devices: []*DeviceInfo, host: *HostInfo, history: HistoryBuffers = .{}, + /// Charts plot one sample per braille dot, two per cell: the history + /// keeps enough of them to fill the widest terminal seen. + history_len: usize = 0, targets: Targets, processes: *const ProcessDoubleBuffer, gpa: std.mem.Allocator, @@ -125,23 +98,30 @@ pub const SystemState = struct { switch (self.devices[i].*) { .cuda, .rocm, .oneapi => |*sv| { const gpu = sv.front().*; - self.history.util[i].push(gpu.util_percent orelse 0); + self.record(&self.history.util[i], gpu.util_percent orelse 0); const used = gpu.mem_used_bytes orelse 0; const total = gpu.mem_total_bytes orelse 0; - self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); - self.history.temp[i].push(gpu.temperature orelse 0); - self.history.power[i].push(gpu.power_mw orelse 0); + self.record(&self.history.mem_util[i], if (total > 0) used * 100 / total else 0); + self.record(&self.history.temp[i], gpu.temperature orelse 0); + self.record(&self.history.power[i], gpu.power_mw orelse 0); }, inline else => |*sv| { const info = sv.front().*; - self.history.util[i].push(info.util_percent orelse 0); + self.record(&self.history.util[i], info.util_percent orelse 0); const used = info.mem_used_bytes orelse 0; const total = info.mem_total_bytes orelse 0; - self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); - self.history.temp[i].push(0); - self.history.power[i].push(0); + self.record(&self.history.mem_util[i], if (total > 0) used * 100 / total else 0); + self.record(&self.history.temp[i], 0); + self.record(&self.history.power[i], 0); }, } } } + + fn record(self: *SystemState, history: *History, value: u64) void { + if (history.items.len > 0 and history.items.len >= self.history_len) { + _ = history.orderedRemove(0); + } + history.append(self.arena, value) catch {}; + } }; diff --git a/bin/zml-smi/tui/pages/detail/detail_common.zig b/bin/zml-smi/tui/pages/detail/detail_common.zig index 93e8addb..8720c6fc 100644 --- a/bin/zml-smi/tui/pages/detail/detail_common.zig +++ b/bin/zml-smi/tui/pages/detail/detail_common.zig @@ -35,7 +35,7 @@ pub fn historyChart( ) std.mem.Allocator.Error!Chart { return .{ .title = opts.title, - .data = try utils.normalizeRange(ctx.arena, try history[id].sliceLast(ctx.arena, data.history_len), opts.y_min, opts.y_max), + .data = try utils.normalizeRange(ctx.arena, history[id].items, opts.y_min, opts.y_max), .value_label = opts.value_label, .info_line = opts.info_line, .y_min = opts.y_min, diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig index 7884d6f8..ef889acd 100644 --- a/bin/zml-smi/tui/pages/detail/gpu_detail.zig +++ b/bin/zml-smi/tui/pages/detail/gpu_detail.zig @@ -70,7 +70,7 @@ pub fn draw( const power_limit_mw = gpu.power_limit_mw orelse 0; const power_limit_w = power_limit_mw / 1000; - const power_raw = try state.history.power[id].sliceLast(ctx.arena, data.history_len); + const power_raw = state.history.power[id].items; const power_watts = try ctx.arena.alloc(u64, power_raw.len); for (power_raw, 0..) |p, j| { power_watts[j] = p / 1000; diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig index 74b3796b..96438028 100644 --- a/bin/zml-smi/tui/top.zig +++ b/bin/zml-smi/tui/top.zig @@ -111,6 +111,7 @@ const Model = struct { pub fn draw(self: *Model, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const screen = ctx.max.size(); const content_w = @max(screen.width, min_width); + self.state.history_len = @max(self.state.history_len, 2 * @as(usize, content_w)); if (self.viewing_device != self.prev_viewing_device) { self.scroll.reset(); diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig index c7f7ac38..2c8c3894 100644 --- a/bin/zml-smi/tui/widgets/device_card.zig +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -122,10 +122,8 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx .highlighted = self.highlighted, }; - const util_hist = try self.state.history.util[i].sliceLast(ctx.arena, data.history_len); - const mem_hist = try self.state.history.mem_util[i].sliceLast(ctx.arena, data.history_len); - const util_chart_data = try utils.normalizeRange(ctx.arena, util_hist, 0, 100); - const mem_chart_data = try utils.normalizeRange(ctx.arena, mem_hist, 0, 100); + const util_chart_data = try utils.normalizeRange(ctx.arena, self.state.history.util[i].items, 0, 100); + const mem_chart_data = try utils.normalizeRange(ctx.arena, self.state.history.mem_util[i].items, 0, 100); const chart_entries = [2]MetricCard.ChartEntry{ .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix, .data = util_chart_data }, From d25e9c302cba36fc34dd37e72f82b6d389a6c1ab Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 10 Nov 2026 14:34:26 +0100 Subject: [PATCH 04/19] bin/zml-smi: round image width to the nearest column The terminal stretches an image over the cells it is given, so rounding the width up widened icons by up to a cell. The image tests now run as part of the tui test target, which was running none. --- bin/zml-smi/tui/lib/image.zig | 24 +++++++++++++++++------- bin/zml-smi/tui/lib/ui.zig | 4 +++- bin/zml-smi/tui/tui.zig | 4 ++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/bin/zml-smi/tui/lib/image.zig b/bin/zml-smi/tui/lib/image.zig index a365ac4d..18b6bf90 100644 --- a/bin/zml-smi/tui/lib/image.zig +++ b/bin/zml-smi/tui/lib/image.zig @@ -24,13 +24,7 @@ pub fn draw(self: *const Image, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surf }; } - // Compute cell columns from image aspect ratio and terminal cell pixel size - const cell_h: u32 = if (ctx.cell_size.height > 0) ctx.cell_size.height else 20; - const cell_w: u32 = if (ctx.cell_size.width > 0) ctx.cell_size.width else 10; - const height_px = @as(u32, target_rows) * cell_h; - const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(self.image.height)); - const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(self.image.width)) * scale_f); - const cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); + const cols = ui.imageCellSize(self.image, target_rows, ctx.cell_size).cols; const size: vxfw.Size = .{ .width = cols, .height = target_rows }; var surf = try vxfw.Surface.init(ctx.arena, ui.widget(self), size); @@ -79,6 +73,22 @@ test Image { try std.testing.expectEqual(@as(?u16, 20), size.cols); } +test "Image rounds its width to the nearest column" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + vxfw.DrawContext.init(.unicode); + + // One 17px row scales a square icon to 17px: closer to 2 columns of 8px than to 3. + const img: Image = .{ .image = .{ .id = 1, .width = 64, .height = 64 }, .rows = 1 }; + const surface = try img.draw(.{ + .arena = arena.allocator(), + .min = .{}, + .max = .{ .width = 40, .height = 10 }, + .cell_size = .{ .width = 8, .height = 17 }, + }); + try std.testing.expectEqual(@as(u16, 2), surface.size.width); +} + test "refAllDecls" { std.testing.refAllDecls(@This()); } diff --git a/bin/zml-smi/tui/lib/ui.zig b/bin/zml-smi/tui/lib/ui.zig index 6008d1e0..4a533903 100644 --- a/bin/zml-smi/tui/lib/ui.zig +++ b/bin/zml-smi/tui/lib/ui.zig @@ -76,12 +76,14 @@ pub const ImageCellSize = struct { /// Compute the cell dimensions for an image scaled to a target row count, /// maintaining aspect ratio based on the terminal's cell pixel size. +/// The terminal stretches the image over exactly these cells, so the width +/// is rounded to the nearest column: rounding up would widen the image. pub fn imageCellSize(img: vaxis.Image, target_rows: u16, cell_size: vxfw.Size) ImageCellSize { const cell_h: u32 = if (cell_size.height > 0) cell_size.height else 20; const cell_w: u32 = if (cell_size.width > 0) cell_size.width else 10; const height_px = @as(u32, target_rows) * cell_h; const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(img.height)); const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(img.width)) * scale_f); - const img_cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); + const img_cols: u16 = @intCast(@max(1, (width_px + cell_w / 2) / cell_w)); return .{ .cols = img_cols, .rows = target_rows }; } diff --git a/bin/zml-smi/tui/tui.zig b/bin/zml-smi/tui/tui.zig index 015e3ce2..0dce23b5 100644 --- a/bin/zml-smi/tui/tui.zig +++ b/bin/zml-smi/tui/tui.zig @@ -1,3 +1,7 @@ pub const data = @import("data.zig"); pub const print = @import("print.zig"); pub const top = @import("top.zig"); + +test { + _ = @import("lib/image.zig"); +} From 7ec18025f53183f5c139c774c2675299078df027 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 19:44:08 +0100 Subject: [PATCH 05/19] bin/zml-smi: delete the platform sources The platforms are rewritten against a new core, one file each. The new file names differ from the old ones only by case, which the case-insensitive checkouts mirrored to the build boxes cannot hold side by side. The ports read the old sources from history. --- bin/zml-smi/platforms/amdsmi/amdsmi.zig | 295 -------- bin/zml-smi/platforms/amdsmi/metrics.zig | 155 ---- bin/zml-smi/platforms/amdsmi/process.zig | 183 ----- bin/zml-smi/platforms/linux/BUILD.bazel | 18 - bin/zml-smi/platforms/linux/linux.zig | 2 - bin/zml-smi/platforms/linux/metrics.zig | 97 --- bin/zml-smi/platforms/linux/process.zig | 122 ---- bin/zml-smi/platforms/macos/BUILD.bazel | 17 - bin/zml-smi/platforms/macos/macos.zig | 2 - bin/zml-smi/platforms/macos/metrics.zig | 161 ----- bin/zml-smi/platforms/macos/process.zig | 30 - bin/zml-smi/platforms/neuron/metrics.zig | 201 ------ bin/zml-smi/platforms/neuron/nrt.zig | 230 ------ bin/zml-smi/platforms/neuron/process.zig | 108 --- bin/zml-smi/platforms/nvml/metrics.zig | 222 ------ bin/zml-smi/platforms/nvml/nvml.zig | 349 --------- bin/zml-smi/platforms/nvml/process.zig | 54 -- bin/zml-smi/platforms/oneapi/metrics.zig | 134 ---- bin/zml-smi/platforms/oneapi/monitor.zig | 877 ----------------------- bin/zml-smi/platforms/oneapi/oneapi.zig | 248 ------- bin/zml-smi/platforms/oneapi/process.zig | 45 -- bin/zml-smi/platforms/tpu/metrics.zig | 168 ----- bin/zml-smi/platforms/tpu/process.zig | 73 -- bin/zml-smi/platforms/tpu/tpuinfo.zig | 37 - 24 files changed, 3828 deletions(-) delete mode 100644 bin/zml-smi/platforms/amdsmi/amdsmi.zig delete mode 100644 bin/zml-smi/platforms/amdsmi/metrics.zig delete mode 100644 bin/zml-smi/platforms/amdsmi/process.zig delete mode 100644 bin/zml-smi/platforms/linux/BUILD.bazel delete mode 100644 bin/zml-smi/platforms/linux/linux.zig delete mode 100644 bin/zml-smi/platforms/linux/metrics.zig delete mode 100644 bin/zml-smi/platforms/linux/process.zig delete mode 100644 bin/zml-smi/platforms/macos/BUILD.bazel delete mode 100644 bin/zml-smi/platforms/macos/macos.zig delete mode 100644 bin/zml-smi/platforms/macos/metrics.zig delete mode 100644 bin/zml-smi/platforms/macos/process.zig delete mode 100644 bin/zml-smi/platforms/neuron/metrics.zig delete mode 100644 bin/zml-smi/platforms/neuron/nrt.zig delete mode 100644 bin/zml-smi/platforms/neuron/process.zig delete mode 100644 bin/zml-smi/platforms/nvml/metrics.zig delete mode 100644 bin/zml-smi/platforms/nvml/nvml.zig delete mode 100644 bin/zml-smi/platforms/nvml/process.zig delete mode 100644 bin/zml-smi/platforms/oneapi/metrics.zig delete mode 100644 bin/zml-smi/platforms/oneapi/monitor.zig delete mode 100644 bin/zml-smi/platforms/oneapi/oneapi.zig delete mode 100644 bin/zml-smi/platforms/oneapi/process.zig delete mode 100644 bin/zml-smi/platforms/tpu/metrics.zig delete mode 100644 bin/zml-smi/platforms/tpu/process.zig delete mode 100644 bin/zml-smi/platforms/tpu/tpuinfo.zig diff --git a/bin/zml-smi/platforms/amdsmi/amdsmi.zig b/bin/zml-smi/platforms/amdsmi/amdsmi.zig deleted file mode 100644 index 2d89bc7c..00000000 --- a/bin/zml-smi/platforms/amdsmi/amdsmi.zig +++ /dev/null @@ -1,295 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const stdx = @import("stdx"); -const DynLib = @import("zml-smi/dynlib"); -const sandbox = @import("zml-smi/sandbox"); - -const AmdSmi = @This(); - -pub const Error = ReturnError || error{AmdSmiUnavailable}; -pub const Handle = c.amdsmi_processor_handle; -pub const ProcInfo = c.amdsmi_proc_info_t; - -lib: Fns, -gpu_handles: []c.amdsmi_processor_handle, - -const Fns = struct { - amdsmi_init: *const @TypeOf(c.amdsmi_init), - amdsmi_get_socket_handles: *const @TypeOf(c.amdsmi_get_socket_handles), - amdsmi_get_processor_handles: *const @TypeOf(c.amdsmi_get_processor_handles), - amdsmi_get_gpu_asic_info: *const @TypeOf(c.amdsmi_get_gpu_asic_info), - amdsmi_get_gpu_metrics_info: *const @TypeOf(c.amdsmi_get_gpu_metrics_info), - amdsmi_get_power_info: *const @TypeOf(c.amdsmi_get_power_info), - amdsmi_get_temp_metric: *const @TypeOf(c.amdsmi_get_temp_metric), - amdsmi_get_gpu_fan_speed: *const @TypeOf(c.amdsmi_get_gpu_fan_speed), - amdsmi_get_gpu_activity: *const @TypeOf(c.amdsmi_get_gpu_activity), - amdsmi_get_clock_info: *const @TypeOf(c.amdsmi_get_clock_info), - amdsmi_get_gpu_memory_total: *const @TypeOf(c.amdsmi_get_gpu_memory_total), - amdsmi_get_gpu_memory_usage: *const @TypeOf(c.amdsmi_get_gpu_memory_usage), - amdsmi_get_pcie_info: *const @TypeOf(c.amdsmi_get_pcie_info), - amdsmi_get_gpu_bdf_id: *const @TypeOf(c.amdsmi_get_gpu_bdf_id), - amdsmi_get_gpu_process_list: *const @TypeOf(c.amdsmi_get_gpu_process_list), - amdsmi_get_gpu_driver_info: *const @TypeOf(c.amdsmi_get_gpu_driver_info), -}; - -pub fn init(allocator: std.mem.Allocator) !AmdSmi { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = sandbox.path(&path_buf) orelse return error.AmdSmiUnavailable; - - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libamd_smi.so.27" }); - - var dynlib: std.DynLib = .{ .inner = .{ - .handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - if (std.c.dlerror()) |err| std.log.err("amdsmi: dlopen: {s}", .{err}); - return error.AmdSmiUnavailable; - }, - } }; - const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.AmdSmiUnavailable; - - try check(fns.amdsmi_init(c.AMDSMI_INIT_AMD_GPUS)); - - var socket_count: u32 = 0; - try check(fns.amdsmi_get_socket_handles(&socket_count, null)); - - const sockets = try allocator.alloc(c.amdsmi_socket_handle, socket_count); - defer allocator.free(sockets); - try check(fns.amdsmi_get_socket_handles(&socket_count, @ptrCast(sockets.ptr))); - - var handle_list: std.ArrayList(c.amdsmi_processor_handle) = .empty; - defer handle_list.deinit(allocator); - - for (sockets[0..socket_count]) |socket| { - var proc_count: u32 = 0; - try check(fns.amdsmi_get_processor_handles(socket, &proc_count, null)); - - const procs = try allocator.alloc(c.amdsmi_processor_handle, proc_count); - defer allocator.free(procs); - - try check(fns.amdsmi_get_processor_handles(socket, &proc_count, @ptrCast(procs.ptr))); - - for (procs[0..proc_count]) |proc| { - try handle_list.append(allocator, proc); - } - } - - return .{ .lib = fns, .gpu_handles = try handle_list.toOwnedSlice(allocator) }; -} - -pub fn handleByIndex(self: AmdSmi, device_id: u32) Error!Handle { - if (device_id >= self.gpu_handles.len) { - return error.not_found; - } - return self.gpu_handles[device_id]; -} - -pub fn deviceCount(self: AmdSmi) u32 { - return @intCast(self.gpu_handles.len); -} - -pub const name_buf_len = c.AMDSMI_MAX_STRING_LENGTH; - -pub fn name(self: AmdSmi, handle: Handle, buf: *[c.AMDSMI_MAX_STRING_LENGTH]u8) Error![:0]const u8 { - var info: c.amdsmi_asic_info_t = undefined; - try check(self.lib.amdsmi_get_gpu_asic_info(handle, &info)); - @memcpy(buf, &info.market_name); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn powerUsage(self: AmdSmi, handle: Handle) Error!u32 { - var metrics: c.amdsmi_gpu_metrics_t = undefined; - try check(self.lib.amdsmi_get_gpu_metrics_info(handle, &metrics)); - const unsupported16 = std.math.maxInt(u16); - if (metrics.current_socket_power != unsupported16) { - return metrics.current_socket_power; - } - if (metrics.average_socket_power != unsupported16) { - return metrics.average_socket_power; - } - return error.not_supported; -} - -pub fn powerLimit(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_power_info_t = undefined; - try check(self.lib.amdsmi_get_power_info(handle, &info)); - return info.power_limit; -} - -pub fn temperature(self: AmdSmi, handle: Handle) Error!i64 { - var temp: i64 = 0; - try check(self.lib.amdsmi_get_temp_metric(handle, c.AMDSMI_TEMPERATURE_TYPE_EDGE, c.AMDSMI_TEMP_CURRENT, &temp)); - return temp; -} - -pub fn fanSpeed(self: AmdSmi, handle: Handle) Error!i64 { - var speed: i64 = 0; - try check(self.lib.amdsmi_get_gpu_fan_speed(handle, @as(u32, 0), &speed)); - return speed; -} - -pub fn gpuUtil(self: AmdSmi, handle: Handle) Error!u32 { - var usage: c.amdsmi_engine_usage_t = undefined; - try check(self.lib.amdsmi_get_gpu_activity(handle, &usage)); - return usage.gfx_activity; -} - -pub fn clockGraphics(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); - return info.clk; -} - -pub fn maxClockGraphics(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); - return info.max_clk; -} - -pub fn clockMem(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); - return info.clk; -} - -pub fn maxClockMem(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); - return info.max_clk; -} - -pub fn clockSoc(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SOC, &info)); - return info.clk; -} - -pub fn memTotal(self: AmdSmi, handle: Handle) Error!u64 { - var total: u64 = 0; - try check(self.lib.amdsmi_get_gpu_memory_total(handle, c.AMDSMI_MEM_TYPE_VRAM, &total)); - return total; -} - -pub fn memUsed(self: AmdSmi, handle: Handle) Error!u64 { - var used: u64 = 0; - try check(self.lib.amdsmi_get_gpu_memory_usage(handle, c.AMDSMI_MEM_TYPE_VRAM, &used)); - return used; -} - -pub fn pcieWidth(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_pcie_info_t = undefined; - try check(self.lib.amdsmi_get_pcie_info(handle, &info)); - return @intCast(info.pcie_metric.pcie_width); -} - -pub fn pcieBandwidth(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_pcie_info_t = undefined; - try check(self.lib.amdsmi_get_pcie_info(handle, &info)); - return info.pcie_metric.pcie_bandwidth; -} - -pub fn pcieLinkGen(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_pcie_info_t = undefined; - try check(self.lib.amdsmi_get_pcie_info(handle, &info)); - return info.pcie_static.pcie_interface_version; -} - -pub fn bdfId(self: AmdSmi, handle: Handle) Error!u64 { - var bdf_id: u64 = 0; - try check(self.lib.amdsmi_get_gpu_bdf_id(handle, &bdf_id)); - return bdf_id; -} - -pub const driver_version_buf_len = @typeInfo(@TypeOf(@as(c.amdsmi_driver_info_t, undefined).driver_version)).array.len; - -pub fn driverVersion(self: AmdSmi, handle: Handle, buf: *[driver_version_buf_len]u8) Error![:0]const u8 { - var info: c.amdsmi_driver_info_t = std.mem.zeroes(c.amdsmi_driver_info_t); - try check(self.lib.amdsmi_get_gpu_driver_info(handle, &info)); - @memcpy(buf, &info.driver_version); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn processList(self: AmdSmi, allocator: std.mem.Allocator, handle: Handle) (Error || error{OutOfMemory})![]const ProcInfo { - var count: u32 = 0; - try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, null)); - if (count == 0) { - return &.{}; - } - const procs = try allocator.alloc(ProcInfo, count); - errdefer allocator.free(procs); - try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, @ptrCast(procs.ptr))); - return procs; -} - -// Error handling - -pub const ReturnError = error{ - inval, - not_supported, - not_yet_implemented, - fail_load_module, - fail_load_symbol, - drm_error, - api_failed, - timeout, - retry, - no_perm, - interrupt, - io_error, - address_fault, - file_error, - out_of_resources, - internal_exception, - input_out_of_bounds, - init_error, - refcount_overflow, - directory_not_found, - busy, - not_found, - not_init, - no_slot, - driver_not_loaded, - no_data, - insufficient_size, - unexpected_size, - unexpected_data, - unknown_error, -}; - -fn check(ret: c.amdsmi_status_t) Error!void { - if (ret == c.AMDSMI_STATUS_SUCCESS) { - return; - } - - return switch (ret) { - c.AMDSMI_STATUS_INVAL => error.inval, - c.AMDSMI_STATUS_NOT_SUPPORTED => error.not_supported, - c.AMDSMI_STATUS_NOT_YET_IMPLEMENTED => error.not_yet_implemented, - c.AMDSMI_STATUS_FAIL_LOAD_MODULE => error.fail_load_module, - c.AMDSMI_STATUS_FAIL_LOAD_SYMBOL => error.fail_load_symbol, - c.AMDSMI_STATUS_DRM_ERROR => error.drm_error, - c.AMDSMI_STATUS_API_FAILED => error.api_failed, - c.AMDSMI_STATUS_TIMEOUT => error.timeout, - c.AMDSMI_STATUS_RETRY => error.retry, - c.AMDSMI_STATUS_NO_PERM => error.no_perm, - c.AMDSMI_STATUS_INTERRUPT => error.interrupt, - c.AMDSMI_STATUS_IO => error.io_error, - c.AMDSMI_STATUS_ADDRESS_FAULT => error.address_fault, - c.AMDSMI_STATUS_FILE_ERROR => error.file_error, - c.AMDSMI_STATUS_OUT_OF_RESOURCES => error.out_of_resources, - c.AMDSMI_STATUS_INTERNAL_EXCEPTION => error.internal_exception, - c.AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS => error.input_out_of_bounds, - c.AMDSMI_STATUS_INIT_ERROR => error.init_error, - c.AMDSMI_STATUS_REFCOUNT_OVERFLOW => error.refcount_overflow, - c.AMDSMI_STATUS_DIRECTORY_NOT_FOUND => error.directory_not_found, - c.AMDSMI_STATUS_BUSY => error.busy, - c.AMDSMI_STATUS_NOT_FOUND => error.not_found, - c.AMDSMI_STATUS_NOT_INIT => error.not_init, - c.AMDSMI_STATUS_NO_SLOT => error.no_slot, - c.AMDSMI_STATUS_DRIVER_NOT_LOADED => error.driver_not_loaded, - c.AMDSMI_STATUS_NO_DATA => error.no_data, - c.AMDSMI_STATUS_INSUFFICIENT_SIZE => error.insufficient_size, - c.AMDSMI_STATUS_UNEXPECTED_SIZE => error.unexpected_size, - c.AMDSMI_STATUS_UNEXPECTED_DATA => error.unexpected_data, - else => error.unknown_error, - }; -} diff --git a/bin/zml-smi/platforms/amdsmi/metrics.zig b/bin/zml-smi/platforms/amdsmi/metrics.zig deleted file mode 100644 index b443230c..00000000 --- a/bin/zml-smi/platforms/amdsmi/metrics.zig +++ /dev/null @@ -1,155 +0,0 @@ -const std = @import("std"); -const AmdSmi = @import("amdsmi.zig"); -const device_info = @import("zml-smi/info").device_info; -const DeviceInfo = device_info.DeviceInfo; -const GpuInfo = device_info.GpuInfo; -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; -const Collector = @import("zml-smi/collector").Collector; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const process = @import("process.zig"); - -pub fn start(collector: *Collector) !void { - const amdsmi = try collector.arena.create(AmdSmi); - amdsmi.* = try AmdSmi.init(collector.arena); - - const count = amdsmi.deviceCount(); - const dev_offset: u16 = @intCast(collector.device_infos.items.len); - - for (0..count) |i| { - const dev = Device.open(amdsmi, @intCast(i)) catch continue; - const initial: GpuInfo = .{ - .name = dev.name(collector.arena) catch null, - .driver_version = dev.driverVersion(collector.arena) catch null, - }; - const info = try collector.addDevice(.{ .rocm = .{ .values = .{ initial, initial } } }); - try collector.spawnPoll(pollOnce, .{ null, &info.rocm, dev }); - } - - const processes = try collector.createProcessList(); - try process.init(collector, processes, amdsmi, dev_offset); -} - -const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); - -const Device = struct { - amdsmi: *const AmdSmi, - handle: AmdSmi.Handle, - - pub fn open(amdsmi: *const AmdSmi, index: u32) !Device { - return .{ .amdsmi = amdsmi, .handle = try amdsmi.handleByIndex(index) }; - } - - fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [AmdSmi.name_buf_len]u8 = undefined; - const slice = try self.amdsmi.name(self.handle, &buf); - - return try arena.dupe(u8, slice); - } - - fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [AmdSmi.driver_version_buf_len]u8 = undefined; - const slice = try self.amdsmi.driverVersion(self.handle, &buf); - return try arena.dupe(u8, slice); - } - - // Power - pub fn powerUsage(self: Device) !u64 { - const pw = try self.amdsmi.powerUsage(self.handle); - return @as(u64, pw) * 1000; - } - - pub fn powerLimit(self: Device) !u64 { - const limit = try self.amdsmi.powerLimit(self.handle); - if (limit == std.math.maxInt(u32)) { - return error.not_supported; - } - - return @as(u64, limit) / 1000; - } - - // Thermal - pub fn temperature(self: Device) !u64 { - const temp = try self.amdsmi.temperature(self.handle); - return @intCast(temp); - } - - pub fn fanSpeed(self: Device) !u64 { - const speed = try self.amdsmi.fanSpeed(self.handle); - return @intCast(@divTrunc(speed * 100, 255)); - } - - // Utilization - pub fn gpuUtil(self: Device) !u64 { - return try self.amdsmi.gpuUtil(self.handle); - } - - // Clocks - pub fn clockGraphics(self: Device) !u64 { - return try self.amdsmi.clockGraphics(self.handle); - } - - pub fn clockSoc(self: Device) !u64 { - return try self.amdsmi.clockSoc(self.handle); - } - - pub fn clockMem(self: Device) !u64 { - return try self.amdsmi.clockMem(self.handle); - } - - pub fn maxClockGraphics(self: Device) !u64 { - return try self.amdsmi.maxClockGraphics(self.handle); - } - - pub fn maxClockMem(self: Device) !u64 { - return try self.amdsmi.maxClockMem(self.handle); - } - - // Memory - pub fn memUsed(self: Device) !u64 { - return self.amdsmi.memUsed(self.handle); - } - - pub fn memTotal(self: Device) !u64 { - return self.amdsmi.memTotal(self.handle); - } - - // PCIe - pub fn pcieBandwidth(self: Device) !u64 { - const bw = try self.amdsmi.pcieBandwidth(self.handle); - if (bw == std.math.maxInt(u32)) { - return error.not_supported; - } - - return bw; - } - - pub fn pcieLinkGen(self: Device) !u64 { - return try self.amdsmi.pcieLinkGen(self.handle); - } - - pub fn pcieLinkWidth(self: Device) !u64 { - return try self.amdsmi.pcieWidth(self.handle); - } -}; - -const metrics = .{ - .{ .field = "power_mw", .query = Device.powerUsage }, - .{ .field = "power_limit_mw", .query = Device.powerLimit }, - .{ .field = "temperature", .query = Device.temperature }, - .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, - .{ .field = "util_percent", .query = Device.gpuUtil }, - .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, - .{ .field = "clock_soc_mhz", .query = Device.clockSoc }, - .{ .field = "clock_mem_mhz", .query = Device.clockMem }, - .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, - .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, - .{ .field = "mem_used_bytes", .query = Device.memUsed }, - .{ .field = "mem_total_bytes", .query = Device.memTotal }, - .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, - .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, - .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, -}; - -test { - _ = process; -} diff --git a/bin/zml-smi/platforms/amdsmi/process.zig b/bin/zml-smi/platforms/amdsmi/process.zig deleted file mode 100644 index df687eb0..00000000 --- a/bin/zml-smi/platforms/amdsmi/process.zig +++ /dev/null @@ -1,183 +0,0 @@ -const std = @import("std"); -const c = std.c; -const AmdSmi = @import("amdsmi.zig"); -const pi = @import("zml-smi/info").process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; - -const bdf_len = "0000:00:00.0".len; - -/// amdsmi reports the time a process spent on the engines as a running total, -/// so utilization is how much it grew between two polls. -const Sample = struct { - engine_ns: u64, - timestamp_ns: u64, - - fn utilSince(self: Sample, previous: Sample) ?u16 { - if (self.timestamp_ns <= previous.timestamp_ns or self.engine_ns < previous.engine_ns) return null; - return @intCast(@min(100, (self.engine_ns - previous.engine_ns) * 100 / (self.timestamp_ns - previous.timestamp_ns))); - } -}; - -/// Last sample of each process on each device. -const Samples = std.AutoHashMapUnmanaged(Key, Sample); -const Key = struct { pid: u32, device: u32 }; - -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16) !void { - const device_count = amdsmi.deviceCount(); - const pci_slots = try collector.arena.alloc(?[bdf_len]u8, device_count); - - for (pci_slots, 0..) |*slot, i| { - const handle = amdsmi.handleByIndex(@intCast(i)) catch { - slot.* = null; - continue; - }; - - slot.* = formatBdf(amdsmi.bdfId(handle) catch { - slot.* = null; - continue; - }); - } - - const samples = try collector.arena.create([2]Samples); - samples.* = .{ .empty, .empty }; - - const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.gpa, collector.arena, collector.io, list, amdsmi, dev_offset, device_count, pci_slots, samples }; - // A one-shot run polls once: sample beforehand so that poll has a utilization to report. - if (collector.poll_only) { - @call(.auto, pollOnce, args); - collector.io.sleep(.fromMilliseconds(collector.worker.poll_interval_ms), .awake) catch {}; - } - - try collector.spawnPoll(pollOnce, args); -} - -fn pollOnce(allocator: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16, device_count: u32, pci_slots: []?[bdf_len]u8, samples: *[2]Samples) void { - const back = list.back(); - const previous, const current = .{ &samples[0], &samples[1] }; - - back.clearRetainingCapacity(); - current.clearRetainingCapacity(); - - for (0..device_count) |dev_idx| { - const handle = amdsmi.handleByIndex(@intCast(dev_idx)) catch continue; - - const procs = blk: { - const saved = c.dup(c.STDERR_FILENO); - const devnull = c.open("/dev/null", .{ .ACCMODE = .WRONLY }); - - if (devnull >= 0) { - if (saved >= 0) { - _ = c.dup2(devnull, c.STDERR_FILENO); - } - - _ = c.close(devnull); - } - - defer if (saved >= 0) { - _ = c.dup2(saved, c.STDERR_FILENO); - _ = c.close(saved); - }; - - break :blk amdsmi.processList(allocator, handle) catch continue; - }; - defer { - if (procs.len > 0) { - allocator.free(procs); - } - } - const timestamp_ns: u64 = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); - - const pci_slot = if (dev_idx < pci_slots.len) &(pci_slots[dev_idx] orelse continue) else continue; - - for (procs) |proc| { - const key: Key = .{ .pid = proc.pid, .device = @intCast(dev_idx) }; - const sample: Sample = .{ .engine_ns = proc.engine_usage.gfx +| proc.engine_usage.enc, .timestamp_ns = timestamp_ns }; - current.put(arena, key, sample) catch {}; - - back.append(allocator, .{ - .pid = proc.pid, - .device_idx = @intCast(dev_idx + dev_offset), - .dev_util_percent = if (previous.get(key)) |p| sample.utilSince(p) else null, - .dev_mem_kib = readProcessGpuVram(io, proc.pid, pci_slot), - }) catch break; - } - } - - std.mem.swap(Samples, previous, current); - list.swap(); -} - -/// Reads per-GPU VRAM from /proc//fdinfo/, filtered by PCI slot. -/// amdsmi reports global VRAM across all GPUs; fdinfo gives per-GPU values. -fn readProcessGpuVram(io: std.Io, pid: u32, pci_slot: *const [bdf_len]u8) u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const dir_path = std.fmt.bufPrint(&path_buf, "/proc/{d}/fdinfo", .{pid}) catch return 0; - - var dir = std.Io.Dir.openDirAbsolute(io, dir_path, .{ .iterate = true }) catch return 0; - defer dir.close(io); - - var total_vram_kib: u64 = 0; - var it = dir.iterate(); - while (it.next(io) catch null) |entry| { - var file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const file_path = std.fmt.bufPrint(&file_buf, "{s}/{s}", .{ dir_path, entry.name }) catch continue; - - total_vram_kib += parseFdinfoVram(io, file_path, pci_slot); - } - - return total_vram_kib; -} - -fn parseFdinfoVram(io: std.Io, path: []const u8, pci_slot: *const [bdf_len]u8) u64 { - var file = std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }) catch return 0; - defer file.close(io); - - var read_buf: [4096]u8 = undefined; - var reader = file.reader(io, &read_buf); - - var found_pci = false; - var vram_kib: ?u64 = null; - - while (true) { - const line = reader.interface.takeDelimiterExclusive('\n') catch break; - reader.interface.toss(1); - - if (std.mem.indexOf(u8, line, pci_slot) != null) found_pci = true; - - // Support both new and old fdinfo keys (see amdgpu_fdinfo.c) - inline for ([_][]const u8{ "drm-memory-vram:\t", "vram mem:\t" }) |key| { - if (std.mem.startsWith(u8, line, key)) { - const rest = line[key.len..]; - const end = std.mem.indexOfAny(u8, rest, " \t\n") orelse rest.len; - vram_kib = std.fmt.parseInt(u64, rest[0..end], 10) catch 0; - } - } - - if (found_pci and vram_kib != null) return vram_kib.?; - } - - return 0; -} - -fn formatBdf(bdf_id: u64) [bdf_len]u8 { - var buf: [bdf_len]u8 = undefined; - _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ - (bdf_id >> 16) & 0xFFFF, - (bdf_id >> 8) & 0xFF, - (bdf_id >> 3) & 0x1F, - bdf_id & 0x7, - }) catch unreachable; - - return buf; -} - -test "utilSince is the engine time growth over the elapsed time" { - const previous: Sample = .{ .engine_ns = 100, .timestamp_ns = 1000 }; - - try std.testing.expectEqual(50, (Sample{ .engine_ns = 600, .timestamp_ns = 2000 }).utilSince(previous)); - // gfx and enc can both be busy - try std.testing.expectEqual(100, (Sample{ .engine_ns = 2100, .timestamp_ns = 2000 }).utilSince(previous)); - // the pid went to another process - try std.testing.expectEqual(null, (Sample{ .engine_ns = 10, .timestamp_ns = 2000 }).utilSince(previous)); -} diff --git a/bin/zml-smi/platforms/linux/BUILD.bazel b/bin/zml-smi/platforms/linux/BUILD.bazel deleted file mode 100644 index 38cbdab8..00000000 --- a/bin/zml-smi/platforms/linux/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "linux", - import_name = "zml-smi/platforms/linux", - main = "linux.zig", - srcs = [ - "metrics.zig", - "process.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:sysfs", - ], -) diff --git a/bin/zml-smi/platforms/linux/linux.zig b/bin/zml-smi/platforms/linux/linux.zig deleted file mode 100644 index ffab3ec4..00000000 --- a/bin/zml-smi/platforms/linux/linux.zig +++ /dev/null @@ -1,2 +0,0 @@ -pub const metrics = @import("metrics.zig"); -pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/linux/metrics.zig b/bin/zml-smi/platforms/linux/metrics.zig deleted file mode 100644 index 5a69663a..00000000 --- a/bin/zml-smi/platforms/linux/metrics.zig +++ /dev/null @@ -1,97 +0,0 @@ -const std = @import("std"); -const sysfs = @import("zml-smi/sysfs"); -const host_info = @import("zml-smi/info").host_info; -const HostInfo = host_info.HostInfo; -const HostData = host_info.HostData; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const Collector = @import("zml-smi/collector").Collector; - -pub fn init(collector: *Collector, info: *HostInfo) !void { - const poll_arena = try collector.createPollArena(); - const host: Host = .{ .io = collector.io, .arena = poll_arena }; - - // read once metrics - var initial = info.front().*; - initial.hostname = host.hostname(collector.arena) catch null; - initial.kernel = host.kernel(collector.arena) catch null; - initial.cpu_name = host.cpuName(collector.arena) catch null; - initial.cpu_cores = host.cpuCores() catch null; - initial.mem_total_kib = host.memTotal() catch null; - info.back().* = initial; - info.swap(); - - try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); -} - -const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); - -const Host = struct { - io: std.Io, - arena: *std.heap.ArenaAllocator, - - pub fn hostname(self: Host, allocator: std.mem.Allocator) ![]const u8 { - return sysfs.readString(allocator, self.io, "/proc/sys/kernel/hostname"); - } - - pub fn kernel(self: Host, allocator: std.mem.Allocator) ![]const u8 { - return sysfs.readString(allocator, self.io, "/proc/sys/kernel/osrelease"); - } - - pub fn cpuName(self: Host, allocator: std.mem.Allocator) ![]const u8 { - return sysfs.readFieldString(allocator, self.io, "/proc/cpuinfo", "model name"); - } - - pub fn cpuCores(self: Host) !u64 { - const data = try sysfs.readString(self.arena.allocator(), self.io, "/sys/devices/system/cpu/present"); - const dash = std.mem.indexOfScalar(u8, data, '-') orelse return 1; - - return (try std.fmt.parseInt(u64, data[dash + 1 ..], 10)) + 1; - } - - pub fn memTotal(self: Host) !u64 { - return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemTotal"); - } - - pub fn memAvailable(self: Host) !u64 { - return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemAvailable"); - } - - fn parseLoadAvg(self: Host, comptime n: usize) !f32 { - const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/loadavg"); - var iter = std.mem.splitScalar(u8, data, ' '); - - var i: usize = 0; - while (iter.next()) |tok| : (i += 1) { - if (i == n) return std.fmt.parseFloat(f32, tok); - } - - return error.NotFound; - } - - pub fn load1(self: Host) !f32 { - return self.parseLoadAvg(0); - } - - pub fn load5(self: Host) !f32 { - return self.parseLoadAvg(1); - } - - pub fn load15(self: Host) !f32 { - return self.parseLoadAvg(2); - } - - pub fn uptime(self: Host) !u64 { - const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/uptime"); - const dot = std.mem.indexOfScalar(u8, data, '.') orelse data.len; - - return std.fmt.parseInt(u64, data[0..dot], 10); - } -}; - -const metrics = .{ - .{ .field = "mem_available_kib", .query = Host.memAvailable }, - .{ .field = "load_1", .query = Host.load1 }, - .{ .field = "load_5", .query = Host.load5 }, - .{ .field = "load_15", .query = Host.load15 }, - .{ .field = "uptime_seconds", .query = Host.uptime }, -}; diff --git a/bin/zml-smi/platforms/linux/process.zig b/bin/zml-smi/platforms/linux/process.zig deleted file mode 100644 index fcbd6d7d..00000000 --- a/bin/zml-smi/platforms/linux/process.zig +++ /dev/null @@ -1,122 +0,0 @@ -const std = @import("std"); -const sysfs = @import("zml-smi/sysfs"); -const pi = @import("zml-smi/info").process_info; -const Collector = @import("zml-smi/collector").Collector; - -const ProcessInfo = pi.ProcessInfo; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(ProcessInfo)); - -/// Merges every process list into `collector.processes`, completed with what -/// the host knows about each local process. -pub fn start(collector: *Collector) !void { - const enricher = try collector.arena.create(ProcessEnricher); - enricher.* = .{ .arenas = .{ try collector.createPollArena(), try collector.createPollArena() } }; - try collector.spawnPoll(pollOnce, .{ enricher, collector.gpa, collector.io, collector.process_lists.items, &collector.processes }); -} - -fn pollOnce(enricher: *ProcessEnricher, gpa: std.mem.Allocator, io: std.Io, lists: []const *const ProcessDoubleBuffer, processes: *ProcessDoubleBuffer) void { - // Each side of the double buffer owns its strings, in the arena of the same index. - const arena = enricher.arenas[1 - processes.current.load(.acquire)]; - _ = arena.reset(.retain_capacity); - - const back = processes.back(); - back.clearRetainingCapacity(); - for (lists) |list| { - back.appendSlice(gpa, list.front().items) catch break; - } - - enricher.enrich(arena.allocator(), io, back.items); - processes.swap(); -} - -const ProcessEnricher = struct { - arenas: [2]*std.heap.ArenaAllocator, - /// Lives in the arena of the previous poll, which is reset one poll later. - prev_ticks: std.AutoHashMapUnmanaged(u32, u64) = .empty, - prev_total_ticks: u64 = 0, - - fn enrich(self: *ProcessEnricher, alloc: std.mem.Allocator, io: std.Io, procs: []ProcessInfo) void { - const curr_total = readTotalCpuTicks(alloc, io); - const delta_total = curr_total -| self.prev_total_ticks; - - var curr_ticks: std.AutoHashMapUnmanaged(u32, u64) = .empty; - curr_ticks.ensureTotalCapacity(alloc, @intCast(procs.len)) catch {}; - - for (procs) |*info| { - if (info.remote) { - // Already enriched by its host: only the strings are taken over. - info.username = alloc.dupe(u8, info.username) catch ""; - info.comm = alloc.dupe(u8, info.comm) catch ""; - continue; - } - // Read Uid, VmRSS from /proc/{pid}/status - const status_path = std.fmt.allocPrint(alloc, "/proc/{d}/status", .{info.pid}) catch continue; - const uid: u32 = @intCast(sysfs.readFieldInt(alloc, io, status_path, "Uid") catch 0); - info.uid = uid; - info.username = lookupUsername(alloc, uid) catch continue; - info.rss_kib = sysfs.readFieldInt(alloc, io, status_path, "VmRSS") catch 0; - - // Read cmdline - const cmdline_path = std.fmt.allocPrint(alloc, "/proc/{d}/cmdline", .{info.pid}) catch continue; - info.comm = readCmdline(alloc, io, cmdline_path) catch continue; - - // CPU% delta - const ticks = readProcTicks(alloc, io, info.pid) orelse continue; - if (delta_total > 0) { - if (self.prev_ticks.get(info.pid)) |prev| { - const delta_proc = ticks -| prev; - info.cpu_percent = @intCast(@min(delta_proc *| 1000 / delta_total, std.math.maxInt(u16))); - } - } - - curr_ticks.put(alloc, info.pid, ticks) catch continue; - } - - self.prev_ticks = curr_ticks; - self.prev_total_ticks = curr_total; - } -}; - -fn lookupUsername(allocator: std.mem.Allocator, uid: u32) ![]const u8 { - const pw = std.c.getpwuid(uid) orelse return error.NoUser; - const name = std.mem.span(pw.name orelse return error.NoName); - - return try allocator.dupe(u8, name); -} - -/// Extract utime + stime from /proc/{pid}/stat. -fn readProcTicks(allocator: std.mem.Allocator, io: std.Io, pid: u32) ?u64 { - const path = std.fmt.allocPrint(allocator, "/proc/{d}/stat", .{pid}) catch return null; - const data = sysfs.readFirstLine(allocator, io, path) catch return null; - - // Fields after ") ": state=0, ppid=1, ..., utime=11, stime=12 - const rest = data[(std.mem.lastIndexOfScalar(u8, data, ')') orelse return null) + 2 ..]; - - return sysfs.nthTokenInt(rest, 11) + sysfs.nthTokenInt(rest, 12); -} - -fn readCmdline(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { - const data = try sysfs.readFirstLine(allocator, io, path); - - for (data) |*c| { - if (c.* == 0) { - c.* = ' '; - } - } - - return std.mem.trimEnd(u8, data, &std.ascii.whitespace); -} - -fn readTotalCpuTicks(allocator: std.mem.Allocator, io: std.Io) u64 { - const data = sysfs.readFirstLine(allocator, io, "/proc/stat") catch return 0; - - var total: u64 = 0; - var iter = std.mem.tokenizeAny(u8, data, " \t"); - - _ = iter.next(); // skip "cpu" - while (iter.next()) |tok| { - total += std.fmt.parseInt(u64, tok, 10) catch 0; - } - - return total; -} diff --git a/bin/zml-smi/platforms/macos/BUILD.bazel b/bin/zml-smi/platforms/macos/BUILD.bazel deleted file mode 100644 index 57d4709c..00000000 --- a/bin/zml-smi/platforms/macos/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "macos", - import_name = "zml-smi/platforms/macos", - main = "macos.zig", - srcs = [ - "metrics.zig", - "process.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - ], -) diff --git a/bin/zml-smi/platforms/macos/macos.zig b/bin/zml-smi/platforms/macos/macos.zig deleted file mode 100644 index ffab3ec4..00000000 --- a/bin/zml-smi/platforms/macos/macos.zig +++ /dev/null @@ -1,2 +0,0 @@ -pub const metrics = @import("metrics.zig"); -pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/macos/metrics.zig b/bin/zml-smi/platforms/macos/metrics.zig deleted file mode 100644 index 616168cd..00000000 --- a/bin/zml-smi/platforms/macos/metrics.zig +++ /dev/null @@ -1,161 +0,0 @@ -const std = @import("std"); -const host_info = @import("zml-smi/info").host_info; -const HostInfo = host_info.HostInfo; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const Collector = @import("zml-smi/collector").Collector; - -const c = std.c; - -const vm_statistics64 = extern struct { - free_count: c_uint, - active_count: c_uint, - inactive_count: c_uint, - wire_count: c_uint, - zero_fill_count: u64, - reactivations: u64, - pageins: u64, - pageouts: u64, - faults: u64, - cow_faults: u64, - lookups: u64, - hits: u64, - purges: u64, - purgeable_count: c_uint, - speculative_count: c_uint, - decompressions: u64, - compressions: u64, - swapins: u64, - swapouts: u64, - compressor_page_count: c_uint, - throttled_count: c_uint, - external_page_count: c_uint, - internal_page_count: c_uint, - total_uncompressed_pages_in_compressor: u64, -}; - -const HOST_VM_INFO64: c_int = 4; -const HOST_VM_INFO64_COUNT: c_uint = @sizeOf(vm_statistics64) / @sizeOf(c_int); -const KERN_SUCCESS: c_int = 0; - -extern "c" fn host_statistics64(host: c.mach_port_t, flavor: c_int, info: *anyopaque, count: *c_uint) c.kern_return_t; -extern "c" fn getloadavg(loadavg: *[3]f64, nelem: c_int) c_int; - -pub fn init(collector: *Collector, info: *HostInfo) !void { - const poll_arena = try collector.createPollArena(); - const host: Host = .{}; - - var initial = info.front().*; - initial.hostname = sysctlString(collector.arena, "kern.hostname") catch null; - initial.kernel = sysctlString(collector.arena, "kern.osrelease") catch null; - initial.cpu_name = sysctlString(collector.arena, "machdep.cpu.brand_string") catch null; - initial.cpu_cores = host.cpuCores() catch null; - initial.mem_total_kib = host.memTotal() catch null; - info.back().* = initial; - info.swap(); - - try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); -} - -const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); - -const Host = struct { - pub fn cpuCores(_: Host) !u64 { - return @intCast(try sysctlU32("hw.logicalcpu")); - } - - pub fn memTotal(_: Host) !u64 { - return (try sysctlU64("hw.memsize")) / 1024; - } - - pub fn memAvailable(_: Host) !u64 { - var vm_stat: vm_statistics64 = undefined; - var count: c_uint = HOST_VM_INFO64_COUNT; - - if (host_statistics64(c.mach_host_self(), HOST_VM_INFO64, &vm_stat, &count) != KERN_SUCCESS) { - return error.HostStatsFailed; - } - - var page_size: u64 = 0; - if (c._host_page_size(c.mach_host_self(), &page_size) != KERN_SUCCESS) { - return error.PageSizeFailed; - } - - const available_pages: u64 = @as(u64, vm_stat.free_count) + - @as(u64, vm_stat.inactive_count) + - @as(u64, vm_stat.purgeable_count); - - return (available_pages * page_size) / 1024; - } - - fn loadAvg() ![3]f64 { - var avg: [3]f64 = undefined; - if (getloadavg(&avg, 3) != 3) { - return error.LoadAvgFailed; - } - - return avg; - } - - pub fn load1(_: Host) !f32 { - return @floatCast((try loadAvg())[0]); - } - - pub fn load5(_: Host) !f32 { - return @floatCast((try loadAvg())[1]); - } - - pub fn load15(_: Host) !f32 { - return @floatCast((try loadAvg())[2]); - } - - pub fn uptime(_: Host) !u64 { - var boottime: c.timeval = undefined; - var len: usize = @sizeOf(c.timeval); - if (c.sysctlbyname("kern.boottime", std.mem.asBytes(&boottime), &len, null, 0) != 0) { - return error.SysctlFailed; - } - - var now: c.timeval = undefined; - _ = c.gettimeofday(&now, null); - - return @intCast(now.sec - boottime.sec); - } -}; - -fn sysctlString(allocator: std.mem.Allocator, name: [*:0]const u8) ![]const u8 { - var buf: [256]u8 = undefined; - var len: usize = buf.len; - if (c.sysctlbyname(name, &buf, &len, null, 0) != 0) { - return error.SysctlFailed; - } - - return allocator.dupe(u8, std.mem.span(@as([*:0]const u8, @ptrCast(buf[0..len])))); -} - -fn sysctlU64(name: [*:0]const u8) !u64 { - var value: u64 = 0; - var len: usize = @sizeOf(u64); - if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { - return error.SysctlFailed; - } - - return value; -} - -fn sysctlU32(name: [*:0]const u8) !u32 { - var value: u32 = 0; - var len: usize = @sizeOf(u32); - if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { - return error.SysctlFailed; - } - - return value; -} - -const metrics = .{ - .{ .field = "mem_available_kib", .query = Host.memAvailable }, - .{ .field = "load_1", .query = Host.load1 }, - .{ .field = "load_5", .query = Host.load5 }, - .{ .field = "load_15", .query = Host.load15 }, - .{ .field = "uptime_seconds", .query = Host.uptime }, -}; diff --git a/bin/zml-smi/platforms/macos/process.zig b/bin/zml-smi/platforms/macos/process.zig deleted file mode 100644 index fd9c2fc4..00000000 --- a/bin/zml-smi/platforms/macos/process.zig +++ /dev/null @@ -1,30 +0,0 @@ -const std = @import("std"); -const pi = @import("zml-smi/info").process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; - -/// Merges every process list into `collector.processes`. No backend runs on macOS: -/// the processes all come from remotes, already enriched by their host. -pub fn start(collector: *Collector) !void { - const arenas: [2]*std.heap.ArenaAllocator = .{ try collector.createPollArena(), try collector.createPollArena() }; - try collector.spawnPoll(pollOnce, .{ arenas, collector.gpa, collector.process_lists.items, &collector.processes }); -} - -fn pollOnce(arenas: [2]*std.heap.ArenaAllocator, gpa: std.mem.Allocator, lists: []const *const ProcessDoubleBuffer, processes: *ProcessDoubleBuffer) void { - // Each side of the double buffer owns its strings, in the arena of the same index. - const arena = arenas[1 - processes.current.load(.acquire)]; - _ = arena.reset(.retain_capacity); - - const back = processes.back(); - back.clearRetainingCapacity(); - for (lists) |list| { - back.appendSlice(gpa, list.front().items) catch break; - } - - for (back.items) |*info| { - info.username = arena.allocator().dupe(u8, info.username) catch ""; - info.comm = arena.allocator().dupe(u8, info.comm) catch ""; - } - - processes.swap(); -} diff --git a/bin/zml-smi/platforms/neuron/metrics.zig b/bin/zml-smi/platforms/neuron/metrics.zig deleted file mode 100644 index 0b5a5ddd..00000000 --- a/bin/zml-smi/platforms/neuron/metrics.zig +++ /dev/null @@ -1,201 +0,0 @@ -const std = @import("std"); -const sysfs = @import("zml-smi/sysfs"); -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; -const device_info = @import("zml-smi/info").device_info; -const DeviceInfo = device_info.DeviceInfo; -const NeuronInfo = device_info.NeuronInfo; -const Collector = @import("zml-smi/collector").Collector; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const Nrt = @import("nrt.zig"); -const process = @import("process.zig"); - -const base_path = "/sys/devices/virtual/neuron_device"; - -pub const UtilAtomic = std.atomic.Value(u64); - -pub fn start(collector: *Collector) !void { - const nrt = try collector.arena.create(Nrt); - nrt.* = try Nrt.init(collector.arena, collector.io); - - if (nrt.handles.len == 0) { - return; - } - - const total_nc = nrt.totalNcCount() catch return; - const nc_per_device = total_nc / @as(u32, @intCast(nrt.handles.len)); - - const total_cores = @as(usize, nrt.handles.len) * nc_per_device; - const util_buf = try collector.arena.alloc(UtilAtomic, total_cores); - for (util_buf) |*u| u.* = .init(0); - - const GiB: u64 = 1024 * 1024 * 1024; - var neuron_infos: std.ArrayList(*DeviceInfo) = .empty; - const dev_offset: u8 = @intCast(collector.device_infos.items.len); - - var ver_buf: [Nrt.version_buf_len]u8 = undefined; - const nrt_ver: ?[]const u8 = if (nrt.version(&ver_buf)) |v| - collector.arena.dupe(u8, v) catch null - else |_| - null; - - var flat_core: usize = 0; - for (nrt.handles, nrt.device_indexes) |dev_handle, device_idx| { - const device_type = Nrt.deviceType(dev_handle); - const mem_per_device: u64 = switch (device_type) { - .inf1 => 8 * GiB, - .inf2_trn1 => 32 * GiB, - .trn2 => 96 * GiB, - .trn3 => 144 * GiB, - else => 0, - }; - - for (0..nc_per_device) |ci| { - const poll_arena = try collector.createPollArena(); - - const dev: Device = .{ - .io = collector.io, - .arena = poll_arena, - .device_idx = @intCast(device_idx), - .core_idx = @intCast(ci), - .mem_per_device = mem_per_device, - .nc_per_device = nc_per_device, - .util_src = &util_buf[flat_core], - }; - - const initial: NeuronInfo = .{ - .name = dev.name(collector.arena) catch null, - .driver_version = nrt_ver, - }; - const info = try collector.addDevice(.{ .neuron = .{ .values = .{ initial, initial } } }); - try neuron_infos.append(collector.arena, info); - try collector.spawnPoll(pollOnce, .{ poll_arena, &info.neuron, dev }); - flat_core += 1; - } - } - - if (neuron_infos.items.len > 0) { - const processes = try collector.createProcessList(); - const warmup = collector.poll_only; - try process.init(collector, processes, nrt, nc_per_device, util_buf, dev_offset, warmup); - - if (warmup) { - for (neuron_infos.items, util_buf) |info, *u| { - const back = info.neuron.back(); - back.* = info.neuron.front().*; - back.util_percent = u.load(.acquire); - info.neuron.swap(); - } - } - } -} - -const pollOnce = poll_metrics.poll(*DoubleBuffer(NeuronInfo), Device, metrics); - -const Device = struct { - io: std.Io, - arena: *std.heap.ArenaAllocator, - device_idx: u32, - core_idx: u32, - mem_per_device: u64 = 0, - nc_per_device: u32 = 1, - util_src: *const UtilAtomic, - - pub fn utilPercent(self: Device) !u64 { - return self.util_src.load(.acquire); - } - - fn devicePath(self: Device, sub_path: []const u8) ![]const u8 { - return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/{s}", .{ self.device_idx, sub_path }); - } - - fn corePath(self: Device, sub_path: []const u8) ![]const u8 { - return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/{s}", .{ self.device_idx, self.core_idx, sub_path }); - } - - fn name(self: Device, allocator: std.mem.Allocator) ![]const u8 { - return sysfs.readString(allocator, self.io, try self.devicePath("info/architecture/device_name")); - } - - fn readCoreMem(self: Device, comptime subdir: []const u8) !u64 { - return sysfs.readInt(self.arena.allocator(), self.io, try self.corePath("stats/memory_usage/device_mem/" ++ subdir ++ "/total")); - } - - pub fn memTotal(self: Device) !u64 { - if (self.mem_per_device == 0) { - return error.NrtUnavailable; - } - return self.mem_per_device; - } - - // Sysfs-based metrics — RAM is shared across cores, so sum all cores on the device. - pub fn memUsed(self: Device) !u64 { - var total: u64 = 0; - for (0..self.nc_per_device) |ci| { - const path = try std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/stats/memory_usage/device_mem/total", .{ self.device_idx, ci }); - total += try sysfs.readInt(self.arena.allocator(), self.io, path); - } - - return total; - } - - pub fn tensors(self: Device) !u64 { - return self.readCoreMem("tensors"); - } - - pub fn constants(self: Device) !u64 { - return self.readCoreMem("constants"); - } - - pub fn modelCode(self: Device) !u64 { - return self.readCoreMem("model_code"); - } - - pub fn sharedScratchpad(self: Device) !u64 { - return self.readCoreMem("model_shared_scratchpad"); - } - - pub fn nonsharedScratchpad(self: Device) !u64 { - return self.readCoreMem("nonshared_scratchpad"); - } - - pub fn runtime(self: Device) !u64 { - return self.readCoreMem("runtime_memory"); - } - - pub fn driver(self: Device) !u64 { - return self.readCoreMem("driver_memory"); - } - - pub fn dmaRings(self: Device) !u64 { - return self.readCoreMem("dma_rings"); - } - - pub fn collectives(self: Device) !u64 { - return self.readCoreMem("collectives"); - } - - pub fn notifications(self: Device) !u64 { - return self.readCoreMem("notifications"); - } - - pub fn uncategorized(self: Device) !u64 { - return self.readCoreMem("uncategorized"); - } -}; - -const metrics = .{ - .{ .field = "util_percent", .query = Device.utilPercent }, - .{ .field = "mem_total_bytes", .query = Device.memTotal }, - .{ .field = "mem_used_bytes", .query = Device.memUsed }, - .{ .field = "nc_tensors", .query = Device.tensors }, - .{ .field = "nc_constants", .query = Device.constants }, - .{ .field = "nc_model_code", .query = Device.modelCode }, - .{ .field = "nc_shared_scratchpad", .query = Device.sharedScratchpad }, - .{ .field = "nc_nonshared_scratchpad", .query = Device.nonsharedScratchpad }, - .{ .field = "nc_runtime", .query = Device.runtime }, - .{ .field = "nc_driver", .query = Device.driver }, - .{ .field = "nc_dma_rings", .query = Device.dmaRings }, - .{ .field = "nc_collectives", .query = Device.collectives }, - .{ .field = "nc_notifications", .query = Device.notifications }, - .{ .field = "nc_uncategorized", .query = Device.uncategorized }, -}; diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig deleted file mode 100644 index cd77b3c0..00000000 --- a/bin/zml-smi/platforms/neuron/nrt.zig +++ /dev/null @@ -1,230 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const elf = std.elf; -const stdx = @import("stdx"); -const DynLib = @import("zml-smi/dynlib"); -const sandbox = @import("zml-smi/sandbox"); - -const Nrt = @This(); - -pub const Error = error{ nrt_error, NrtUnavailable }; -pub const AppInfo = c.struct_neuron_app_info; -pub const DeviceType = enum(c_int) { - inf1 = 1, - inf2_trn1 = 2, - trn2 = 3, - trn3 = 4, - _, -}; - -lib: Fns, -private_lib: PrivateFns, -handles: []const *c.ndl_device_t, -device_indexes: []const c_int, - -const Fns = struct { - nrt_get_total_nc_count: *const @TypeOf(c.nrt_get_total_nc_count), - nrt_get_version: *const @TypeOf(c.nrt_get_version), -}; - -const PrivateFns = struct { - nds_open: *const @TypeOf(c.nds_open), - ndl_available_devices: *const @TypeOf(c.ndl_available_devices), - ndl_open_device: *const @TypeOf(c.ndl_open_device), - ndl_close_device: *const @TypeOf(c.ndl_close_device), - ndl_get_all_apps_info: *const @TypeOf(c.ndl_get_all_apps_info), - nds_close: *const @TypeOf(c.nds_close), - nds_get_nc_counter: *const @TypeOf(c.nds_get_nc_counter), -}; - -pub fn init(allocator: std.mem.Allocator, io: std.Io) !Nrt { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = sandbox.path(&path_buf) orelse { - std.log.err("neuron: sandbox path unavailable", .{}); - return error.NrtUnavailable; - }; - - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnrt.so.1" }) catch { - std.log.err("neuron: failed to construct libnrt.so.1 path", .{}); - return error.NrtUnavailable; - }; - - const dl_handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - if (std.c.dlerror()) |err| std.log.err("neuron: dlopen: {s}", .{err}); - return error.NrtUnavailable; - }; - var dynlib: std.DynLib = .{ .inner = .{ .handle = dl_handle } }; - - const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NrtUnavailable; - const private_fns = resolveFromElf(dl_handle, io, "nrt_init") catch |err| { - std.log.err("neuron: ELF resolution failed: {s}", .{@errorName(err)}); - return error.NrtUnavailable; - }; - - var dev_index_buf: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; - const count: usize = @intCast(@max(0, private_fns.ndl_available_devices(&dev_index_buf, c.MAX_NEURON_DEVICE_COUNT))); - - var handle_list: std.ArrayList(*c.ndl_device_t) = .empty; - var index_list: std.ArrayList(c_int) = .empty; - - for (dev_index_buf[0..count]) |device_idx| { - var dev: ?*c.ndl_device_t = null; - var t: c.struct_ndl_device_init_param = .{ - .initialize_device = false, - .num_dram_regions = 0, - .map_hbm = false, - .skip_copy_bufs = true, - }; - - if (private_fns.ndl_open_device(device_idx, &t, &dev) == 0) { - if (dev) |d| { - errdefer _ = private_fns.ndl_close_device(d); - try handle_list.append(allocator, d); - try index_list.append(allocator, device_idx); - } - } - } - - return .{ - .lib = fns, - .private_lib = private_fns, - .handles = try handle_list.toOwnedSlice(allocator), - .device_indexes = try index_list.toOwnedSlice(allocator), - }; -} - -pub fn deviceCount(self: Nrt) u32 { - return @intCast(self.handles.len); -} - -pub fn allAppsInfo(self: Nrt, dev: *c.ndl_device_t) Error!struct { ptr: ?[*]AppInfo, count: usize } { - var info: ?[*]AppInfo = null; - var count: usize = 0; - if (self.private_lib.ndl_get_all_apps_info(dev, &info, &count, c.APP_INFO_ALL) != 0) { - return error.nrt_error; - } - return .{ .ptr = info, .count = count }; -} - -pub fn ndsOpen(self: Nrt, dev: *c.ndl_device_t, pid: c.pid_t) Error!*c.nds_instance_t { - var inst: ?*c.nds_instance_t = null; - if (self.private_lib.nds_open(dev, pid, &inst) != 0) { - return error.nrt_error; - } - return inst orelse error.nrt_error; -} - -pub fn ndsClose(self: Nrt, inst: *c.nds_instance_t) void { - _ = self.private_lib.nds_close(inst); -} - -pub fn ncCounter(self: Nrt, inst: *c.nds_instance_t, pnc_index: c_int, counter_index: u32) Error!u64 { - var value: u64 = 0; - if (self.private_lib.nds_get_nc_counter(inst, pnc_index, counter_index, &value) != 0) { - return error.nrt_error; - } - return value; -} - -pub const version_buf_len = c.RT_VERSION_DETAIL_LEN; - -pub fn version(self: Nrt, buf: *[version_buf_len]u8) Error![:0]const u8 { - var ver: c.nrt_version_t = std.mem.zeroes(c.nrt_version_t); - if (self.lib.nrt_get_version(&ver, @sizeOf(c.nrt_version_t)) != 0) { - return error.nrt_error; - } - @memcpy(buf, &ver.rt_detail); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn totalNcCount(self: Nrt) Error!u32 { - var count: u32 = 0; - if (self.lib.nrt_get_total_nc_count(&count) != 0) { - return error.nrt_error; - } - return count; -} - -pub fn hbmSize(dev: *c.ndl_device_t) usize { - return dev.hbm_size; -} - -pub fn deviceType(dev: *c.ndl_device_t) DeviceType { - return @fromBackingInt(@intCast(dev.device_type)); -} - -fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const u8) !PrivateFns { - const field_names = comptime std.meta.fieldNames(PrivateFns); - - const sym_addr = std.c.dlsym(handle, known_sym) orelse - return error.SymbolNotFound; - - var dl_info: DlInfo = undefined; - if (dladdr(sym_addr, &dl_info) == 0) return error.DladdrFailed; - - const elf_base = @intFromPtr(dl_info.dli_fbase); - const elf_path = std.mem.span(dl_info.dli_fname); - - const file = try std.Io.Dir.openFileAbsolute(io, elf_path, .{}); - defer file.close(io); - - const stat = try file.stat(io); - const mmap = try std.posix.mmap( - null, - @intCast(stat.size), - .{ .READ = true }, - .{ .TYPE = .PRIVATE }, - file.handle, - 0, - ); - defer std.posix.munmap(mmap); - - var result: PrivateFns = undefined; - var remaining: usize = field_names.len; - - const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(mmap)); - const shdrs: []const elf.ElfN.Shdr = @as([*]const elf.ElfN.Shdr, @ptrCast(@alignCast(mmap[ehdr.shoff..])))[0..ehdr.shnum]; - - for (shdrs) |sh| { - if (sh.type != .SYMTAB) { - continue; - } - - const strtab_sh = shdrs[sh.link]; - const strtab: []const u8 = mmap[strtab_sh.offset..][0..strtab_sh.size]; - - for (std.mem.bytesAsSlice(elf.ElfN.Sym, mmap[sh.offset..][0..sh.size])) |s| { - if (s.value == 0) { - continue; - } - - const sym_name = std.mem.span(@as([*c]const u8, @ptrCast(strtab[@as(usize, s.name)..]))); - - inline for (field_names) |field_name| { - if (std.mem.eql(u8, sym_name, field_name)) { - @field(result, field_name) = @ptrFromInt(elf_base + @as(usize, s.value)); - remaining -= 1; - } - } - - if (remaining == 0) { - return result; - } - } - } - - if (remaining > 0) { - return error.SymbolResolutionFailed; - } - return result; -} - -const DlInfo = extern struct { - dli_fname: [*:0]const u8, - dli_fbase: *anyopaque, - dli_sname: [*c]const u8, - dli_saddr: ?*anyopaque, -}; - -extern fn dladdr(addr: *anyopaque, info: *DlInfo) c_int; diff --git a/bin/zml-smi/platforms/neuron/process.zig b/bin/zml-smi/platforms/neuron/process.zig deleted file mode 100644 index 2dbf363c..00000000 --- a/bin/zml-smi/platforms/neuron/process.zig +++ /dev/null @@ -1,108 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const Nrt = @import("nrt.zig"); -const smi_info = @import("zml-smi/info"); -const pi = smi_info.process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; -const UtilAtomic = @import("metrics.zig").UtilAtomic; - -const DeltaState = struct { - prev_total_us: u64 = 0, - prev_timestamp: ?std.Io.Timestamp = null, -}; - -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, warmup: bool) !void { - if (nrt.handles.len == 0) return; - const delta_states = try collector.arena.alloc(DeltaState, util_buf.len); - @memset(delta_states, .{}); - const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.gpa, collector.io, list, nrt, nc_per_device, util_buf, dev_offset, delta_states }; - if (warmup) { - @call(.auto, pollOnce, args); - collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; - @call(.auto, pollOnce, args); - collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; - } - - try collector.spawnPoll(pollOnce, args); -} - -fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, delta_states: []DeltaState) void { - const back = list.back(); - - back.clearRetainingCapacity(); - - for (nrt.handles, 0..) |handle, dev_i| { - const apps_result = nrt.allAppsInfo(handle) catch continue; - defer { - if (apps_result.ptr) |ptr| { - std.c.free(@ptrCast(ptr)); - } - } - - const apps = if (apps_result.ptr) |ptr| ptr[0..apps_result.count] else continue; - - var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = @splat(0); - - for (apps) |*app| { - if (app.pid <= 0) { - continue; - } - - const nds = nrt.ndsOpen(handle, app.pid) catch continue; - defer nrt.ndsClose(nds); - - for (0..nc_per_device) |ci| { - const time_in_use = nrt.ncCounter(nds, @intCast(ci), c.NDS_NC_COUNTER_TIME_IN_USE) catch continue; - total_us_per_core[ci] += time_in_use; - - if (time_in_use == 0) { - continue; - } - - const dev_idx: u32 = @intCast(dev_i * nc_per_device + ci); - - const pid = std.math.cast(u32, app.pid) orelse continue; - var info: pi.ProcessInfo = .{ - .pid = pid, - .device_idx = @intCast(dev_idx + dev_offset), - .dev_mem_kib = @intCast(app.device_mem_size / 1024), - }; - - if (dev_idx < util_buf.len) { - info.dev_util_percent = @intCast(util_buf[dev_idx].load(.acquire)); - } - - if (info.dev_mem_kib != null and info.dev_mem_kib.? > 0) - back.append(allocator, info) catch break; - } - } - - const now: std.Io.Timestamp = .now(io, .awake); - - for (0..nc_per_device) |ci| { - const dev_idx = dev_i * nc_per_device + ci; - if (dev_idx >= delta_states.len) { - break; - } - - const ds = &delta_states[dev_idx]; - const prev_us = ds.prev_total_us; - ds.prev_total_us = total_us_per_core[ci]; - - if (ds.prev_timestamp) |prev| { - const elapsed = prev.untilNow(io, .awake); - const elapsed_us: u64 = @intCast(@divFloor(elapsed.nanoseconds, std.time.ns_per_us)); - - if (elapsed_us > 0) { - const delta_us = total_us_per_core[ci] -| prev_us; - util_buf[dev_idx].store(@min(100, delta_us * 100 / elapsed_us), .release); - } - } - - ds.prev_timestamp = now; - } - } - - list.swap(); -} diff --git a/bin/zml-smi/platforms/nvml/metrics.zig b/bin/zml-smi/platforms/nvml/metrics.zig deleted file mode 100644 index 213dd253..00000000 --- a/bin/zml-smi/platforms/nvml/metrics.zig +++ /dev/null @@ -1,222 +0,0 @@ -const std = @import("std"); -const Nvml = @import("nvml.zig"); -const device_info = @import("zml-smi/info").device_info; -const GpuInfo = device_info.GpuInfo; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; -const Collector = @import("zml-smi/collector").Collector; -const process = @import("process.zig"); - -const pcie_every: u8 = 4; - -const Slot = struct { - info: *DoubleBuffer(GpuInfo), - dev: Device, -}; - -const Ctx = struct { - gpa: std.mem.Allocator, - nvml: *const Nvml, - slots: []Slot, - processes: *process.ProcessDoubleBuffer, - last_seen_ts: []u64, - dev_offset: u16, - device_count: u32, - tick: u8 = 0, -}; - -pub fn start(collector: *Collector) !void { - const nvml = try collector.arena.create(Nvml); - nvml.* = try Nvml.init(); - - const count = try nvml.deviceCount(); - const dev_offset: u16 = @intCast(collector.device_infos.items.len); - - var slots: std.ArrayList(Slot) = try .initCapacity(collector.arena, count); - for (0..count) |i| { - const dev = Device.open(nvml, @intCast(i)) catch continue; - const initial: GpuInfo = .{ - .name = dev.name(collector.arena) catch null, - .driver_version = dev.driverVersion(collector.arena) catch null, - .cuda_driver_version = dev.cudaDriverVersion(collector.arena) catch null, - }; - const info = try collector.addDevice(.{ .cuda = .{ .values = .{ initial, initial } } }); - slots.appendAssumeCapacity(.{ .info = &info.cuda, .dev = dev }); - } - - const last_seen_ts = try collector.arena.alloc(u64, count); - @memset(last_seen_ts, 0); - - const ctx = try collector.arena.create(Ctx); - ctx.* = .{ - .gpa = collector.gpa, - .nvml = nvml, - .slots = slots.items, - .processes = try collector.createProcessList(), - .last_seen_ts = last_seen_ts, - .dev_offset = dev_offset, - .device_count = count, - }; - try collector.spawnPoll(pollOnce, .{ctx}); -} - -fn pollOnce(ctx: *Ctx) void { - const pcie = ctx.tick % pcie_every == 0; - ctx.tick +%= 1; - - for (ctx.slots) |slot| { - pollDevice(slot.info, slot.dev, pcie); - } - process.pollOnce(ctx.gpa, ctx.processes, ctx.nvml, ctx.dev_offset, ctx.device_count, ctx.last_seen_ts); -} - -fn pollDevice(db: *DoubleBuffer(GpuInfo), dev: Device, pcie: bool) void { - const back = db.back(); - back.* = db.front().*; - poll_metrics.apply(metrics, back, dev); - if (pcie) poll_metrics.apply(pcie_metrics, back, dev); - db.swap(); -} - -const Device = struct { - nvml: *const Nvml, - handle: Nvml.Handle, - - pub fn open(nvml: *const Nvml, index: u32) !Device { - return .{ .nvml = nvml, .handle = try nvml.handleByIndex(index) }; - } - - fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [Nvml.name_buf_len]u8 = undefined; - const slice = try self.nvml.name(self.handle, &buf); - - return try arena.dupe(u8, slice); - } - - fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [Nvml.driver_version_buf_size]u8 = undefined; - const slice = try self.nvml.driverVersion(&buf); - - return try arena.dupe(u8, slice); - } - - fn cudaDriverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { - const v = try self.nvml.cudaDriverVersion(); - return try std.fmt.allocPrint(arena, "{}.{}", .{ - Nvml.cudaDriverVersionMajor(v), - Nvml.cudaDriverVersionMinor(v), - }); - } - - // Power - pub fn powerUsage(self: Device) !u64 { - return @intCast(try self.nvml.powerUsage(self.handle)); - } - - pub fn powerLimit(self: Device) !u64 { - return @intCast(try self.nvml.powerLimit(self.handle)); - } - - // Thermal - pub fn temperature(self: Device) !u64 { - return @intCast(try self.nvml.temperature(self.handle)); - } - - pub fn fanSpeed(self: Device) !u64 { - return @intCast(try self.nvml.fanSpeed(self.handle)); - } - - // Utilization - pub fn gpuUtil(self: Device) !u64 { - return @intCast(try self.nvml.utilizationGpu(self.handle)); - } - - pub fn encoderUtil(self: Device) !u64 { - return @intCast(try self.nvml.encoderUtil(self.handle)); - } - - pub fn decoderUtil(self: Device) !u64 { - return @intCast(try self.nvml.decoderUtil(self.handle)); - } - - // Clocks - pub fn clockGraphics(self: Device) !u64 { - return @intCast(try self.nvml.clockGraphics(self.handle)); - } - - pub fn clockSm(self: Device) !u64 { - return @intCast(try self.nvml.clockSm(self.handle)); - } - - pub fn clockMem(self: Device) !u64 { - return @intCast(try self.nvml.clockMem(self.handle)); - } - - pub fn maxClockGraphics(self: Device) !u64 { - return @intCast(try self.nvml.maxClockGraphics(self.handle)); - } - - pub fn maxClockMem(self: Device) !u64 { - return @intCast(try self.nvml.maxClockMem(self.handle)); - } - - // Memory - pub fn memUsed(self: Device) !u64 { - return self.nvml.memUsed(self.handle); - } - - pub fn memTotal(self: Device) !u64 { - return self.nvml.memTotal(self.handle); - } - - pub fn memBusWidth(self: Device) !u64 { - return @intCast(try self.nvml.memBusWidth(self.handle)); - } - - // PCIe - pub fn pcieTx(self: Device) !u64 { - return @intCast(try self.nvml.pcieTxKBps(self.handle)); - } - - pub fn pcieRx(self: Device) !u64 { - return @intCast(try self.nvml.pcieRxKBps(self.handle)); - } - - pub fn pcieBandwidth(self: Device) !u64 { - return @intCast(try self.nvml.pcieSpeed(self.handle)); - } - - pub fn pcieLinkGen(self: Device) !u64 { - return @intCast(try self.nvml.pcieLinkGen(self.handle)); - } - - pub fn pcieLinkWidth(self: Device) !u64 { - return @intCast(try self.nvml.pcieLinkWidth(self.handle)); - } -}; - -const metrics = .{ - .{ .field = "power_mw", .query = Device.powerUsage }, - .{ .field = "power_limit_mw", .query = Device.powerLimit }, - .{ .field = "temperature", .query = Device.temperature }, - .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, - .{ .field = "util_percent", .query = Device.gpuUtil }, - .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, - .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, - .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, - .{ .field = "clock_sm_mhz", .query = Device.clockSm }, - .{ .field = "clock_mem_mhz", .query = Device.clockMem }, - .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, - .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, - .{ .field = "mem_used_bytes", .query = Device.memUsed }, - .{ .field = "mem_total_bytes", .query = Device.memTotal }, - .{ .field = "mem_bus_width", .query = Device.memBusWidth }, - .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, - .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, - .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, -}; - -const pcie_metrics = .{ - .{ .field = "pcie_tx_kbps", .query = Device.pcieTx }, - .{ .field = "pcie_rx_kbps", .query = Device.pcieRx }, -}; diff --git a/bin/zml-smi/platforms/nvml/nvml.zig b/bin/zml-smi/platforms/nvml/nvml.zig deleted file mode 100644 index e2b36982..00000000 --- a/bin/zml-smi/platforms/nvml/nvml.zig +++ /dev/null @@ -1,349 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const DynLib = @import("zml-smi/dynlib"); - -const Nvml = @This(); - -pub const driver_version_buf_size = c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE; -pub const Error = ReturnError || error{NvmlUnavailable}; -pub const Handle = c.nvmlDevice_t; -pub const ProcessInfo_t = c.nvmlProcessInfo_t; - -lib: Fns, - -const Fns = struct { - nvmlInit_v2: *const @TypeOf(c.nvmlInit_v2), - nvmlDeviceGetHandleByIndex_v2: *const @TypeOf(c.nvmlDeviceGetHandleByIndex_v2), - nvmlDeviceGetCount_v2: *const @TypeOf(c.nvmlDeviceGetCount_v2), - nvmlDeviceGetName: *const @TypeOf(c.nvmlDeviceGetName), - nvmlDeviceGetPowerUsage: *const @TypeOf(c.nvmlDeviceGetPowerUsage), - nvmlDeviceGetTemperature: *const @TypeOf(c.nvmlDeviceGetTemperature), - nvmlDeviceGetUtilizationRates: *const @TypeOf(c.nvmlDeviceGetUtilizationRates), - nvmlDeviceGetClockInfo: *const @TypeOf(c.nvmlDeviceGetClockInfo), - nvmlDeviceGetMaxClockInfo: *const @TypeOf(c.nvmlDeviceGetMaxClockInfo), - nvmlDeviceGetMemoryInfo: *const @TypeOf(c.nvmlDeviceGetMemoryInfo), - nvmlDeviceGetFanSpeed: *const @TypeOf(c.nvmlDeviceGetFanSpeed), - nvmlDeviceGetEnforcedPowerLimit: *const @TypeOf(c.nvmlDeviceGetEnforcedPowerLimit), - nvmlDeviceGetPcieThroughput: *const @TypeOf(c.nvmlDeviceGetPcieThroughput), - nvmlDeviceGetEncoderUtilization: *const @TypeOf(c.nvmlDeviceGetEncoderUtilization), - nvmlDeviceGetDecoderUtilization: *const @TypeOf(c.nvmlDeviceGetDecoderUtilization), - nvmlDeviceGetCurrPcieLinkGeneration: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkGeneration), - nvmlDeviceGetCurrPcieLinkWidth: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkWidth), - nvmlDeviceGetMemoryBusWidth: *const @TypeOf(c.nvmlDeviceGetMemoryBusWidth), - nvmlDeviceGetPcieSpeed: *const @TypeOf(c.nvmlDeviceGetPcieSpeed), - nvmlDeviceGetComputeRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetComputeRunningProcesses_v3), - nvmlDeviceGetGraphicsRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetGraphicsRunningProcesses_v3), - nvmlDeviceGetProcessUtilization: *const @TypeOf(c.nvmlDeviceGetProcessUtilization), - nvmlSystemGetCudaDriverVersion_v2: *const @TypeOf(c.nvmlSystemGetCudaDriverVersion_v2), - nvmlSystemGetDriverVersion: *const @TypeOf(c.nvmlSystemGetDriverVersion), -}; - -pub fn init() Error!Nvml { - var dynlib: std.DynLib = .{ .inner = .{ - .handle = std.c.dlopen("libnvidia-ml.so.1", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse - std.c.dlopen("libnvidia-ml.so", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - if (std.c.dlerror()) |err| std.log.err("nvml: dlopen: {s}", .{err}); - return error.NvmlUnavailable; - }, - } }; - const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NvmlUnavailable; - try check(fns.nvmlInit_v2()); - return .{ .lib = fns }; -} - -pub fn handleByIndex(self: Nvml, device_id: u32) Error!c.nvmlDevice_t { - var handle: c.nvmlDevice_t = undefined; - try check(self.lib.nvmlDeviceGetHandleByIndex_v2(device_id, &handle)); - return handle; -} - -pub fn deviceCount(self: Nvml) Error!u32 { - var count: c_uint = 0; - try check(self.lib.nvmlDeviceGetCount_v2(&count)); - return @intCast(count); -} - -pub const name_buf_len = c.NVML_DEVICE_NAME_V2_BUFFER_SIZE; - -pub fn name(self: Nvml, handle: c.nvmlDevice_t, buf: *[c.NVML_DEVICE_NAME_V2_BUFFER_SIZE]u8) Error![:0]const u8 { - try check(self.lib.nvmlDeviceGetName(handle, buf, buf.len)); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn powerUsage(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var power_mw: c_uint = 0; - try check(self.lib.nvmlDeviceGetPowerUsage(handle, &power_mw)); - return power_mw; -} - -pub fn temperature(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var temp: c_uint = 0; - try check(self.lib.nvmlDeviceGetTemperature(handle, c.NVML_TEMPERATURE_GPU, &temp)); - return temp; -} - -pub fn utilizationGpu(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var util: c.nvmlUtilization_t = undefined; - try check(self.lib.nvmlDeviceGetUtilizationRates(handle, &util)); - return util.gpu; -} - -pub fn clockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); - return clock; -} - -pub fn clockSm(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_SM, &clock)); - return clock; -} - -pub fn clockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); - return clock; -} - -pub fn maxClockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); - return clock; -} - -pub fn maxClockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); - return clock; -} - -pub fn memTotal(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { - var mem: c.nvmlMemory_t = undefined; - try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); - return @intCast(mem.total); -} - -pub fn memUsed(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { - var mem: c.nvmlMemory_t = undefined; - try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); - return @intCast(mem.used); -} - -pub fn fanSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var speed: c_uint = 0; - try check(self.lib.nvmlDeviceGetFanSpeed(handle, &speed)); - return speed; -} - -pub fn powerLimit(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var limit: c_uint = 0; - try check(self.lib.nvmlDeviceGetEnforcedPowerLimit(handle, &limit)); - return limit; -} - -pub fn pcieTxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var value: c_uint = 0; - try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_TX_BYTES, &value)); - return value; -} - -pub fn pcieRxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var value: c_uint = 0; - try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_RX_BYTES, &value)); - return value; -} - -pub fn encoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var util: c_uint = 0; - var sampling: c_uint = 0; - try check(self.lib.nvmlDeviceGetEncoderUtilization(handle, &util, &sampling)); - return util; -} - -pub fn decoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var util: c_uint = 0; - var sampling: c_uint = 0; - try check(self.lib.nvmlDeviceGetDecoderUtilization(handle, &util, &sampling)); - return util; -} - -pub fn pcieLinkGen(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var gen: c_uint = 0; - try check(self.lib.nvmlDeviceGetCurrPcieLinkGeneration(handle, &gen)); - return gen; -} - -pub fn pcieLinkWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var width: c_uint = 0; - try check(self.lib.nvmlDeviceGetCurrPcieLinkWidth(handle, &width)); - return width; -} - -pub fn pcieSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var speed: c_uint = 0; - try check(self.lib.nvmlDeviceGetPcieSpeed(handle, &speed)); - return speed; -} - -pub fn driverVersion(self: Nvml, buf: *[c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE]u8) Error![:0]const u8 { - try check(self.lib.nvmlSystemGetDriverVersion(buf, buf.len)); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn cudaDriverVersionMajor(v: c_int) c_int { - return @divTrunc(v, 1000); -} - -pub fn cudaDriverVersionMinor(v: c_int) c_int { - return @divTrunc(@mod(v, 1000), 10); -} - -pub fn cudaDriverVersion(self: Nvml) Error!c_int { - var version: c_int = 0; - try check(self.lib.nvmlSystemGetCudaDriverVersion_v2(&version)); - return version; -} - -pub fn memBusWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var width: c_uint = 0; - try check(self.lib.nvmlDeviceGetMemoryBusWidth(handle, &width)); - return width; -} - -const max_query = 4096; - -fn sizedCount(count: c_uint) ?c_uint { - if (count == 0 or count > max_query) return null; - return count; -} - -pub fn computeRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]c.nvmlProcessInfo_t { - var count: c_uint = 0; - const ret = self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, null); - if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { - if (ret == c.NVML_SUCCESS) { - return &.{}; - } - try check(ret); - } - count = sizedCount(count) orelse return &.{}; - const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); - errdefer allocator.free(infos); - try check(self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); - return infos; -} - -pub fn graphicsRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]const c.nvmlProcessInfo_t { - var count: c_uint = 0; - const ret = self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, null); - - if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { - if (ret == c.NVML_SUCCESS) { - return &.{}; - } - try check(ret); - } - - count = sizedCount(count) orelse return &.{}; - const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); - errdefer allocator.free(infos); - - try check(self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); - return infos; -} - -pub fn processUtilization(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t, last_seen: u64) (Error || error{OutOfMemory})![]const c.nvmlProcessUtilizationSample_t { - var count: c_uint = 0; - const ret = self.lib.nvmlDeviceGetProcessUtilization(handle, null, &count, last_seen); - - if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { - if (ret == c.NVML_SUCCESS) { - return &.{}; - } - try check(ret); - } - - count = sizedCount(count) orelse return &.{}; - const samples = try allocator.alloc(c.nvmlProcessUtilizationSample_t, count); - errdefer allocator.free(samples); - - try check(self.lib.nvmlDeviceGetProcessUtilization(handle, @ptrCast(samples.ptr), &count, last_seen)); - return samples; -} - -// Error handling - -pub const ReturnError = error{ - error_uninitialized, - error_invalid_argument, - error_not_supported, - error_no_permission, - error_already_initialized, - error_not_found, - error_insufficient_size, - error_insufficient_power, - error_driver_not_loaded, - error_timeout, - error_irq_issue, - error_library_not_found, - error_function_not_found, - error_corrupted_inforom, - error_gpu_is_lost, - error_reset_required, - error_operating_system, - error_lib_rm_version_mismatch, - error_in_use, - error_memory, - error_no_data, - error_vgpu_ecc_not_supported, - error_insufficient_resources, - error_freq_not_supported, - error_argument_version_mismatch, - error_deprecated, - error_not_ready, - error_gpu_not_found, - error_invalid_state, - error_reset_type_not_supported, - error_unknown, -}; - -fn check(ret: c_uint) Error!void { - if (ret == c.NVML_SUCCESS) { - return; - } - return switch (ret) { - c.NVML_ERROR_UNINITIALIZED => error.error_uninitialized, - c.NVML_ERROR_INVALID_ARGUMENT => error.error_invalid_argument, - c.NVML_ERROR_NOT_SUPPORTED => error.error_not_supported, - c.NVML_ERROR_NO_PERMISSION => error.error_no_permission, - c.NVML_ERROR_ALREADY_INITIALIZED => error.error_already_initialized, - c.NVML_ERROR_NOT_FOUND => error.error_not_found, - c.NVML_ERROR_INSUFFICIENT_SIZE => error.error_insufficient_size, - c.NVML_ERROR_INSUFFICIENT_POWER => error.error_insufficient_power, - c.NVML_ERROR_DRIVER_NOT_LOADED => error.error_driver_not_loaded, - c.NVML_ERROR_TIMEOUT => error.error_timeout, - c.NVML_ERROR_IRQ_ISSUE => error.error_irq_issue, - c.NVML_ERROR_LIBRARY_NOT_FOUND => error.error_library_not_found, - c.NVML_ERROR_FUNCTION_NOT_FOUND => error.error_function_not_found, - c.NVML_ERROR_CORRUPTED_INFOROM => error.error_corrupted_inforom, - c.NVML_ERROR_GPU_IS_LOST => error.error_gpu_is_lost, - c.NVML_ERROR_RESET_REQUIRED => error.error_reset_required, - c.NVML_ERROR_OPERATING_SYSTEM => error.error_operating_system, - c.NVML_ERROR_LIB_RM_VERSION_MISMATCH => error.error_lib_rm_version_mismatch, - c.NVML_ERROR_IN_USE => error.error_in_use, - c.NVML_ERROR_MEMORY => error.error_memory, - c.NVML_ERROR_NO_DATA => error.error_no_data, - c.NVML_ERROR_VGPU_ECC_NOT_SUPPORTED => error.error_vgpu_ecc_not_supported, - c.NVML_ERROR_INSUFFICIENT_RESOURCES => error.error_insufficient_resources, - c.NVML_ERROR_FREQ_NOT_SUPPORTED => error.error_freq_not_supported, - c.NVML_ERROR_ARGUMENT_VERSION_MISMATCH => error.error_argument_version_mismatch, - c.NVML_ERROR_DEPRECATED => error.error_deprecated, - c.NVML_ERROR_NOT_READY => error.error_not_ready, - c.NVML_ERROR_GPU_NOT_FOUND => error.error_gpu_not_found, - c.NVML_ERROR_INVALID_STATE => error.error_invalid_state, - c.NVML_ERROR_RESET_TYPE_NOT_SUPPORTED => error.error_reset_type_not_supported, - c.NVML_ERROR_UNKNOWN => error.error_unknown, - else => error.error_unknown, - }; -} diff --git a/bin/zml-smi/platforms/nvml/process.zig b/bin/zml-smi/platforms/nvml/process.zig deleted file mode 100644 index 5e18cdcb..00000000 --- a/bin/zml-smi/platforms/nvml/process.zig +++ /dev/null @@ -1,54 +0,0 @@ -const std = @import("std"); -const Nvml = @import("nvml.zig"); -const pi = @import("zml-smi/info").process_info; - -pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); - -pub fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16, device_count: u32, last_seen_ts: []u64) void { - const back = list.back(); - - back.clearRetainingCapacity(); - - for (0..device_count) |dev_idx| { - const handle = nvml.handleByIndex(@intCast(dev_idx)) catch continue; - const idx: u16 = @intCast(dev_idx + dev_offset); - - const compute = nvml.computeRunningProcesses(allocator, handle) catch &.{}; - defer if (compute.len > 0) allocator.free(compute); - collectFromQuery(allocator, back, idx, compute); - - const graphics = nvml.graphicsRunningProcesses(allocator, handle) catch &.{}; - defer if (graphics.len > 0) allocator.free(graphics); - collectFromQuery(allocator, back, idx, graphics); - - const last_ts = last_seen_ts[dev_idx]; - const utils = nvml.processUtilization(allocator, handle, last_ts) catch &.{}; - defer if (utils.len > 0) allocator.free(utils); - - for (utils) |sample| { - if (sample.smUtil > 100 or sample.timeStamp <= last_ts) { - continue; - } - - last_seen_ts[dev_idx] = @max(last_seen_ts[dev_idx], sample.timeStamp); - - for (back.items) |*entry| { - if (entry.pid == sample.pid and entry.device_idx == idx) { - entry.dev_util_percent = @intCast(sample.smUtil); - } - } - } - } - - list.swap(); -} - -fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u16, procs: []const Nvml.ProcessInfo_t) void { - for (procs) |gp| { - back.append(allocator, .{ - .pid = gp.pid, - .device_idx = dev_idx, - .dev_mem_kib = @intCast(gp.usedGpuMemory / 1024), - }) catch return; - } -} diff --git a/bin/zml-smi/platforms/oneapi/metrics.zig b/bin/zml-smi/platforms/oneapi/metrics.zig deleted file mode 100644 index ae2a86a3..00000000 --- a/bin/zml-smi/platforms/oneapi/metrics.zig +++ /dev/null @@ -1,134 +0,0 @@ -const std = @import("std"); -const OneApi = @import("oneapi.zig"); -const device_info = @import("zml-smi/info").device_info; -const GpuInfo = device_info.GpuInfo; -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; -const Collector = @import("zml-smi/collector").Collector; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const process = @import("process.zig"); - -pub fn start(collector: *Collector) !void { - const oneapi = try collector.arena.create(OneApi); - oneapi.* = try OneApi.init(collector.arena, collector.io); - - const count = oneapi.monitor.devices.len; - const dev_offset: u16 = @intCast(collector.device_infos.items.len); - - for (0..count) |i| { - const poll_arena = try collector.createPollArena(); - const dev = Device.open(oneapi, i, poll_arena) catch continue; - const initial: GpuInfo = .{ - .name = dev.name(collector.arena) catch null, - .driver_version = dev.driverVersion(collector.arena) catch null, - .power_limit_mw = dev.powerLimit() catch null, - .clock_graphics_max_mhz = dev.maxClockGraphics() catch null, - .mem_total_bytes = dev.memTotal() catch null, - .pcie_link_gen = dev.pcieLinkGen() catch null, - .pcie_link_width = dev.pcieLinkWidth() catch null, - .pcie_bandwidth_mbps = dev.pcieBandwidth() catch null, - }; - const info = try collector.addDevice(.{ .oneapi = .{ .values = .{ initial, initial } } }); - - try collector.spawnPoll(pollOnce, .{ poll_arena, &info.oneapi, dev }); - } - - const processes = try collector.createProcessList(); - try process.init(collector, processes, oneapi, dev_offset); -} - -const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); - -const Device = struct { - oneapi: *OneApi, - arena: *std.heap.ArenaAllocator, - handle: OneApi.Handle, - - fn open(oneapi: *OneApi, index: usize, arena: *std.heap.ArenaAllocator) !Device { - return .{ - .oneapi = oneapi, - .arena = arena, - .handle = try oneapi.handleByIndex(index), - }; - } - - fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { - return self.oneapi.name(arena, self.handle); - } - - fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { - return self.oneapi.driverVersion(arena); - } - - pub fn powerUsage(self: Device) !u64 { - return self.oneapi.powerUsage(self.arena.allocator(), self.handle); - } - - pub fn temperature(self: Device) !u64 { - return self.oneapi.temperature(self.arena.allocator(), self.handle); - } - - pub fn fanSpeed(self: Device) !u64 { - return self.oneapi.fanSpeed(self.arena.allocator(), self.handle); - } - - pub fn gpuUtil(self: Device) !u64 { - return self.oneapi.gpuUtil(self.arena.allocator(), self.handle); - } - - pub fn encoderUtil(self: Device) !u64 { - return self.oneapi.encoderUtil(self.arena.allocator(), self.handle); - } - - pub fn decoderUtil(self: Device) !u64 { - return self.oneapi.decoderUtil(self.arena.allocator(), self.handle); - } - - pub fn powerLimit(self: Device) !u64 { - return self.oneapi.powerLimit(self.arena.allocator(), self.handle); - } - - pub fn clockGraphics(self: Device) !u64 { - return self.oneapi.clockGraphics(self.arena.allocator(), self.handle); - } - - pub fn maxClockGraphics(self: Device) !u64 { - return self.oneapi.maxClockGraphics(self.arena.allocator(), self.handle); - } - - pub fn memUsed(self: Device) !u64 { - return self.oneapi.memUsed(self.arena.allocator(), self.handle); - } - - pub fn memTotal(self: Device) !u64 { - return self.oneapi.memTotal(self.arena.allocator(), self.handle); - } - - pub fn pcieLinkGen(self: Device) !u64 { - return self.oneapi.pcieLinkGen(self.arena.allocator(), self.handle); - } - - pub fn pcieLinkWidth(self: Device) !u64 { - return self.oneapi.pcieLinkWidth(self.arena.allocator(), self.handle); - } - - pub fn pcieBandwidth(self: Device) !u64 { - return self.oneapi.pcieBandwidth(self.arena.allocator(), self.handle); - } -}; - -const metrics = .{ - .{ .field = "power_mw", .query = Device.powerUsage }, - .{ .field = "power_limit_mw", .query = Device.powerLimit }, - .{ .field = "temperature", .query = Device.temperature }, - .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, - .{ .field = "util_percent", .query = Device.gpuUtil }, - .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, - .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, - .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, - .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, - .{ .field = "mem_used_bytes", .query = Device.memUsed }, - .{ .field = "mem_total_bytes", .query = Device.memTotal }, - .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, - .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, - .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, -}; diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig deleted file mode 100644 index dd5f97a9..00000000 --- a/bin/zml-smi/platforms/oneapi/monitor.zig +++ /dev/null @@ -1,877 +0,0 @@ -const std = @import("std"); -const smi_sysfs = @import("zml-smi/sysfs"); -const Monitor = @This(); - -pub const bus_device_identifier_len = "0000:00:00.0".len; -pub const Handle = enum(u32) { _ }; - -const Device = struct { - render_idx: usize, - drm_path: []const u8, - dev_path: []const u8, - pcie_path: []const u8, - hwmon_path: ?[]const u8, - bus_device_identifier: ?[bus_device_identifier_len]u8, - activity_prev: ?EngineSample = null, - encoder_prev: ?EngineSample = null, - decoder_prev: ?EngineSample = null, - energy_prev: ?EnergySample = null, -}; - -const EnergySample = struct { - micro_joules: u64, - timestamp_ns: u64, -}; - -pub const EngineSample = struct { - engine_ns: u64, - timestamp_ns: u64, -}; - -const DeviceSample = struct { - mem_kib: u64 = 0, - engine: ?EngineSample = null, - encoder: ?EngineSample = null, - decoder: ?EngineSample = null, -}; - -pub const ProcessSample = struct { - pid: u32, - device_idx: u16, - mem_kib: ?u64 = null, - engine: ?EngineSample = null, - encoder: ?EngineSample = null, - decoder: ?EngineSample = null, -}; - -pub const ProcessUsage = std.AutoHashMapUnmanaged(u64, ProcessSample); -pub const ProcessPrevious = std.AutoHashMapUnmanaged(u64, EngineSample); -const DeviceUsage = std.AutoHashMapUnmanaged(u16, DeviceSample); - -allocator: std.mem.Allocator, -io: std.Io, -devices: []Device, - -pub fn init(allocator: std.mem.Allocator, io: std.Io) !Monitor { - var monitor: Monitor = .{ - .allocator = allocator, - .io = io, - .devices = try discoverIntelDevices(allocator, io), - }; - try monitor.seedDeviceMetrics(); - return monitor; -} - -pub fn deviceId(self: *const Monitor, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { - const dev = try self.device(handle); - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/device", .{dev.dev_path}); - const raw = try smi_sysfs.readString(allocator, self.io, path); - return std.mem.trim(u8, raw, &std.ascii.whitespace); -} - -inline fn seedDeviceMetrics(self: *Monitor) !void { - var device_usage = collectDeviceUsage(self.allocator, self.io, self.devices) catch DeviceUsage{}; - defer device_usage.deinit(self.allocator); - - for (self.devices, 0..) |*dev, idx| { - dev.energy_prev = readEnergy(self.allocator, self.io, dev) catch null; - if (device_usage.get(@intCast(idx))) |sample| { - dev.activity_prev = sample.engine; - dev.encoder_prev = sample.encoder; - dev.decoder_prev = sample.decoder; - } - } -} - -pub fn saveProcessPrevious(allocator: std.mem.Allocator, previous: *ProcessPrevious, usage: *const ProcessUsage) !void { - previous.clearRetainingCapacity(); - try previous.ensureTotalCapacity(allocator, @intCast(usage.count())); - - var it = usage.iterator(); - while (it.next()) |entry| { - if (entry.value_ptr.engine) |engine| { - previous.putAssumeCapacity(entry.key_ptr.*, engine); - } - } -} - -inline fn discoverIntelDevices(allocator: std.mem.Allocator, io: std.Io) ![]Device { - var devices: std.ArrayList(Device) = .empty; - errdefer devices.deinit(allocator); - - var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch |err| switch (err) { - error.FileNotFound => return &.{}, - else => |e| return e, - }; - defer dir.close(io); - - var it = dir.iterate(); - while (it.next(io) catch null) |entry| { - const render_idx = parseRenderIndex(entry.name) orelse continue; - const dev = openDevice(allocator, io, render_idx) catch continue; - try devices.append(allocator, dev); - } - - std.mem.sort(Device, devices.items, {}, struct { - fn lessThan(_: void, lhs: Device, rhs: Device) bool { - return lhs.render_idx < rhs.render_idx; - } - }.lessThan); - return devices.toOwnedSlice(allocator); -} - -inline fn openDevice(allocator: std.mem.Allocator, io: std.Io, render_idx: usize) !Device { - const drm_path = try std.fmt.allocPrint(allocator, "/sys/class/drm/renderD{d}", .{render_idx}); - errdefer allocator.free(drm_path); - - const dev_path = try std.fmt.allocPrint(allocator, "{s}/device", .{drm_path}); - errdefer allocator.free(dev_path); - - var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const vendor_path = try std.fmt.bufPrint(&vendor_path_buf, "{s}/vendor", .{dev_path}); - const vendor = try smi_sysfs.readString(allocator, io, vendor_path); - if (!std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), "0x8086")) { - return error.not_intel; - } - - const pcie_path = try findPcieLinkPath(allocator, io, dev_path); - errdefer if (pcie_path.ptr != dev_path.ptr) allocator.free(pcie_path); - - return .{ - .render_idx = render_idx, - .drm_path = drm_path, - .dev_path = dev_path, - .pcie_path = pcie_path, - .hwmon_path = findHwmon(allocator, io, dev_path) catch null, - .bus_device_identifier = readBusDeviceId(allocator, io, dev_path) catch null, - }; -} - -pub fn device(self: *const Monitor, handle: Handle) !*const Device { - const idx = @backingInt(handle); - if (idx >= self.devices.len) return error.NotFound; - return &self.devices[idx]; -} - -pub fn readTemperature(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - const hwmon = dev.hwmon_path orelse return error.NotFound; - const milli_c = readHwmonInput(allocator, io, hwmon, "temp", "pkg") catch - readHwmonInput(allocator, io, hwmon, "temp", "gpu") catch - readNumberedInput(allocator, io, hwmon, "temp", 2) catch - try readNumberedInput(allocator, io, hwmon, "temp", 1); - return (milli_c + 500) / 1000; -} - -pub fn readEnergy(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !EnergySample { - const hwmon = dev.hwmon_path orelse return error.NotFound; - const micro_joules = readHwmonInput(allocator, io, hwmon, "energy", "card") catch - readHwmonInput(allocator, io, hwmon, "energy", "pkg") catch - try readNumberedInput(allocator, io, hwmon, "energy", 1); - return .{ - .micro_joules = micro_joules, - .timestamp_ns = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds), - }; -} -pub fn readPowerLimit(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - const hwmon = dev.hwmon_path orelse return error.NotFound; - const microwatts = readNumberedSuffix(allocator, io, hwmon, "power", 1, "cap") catch - try readNumberedSuffix(allocator, io, hwmon, "power", 1, "max"); - return microwatts / 1000; -} - -pub fn readFanSpeed(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - const hwmon = dev.hwmon_path orelse return error.NotFound; - return readPwmFanSpeed(allocator, io, hwmon) catch readRpmFanSpeedPercent(allocator, io, hwmon); -} - -pub fn readClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - return readFreq(allocator, io, dev, "act_freq") catch readFreq(allocator, io, dev, "cur_freq"); -} - -pub fn readMaxClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - return readFreq(allocator, io, dev, "max_freq"); -} - -pub fn readMemTotal(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/resource", .{dev.dev_path}); - return readLargestResource(allocator, io, path); -} - -pub fn readPcieLinkGen(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_speed", .{dev.pcie_path}); - const raw = try smi_sysfs.readString(allocator, io, path); - return pcieGenFromSpeed(raw); -} - -pub fn readPcieLinkWidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_width", .{dev.pcie_path}); - return smi_sysfs.readInt(allocator, io, path); -} - -pub fn readPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - const gen = try readPcieLinkGen(allocator, io, dev); - const width = try readPcieLinkWidth(allocator, io, dev); - return pcieBandwidthFromLink(gen, width) orelse error.NotFound; -} - -pub fn collectDeviceUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !DeviceUsage { - var processes = try collectProcessUsage(allocator, io, devices); - errdefer processes.deinit(allocator); - - var result: DeviceUsage = .{}; - errdefer result.deinit(allocator); - - var it = processes.iterator(); - while (it.next()) |entry| { - const sample = entry.value_ptr.*; - const gop = try result.getOrPut(allocator, sample.device_idx); - if (!gop.found_existing) gop.value_ptr.* = .{}; - if (sample.mem_kib) |mem| gop.value_ptr.mem_kib += mem; - addEngineSample(&gop.value_ptr.engine, sample.engine); - addEngineSample(&gop.value_ptr.encoder, sample.encoder); - addEngineSample(&gop.value_ptr.decoder, sample.decoder); - } - - return result; -} - -pub fn collectProcessUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !ProcessUsage { - var result: ProcessUsage = .{}; - errdefer result.deinit(allocator); - - var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; - defer seen_device_clients.deinit(allocator); - - var proc_dir = try std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }); - defer proc_dir.close(io); - - const now_ns: u64 = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); - var proc_it = proc_dir.iterate(); - while (proc_it.next(io) catch null) |proc_entry| { - const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; - - var fdinfo_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const fdinfo_path = std.fmt.bufPrint(&fdinfo_path_buf, "/proc/{d}/fdinfo", .{pid}) catch continue; - var fdinfo_dir = std.Io.Dir.openDirAbsolute(io, fdinfo_path, .{ .iterate = true }) catch continue; - defer fdinfo_dir.close(io); - - var fd_it = fdinfo_dir.iterate(); - while (fd_it.next(io) catch null) |fd_entry| { - var file_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const file_path = std.fmt.bufPrint(&file_path_buf, "{s}/{s}", .{ fdinfo_path, fd_entry.name }) catch continue; - const parsed = parseFdinfoFile(io, file_path, now_ns) catch continue; - const bus_device_identifier = parsed.bus_device_identifier orelse continue; - const dev_idx = matchDevice(devices, bus_device_identifier) orelse continue; - - // Skip fd if DRM client has been seen for this device. - // Summing after dup() / fork() over-reports memory and compute. - if (parsed.drm_client_id) |cid| { - if ((try seen_device_clients.getOrPut(allocator, .{ dev_idx, cid })).found_existing) continue; - } - - const key = processKey(pid, dev_idx); - const gop = try result.getOrPut(allocator, key); - if (!gop.found_existing) { - gop.value_ptr.* = .{ .pid = pid, .device_idx = dev_idx }; - } - mergeProcessSample(gop.value_ptr, parsed.sample); - } - } - - return result; -} - -pub fn processUtil(previous: ?EngineSample, current: ?EngineSample) ?u16 { - const prev = previous orelse return null; - const cur = current orelse return null; - if (cur.timestamp_ns <= prev.timestamp_ns or cur.engine_ns < prev.engine_ns) return null; - const engine_delta = cur.engine_ns - prev.engine_ns; - const time_delta = cur.timestamp_ns - prev.timestamp_ns; - if (time_delta == 0) return null; - const pct = @min(engine_delta * 100 / time_delta, 100); - return @intCast(if (pct == 0 and engine_delta > 0) 0 else pct); -} - -pub fn powerMilliwatts(previous: ?EnergySample, current: ?EnergySample) ?u64 { - const prev = previous orelse return null; - const cur = current orelse return null; - return milliwattsFromEnergyDelta(prev.micro_joules, cur.micro_joules, prev.timestamp_ns, cur.timestamp_ns); -} - -inline fn milliwattsFromEnergyDelta(prev_uj: u64, cur_uj: u64, prev_ns: u64, cur_ns: u64) ?u64 { - if (cur_uj < prev_uj or cur_ns <= prev_ns) return null; - const energy_delta = cur_uj - prev_uj; - const time_delta = cur_ns - prev_ns; - if (time_delta == 0) return null; - return energy_delta * 1_000_000 / time_delta; -} - -inline fn readFreq(allocator: std.mem.Allocator, io: std.Io, dev: *const Device, file: []const u8) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/tile0/gt0/freq0/{s}", .{ dev.dev_path, file }); - return smi_sysfs.readInt(allocator, io, path); -} - -inline fn parseRenderIndex(name: []const u8) ?usize { - if (!std.mem.startsWith(u8, name, "renderD")) return null; - return std.fmt.parseInt(usize, name["renderD".len..], 10) catch null; -} - -inline fn findHwmon(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !?[]const u8 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const hwmon_root = try std.fmt.bufPrint(&path_buf, "{s}/hwmon", .{dev_path}); - var dir = std.Io.Dir.openDirAbsolute(io, hwmon_root, .{ .iterate = true }) catch return null; - defer dir.close(io); - - var it = dir.iterate(); - while (it.next(io) catch null) |entry| { - if (!std.mem.startsWith(u8, entry.name, "hwmon")) continue; - const candidate = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ hwmon_root, entry.name }); - var name_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const name_path = try std.fmt.bufPrint(&name_buf, "{s}/name", .{candidate}); - const name = smi_sysfs.readString(allocator, io, name_path) catch return candidate; - const trimmed = std.mem.trim(u8, name, &std.ascii.whitespace); - if (std.mem.eql(u8, trimmed, "xe") or std.mem.eql(u8, trimmed, "i915")) return candidate; - } - return null; -} - -inline fn readBusDeviceId(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![bus_device_identifier_len]u8 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/uevent", .{dev_path}); - const raw = try smi_sysfs.readFieldString(allocator, io, path, "PCI_SLOT_NAME="); - const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); - if (trimmed.len != bus_device_identifier_len) return error.NotFound; - var out: [bus_device_identifier_len]u8 = undefined; - @memcpy(&out, trimmed); - return out; -} - -inline fn findPcieLinkPath(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![]const u8 { - var candidate = dev_path; - var best_path: ?[]const u8 = null; - - var best_bandwidth: u64 = 0; - var depth: usize = 0; - while (depth < 8) : (depth += 1) { - if (!try isIntelPciDevice(allocator, io, candidate)) break; - - const bandwidth = readMaxPcieBandwidth(allocator, io, candidate) catch 0; - if (bandwidth > best_bandwidth) { - best_path = candidate; - best_bandwidth = bandwidth; - } - - const parent = try std.fmt.allocPrint(allocator, "{s}/..", .{candidate}); - candidate = parent; - } - - return best_path orelse dev_path; -} - -inline fn isIntelPciDevice(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !bool { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const vendor_path = try std.fmt.bufPrint(&path_buf, "{s}/vendor", .{dev_path}); - const vendor_raw = smi_sysfs.readString(allocator, io, vendor_path) catch return false; - - const vendor = std.mem.trim(u8, vendor_raw, &std.ascii.whitespace); - return std.mem.eql(u8, vendor, "0x8086"); -} - -inline fn readMaxPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !u64 { - var speed_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const speed_path = try std.fmt.bufPrint(&speed_path_buf, "{s}/max_link_speed", .{dev_path}); - const raw_speed = try smi_sysfs.readString(allocator, io, speed_path); - - var width_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const width_path = try std.fmt.bufPrint(&width_path_buf, "{s}/max_link_width", .{dev_path}); - const width = try smi_sysfs.readInt(allocator, io, width_path); - const gen = try pcieGenFromSpeed(raw_speed); - return pcieBandwidthFromLink(gen, width) orelse error.NotFound; -} - -inline fn readHwmonInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, label: []const u8) !u64 { - for (1..16) |idx| { - var label_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const label_path = try std.fmt.bufPrint(&label_path_buf, "{s}/{s}{d}_label", .{ hwmon, prefix, idx }); - const raw = smi_sysfs.readString(allocator, io, label_path) catch continue; - const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); - if (!std.ascii.eqlIgnoreCase(trimmed, label)) continue; - return readNumberedInput(allocator, io, hwmon, prefix, idx); - } - return error.NotFound; -} - -inline fn readNumberedInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { - return readNumberedSuffix(allocator, io, hwmon, prefix, idx, "input"); -} - -inline fn readNumberedSuffix(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize, suffix: []const u8) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}_{s}", .{ hwmon, prefix, idx, suffix }); - return smi_sysfs.readInt(allocator, io, path); -} - -inline fn readNumberedRaw(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}", .{ hwmon, prefix, idx }); - return smi_sysfs.readInt(allocator, io, path); -} - -inline fn readPwmFanSpeed(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { - for (1..16) |idx| { - const pwm = readNumberedRaw(allocator, io, hwmon, "pwm", idx) catch continue; - return pwmToPercent(pwm); - } - return error.NotFound; -} - -inline fn readRpmFanSpeedPercent(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { - for (1..16) |idx| { - const input = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "input") catch continue; - const max = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "max") catch continue; - if (max == 0) continue; - return @min(input * 100 / max, 100); - } - return error.NotFound; -} - -inline fn pwmToPercent(pwm: u64) u64 { - return @min((@min(pwm, 255) * 100 + 127) / 255, 100); -} - -inline fn readLargestResource(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { - var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); - defer file.close(io); - - var read_buf: [4096]u8 = undefined; - var reader = file.reader(io, &read_buf); - var line_writer: std.Io.Writer.Allocating = .init(allocator); - defer line_writer.deinit(); - - var largest: u64 = 0; - while (true) { - line_writer.clearRetainingCapacity(); - _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch |err| switch (err) { - error.EndOfStream => break, - else => |e| return e, - }; - reader.interface.toss(1); - largest = @max(largest, resourceLineSize(line_writer.written()) orelse 0); - } - if (largest == 0) return error.NotFound; - return largest; -} - -inline fn resourceLineSize(line: []const u8) ?u64 { - var it = std.mem.tokenizeAny(u8, line, " \t"); - const base = parseHex(it.next() orelse return null) orelse return null; - const end = parseHex(it.next() orelse return null) orelse return null; - if (end < base) return null; - const size = end - base + 1; - return if (size >= 256 * 1024 * 1024) size else null; -} - -inline fn parseHex(raw: []const u8) ?u64 { - const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); - const digits = if (std.mem.startsWith(u8, trimmed, "0x")) trimmed[2..] else trimmed; - return std.fmt.parseInt(u64, digits, 16) catch null; -} - -inline fn pcieGenFromSpeed(raw: []const u8) !u64 { - const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); - if (std.mem.startsWith(u8, trimmed, "2.5")) return 1; - if (std.mem.startsWith(u8, trimmed, "5.0") or std.mem.startsWith(u8, trimmed, "5 ")) return 2; - if (std.mem.startsWith(u8, trimmed, "8.0") or std.mem.startsWith(u8, trimmed, "8 ")) return 3; - if (std.mem.startsWith(u8, trimmed, "16.0") or std.mem.startsWith(u8, trimmed, "16 ")) return 4; - if (std.mem.startsWith(u8, trimmed, "32.0") or std.mem.startsWith(u8, trimmed, "32 ")) return 5; - if (std.mem.startsWith(u8, trimmed, "64.0") or std.mem.startsWith(u8, trimmed, "64 ")) return 6; - return error.NotFound; -} - -inline fn pcieBandwidthFromLink(gen: u64, width: u64) ?u64 { - const lane_mb_s: u64 = switch (gen) { - 1 => 250, - 2 => 500, - 3 => 985, - 4 => 1969, - 5 => 3938, - 6 => 7563, - else => return null, - }; - return lane_mb_s * width; -} - -const FdinfoSample = struct { - mem_kib: ?u64 = null, - engine: ?EngineSample = null, - encoder: ?EngineSample = null, - decoder: ?EngineSample = null, -}; - -const ParsedFdinfo = struct { - bus_device_identifier: ?[bus_device_identifier_len]u8 = null, - drm_client_id: ?u64 = null, - sample: FdinfoSample = .{}, - resident_vram_kib: u64 = 0, - total_vram_kib: u64 = 0, - resident_memory_kib: u64 = 0, - total_memory_kib: u64 = 0, - cycle_counter: u64 = 0, - total_cycle_counter: u64 = 0, - engine_time_seen: bool = false, -}; - -inline fn parseFdinfoFile(io: std.Io, path: []const u8, timestamp_ns: u64) !ParsedFdinfo { - var read_buf: [8192]u8 = undefined; - const data = std.Io.Dir.readFile(.cwd(), io, path, &read_buf) catch return error.FileUnavailable; - - var parsed: ParsedFdinfo = .{}; - - var lines = std.mem.tokenizeScalar(u8, data, '\n'); - while (lines.next()) |line| { - parseFdinfoLine(&parsed, line, timestamp_ns); - } - - finishFdinfo(&parsed); - return parsed; -} - -inline fn finishFdinfo(self: *ParsedFdinfo) void { - if (self.resident_vram_kib > 0) { - self.sample.mem_kib = self.resident_vram_kib; - } else if (self.total_vram_kib > 0) { - self.sample.mem_kib = self.total_vram_kib; - } else if (self.resident_memory_kib > 0) { - self.sample.mem_kib = self.resident_memory_kib; - } else if (self.total_memory_kib > 0) { - self.sample.mem_kib = self.total_memory_kib; - } - - if (!self.engine_time_seen and self.total_cycle_counter > 0) { - self.sample.engine = .{ - .engine_ns = self.cycle_counter, - .timestamp_ns = self.total_cycle_counter, - }; - } -} - -inline fn parseFdinfoLine(parsed: *ParsedFdinfo, line: []const u8, timestamp_ns: u64) void { - if (std.mem.startsWith(u8, line, "drm-pdev:")) { - parsed.bus_device_identifier = parseBdf(line["drm-pdev:".len..]); - return; - } - - if (std.mem.startsWith(u8, line, "drm-client-id:")) { - parsed.drm_client_id = firstInt(line["drm-client-id:".len..]); - return; - } - - if (std.mem.startsWith(u8, line, "vram mem:")) { - parsed.total_vram_kib += memoryKiB(line["vram mem:".len..]); - return; - } - - if (std.mem.startsWith(u8, line, "drm-cycles-")) { - const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; - parsed.cycle_counter += firstInt(line[colon + 1 ..]); - return; - } - - if (std.mem.startsWith(u8, line, "drm-total-cycles-")) { - const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; - parsed.total_cycle_counter = @max(parsed.total_cycle_counter, firstInt(line[colon + 1 ..])); - return; - } - - if (std.mem.startsWith(u8, line, "drm-engine-capacity-")) { - return; - } - - if (std.mem.startsWith(u8, line, "drm-engine-")) { - const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; - const engine_label = line["drm-engine-".len..colon]; - const value = firstInt(line[colon + 1 ..]); - parsed.engine_time_seen = true; - addEngineTime(&parsed.sample.engine, value, timestamp_ns); - if (containsAsciiIgnoreCase(engine_label, "encode")) { - addEngineTime(&parsed.sample.encoder, value, timestamp_ns); - } else if (containsAsciiIgnoreCase(engine_label, "decode")) { - addEngineTime(&parsed.sample.decoder, value, timestamp_ns); - } - return; - } - - if (std.mem.startsWith(u8, line, "drm-")) { - const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; - const key = line[0..colon]; - const value = memoryKiB(line[colon + 1 ..]); - if (value == 0) return; - - if (std.mem.startsWith(u8, key, "drm-resident-vram")) { - parsed.resident_vram_kib += value; - return; - } - if (std.mem.startsWith(u8, key, "drm-total-vram") or std.mem.startsWith(u8, key, "drm-memory-vram")) { - parsed.total_vram_kib += value; - return; - } - if (std.mem.startsWith(u8, key, "drm-resident-")) { - parsed.resident_memory_kib += value; - return; - } - if (std.mem.startsWith(u8, key, "drm-total-")) { - parsed.total_memory_kib += value; - return; - } - if (std.mem.eql(u8, key, "drm-resident-memory")) { - parsed.resident_memory_kib += value; - return; - } - if (std.mem.eql(u8, key, "drm-total-memory")) { - parsed.total_memory_kib += value; - return; - } - } -} - -fn mergeProcessSample(dst: *ProcessSample, src: FdinfoSample) void { - if (src.mem_kib) |mem| dst.mem_kib = (dst.mem_kib orelse 0) + mem; - addEngineSample(&dst.engine, src.engine); - addEngineSample(&dst.encoder, src.encoder); - addEngineSample(&dst.decoder, src.decoder); -} - -fn addEngineSample(dst: *?EngineSample, src: ?EngineSample) void { - if (src) |engine| addEngineTime(dst, engine.engine_ns, engine.timestamp_ns); -} - -fn addEngineTime(dst: *?EngineSample, engine_ns: u64, timestamp_ns: u64) void { - const current = dst.* orelse EngineSample{ .engine_ns = 0, .timestamp_ns = timestamp_ns }; - dst.* = .{ - .engine_ns = current.engine_ns + engine_ns, - .timestamp_ns = timestamp_ns, - }; -} - -fn matchDevice(devices: []const Device, bus_device_identifier: [bus_device_identifier_len]u8) ?u16 { - for (devices, 0..) |dev, idx| { - if (dev.bus_device_identifier) |device_bus_device_identifier| { - if (std.mem.eql(u8, &device_bus_device_identifier, &bus_device_identifier)) return @intCast(idx); - } - } - return null; -} - -fn processKey(pid: u32, device_idx: u16) u64 { - return (@as(u64, device_idx) << 32) | pid; -} - -pub const BdfParts = struct { - domain: u32, - bus: u32, - device: u32, - function: u32, -}; - -pub fn formatBdf(parts: BdfParts) [bus_device_identifier_len]u8 { - var buf: [bus_device_identifier_len]u8 = undefined; - _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ - parts.domain, - parts.bus, - parts.device, - parts.function, - }) catch unreachable; - return buf; -} - -inline fn parseBdf(raw: []const u8) ?[bus_device_identifier_len]u8 { - const trimmed = std.mem.trim(u8, raw, " \t\n"); - var out: [bus_device_identifier_len]u8 = undefined; - if (trimmed.len == bus_device_identifier_len) { - @memcpy(&out, trimmed[0..bus_device_identifier_len]); - return out; - } - if (trimmed.len == "00:00.0".len) { - _ = std.fmt.bufPrint(&out, "0000:{s}", .{trimmed}) catch return null; - return out; - } - return null; -} - -inline fn firstInt(raw: []const u8) u64 { - var iter = std.mem.tokenizeAny(u8, raw, " \t"); - return std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch 0; -} - -inline fn memoryKiB(raw: []const u8) u64 { - var iter = std.mem.tokenizeAny(u8, raw, " \t"); - const value = std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch return 0; - const unit = iter.next() orelse return value; - if (std.ascii.eqlIgnoreCase(unit, "KiB")) return value; - if (std.ascii.eqlIgnoreCase(unit, "MiB")) return value * 1024; - if (std.ascii.eqlIgnoreCase(unit, "GiB")) return value * 1024 * 1024; - if (std.ascii.eqlIgnoreCase(unit, "B")) return value / 1024; - return value; -} - -inline fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { - if (needle.len == 0) return true; - if (needle.len > haystack.len) return false; - for (0..haystack.len - needle.len + 1) |i| { - if (std.ascii.eqlIgnoreCase(haystack[i .. i + needle.len], needle)) return true; - } - return false; -} - -test "oneAPI fdinfo parser" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); - parseFdinfoLine(&parsed, "drm-client-id:\t42", 10); - parseFdinfoLine(&parsed, "drm-total-memory:\t2048 KiB", 10); - parseFdinfoLine(&parsed, "drm-engine-render:\t100 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-compute:\t50 ns", 10); - finishFdinfo(&parsed); - - try std.testing.expect(parsed.bus_device_identifier != null); - try std.testing.expectEqualSlices(u8, "0000:03:00.0", &parsed.bus_device_identifier.?); - try std.testing.expectEqual(@as(?u64, 42), parsed.drm_client_id); - try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); - try std.testing.expectEqual(@as(u64, 150), parsed.sample.engine.?.engine_ns); - try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); - try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); -} - -test "oneAPI fdinfo parser classifies explicit media engines" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-engine-video-decode:\t100 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-media-encode:\t50 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-VIDEO-DECODE1:\t25 ns", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); - try std.testing.expectEqual(@as(u64, 50), parsed.sample.encoder.?.engine_ns); - try std.testing.expectEqual(@as(u64, 125), parsed.sample.decoder.?.engine_ns); -} - -test "oneAPI fdinfo parser leaves generic video engines unclassified" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-engine-video:\t100 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-vcs0:\t50 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-vecs0:\t25 ns", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); - try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); - try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); -} - -test "oneAPI fdinfo parser cycle counters" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); - parseFdinfoLine(&parsed, "drm-cycles-rcs:\t10", 10); - parseFdinfoLine(&parsed, "drm-total-cycles-rcs:\t100", 10); - parseFdinfoLine(&parsed, "drm-engine-capacity-vcs:\t2", 10); - parseFdinfoLine(&parsed, "drm-cycles-ccs:\t30", 10); - parseFdinfoLine(&parsed, "drm-total-cycles-ccs:\t100", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(u64, 40), parsed.sample.engine.?.engine_ns); - try std.testing.expectEqual(@as(u64, 100), parsed.sample.engine.?.timestamp_ns); -} - -test "oneAPI fdinfo parser xe vram regions" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); - parseFdinfoLine(&parsed, "drm-total-system:\t184 MiB", 10); - parseFdinfoLine(&parsed, "drm-total-vram0:\t4096 KiB", 10); - parseFdinfoLine(&parsed, "drm-resident-vram0:\t3072 KiB", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(?u64, 3072), parsed.sample.mem_kib); -} - -test "oneAPI fdinfo parser falls back to total vram" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-total-vram0:\t2 MiB", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); -} - -test "oneAPI DRM clients dedup per device" { - var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; - defer seen_device_clients.deinit(std.testing.allocator); - - try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); - try std.testing.expectEqual(true, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); - try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 1, 42 })).found_existing); - try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 43 })).found_existing); -} - -test "oneAPI process utilization delta" { - try std.testing.expectEqual(@as(?u16, 50), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 600, .timestamp_ns = 2000 })); - try std.testing.expectEqual(@as(?u16, 1), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 101, .timestamp_ns = 2000 })); - try std.testing.expectEqual(@as(?u16, null), processUtil(null, .{ .engine_ns = 600, .timestamp_ns = 2000 })); -} - -test "oneAPI PWM fan speed conversion" { - try std.testing.expectEqual(@as(u64, 0), pwmToPercent(0)); - try std.testing.expectEqual(@as(u64, 50), pwmToPercent(128)); - try std.testing.expectEqual(@as(u64, 100), pwmToPercent(255)); - try std.testing.expectEqual(@as(u64, 100), pwmToPercent(300)); -} - -test "oneAPI process previous keeps engine samples" { - var usage: ProcessUsage = .{}; - defer usage.deinit(std.testing.allocator); - - const engine_key = processKey(10, 1); - const mem_only_key = processKey(20, 1); - try usage.put(std.testing.allocator, engine_key, .{ - .pid = 10, - .device_idx = 1, - .engine = .{ .engine_ns = 100, .timestamp_ns = 1000 }, - }); - try usage.put(std.testing.allocator, mem_only_key, .{ - .pid = 20, - .device_idx = 1, - .mem_kib = 2048, - }); - - var previous: ProcessPrevious = .{}; - defer previous.deinit(std.testing.allocator); - - try saveProcessPrevious(std.testing.allocator, &previous, &usage); - - try std.testing.expectEqual(@as(u32, 1), previous.count()); - try std.testing.expectEqual(@as(?EngineSample, .{ .engine_ns = 100, .timestamp_ns = 1000 }), previous.get(engine_key)); - try std.testing.expectEqual(@as(?EngineSample, null), previous.get(mem_only_key)); -} - -test "oneAPI power delta from hwmon energy" { - try std.testing.expectEqual(@as(?u64, 20_000), milliwattsFromEnergyDelta(1_000, 11_000, 1_000, 501_000)); - try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(11_000, 1_000, 1_000, 501_000)); - try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(1_000, 11_000, 501_000, 1_000)); -} - -test "oneAPI resource line chooses large memory BARs" { - try std.testing.expectEqual(@as(?u64, 34_359_738_368), resourceLineSize("0x0000048800000000 0x0000048fffffffff 0x000000000014220c")); - try std.testing.expectEqual(@as(?u64, null), resourceLineSize("0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200")); -} - -test "oneAPI PCIe speed maps to generation" { - try std.testing.expectEqual(@as(u64, 1), try pcieGenFromSpeed("2.5 GT/s PCIe")); - try std.testing.expectEqual(@as(u64, 4), try pcieGenFromSpeed("16.0 GT/s PCIe")); - try std.testing.expectError(error.NotFound, pcieGenFromSpeed("Unknown")); -} - -test "oneAPI PCIe bandwidth derives from generation and width" { - try std.testing.expectEqual(@as(?u64, 250), pcieBandwidthFromLink(1, 1)); - try std.testing.expectEqual(@as(?u64, 63_008), pcieBandwidthFromLink(5, 16)); - try std.testing.expectEqual(@as(?u64, null), pcieBandwidthFromLink(0, 16)); -} diff --git a/bin/zml-smi/platforms/oneapi/oneapi.zig b/bin/zml-smi/platforms/oneapi/oneapi.zig deleted file mode 100644 index 2e1e5deb..00000000 --- a/bin/zml-smi/platforms/oneapi/oneapi.zig +++ /dev/null @@ -1,248 +0,0 @@ -const std = @import("std"); -const Monitor = @import("monitor.zig"); -const smi_sysfs = @import("zml-smi/sysfs"); - -const OneApi = @This(); - -pub const Handle = Monitor.Handle; - -monitor: Monitor, -io: std.Io, - -pub fn init(allocator: std.mem.Allocator, io: std.Io) !OneApi { - return .{ - .monitor = try Monitor.init(allocator, io), - .io = io, - }; -} - -pub fn handleByIndex(self: *const OneApi, device_id: usize) !Handle { - if (device_id >= self.monitor.devices.len) return error.NotFound; - return @fromBackingInt(@intCast(@as(u32, @intCast(device_id)))); -} - -pub fn name(self: *const OneApi, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { - if (self.monitor.device(handle)) |dev| { - if (dev.bus_device_identifier) |bdf| { - if (try lspciName(allocator, self.io, bdf[0..])) |display_name| { - return display_name; - } - } - } else |_| {} - - const id = try self.monitor.deviceId(allocator, handle); - return try std.fmt.allocPrint(allocator, "Intel GPU {s}", .{id}); -} - -fn lspciName(allocator: std.mem.Allocator, io: std.Io, bdf: []const u8) !?[]const u8 { - const result = std.process.run(allocator, io, .{ - .argv = &.{ "lspci", "-Dnn", "-s", bdf }, - .stdout_limit = .limited(4096), - .stderr_limit = .limited(1024), - .reserve_amount = 256, - }) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => return null, - }; - defer allocator.free(result.stdout); - defer allocator.free(result.stderr); - - switch (result.term) { - .exited => |code| if (code != 0) return null, - else => return null, - } - - return parseLspciName(allocator, result.stdout); -} - -fn parseLspciName(allocator: std.mem.Allocator, output: []const u8) !?[]const u8 { - var lines = std.mem.tokenizeScalar(u8, output, '\n'); - while (lines.next()) |line| { - const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace); - const header_end = std.mem.indexOf(u8, trimmed, "]: ") orelse continue; - const header = trimmed[0 .. header_end + 1]; - if (!containsAsciiIgnoreCase(header, "VGA") and !containsAsciiIgnoreCase(header, "Display")) continue; - - const name_start = header_end + "]: ".len; - const name_end = std.mem.lastIndexOf(u8, trimmed[name_start..], " [8086:") orelse - std.mem.indexOf(u8, trimmed[name_start..], " (rev ") orelse - trimmed[name_start..].len; - const raw_name = std.mem.trim(u8, trimmed[name_start .. name_start + name_end], &std.ascii.whitespace); - if (raw_name.len == 0 or std.mem.startsWith(u8, raw_name, "Intel Corporation Device ")) return null; - return try allocator.dupe(u8, raw_name); - } - return null; -} - -fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { - if (needle.len == 0) return true; - if (needle.len > haystack.len) return false; - for (0..haystack.len - needle.len + 1) |idx| { - if (std.ascii.eqlIgnoreCase(haystack[idx .. idx + needle.len], needle)) return true; - } - return false; -} - -test "parse lspci display name" { - const output = - \\0000:84:00.0 VGA compatible controller [0300]: Intel Corporation Battlemage G31 [Arc Pro B70] [8086:e223] (rev 04) - ; - const name_ = (try parseLspciName(std.testing.allocator, output)).?; - defer std.testing.allocator.free(name_); - try std.testing.expectEqualStrings("Intel Corporation Battlemage G31 [Arc Pro B70]", name_); -} - -test "parse lspci display name skips generic device ids" { - const output = - \\0000:84:00.0 Display controller [0380]: Intel Corporation Device e223 [8086:e223] (rev 04) - ; - try std.testing.expect(try parseLspciName(std.testing.allocator, output) == null); -} - -pub fn driverVersion(self: *const OneApi, allocator: std.mem.Allocator) ![]const u8 { - const version = smi_sysfs.readString(allocator, self.io, "/sys/module/xe/version") catch - smi_sysfs.readString(allocator, self.io, "/sys/module/xe/srcversion") catch - smi_sysfs.readString(allocator, self.io, "/sys/module/i915/version") catch - try smi_sysfs.readString(allocator, self.io, "/sys/module/i915/srcversion"); - const trimmed = std.mem.trim(u8, version, &std.ascii.whitespace); - if (trimmed.len == 0) return error.NotFound; - return trimmed; -} - -pub fn powerUsage(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = block: { - const idx = @backingInt(handle); - if (idx >= self.monitor.devices.len) return error.NotFound; - break :block &self.monitor.devices[idx]; - }; - const current = Monitor.readEnergy(allocator, self.io, dev) catch |err| { - dev.energy_prev = null; - return err; - }; - const power = Monitor.powerMilliwatts(dev.energy_prev, current) orelse { - dev.energy_prev = current; - return error.NotFound; - }; - dev.energy_prev = current; - return power; -} - -pub fn powerLimit(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readPowerLimit(allocator, self.io, dev); -} - -pub fn temperature(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readTemperature(allocator, self.io, dev); -} - -pub fn fanSpeed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readFanSpeed(allocator, self.io, dev); -} - -pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = block: { - const idx = @backingInt(handle); - if (idx >= self.monitor.devices.len) return error.NotFound; - break :block &self.monitor.devices[idx]; - }; - - var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch { - dev.activity_prev = null; - return 0; - }; - defer usage.deinit(allocator); - - const dev_idx: u16 = @intCast(@backingInt(handle)); - const current = if (usage.get(dev_idx)) |sample| sample.engine else null; - const util = Monitor.processUtil(dev.activity_prev, current) orelse 0; - dev.activity_prev = current; - return util; -} - -pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = block: { - const idx = @backingInt(handle); - if (idx >= self.monitor.devices.len) return error.NotFound; - break :block &self.monitor.devices[idx]; - }; - - var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { - dev.encoder_prev = null; - return err; - }; - defer usage.deinit(allocator); - - const dev_idx: u16 = @intCast(@backingInt(handle)); - const current = if (usage.get(dev_idx)) |sample| sample.encoder else null; - const util = Monitor.processUtil(dev.encoder_prev, current) orelse { - dev.encoder_prev = current; - return error.NotFound; - }; - dev.encoder_prev = current; - return util; -} - -pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = block: { - const idx = @backingInt(handle); - if (idx >= self.monitor.devices.len) return error.NotFound; - break :block &self.monitor.devices[idx]; - }; - - var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { - dev.decoder_prev = null; - return err; - }; - defer usage.deinit(allocator); - - const dev_idx: u16 = @intCast(@backingInt(handle)); - const current = if (usage.get(dev_idx)) |sample| sample.decoder else null; - const util = Monitor.processUtil(dev.decoder_prev, current) orelse { - dev.decoder_prev = current; - return error.NotFound; - }; - dev.decoder_prev = current; - return util; -} - -pub fn clockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readClockGraphics(allocator, self.io, dev); -} - -pub fn maxClockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readMaxClockGraphics(allocator, self.io, dev); -} - -pub fn memUsed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev_idx: u16 = @intCast(@backingInt(handle)); - - var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch return 0; - defer usage.deinit(allocator); - - return if (usage.get(dev_idx)) |sample| sample.mem_kib * 1024 else 0; -} - -pub fn memTotal(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readMemTotal(allocator, self.io, dev); -} - -pub fn pcieLinkGen(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readPcieLinkGen(allocator, self.io, dev); -} - -pub fn pcieLinkWidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readPcieLinkWidth(allocator, self.io, dev); -} - -pub fn pcieBandwidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readPcieBandwidth(allocator, self.io, dev); -} diff --git a/bin/zml-smi/platforms/oneapi/process.zig b/bin/zml-smi/platforms/oneapi/process.zig deleted file mode 100644 index c4d0c719..00000000 --- a/bin/zml-smi/platforms/oneapi/process.zig +++ /dev/null @@ -1,45 +0,0 @@ -const std = @import("std"); -const OneApi = @import("oneapi.zig"); -const Monitor = @import("monitor.zig"); -const pi = @import("zml-smi/info").process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; - -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16) !void { - const state = try collector.arena.create(Monitor.ProcessPrevious); - state.* = .{}; - - const previous_usage: ?Monitor.ProcessUsage = Monitor.collectProcessUsage(collector.gpa, collector.io, oneapi.monitor.devices) catch null; - if (previous_usage) |usage_value| { - var usage = usage_value; - defer usage.deinit(collector.gpa); - Monitor.saveProcessPrevious(collector.arena, state, &usage) catch {}; - } - - try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.arena, collector.io, list, oneapi, dev_offset, state }); -} - -fn pollOnce(allocator: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16, state: *Monitor.ProcessPrevious) void { - const back = list.back(); - back.clearRetainingCapacity(); - - var usage = Monitor.collectProcessUsage(allocator, io, oneapi.monitor.devices) catch { - list.swap(); - return; - }; - defer usage.deinit(allocator); - - var it = usage.iterator(); - while (it.next()) |entry| { - const sample = entry.value_ptr.*; - back.append(allocator, .{ - .pid = sample.pid, - .device_idx = @intCast(sample.device_idx + dev_offset), - .dev_mem_kib = sample.mem_kib, - .dev_util_percent = Monitor.processUtil(state.get(entry.key_ptr.*), sample.engine), - }) catch break; - } - - Monitor.saveProcessPrevious(arena, state, &usage) catch {}; - list.swap(); -} diff --git a/bin/zml-smi/platforms/tpu/metrics.zig b/bin/zml-smi/platforms/tpu/metrics.zig deleted file mode 100644 index 9410b3f8..00000000 --- a/bin/zml-smi/platforms/tpu/metrics.zig +++ /dev/null @@ -1,168 +0,0 @@ -const std = @import("std"); -const sysfs = @import("zml-smi/sysfs"); -const tpuinfo = @import("tpuinfo.zig"); -const device_info = @import("zml-smi/info").device_info; -const DeviceInfo = device_info.DeviceInfo; -const TpuInfo = device_info.TpuInfo; -const Collector = @import("zml-smi/collector").Collector; -const tpu_process = @import("process.zig"); - -const address = "localhost:8431"; - -pub fn start(collector: *Collector) !void { - const chip = scanPciChips(collector.arena, collector.io) orelse return error.TpuUnavailable; - const device_count = chip.chip_count * chip.devices_per_chip; - var tpu_infos: std.ArrayList(*DeviceInfo) = .empty; - const dev_offset: u8 = @intCast(collector.device_infos.items.len); - - for (0..device_count) |_| { - const chip_name = try collector.arena.dupe(u8, chip.name); - const initial: TpuInfo = .{ .name = chip_name }; - const info = try collector.addDevice(.{ .tpu = .{ .values = .{ initial, initial } } }); - try tpu_infos.append(collector.arena, info); - } - - try collector.spawnPoll(pollAllDevices, .{tpu_infos.items}); - - if (tpu_infos.items.len > 0) { - const processes = try collector.createProcessList(); - try tpu_process.init(collector, chip.devices_per_chip, tpu_infos.items, processes, dev_offset); - } -} - -const MetricKind = enum { int, double }; - -fn pollAllDevices(devs: []*DeviceInfo) void { - var mem_used: [tpuinfo.max_devices]?u64 = undefined; - var mem_total: [tpuinfo.max_devices]?u64 = undefined; - var util: [tpuinfo.max_devices]?u64 = undefined; - - for (devs, 0..) |info, i| { - const front = info.tpu.front(); - mem_used[i] = front.mem_used_bytes; - mem_total[i] = front.mem_total_bytes; - util[i] = front.util_percent; - } - - queryRaw("tpu.runtime.hbm.memory.usage.bytes", .int, mem_used[0..devs.len]); - queryRaw("tpu.runtime.hbm.memory.total.bytes", .int, mem_total[0..devs.len]); - queryRaw("tpu.runtime.tensorcore.dutycycle.percent", .double, util[0..devs.len]); - - for (devs, 0..) |info, i| { - const back = info.tpu.back(); - back.* = info.tpu.front().*; - back.mem_used_bytes = mem_used[i]; - back.mem_total_bytes = mem_total[i]; - back.util_percent = util[i]; - info.tpu.swap(); - } -} - -fn queryRaw(metric_name: [:0]const u8, comptime kind: MetricKind, out: []?u64) void { - var ids: [tpuinfo.max_devices]c_longlong = undefined; - switch (kind) { - .int => { - var vals: [tpuinfo.max_devices]c_longlong = undefined; - const n = tpuinfo.queryInt(address, metric_name, &ids, &vals) catch { - @memset(out, null); - return; - }; - - for (ids[0..n], vals[0..n]) |id, val| { - if (id >= 0 and id < out.len) out[@intCast(id)] = @intCast(val); - } - }, - .double => { - var vals: [tpuinfo.max_devices]f64 = undefined; - const n = tpuinfo.queryDouble(address, metric_name, &ids, &vals) catch { - @memset(out, null); - return; - }; - - for (ids[0..n], vals[0..n]) |id, val| { - if (id >= 0 and id < out.len and std.math.isFinite(val)) - out[@intCast(id)] = @intFromFloat(@round(val)); - } - }, - } -} - -// -- Discover available chips -// We need to scan /sys/bus/pci/devices for Google TPU chips. -// From: https://github.com/AI-Hypercomputer/cloud-accelerator-diagnostics/blob/c94f463dd3bf99eaecf4f5af46f87cc2a5ff1b22/tpu_info/tpu_info/device.py#L43-L92 - -const ChipInfo = struct { - name: []const u8, - devices_per_chip: u32, - chip_count: u32, -}; - -const google_pci_vendor_id = "0x1ae0"; -const pci_base = "/sys/bus/pci/devices"; - -const ChipEntry = struct { - device_id: []const u8, - subsystem_id: ?[]const u8 = null, - name: []const u8, - devices_per_chip: u32, -}; - -const chip_table = [_]ChipEntry{ - .{ .device_id = "0x0027", .subsystem_id = "0x004e", .name = "TPU v2", .devices_per_chip = 2 }, - .{ .device_id = "0x0027", .subsystem_id = "0x004f", .name = "TPU v3", .devices_per_chip = 2 }, - .{ .device_id = "0x005e", .name = "TPU v4", .devices_per_chip = 1 }, - .{ .device_id = "0x0063", .name = "TPU v5e", .devices_per_chip = 1 }, - .{ .device_id = "0x0062", .name = "TPU v5p", .devices_per_chip = 1 }, - .{ .device_id = "0x006f", .name = "TPU v6e", .devices_per_chip = 1 }, - .{ .device_id = "0x0076", .name = "TPU7x", .devices_per_chip = 2 }, -}; - -fn scanPciChips(allocator: std.mem.Allocator, io: std.Io) ?ChipInfo { - var pci_dir = std.Io.Dir.openDir(.cwd(), io, pci_base, .{ .iterate = true }) catch return null; - defer pci_dir.close(io); - - var it = pci_dir.iterate(); - var chip_count: u32 = 0; - var result: ChipInfo = undefined; - - while (it.next(io) catch null) |entry| { - const vendor = readSysfs(allocator, io, entry.name, "vendor") catch continue; - if (!std.mem.eql(u8, vendor, google_pci_vendor_id)) { - continue; - } - - const device_raw = readSysfs(allocator, io, entry.name, "device") catch continue; - const subsystem_raw: ?[]const u8 = readSysfs(allocator, io, entry.name, "subsystem_device") catch null; - - for (chip_table) |chip| { - const dev_match = std.mem.eql(u8, device_raw, chip.device_id); - const sub_match = if (chip.subsystem_id) |expected_sub| - if (subsystem_raw) |actual_sub| std.mem.eql(u8, actual_sub, expected_sub) else false - else - true; - - if (dev_match and sub_match) { - result = .{ - .name = chip.name, - .devices_per_chip = chip.devices_per_chip, - .chip_count = undefined, - }; - - chip_count += 1; - break; - } - } - } - - if (chip_count > 0) { - result.chip_count = chip_count; - return result; - } - - return null; -} - -fn readSysfs(allocator: std.mem.Allocator, io: std.Io, slot: []const u8, file: []const u8) ![]const u8 { - const path = try std.fmt.allocPrint(allocator, pci_base ++ "/{s}/{s}", .{ slot, file }); - return sysfs.readString(allocator, io, path); -} diff --git a/bin/zml-smi/platforms/tpu/process.zig b/bin/zml-smi/platforms/tpu/process.zig deleted file mode 100644 index cca8f6a1..00000000 --- a/bin/zml-smi/platforms/tpu/process.zig +++ /dev/null @@ -1,73 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const pi = smi_info.process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const Collector = @import("zml-smi/collector").Collector; - -pub fn init(collector: *Collector, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u16) !void { - try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, devices_per_chip, device_infos, dev_offset }); -} - -fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u16) void { - scan(allocator, io, list.back(), devices_per_chip, device_infos, dev_offset); - list.swap(); -} - -fn scan(allocator: std.mem.Allocator, io: std.Io, back: *std.ArrayList(pi.ProcessInfo), devices_per_chip: u32, infos: []*DeviceInfo, dev_offset: u16) void { - back.clearRetainingCapacity(); - - var proc_dir = std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }) catch return; - defer proc_dir.close(io); - - var proc_it = proc_dir.iterate(); - while (proc_it.next(io) catch null) |proc_entry| { - const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; - - var fd_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const fd_sub = std.fmt.bufPrint(&fd_path_buf, "{d}/fd", .{pid}) catch continue; - var fd_dir = proc_dir.openDir(io, fd_sub, .{ .iterate = true }) catch continue; - defer fd_dir.close(io); - - var fd_it = fd_dir.iterate(); - while (fd_it.next(io) catch null) |fd_entry| { - var link_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const len = fd_dir.readLink(io, fd_entry.name, &link_buf) catch continue; - const target = link_buf[0..len]; - - if (parseChipIndex(target)) |chip_idx| { - const base: u16 = @intCast(chip_idx * devices_per_chip); - for (0..devices_per_chip) |d| { - const local_idx: u16 = @intCast(base + d); - var info: pi.ProcessInfo = .{ .pid = pid, .device_idx = local_idx + dev_offset }; - - if (local_idx < infos.len) { - const tpu = infos[local_idx].tpu.front().*; - - if (tpu.mem_used_bytes) |mem| { - info.dev_mem_kib = @intCast(mem / 1024); - } - - if (tpu.util_percent) |util| { - info.dev_util_percent = @intCast(util); - } - } - - back.append(allocator, info) catch return; - } - } - } - } -} - -fn parseChipIndex(target: []const u8) ?u32 { - if (std.mem.startsWith(u8, target, "/dev/accel")) { - return std.fmt.parseInt(u32, target["/dev/accel".len..], 10) catch null; - } - - if (std.mem.startsWith(u8, target, "/dev/vfio/")) { - return std.fmt.parseInt(u32, target["/dev/vfio/".len..], 10) catch null; - } - - return null; -} diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.zig b/bin/zml-smi/platforms/tpu/tpuinfo.zig deleted file mode 100644 index 1baf62df..00000000 --- a/bin/zml-smi/platforms/tpu/tpuinfo.zig +++ /dev/null @@ -1,37 +0,0 @@ -const std = @import("std"); -const c = @import("c"); - -pub const max_devices: usize = c.TPUINFO_MAX_DEVICES; - -pub const Error = error{ - TpuUnavailable, - QueryFailed, -}; - -pub fn queryInt( - address: [:0]const u8, - metric_name: [:0]const u8, - device_ids: []c_longlong, - values: []c_longlong, -) Error!u32 { - const n = c.tpu_query_int(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); - if (n < 0) { - return error.QueryFailed; - } - - return @intCast(n); -} - -pub fn queryDouble( - address: [:0]const u8, - metric_name: [:0]const u8, - device_ids: []c_longlong, - values: []f64, -) Error!u32 { - const n = c.tpu_query_double(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); - if (n < 0) { - return error.QueryFailed; - } - - return @intCast(n); -} From a3b5876b2e67f72c9865ce34e017eb91161558ab Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 20:16:06 +0100 Subject: [PATCH 06/19] bin/zml-smi: add the core module zml-smi/smi, imported as "zml-smi", replaces info, collector and utils. A device is written once and a sample is a union keyed by its target: named ?u64 fields in one unit each, no strings. Platforms implement smi.Platform, an interface with one batch poll and an optional process list per interval, over samples that start empty. The collector owns the tasks, the global device indices and the snapshots: one task per source, a host task that merges and enriches the process rows once per round, and one round (two when a source derives values from two polls) for the one-shot outputs. Remote hosts are sources too: their fetch races a timer, and their devices are matched by row, target and name. The blocks the platforms share, each with its tests: sysfs reads into caller buffers through directories kept open, hwmon, pci, fdinfo, FdHolders, dynlib with Checked, units and Counter. --- bin/zml-smi/BUILD.bazel | 1 + bin/zml-smi/collector/BUILD.bazel | 13 -- bin/zml-smi/collector/collector.zig | 97 ---------- bin/zml-smi/collector/worker.zig | 25 --- bin/zml-smi/info/BUILD.bazel | 15 -- bin/zml-smi/info/device_info.zig | 95 ---------- bin/zml-smi/info/host_info.zig | 16 -- bin/zml-smi/info/info.zig | 9 - bin/zml-smi/info/poll_metrics.zig | 20 --- bin/zml-smi/info/process_info.zig | 12 -- bin/zml-smi/smi/BUILD.bazel | 33 ++++ bin/zml-smi/smi/collector.zig | 262 ++++++++++++++++++++++++++++ bin/zml-smi/smi/counter.zig | 139 +++++++++++++++ bin/zml-smi/smi/double_buffer.zig | 59 +++++++ bin/zml-smi/smi/dynlib.zig | 108 ++++++++++++ bin/zml-smi/smi/fdinfo.zig | 170 ++++++++++++++++++ bin/zml-smi/smi/host/linux.zig | 140 +++++++++++++++ bin/zml-smi/smi/host/macos.zig | 99 +++++++++++ bin/zml-smi/smi/hwmon.zig | 90 ++++++++++ bin/zml-smi/smi/model.zig | 178 +++++++++++++++++++ bin/zml-smi/smi/pci.zig | 164 +++++++++++++++++ bin/zml-smi/smi/proc.zig | 94 ++++++++++ bin/zml-smi/smi/remote.zig | 147 ++++++++++++++++ bin/zml-smi/smi/sandbox.zig | 30 ++++ bin/zml-smi/smi/smi.zig | 50 ++++++ bin/zml-smi/smi/sysfs.zig | 148 ++++++++++++++++ bin/zml-smi/smi/units.zig | 30 ++++ bin/zml-smi/utils/BUILD.bazel | 30 ---- bin/zml-smi/utils/double_buffer.zig | 32 ---- bin/zml-smi/utils/dynlib.zig | 10 -- bin/zml-smi/utils/sandbox.zig | 44 ----- bin/zml-smi/utils/sysfs.zig | 70 -------- 32 files changed, 1942 insertions(+), 488 deletions(-) delete mode 100644 bin/zml-smi/collector/BUILD.bazel delete mode 100644 bin/zml-smi/collector/collector.zig delete mode 100644 bin/zml-smi/collector/worker.zig delete mode 100644 bin/zml-smi/info/BUILD.bazel delete mode 100644 bin/zml-smi/info/device_info.zig delete mode 100644 bin/zml-smi/info/host_info.zig delete mode 100644 bin/zml-smi/info/info.zig delete mode 100644 bin/zml-smi/info/poll_metrics.zig delete mode 100644 bin/zml-smi/info/process_info.zig create mode 100644 bin/zml-smi/smi/BUILD.bazel create mode 100644 bin/zml-smi/smi/collector.zig create mode 100644 bin/zml-smi/smi/counter.zig create mode 100644 bin/zml-smi/smi/double_buffer.zig create mode 100644 bin/zml-smi/smi/dynlib.zig create mode 100644 bin/zml-smi/smi/fdinfo.zig create mode 100644 bin/zml-smi/smi/host/linux.zig create mode 100644 bin/zml-smi/smi/host/macos.zig create mode 100644 bin/zml-smi/smi/hwmon.zig create mode 100644 bin/zml-smi/smi/model.zig create mode 100644 bin/zml-smi/smi/pci.zig create mode 100644 bin/zml-smi/smi/proc.zig create mode 100644 bin/zml-smi/smi/remote.zig create mode 100644 bin/zml-smi/smi/sandbox.zig create mode 100644 bin/zml-smi/smi/smi.zig create mode 100644 bin/zml-smi/smi/sysfs.zig create mode 100644 bin/zml-smi/smi/units.zig delete mode 100644 bin/zml-smi/utils/BUILD.bazel delete mode 100644 bin/zml-smi/utils/double_buffer.zig delete mode 100644 bin/zml-smi/utils/dynlib.zig delete mode 100644 bin/zml-smi/utils/sandbox.zig delete mode 100644 bin/zml-smi/utils/sysfs.zig diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index 077f5b34..cfefedf2 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -61,6 +61,7 @@ copy_to_directory( "libnrt.so.1": "lib/libnrt.so.1", "libncfw.so.2": "lib/libncfw.so.2", }, + visibility = ["//bin/zml-smi:__subpackages__"], ) run_binary( diff --git a/bin/zml-smi/collector/BUILD.bazel b/bin/zml-smi/collector/BUILD.bazel deleted file mode 100644 index 0bf5fa8e..00000000 --- a/bin/zml-smi/collector/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "collector", - import_name = "zml-smi/collector", - main = "collector.zig", - srcs = ["worker.zig"], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - ], -) diff --git a/bin/zml-smi/collector/collector.zig b/bin/zml-smi/collector/collector.zig deleted file mode 100644 index 3453e942..00000000 --- a/bin/zml-smi/collector/collector.zig +++ /dev/null @@ -1,97 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const pi = smi_info.process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Worker = @import("worker.zig").Worker; - -pub const Collector = struct { - device_infos: std.ArrayList(*DeviceInfo) = .empty, - process_lists: std.ArrayList(*ProcessDoubleBuffer) = .empty, - /// Every process list merged into one, written by the process poll of the host. - processes: ProcessDoubleBuffer = .{ .values = .{ .empty, .empty } }, - poll_arenas: std.ArrayList(*std.heap.ArenaAllocator) = .empty, - arena: std.mem.Allocator, - gpa: std.mem.Allocator, - worker: Worker, - io: std.Io, - poll_only: bool, - - pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, options: struct { poll_interval_ms: u16, poll_only: bool = false }) Collector { - return .{ - .arena = arena, - .gpa = gpa, - .io = io, - .worker = .{ .poll_interval_ms = options.poll_interval_ms }, - .poll_only = options.poll_only, - }; - } - - pub fn addDevice(self: *Collector, initial: DeviceInfo) !*DeviceInfo { - const info = try self.arena.create(DeviceInfo); - info.* = initial; - try self.device_infos.append(self.arena, info); - - return info; - } - - pub fn createProcessList(self: *Collector) !*ProcessDoubleBuffer { - const list = try self.arena.create(ProcessDoubleBuffer); - list.* = .{ .values = .{ .empty, .empty } }; - try self.process_lists.append(self.arena, list); - - return list; - } - - pub fn createPollArena(self: *Collector) !*std.heap.ArenaAllocator { - const poll_arena = try self.arena.create(std.heap.ArenaAllocator); - poll_arena.* = std.heap.ArenaAllocator.init(self.gpa); - try self.poll_arenas.append(self.arena, poll_arena); - - return poll_arena; - } - - pub fn spawnPoll(self: *Collector, comptime pollOnce: anytype, args: std.meta.ArgsTuple(@TypeOf(pollOnce))) !void { - @call(.auto, pollOnce, args); - if (self.poll_only) return; - - const Args = std.meta.ArgsTuple(@TypeOf(pollOnce)); - try self.worker.spawn(self.io, struct { - fn f(io: std.Io, w: *const Worker, a: Args) void { - const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); - while (w.isRunning()) { - const start: std.Io.Timestamp = .now(io, .awake); - - @call(.auto, pollOnce, a); - - const elapsed = start.untilNow(io, .awake); - if (elapsed.nanoseconds < interval.nanoseconds) { - io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; - } - } - } - }.f, .{ self.io, &self.worker, args }); - } - - pub fn deinit(self: *Collector) void { - self.worker.shutdown(self.io); - - for (self.device_infos.items) |info| { - self.arena.destroy(info); - } - self.device_infos.deinit(self.arena); - - for (self.process_lists.items) |list| { - list.values[0].deinit(self.gpa); - list.values[1].deinit(self.gpa); - } - self.process_lists.deinit(self.arena); - self.processes.values[0].deinit(self.gpa); - self.processes.values[1].deinit(self.gpa); - - for (self.poll_arenas.items) |poll_arena| { - poll_arena.deinit(); - } - self.poll_arenas.deinit(self.arena); - } -}; diff --git a/bin/zml-smi/collector/worker.zig b/bin/zml-smi/collector/worker.zig deleted file mode 100644 index 125440c5..00000000 --- a/bin/zml-smi/collector/worker.zig +++ /dev/null @@ -1,25 +0,0 @@ -const std = @import("std"); - -pub const Worker = struct { - poll_interval_ms: u16, - should_stop: std.atomic.Value(bool) = .init(false), - group: std.Io.Group = .init, - - pub fn shutdown(self: *Worker, io: std.Io) void { - self.should_stop.store(true, .release); - self.group.await(io) catch {}; - } - - pub fn isRunning(self: *const Worker) bool { - return !self.should_stop.load(.acquire); - } - - pub fn spawn( - self: *Worker, - io: std.Io, - comptime runFn: anytype, - args: std.meta.ArgsTuple(@TypeOf(runFn)), - ) !void { - try self.group.concurrent(io, runFn, args); - } -}; diff --git a/bin/zml-smi/info/BUILD.bazel b/bin/zml-smi/info/BUILD.bazel deleted file mode 100644 index d035b161..00000000 --- a/bin/zml-smi/info/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "info", - import_name = "zml-smi/info", - main = "info.zig", - srcs = [ - "device_info.zig", - "host_info.zig", - "poll_metrics.zig", - "process_info.zig", - ], - visibility = ["//visibility:public"], - deps = ["//bin/zml-smi/utils:double_buffer"], -) diff --git a/bin/zml-smi/info/device_info.zig b/bin/zml-smi/info/device_info.zig deleted file mode 100644 index 4b01c728..00000000 --- a/bin/zml-smi/info/device_info.zig +++ /dev/null @@ -1,95 +0,0 @@ -const std = @import("std"); -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; - -pub const Target = enum { - cuda, - rocm, - oneapi, - neuron, - tpu, -}; - -pub const DeviceInfo = union(Target) { - cuda: DoubleBuffer(GpuInfo), - rocm: DoubleBuffer(GpuInfo), - oneapi: DoubleBuffer(GpuInfo), - neuron: DoubleBuffer(NeuronInfo), - tpu: DoubleBuffer(TpuInfo), - - pub fn isRemote(self: *const DeviceInfo) bool { - switch (self.*) { - inline else => |*db| return db.front().remote, - } - } -}; - -pub const GpuInfo = struct { - remote: bool = false, - name: ?[]const u8 = null, - driver_version: ?[]const u8 = null, - cuda_driver_version: ?[]const u8 = null, - - // Utilization - util_percent: ?u64 = null, - encoder_util_percent: ?u64 = null, - decoder_util_percent: ?u64 = null, - - // Power - power_mw: ?u64 = null, - power_limit_mw: ?u64 = null, - // Thermal - temperature: ?u64 = null, - fan_speed_percent: ?u64 = null, - - // Clocks - clock_graphics_mhz: ?u64 = null, - clock_sm_mhz: ?u64 = null, - clock_soc_mhz: ?u64 = null, - clock_mem_mhz: ?u64 = null, - clock_graphics_max_mhz: ?u64 = null, - clock_mem_max_mhz: ?u64 = null, - - // Memory - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, - mem_bus_width: ?u64 = null, - - // PCIe - pcie_tx_kbps: ?u64 = null, - pcie_rx_kbps: ?u64 = null, - pcie_bandwidth_mbps: ?u64 = null, - pcie_link_gen: ?u64 = null, - pcie_link_width: ?u64 = null, -}; - -pub const NeuronInfo = struct { - remote: bool = false, - name: ?[]const u8 = null, - driver_version: ?[]const u8 = null, - - util_percent: ?u64 = null, - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, - - // Neuron per-core HBM breakdown (bytes) - nc_tensors: ?u64 = null, - nc_constants: ?u64 = null, - nc_model_code: ?u64 = null, - nc_shared_scratchpad: ?u64 = null, - nc_nonshared_scratchpad: ?u64 = null, - nc_runtime: ?u64 = null, - nc_driver: ?u64 = null, - nc_dma_rings: ?u64 = null, - nc_collectives: ?u64 = null, - nc_notifications: ?u64 = null, - nc_uncategorized: ?u64 = null, -}; - -pub const TpuInfo = struct { - remote: bool = false, - name: ?[]const u8 = null, - - util_percent: ?u64 = null, - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, -}; diff --git a/bin/zml-smi/info/host_info.zig b/bin/zml-smi/info/host_info.zig deleted file mode 100644 index f33f6fd8..00000000 --- a/bin/zml-smi/info/host_info.zig +++ /dev/null @@ -1,16 +0,0 @@ -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; - -pub const HostInfo = DoubleBuffer(HostData); - -pub const HostData = struct { - hostname: ?[]const u8 = null, - kernel: ?[]const u8 = null, - cpu_name: ?[]const u8 = null, - cpu_cores: ?u64 = null, - mem_total_kib: ?u64 = null, - mem_available_kib: ?u64 = null, - load_1: ?f32 = null, - load_5: ?f32 = null, - load_15: ?f32 = null, - uptime_seconds: ?u64 = null, -}; diff --git a/bin/zml-smi/info/info.zig b/bin/zml-smi/info/info.zig deleted file mode 100644 index 42e1b1b3..00000000 --- a/bin/zml-smi/info/info.zig +++ /dev/null @@ -1,9 +0,0 @@ -const std = @import("std"); - -pub const device_info = @import("device_info.zig"); -pub const host_info = @import("host_info.zig"); -pub const poll_metrics = @import("poll_metrics.zig"); -pub const process_info = @import("process_info.zig"); - -pub const Target = device_info.Target; -pub const Targets = std.EnumSet(Target); diff --git a/bin/zml-smi/info/poll_metrics.zig b/bin/zml-smi/info/poll_metrics.zig deleted file mode 100644 index 62ed94a7..00000000 --- a/bin/zml-smi/info/poll_metrics.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); - -pub fn apply(comptime table: anytype, dest: anytype, src: anytype) void { - inline for (table) |m| { - @field(dest, m.field) = m.query(src) catch null; - } -} - -pub fn poll(comptime DB: type, comptime Dev: type, comptime table: anytype) fn (?*std.heap.ArenaAllocator, DB, Dev) void { - return struct { - fn f(arena: ?*std.heap.ArenaAllocator, db: DB, dev: Dev) void { - if (arena) |a| _ = a.reset(.retain_capacity); - - const back = db.back(); - back.* = db.front().*; - apply(table, back, dev); - db.swap(); - } - }.f; -} diff --git a/bin/zml-smi/info/process_info.zig b/bin/zml-smi/info/process_info.zig deleted file mode 100644 index 73504656..00000000 --- a/bin/zml-smi/info/process_info.zig +++ /dev/null @@ -1,12 +0,0 @@ -pub const ProcessInfo = struct { - pid: u32 = 0, - device_idx: u16 = 0, - dev_util_percent: ?u16 = null, - dev_mem_kib: ?u64 = null, - uid: u32 = 0, - username: []const u8 = "", - comm: []const u8 = "", - cpu_percent: u16 = 0, // *10, e.g. 125 = 12.5% - rss_kib: u64 = 0, - remote: bool = false, -}; diff --git a/bin/zml-smi/smi/BUILD.bazel b/bin/zml-smi/smi/BUILD.bazel new file mode 100644 index 00000000..fd3bf901 --- /dev/null +++ b/bin/zml-smi/smi/BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "smi", + srcs = [ + "collector.zig", + "counter.zig", + "double_buffer.zig", + "dynlib.zig", + "fdinfo.zig", + "host/linux.zig", + "host/macos.zig", + "hwmon.zig", + "model.zig", + "pci.zig", + "proc.zig", + "remote.zig", + "sandbox.zig", + "sysfs.zig", + "units.zig", + ], + import_name = "zml-smi", + main = "smi.zig", + visibility = ["//bin/zml-smi:__subpackages__"], + deps = ["@rules_zig//zig/runfiles"], +) + +zig_test( + name = "test", + # The sandbox test looks the directory up through the runfiles. + data = ["//bin/zml-smi:sandbox"], + deps = [":smi"], +) diff --git a/bin/zml-smi/smi/collector.zig b/bin/zml-smi/smi/collector.zig new file mode 100644 index 00000000..3cd2053b --- /dev/null +++ b/bin/zml-smi/smi/collector.zig @@ -0,0 +1,262 @@ +const std = @import("std"); +const smi = @import("smi.zig"); + +/// Owns everything a platform does not: tasks, global device indices, snapshots +/// and the merged process list. +pub const Collector = struct { + /// Every device of every platform and remote host, in the order they were added. + devices: std.ArrayList(smi.Device) = .empty, + /// Every process list merged into one, with host details. Written by the host step. + processes: smi.DoubleBuffer(smi.ProcessList), + host: smi.DoubleBuffer(smi.HostData), + /// A source derives values from two polls, so a one-shot print runs two rounds. + two_readings: bool = false, + + sources: std.ArrayList(*Source) = .empty, + host_reader: smi.Host, + host_scratch: std.heap.ArenaAllocator, + arena: std.mem.Allocator, + gpa: std.mem.Allocator, + io: std.Io, + interval: std.Io.Duration, + tasks: std.Io.Group = .init, + stopping: std.atomic.Value(bool) = .init(false), + + pub const Options = struct { poll_interval_ms: u16 = 500 }; + + const Source = struct { + platform: *smi.Platform, + /// The global index of its first device. + first: u16, + sides: smi.DoubleBuffer(Side), + scratch: std.heap.ArenaAllocator, + }; + + const Side = struct { samples: []smi.Sample, processes: smi.ProcessList }; + + pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, options: Options) Collector { + var host: smi.HostData = .{}; + const host_reader: smi.Host = .init(arena, gpa, io, &host); + return .{ + .processes = .{ .values = .{ .init(gpa), .init(gpa) } }, + .host = .{ .values = .{ host, host } }, + .host_reader = host_reader, + .host_scratch = .init(gpa), + .arena = arena, + .gpa = gpa, + .io = io, + .interval = .fromMilliseconds(options.poll_interval_ms), + }; + } + + /// Stops the tasks, then calls `deinit` on the platforms that have one. + pub fn deinit(self: *Collector) void { + self.stopping.store(true, .release); + self.tasks.cancel(self.io); + for (self.sources.items) |source| { + if (source.platform.vtable.deinit) |deinitPlatform| deinitPlatform(source.platform); + for (&source.sides.values) |*side| side.processes.deinit(); + source.scratch.deinit(); + } + for (&self.processes.values) |*list| list.deinit(); + self.host_reader.deinit(); + self.host_scratch.deinit(); + } + + /// Registers an opened platform or remote host. Both sides of its samples start empty. + pub fn add(self: *Collector, platform: *smi.Platform) !void { + const count = platform.devices.len; + const source = try self.arena.create(Source); + source.* = .{ + .platform = platform, + .first = std.math.cast(u16, self.devices.items.len) orelse return error.TooManyDevices, + .sides = .{ .values = .{ + .{ .samples = try self.arena.alloc(smi.Sample, count), .processes = .init(self.gpa) }, + .{ .samples = try self.arena.alloc(smi.Sample, count), .processes = .init(self.gpa) }, + } }, + .scratch = .init(self.gpa), + }; + for (&source.sides.values) |side| { + for (side.samples, platform.devices) |*sample_, device| sample_.* = .init(device.target); + } + try self.devices.appendSlice(self.arena, platform.devices); + try self.sources.append(self.arena, source); + self.two_readings = self.two_readings or platform.two_readings; + } + + /// One round: every source once, then the host step. + pub fn round(self: *Collector) void { + for (self.sources.items) |source| self.poll(source); + self.hostStep(); + } + + /// Runs one round, then starts one task per source and the host task. No `add` may follow. + pub fn start(self: *Collector) !void { + const begin: std.Io.Timestamp = .now(self.io, .awake); + self.round(); + for (self.sources.items) |source| try self.tasks.concurrent(self.io, run, .{ self, source, begin }); + try self.tasks.concurrent(self.io, run, .{ self, null, begin }); + } + + /// The latest sample of a device, by value. + pub fn sample(self: *Collector, device: smi.Device.Index) smi.Sample { + const index = @backingInt(device); + for (self.sources.items) |source| { + const position = index -% source.first; + if (index < source.first or position >= source.platform.devices.len) continue; + defer source.sides.unlock(self.io); + return source.sides.lock(self.io).samples[position]; + } + unreachable; + } + + fn run(self: *Collector, source: ?*Source, first_round: std.Io.Timestamp) std.Io.Cancelable!void { + var begin = first_round; + while (true) { + const elapsed = begin.untilNow(self.io, .awake); + if (elapsed.nanoseconds < self.interval.nanoseconds) { + try self.io.sleep(.fromNanoseconds(self.interval.nanoseconds - elapsed.nanoseconds), .awake); + } + begin = .now(self.io, .awake); + if (source) |s| self.poll(s) else self.hostStep(); + if (self.stopping.load(.acquire)) return; + } + } + + fn poll(self: *Collector, source: *Source) void { + const platform = source.platform; + const back = source.sides.back(); + const scratch = source.scratch.allocator(); + defer _ = source.scratch.reset(.retain_capacity); + + for (back.samples, platform.devices) |*sample_, device| sample_.* = .init(device.target); + back.processes.clear(); + + platform.vtable.poll(platform, scratch, back.samples); + for (back.samples, platform.devices) |*sample_, device| { + if (std.meta.activeTag(sample_.*) != device.target) sample_.* = .init(device.target); + } + + if (platform.vtable.processes) |processes| { + if (processes(platform, scratch, back.samples, &back.processes)) { + // A row that names a device the source does not have is dropped. + const rows = &back.processes.rows; + var kept: usize = 0; + for (rows.items) |row| { + if (row.device_idx >= platform.devices.len) continue; + rows.items[kept] = row; + rows.items[kept].device_idx += source.first; + kept += 1; + } + rows.shrinkRetainingCapacity(kept); + } else |_| { + back.processes.clear(); + } + } + source.sides.swap(self.io); + } + + /// Merges the rows of every source, adds the host details, and reads the host. + fn hostStep(self: *Collector) void { + const merged = self.processes.back(); + merged.clear(); + for (self.sources.items) |source| { + const side = source.sides.lock(self.io); + defer source.sides.unlock(self.io); + for (side.processes.rows.items) |row| { + var copy = row; + copy.username = merged.dupe(row.username) catch ""; + copy.comm = merged.dupe(row.comm) catch ""; + merged.append(copy) catch break; + } + } + self.host_reader.enrich(merged.strings.allocator(), self.host_scratch.allocator(), merged.rows.items); + _ = self.host_scratch.reset(.retain_capacity); + + const host = self.host.back(); + host.* = self.host.front().*; + self.host_reader.sample(host); + + self.processes.swap(self.io); + self.host.swap(self.io); + } +}; + +const Fake = struct { + platform: smi.Platform, + polls: u64 = 0, + + const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes }; + + fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { + const self: *Fake = @alignCast(@fieldParentPtr("platform", platform)); + self.polls += 1; + samples[0].cuda.util_percent = self.polls; + // A tag that is not the device's target. + if (samples.len > 1) samples[1] = .{ .tpu = .{ .util_percent = 1 } }; + } + + fn processes(platform: *smi.Platform, _: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { + const self: *Fake = @alignCast(@fieldParentPtr("platform", platform)); + if (self.polls % 2 == 0) return error.Unavailable; + // A device the platform does not have. + try out.append(.{ .pid = 7, .device_idx = @intCast(self.platform.devices.len) }); + try out.append(.{ .pid = 42, .device_idx = @intCast(self.platform.devices.len - 1), .remote = true, .username = try out.dupe("alice") }); + } +}; + +test "collector rounds" { + const io = std.testing.io; + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + var collector: Collector = .init(arena.allocator(), std.testing.allocator, io, .{}); + defer collector.deinit(); + + var gpus: Fake = .{ .platform = .{ .devices = &.{ .{ .target = .cuda }, .{ .target = .cuda } }, .vtable = &Fake.vtable } }; + var other: Fake = .{ .platform = .{ .devices = &.{.{ .target = .cuda }}, .two_readings = true, .vtable = &Fake.vtable } }; + try collector.add(&gpus.platform); + try std.testing.expect(!collector.two_readings); + try collector.add(&other.platform); + try std.testing.expect(collector.two_readings); + try std.testing.expectEqual(3, collector.devices.items.len); + + // Before any round, samples are empty. + try std.testing.expectEqual(null, collector.sample(@fromBackingInt(0)).cuda.util_percent); + + collector.round(); + try std.testing.expectEqual(1, collector.sample(@fromBackingInt(0)).cuda.util_percent); + // The wrong tag was replaced with an empty sample of the device's target. + try std.testing.expectEqual(null, collector.sample(@fromBackingInt(1)).cuda.util_percent); + try std.testing.expectEqual(1, collector.sample(@fromBackingInt(2)).cuda.util_percent); + { + const merged = collector.processes.lock(io); + defer collector.processes.unlock(io); + // A row naming a missing device is dropped. The others get the global index + // of their device, and keep their strings. + try std.testing.expectEqual(2, merged.rows.items.len); + try std.testing.expectEqual(1, merged.rows.items[0].device_idx); + try std.testing.expectEqual(2, merged.rows.items[1].device_idx); + try std.testing.expectEqualStrings("alice", merged.rows.items[1].username); + } + + // `processes` fails: no rows this round, not the previous ones. + collector.round(); + try std.testing.expectEqual(2, collector.sample(@fromBackingInt(0)).cuda.util_percent); + const merged = collector.processes.lock(io); + defer collector.processes.unlock(io); + try std.testing.expectEqual(0, merged.rows.items.len); +} + +test "collector tasks stop" { + const io = std.testing.io; + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + var collector: Collector = .init(arena.allocator(), std.testing.allocator, io, .{ .poll_interval_ms = 1 }); + var fake: Fake = .{ .platform = .{ .devices = &.{.{ .target = .cuda }}, .vtable = &Fake.vtable } }; + try collector.add(&fake.platform); + try collector.start(); + while (collector.sample(@fromBackingInt(0)).cuda.util_percent.? < 3) try io.sleep(.fromMilliseconds(1), .awake); + collector.deinit(); +} diff --git a/bin/zml-smi/smi/counter.zig b/bin/zml-smi/smi/counter.zig new file mode 100644 index 00000000..cd0996f3 --- /dev/null +++ b/bin/zml-smi/smi/counter.zig @@ -0,0 +1,139 @@ +const std = @import("std"); + +/// A running total (energy, busy time, CPU ticks) read at two polls. +pub const Counter = struct { + last: ?Reading = null, + + pub const Reading = struct { total: u64, at: std.Io.Timestamp }; + + /// Reads the monotonic clock, so the time is that of the read. + /// Null on the first reading, if the total went backwards, or if no time passed. + pub fn advance(self: *Counter, io: std.Io, total: u64) ?Growth { + return self.step(.{ .total = total, .at = .now(io, .awake) }); + } + + fn step(self: *Counter, reading: Reading) ?Growth { + const last = self.last; + self.last = reading; + const previous = last orelse return null; + if (reading.total < previous.total or reading.at.nanoseconds <= previous.at.nanoseconds) return null; + return .{ .amount = reading.total - previous.total, .elapsed = previous.at.durationTo(reading.at) }; + } +}; + +pub const Growth = struct { + amount: u64, + elapsed: std.Io.Duration, + + /// Growth per second: microjoules become microwatts. A result that does not fit is the largest u64. + pub fn perSecond(self: Growth) u64 { + const per_second = @as(u128, self.amount) * std.time.ns_per_s / @as(u128, @intCast(self.elapsed.nanoseconds)); + return std.math.cast(u64, per_second) orelse std.math.maxInt(u64); + } + + /// For a total of busy nanoseconds: its share of the elapsed time, at most 100. + pub fn percentOfTime(self: Growth) u8 { + return percent(self.amount, @intCast(self.elapsed.nanoseconds)); + } + + /// Its share of another growth, at most 100: busy cycles over total cycles. 0 when the other did not grow. + pub fn percentOf(self: Growth, whole: Growth) u8 { + return percent(self.amount, whole.amount); + } + + /// Adds a growth to `sum`, which starts null: the busy time of several processes over one poll. + pub fn add(sum: *?Growth, growth: Growth) void { + if (sum.*) |*total| total.amount +|= growth.amount else sum.* = growth; + } +}; + +fn percent(part: u128, whole: u128) u8 { + if (whole == 0) return 0; + return @intCast(@min(part * 100 / whole, 100)); +} + +/// One counter per key, for totals kept per process. +pub fn Counters(comptime Key: type) type { + return struct { + const Self = @This(); + + map: std.AutoHashMapUnmanaged(Key, Entry) = .empty, + gpa: std.mem.Allocator, + + const Entry = struct { counter: Counter = .{}, seen: bool = false }; + + pub fn init(gpa: std.mem.Allocator) Self { + return .{ .gpa = gpa }; + } + + pub fn deinit(self: *Self) void { + self.map.deinit(self.gpa); + } + + /// Advances the counter of `key` and marks the key as seen. + pub fn advance(self: *Self, key: Key, io: std.Io, total: u64) error{OutOfMemory}!?Growth { + const entry = try self.map.getOrPutValue(self.gpa, key, .{}); + entry.value_ptr.seen = true; + return entry.value_ptr.counter.advance(io, total); + } + + /// Forgets the keys not advanced since the last sweep. Called once per poll. + pub fn sweep(self: *Self) void { + var it = self.map.iterator(); + while (it.next()) |entry| { + if (entry.value_ptr.seen) { + entry.value_ptr.seen = false; + } else { + self.map.removeByPtr(entry.key_ptr); + } + } + } + }; +} + +fn at(ms: i64) std.Io.Timestamp { + return .fromNanoseconds(@as(i96, ms) * std.time.ns_per_ms); +} + +test Counter { + var counter: Counter = .{}; + try std.testing.expectEqual(null, counter.step(.{ .total = 100, .at = at(0) })); + + const growth = counter.step(.{ .total = 600, .at = at(500) }).?; + try std.testing.expectEqual(500, growth.amount); + try std.testing.expectEqual(500, growth.elapsed.toMilliseconds()); + + // The total went backwards, then no time passed. + try std.testing.expectEqual(null, counter.step(.{ .total = 10, .at = at(1000) })); + try std.testing.expectEqual(null, counter.step(.{ .total = 20, .at = at(1000) })); +} + +test Growth { + const half_second: std.Io.Duration = .fromMilliseconds(500); + // 10 mJ in 500 ms is 20 mW. + try std.testing.expectEqual(20_000, (Growth{ .amount = 10_000, .elapsed = half_second }).perSecond()); + try std.testing.expectEqual(std.math.maxInt(u64), (Growth{ .amount = std.math.maxInt(u64), .elapsed = .fromNanoseconds(1) }).perSecond()); + + try std.testing.expectEqual(50, (Growth{ .amount = 250_000_000, .elapsed = half_second }).percentOfTime()); + try std.testing.expectEqual(100, (Growth{ .amount = 2_000_000_000, .elapsed = half_second }).percentOfTime()); + + const cycles: Growth = .{ .amount = 30, .elapsed = half_second }; + try std.testing.expectEqual(30, cycles.percentOf(.{ .amount = 100, .elapsed = half_second })); + try std.testing.expectEqual(0, cycles.percentOf(.{ .amount = 0, .elapsed = half_second })); +} + +test Counters { + const io = std.testing.io; + var counters: Counters(u32) = .init(std.testing.allocator); + defer counters.deinit(); + + try std.testing.expectEqual(null, try counters.advance(1, io, 10)); + try std.testing.expectEqual(null, try counters.advance(2, io, 10)); + counters.sweep(); + + _ = try counters.advance(1, io, 20); + counters.sweep(); + // Key 2 was not advanced since the last sweep, so it starts over. + try std.testing.expect(counters.map.contains(1)); + try std.testing.expect(!counters.map.contains(2)); +} diff --git a/bin/zml-smi/smi/double_buffer.zig b/bin/zml-smi/smi/double_buffer.zig new file mode 100644 index 00000000..c8e3d5cc --- /dev/null +++ b/bin/zml-smi/smi/double_buffer.zig @@ -0,0 +1,59 @@ +const std = @import("std"); + +/// Two copies of a value: one writer fills the back while readers copy the front. +/// The writer holds the lock only to swap; a reader holds it while it copies. +pub fn DoubleBuffer(comptime T: type) type { + return struct { + const Self = @This(); + + values: [2]T, + front_index: u1 = 0, + mutex: std.Io.Mutex = .init, + + /// The side the writer fills. Only the writer calls it. + pub fn back(self: *Self) *T { + return &self.values[~self.front_index]; + } + + /// The side readers see. Only the writer reads it without the lock. + pub fn front(self: *Self) *T { + return &self.values[self.front_index]; + } + + /// Publishes the back. + pub fn swap(self: *Self, io: std.Io) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.front_index = ~self.front_index; + } + + /// Locks the front until `unlock`. + pub fn lock(self: *Self, io: std.Io) *const T { + self.mutex.lockUncancelable(io); + return &self.values[self.front_index]; + } + + pub fn unlock(self: *Self, io: std.Io) void { + self.mutex.unlock(io); + } + + /// A copy of the front. + pub fn read(self: *Self, io: std.Io) T { + defer self.unlock(io); + return self.lock(io).*; + } + }; +} + +test "swap publishes the back" { + const io = std.testing.io; + var buffer: DoubleBuffer(u32) = .{ .values = .{ 1, 2 } }; + + buffer.back().* = 3; + try std.testing.expectEqual(1, buffer.lock(io).*); + buffer.unlock(io); + + buffer.swap(io); + try std.testing.expectEqual(3, buffer.lock(io).*); + buffer.unlock(io); +} diff --git a/bin/zml-smi/smi/dynlib.zig b/bin/zml-smi/smi/dynlib.zig new file mode 100644 index 00000000..e24e131a --- /dev/null +++ b/bin/zml-smi/smi/dynlib.zig @@ -0,0 +1,108 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const sandbox = @import("sandbox.zig"); + +const log = std.log.scoped(.@"zml-smi/dynlib"); + +const flags: std.c.RTLD = .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }; + +/// Opens the first system library that loads and resolves every field of `Api` by its name. +/// A field is a `Checked`, a function pointer, or a pointer to a data symbol. +pub fn open(comptime Api: type, names: []const [:0]const u8) !Api { + for (names) |name| { + const handle = std.c.dlopen(name, flags) orelse continue; + return resolve(Api, handle, name); + } + log.warn("{s}", .{lastError()}); + return error.LibraryNotFound; +} + +/// Opens a library shipped with zml-smi, from its sandbox directory. +pub fn openBundled(comptime Api: type, io: std.Io, name: []const u8) !Api { + var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const directory = sandbox.path(io, &buffer) orelse { + log.warn("{s}: no sandbox directory", .{name}); + return error.LibraryNotFound; + }; + var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrintSentinel(&path_buffer, "{s}/lib/{s}", .{ directory, name }, 0); + const handle = std.c.dlopen(path, flags) orelse { + log.warn("{s}", .{lastError()}); + return error.LibraryNotFound; + }; + return resolve(Api, handle, name); +} + +fn lastError() []const u8 { + return std.mem.span(std.c.dlerror() orelse return "no library to open"); +} + +fn resolve(comptime Api: type, handle: *anyopaque, library: []const u8) !Api { + var api: Api = undefined; + const info = @typeInfo(Api).@"struct"; + inline for (info.field_names, info.field_types) |name, Field| { + const symbol = std.c.dlsym(handle, name) orelse { + log.warn("{s} has no symbol {s}", .{ library, name }); + return error.MissingSymbol; + }; + @field(api, name) = if (@typeInfo(Field) == .@"struct") .{ .function = @ptrCast(@alignCast(symbol)) } else @ptrCast(@alignCast(symbol)); + } + return api; +} + +/// A C function that reports failure through its return value, turned into an error by `check`. +pub fn Checked(comptime function: anytype, comptime check: anytype) type { + const Function = @TypeOf(function); + const Error = @typeInfo(@typeInfo(@TypeOf(check)).@"fn".return_type.?).error_union.error_set; + + return struct { + function: *const Function, + + const params = @typeInfo(Function).@"fn".param_types; + const Result = @typeInfo(params[params.len - 1].?).pointer.child; + const Inputs = inputs: { + var types: [params.len - 1]type = undefined; + for (&types, params[0 .. params.len - 1]) |*T, param| T.* = param.?; + const final = types; + break :inputs @Tuple(&final); + }; + + pub fn call(self: @This(), args: std.meta.ArgsTuple(Function)) Error!void { + return check(@call(.auto, self.function, args)); + } + + /// For a function whose only result is its last parameter: returns that result. + pub fn get(self: @This(), args: Inputs) Error!Result { + var result: Result = std.mem.zeroes(Result); + try check(@call(.auto, self.function, args ++ .{&result})); + return result; + } + }; +} + +const Libc = struct { + getpid: *const @TypeOf(std.c.getpid), + clock_gettime: Checked(std.c.clock_gettime, checkLibc), + close: Checked(std.c.close, checkLibc), + + fn checkLibc(status: c_int) error{Failed}!void { + if (status != 0) return error.Failed; + } +}; + +const libc_names: []const [:0]const u8 = switch (builtin.os.tag) { + .macos => &.{"/usr/lib/libSystem.B.dylib"}, + else => &.{ "libc.so.6", "libc.so" }, +}; + +test open { + const libc = try open(Libc, libc_names); + try std.testing.expectEqual(std.c.getpid(), libc.getpid()); + try std.testing.expect((try libc.clock_gettime.get(.{.MONOTONIC})).sec > 0); + try std.testing.expectError(error.Failed, libc.close.call(.{-1})); + try std.testing.expectError(error.LibraryNotFound, open(Libc, &.{"libmissing.so"})); +} + +test openBundled { + try std.testing.expectError(error.LibraryNotFound, openBundled(Libc, std.testing.io, "libmissing.so")); +} diff --git a/bin/zml-smi/smi/fdinfo.zig b/bin/zml-smi/smi/fdinfo.zig new file mode 100644 index 00000000..a98f83c4 --- /dev/null +++ b/bin/zml-smi/smi/fdinfo.zig @@ -0,0 +1,170 @@ +const std = @import("std"); +const sysfs = @import("sysfs.zig"); + +/// One /proc//fdinfo/ file of a DRM client. Its strings point into the text it was parsed from. +pub const Fdinfo = struct { + /// The PCI address of the device: "0000:03:00.0". + pdev: ?[]const u8 = null, + client_id: ?u64 = null, + region_buffer: [8]Region = undefined, + region_count: u8 = 0, + engine_buffer: [16]Engine = undefined, + engine_count: u8 = 0, + + /// Memory of one region, in bytes. + pub const Region = struct { + name: []const u8 = "", + total: ?u64 = null, + resident: ?u64 = null, + }; + + /// The busy time of one engine class: nanoseconds, or for xe, busy cycles out of total cycles. + pub const Engine = struct { + class: []const u8 = "", + busy_ns: ?u64 = null, + cycles: ?u64 = null, + total_cycles: ?u64 = null, + }; + + pub fn regions(self: *const Fdinfo) []const Region { + return self.region_buffer[0..self.region_count]; + } + + pub fn engines(self: *const Fdinfo) []const Engine { + return self.engine_buffer[0..self.engine_count]; + } + + fn region(self: *Fdinfo, name: []const u8) ?*Region { + return slot(Region, &self.region_buffer, &self.region_count, "name", name); + } + + fn engine(self: *Fdinfo, class: []const u8) ?*Engine { + return slot(Engine, &self.engine_buffer, &self.engine_count, "class", class); + } +}; + +/// Reads one fdinfo file into `buffer`, which the strings of the result point into. +pub fn read(dir: sysfs.Dir, sub_path: []const u8, buffer: []u8) !Fdinfo { + return parse(try dir.read(sub_path, buffer)); +} + +pub fn parse(text: []const u8) Fdinfo { + var info: Fdinfo = .{}; + var lines = std.mem.tokenizeScalar(u8, text, '\n'); + while (lines.next()) |line| { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse continue; + const key = line[0..colon]; + const value = std.mem.trim(u8, line[colon + 1 ..], " \t"); + + if (std.mem.eql(u8, key, "drm-pdev") or std.mem.eql(u8, key, "pdev")) { + info.pdev = value; + } else if (std.mem.eql(u8, key, "drm-client-id")) { + info.client_id = sysfs.firstNumber(value); + } else if (std.mem.startsWith(u8, key, "drm-engine-capacity-")) { + continue; + } else if (cut(key, "drm-engine-")) |class| { + if (info.engine(class)) |engine| engine.busy_ns = sysfs.firstNumber(value); + } else if (cut(key, "drm-total-cycles-")) |class| { + if (info.engine(class)) |engine| engine.total_cycles = sysfs.firstNumber(value); + } else if (cut(key, "drm-cycles-")) |class| { + if (info.engine(class)) |engine| engine.cycles = sysfs.firstNumber(value); + } else if (cut(key, "drm-total-")) |name| { + if (info.region(name)) |region| region.total = bytes(value); + } else if (cut(key, "drm-resident-")) |name| { + if (info.region(name)) |region| region.resident = bytes(value); + } else if (cut(key, "drm-memory-")) |name| { + // The older amdgpu name of resident memory. + if (info.region(name)) |region| region.resident = bytes(value); + } else if (std.mem.endsWith(u8, key, " mem")) { + // The oldest amdgpu one: "vram mem". + if (info.region(key[0 .. key.len - " mem".len])) |region| region.resident = bytes(value); + } + } + return info; +} + +fn cut(key: []const u8, prefix: []const u8) ?[]const u8 { + return if (std.mem.startsWith(u8, key, prefix)) key[prefix.len..] else null; +} + +/// The entry called `name`, added when missing. Null past the capacity. +fn slot(comptime T: type, buffer: []T, count: *u8, comptime key: []const u8, name: []const u8) ?*T { + for (buffer[0..count.*]) |*item| { + if (std.mem.eql(u8, @field(item, key), name)) return item; + } + if (count.* == buffer.len) return null; + const item = &buffer[count.*]; + count.* += 1; + item.* = .{}; + @field(item, key) = name; + return item; +} + +/// A size such as "4096 KiB", in bytes. A size without a unit is in bytes. +fn bytes(value: []const u8) ?u64 { + var words = std.mem.tokenizeAny(u8, value, " \t"); + const size = std.fmt.parseInt(u64, words.next() orelse return null, 10) catch return null; + const unit = words.next() orelse return size; + const shift: u6 = if (std.ascii.eqlIgnoreCase(unit, "KiB") or std.ascii.eqlIgnoreCase(unit, "kB")) + 10 + else if (std.ascii.eqlIgnoreCase(unit, "MiB")) + 20 + else if (std.ascii.eqlIgnoreCase(unit, "GiB")) + 30 + else + 0; + return std.math.mul(u64, size, @as(u64, 1) << shift) catch null; +} + +test "i915 engines and regions" { + const info = parse( + \\pos: 0 + \\drm-driver: i915 + \\drm-pdev: 0000:03:00.0 + \\drm-client-id: 42 + \\drm-total-system0: 2048 KiB + \\drm-resident-local0: 3 MiB + \\drm-engine-render: 100 ns + \\drm-engine-capacity-video: 2 + \\drm-engine-video-decode: 50 ns + ); + try std.testing.expectEqualStrings("0000:03:00.0", info.pdev.?); + try std.testing.expectEqual(42, info.client_id); + try std.testing.expectEqual(2, info.regions().len); + try std.testing.expectEqualStrings("system0", info.regions()[0].name); + try std.testing.expectEqual(2048 * 1024, info.regions()[0].total); + try std.testing.expectEqual(3 * 1024 * 1024, info.regions()[1].resident); + try std.testing.expectEqual(2, info.engines().len); + try std.testing.expectEqualStrings("render", info.engines()[0].class); + try std.testing.expectEqual(100, info.engines()[0].busy_ns); + try std.testing.expectEqualStrings("video-decode", info.engines()[1].class); +} + +test "xe cycles" { + const info = parse( + \\drm-pdev: 0000:03:00.0 + \\drm-cycles-rcs: 10 + \\drm-total-cycles-rcs: 100 + \\drm-cycles-ccs: 30 + \\drm-total-cycles-ccs: 100 + \\drm-total-vram0: 4096 KiB + \\drm-resident-vram0: 3072 KiB + ); + try std.testing.expectEqual(2, info.engines().len); + try std.testing.expectEqual(10, info.engines()[0].cycles); + try std.testing.expectEqual(100, info.engines()[0].total_cycles); + try std.testing.expectEqual(null, info.engines()[0].busy_ns); + try std.testing.expectEqual(4096 * 1024, info.regions()[0].total); + try std.testing.expectEqual(3072 * 1024, info.regions()[0].resident); +} + +test "amdgpu keys" { + const old = parse("pdev:\t0000:0c:00.0\nvram mem:\t1024 kB\n"); + try std.testing.expectEqualStrings("0000:0c:00.0", old.pdev.?); + try std.testing.expectEqualStrings("vram", old.regions()[0].name); + try std.testing.expectEqual(1024 * 1024, old.regions()[0].resident); + + const current = parse("drm-pdev:\t0000:0c:00.0\ndrm-memory-vram:\t2048 KiB\ndrm-engine-gfx:\t5 ns\n"); + try std.testing.expectEqual(2048 * 1024, current.regions()[0].resident); + try std.testing.expectEqual(5, current.engines()[0].busy_ns); +} diff --git a/bin/zml-smi/smi/host/linux.zig b/bin/zml-smi/smi/host/linux.zig new file mode 100644 index 00000000..19f7ec78 --- /dev/null +++ b/bin/zml-smi/smi/host/linux.zig @@ -0,0 +1,140 @@ +//! The host, from /proc and /sys. +const std = @import("std"); +const smi = @import("../smi.zig"); + +pub const Host = struct { + io: std.Io, + proc: ?smi.sysfs.Dir, + /// CPU ticks of every CPU together. + total: smi.Counter = .{}, + /// CPU ticks of each process. + ticks: smi.Counters(u32), + + /// Reads what does not change: hostname, kernel, CPU, core count and total memory. + pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, data: *smi.HostData) Host { + const proc = smi.sysfs.Dir.open(io, "/proc") catch null; + var buffer: [256]u8 = undefined; + if (proc) |dir| { + data.hostname = dupe(arena, dir.read("sys/kernel/hostname", &buffer) catch null); + data.kernel = dupe(arena, dir.read("sys/kernel/osrelease", &buffer) catch null); + data.cpu_name = dupe(arena, dir.field("cpuinfo", "model name", &buffer) catch null); + data.mem_total_kib = smi.sysfs.firstNumber(dir.field("meminfo", "MemTotal", &buffer) catch ""); + } + if (smi.sysfs.Dir.open(io, "/sys/devices/system/cpu")) |cpu| { + defer cpu.close(); + data.cpu_cores = cores(cpu.read("present", &buffer) catch ""); + } else |_| {} + return .{ .io = io, .proc = proc, .ticks = .init(gpa) }; + } + + pub fn deinit(self: *Host) void { + self.ticks.deinit(); + if (self.proc) |proc| proc.close(); + } + + /// Overwrites what changes: available memory, load and uptime. A read that fails gives null. + pub fn sample(self: *Host, data: *smi.HostData) void { + data.mem_available_kib = null; + data.load_1, data.load_5, data.load_15 = .{ null, null, null }; + data.uptime_seconds = null; + const proc = self.proc orelse return; + + var buffer: [256]u8 = undefined; + data.mem_available_kib = smi.sysfs.firstNumber(proc.field("meminfo", "MemAvailable", &buffer) catch ""); + if (proc.read("loadavg", &buffer)) |loadavg| { + var words = std.mem.tokenizeScalar(u8, loadavg, ' '); + inline for (.{ &data.load_1, &data.load_5, &data.load_15 }) |load| { + load.* = std.fmt.parseFloat(f32, words.next() orelse "") catch null; + } + } else |_| {} + if (proc.read("uptime", &buffer)) |uptime| { + data.uptime_seconds = std.fmt.parseInt(u64, std.mem.sliceTo(uptime, '.'), 10) catch null; + } else |_| {} + } + + /// Adds user, command, CPU and RSS to the local rows. Each process is read once, + /// however many rows it has. `strings` holds the strings of the rows. + pub fn enrich(self: *Host, strings: std.mem.Allocator, scratch: std.mem.Allocator, rows: []smi.ProcessInfo) void { + const proc = self.proc orelse return; + defer self.ticks.sweep(); + + var buffer: [512]u8 = undefined; + const total = if (proc.head("stat", &buffer)) |stat| self.total.advance(self.io, totalTicks(stat)) else |_| null; + + var first_row: std.AutoHashMapUnmanaged(u32, usize) = .empty; + for (rows, 0..) |*row, i| { + // A remote host already filled its rows. + if (row.remote) continue; + const entry = first_row.getOrPut(scratch, row.pid) catch continue; + if (entry.found_existing) { + const done = rows[entry.value_ptr.*]; + row.uid, row.username, row.comm, row.cpu_percent, row.rss_kib = .{ done.uid, done.username, done.comm, done.cpu_percent, done.rss_kib }; + continue; + } + entry.value_ptr.* = i; + self.read(proc, strings, row, total); + } + } + + fn read(self: *Host, proc: smi.sysfs.Dir, strings: std.mem.Allocator, row: *smi.ProcessInfo, total: ?smi.Growth) void { + var path: [32]u8 = undefined; + var buffer: [4096]u8 = undefined; + + if (proc.read(processPath(&path, row.pid, "status"), &buffer)) |status| { + var lines = std.mem.tokenizeScalar(u8, status, '\n'); + while (lines.next()) |line| { + if (smi.sysfs.fieldValue(line, "Uid")) |uid| row.uid = @intCast(smi.sysfs.firstNumber(uid) orelse 0); + if (smi.sysfs.fieldValue(line, "VmRSS")) |rss| row.rss_kib = smi.sysfs.firstNumber(rss) orelse 0; + } + if (std.c.getpwuid(row.uid)) |user| { + if (user.name) |name| row.username = strings.dupe(u8, std.mem.span(name)) catch ""; + } + } else |_| {} + + if (proc.head(processPath(&path, row.pid, "cmdline"), &buffer)) |cmdline| { + const command = buffer[0..cmdline.len]; + std.mem.replaceScalar(u8, command, 0, ' '); + row.comm = strings.dupe(u8, std.mem.trimEnd(u8, command, " ")) catch ""; + } else |_| {} + + if (proc.read(processPath(&path, row.pid, "stat"), &buffer)) |stat| { + const ticks = processTicks(stat) orelse return; + const growth = (self.ticks.advance(row.pid, self.io, ticks) catch return) orelse return; + const whole = total orelse return; + if (whole.amount > 0) row.cpu_percent = @intCast(@min(growth.amount *| 1000 / whole.amount, std.math.maxInt(u16))); + } else |_| {} + } +}; + +fn processPath(buffer: []u8, pid: u32, file: []const u8) []const u8 { + return std.fmt.bufPrint(buffer, "{d}/{s}", .{ pid, file }) catch ""; +} + +fn dupe(arena: std.mem.Allocator, string: ?[]const u8) ?[]const u8 { + return arena.dupe(u8, string orelse return null) catch null; +} + +/// The CPUs present, "0-47": the highest index, plus one. +fn cores(present: []const u8) ?u64 { + const dash = std.mem.indexOfScalar(u8, present, '-') orelse return if (present.len > 0) 1 else null; + return (std.fmt.parseInt(u64, present[dash + 1 ..], 10) catch return null) + 1; +} + +/// The sum of the first line of /proc/stat: "cpu user nice system idle ...". +fn totalTicks(stat: []const u8) u64 { + var words = std.mem.tokenizeAny(u8, std.mem.sliceTo(stat, '\n'), " \t"); + _ = words.next(); + var total: u64 = 0; + while (words.next()) |word| total +|= std.fmt.parseInt(u64, word, 10) catch 0; + return total; +} + +/// utime plus stime of /proc//stat, the 12th and 13th fields after the command. +fn processTicks(stat: []const u8) ?u64 { + const end = std.mem.lastIndexOfScalar(u8, stat, ')') orelse return null; + var words = std.mem.tokenizeScalar(u8, stat[end + 1 ..], ' '); + for (0..11) |_| _ = words.next(); + const utime = std.fmt.parseInt(u64, words.next() orelse return null, 10) catch return null; + const stime = std.fmt.parseInt(u64, words.next() orelse return null, 10) catch return null; + return utime + stime; +} diff --git a/bin/zml-smi/smi/host/macos.zig b/bin/zml-smi/smi/host/macos.zig new file mode 100644 index 00000000..487078f1 --- /dev/null +++ b/bin/zml-smi/smi/host/macos.zig @@ -0,0 +1,99 @@ +//! The host, from sysctl and the Mach host statistics. +const std = @import("std"); +const smi = @import("../smi.zig"); + +const c = std.c; + +pub const Host = struct { + /// Reads what does not change: hostname, kernel, CPU, core count and total memory. + pub fn init(arena: std.mem.Allocator, _: std.mem.Allocator, _: std.Io, data: *smi.HostData) Host { + data.hostname = sysctlString(arena, "kern.hostname"); + data.kernel = sysctlString(arena, "kern.osrelease"); + data.cpu_name = sysctlString(arena, "machdep.cpu.brand_string"); + data.cpu_cores = if (sysctl(u32, "hw.logicalcpu")) |cores| cores else null; + data.mem_total_kib = if (sysctl(u64, "hw.memsize")) |bytes| bytes / 1024 else null; + return .{}; + } + + pub fn deinit(_: *Host) void {} + + /// Overwrites what changes: available memory, load and uptime. A read that fails gives null. + pub fn sample(_: *Host, data: *smi.HostData) void { + data.mem_available_kib = memAvailable(); + + var load: [3]f64 = undefined; + const loaded = getloadavg(&load, 3) == 3; + inline for (.{ &data.load_1, &data.load_5, &data.load_15 }, 0..) |field, i| { + field.* = if (loaded) @floatCast(load[i]) else null; + } + + data.uptime_seconds = null; + var now: c.timeval = undefined; + if (sysctl(c.timeval, "kern.boottime")) |boot| { + if (c.gettimeofday(&now, null) == 0) data.uptime_seconds = @intCast(now.sec - boot.sec); + } + } + + /// The processes all come from remote hosts, which filled their rows. + pub fn enrich(_: *Host, _: std.mem.Allocator, _: std.mem.Allocator, _: []smi.ProcessInfo) void {} +}; + +const vm_statistics64 = extern struct { + free_count: c_uint, + active_count: c_uint, + inactive_count: c_uint, + wire_count: c_uint, + zero_fill_count: u64, + reactivations: u64, + pageins: u64, + pageouts: u64, + faults: u64, + cow_faults: u64, + lookups: u64, + hits: u64, + purges: u64, + purgeable_count: c_uint, + speculative_count: c_uint, + decompressions: u64, + compressions: u64, + swapins: u64, + swapouts: u64, + compressor_page_count: c_uint, + throttled_count: c_uint, + external_page_count: c_uint, + internal_page_count: c_uint, + total_uncompressed_pages_in_compressor: u64, +}; + +const HOST_VM_INFO64: c_int = 4; +const KERN_SUCCESS: c_int = 0; + +extern "c" fn host_statistics64(host: c.mach_port_t, flavor: c_int, info: *anyopaque, count: *c_uint) c.kern_return_t; +extern "c" fn getloadavg(loadavg: *[3]f64, nelem: c_int) c_int; + +/// Free, inactive and purgeable pages. +fn memAvailable() ?u64 { + var stats: vm_statistics64 = undefined; + var count: c_uint = @sizeOf(vm_statistics64) / @sizeOf(c_int); + if (host_statistics64(c.mach_host_self(), HOST_VM_INFO64, &stats, &count) != KERN_SUCCESS) return null; + + var page_size: u64 = 0; + if (c._host_page_size(c.mach_host_self(), &page_size) != KERN_SUCCESS) return null; + + const pages = @as(u64, stats.free_count) + stats.inactive_count + stats.purgeable_count; + return pages * page_size / 1024; +} + +fn sysctl(comptime T: type, name: [*:0]const u8) ?T { + var value: T = undefined; + var len: usize = @sizeOf(T); + if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) return null; + return value; +} + +fn sysctlString(arena: std.mem.Allocator, name: [*:0]const u8) ?[]const u8 { + var buffer: [256]u8 = undefined; + var len: usize = buffer.len; + if (c.sysctlbyname(name, &buffer, &len, null, 0) != 0) return null; + return arena.dupe(u8, std.mem.sliceTo(buffer[0..len], 0)) catch null; +} diff --git a/bin/zml-smi/smi/hwmon.zig b/bin/zml-smi/smi/hwmon.zig new file mode 100644 index 00000000..46da85e1 --- /dev/null +++ b/bin/zml-smi/smi/hwmon.zig @@ -0,0 +1,90 @@ +const std = @import("std"); +const sysfs = @import("sysfs.zig"); + +/// The hwmon directory of a device: the one called `name`, or the first when `name` is null. +pub fn find(device: sysfs.Dir, name: ?[]const u8) !sysfs.Dir { + const hwmon = try device.openDir("hwmon"); + defer hwmon.close(); + + var it = hwmon.iterate(); + while (try it.next(device.io)) |entry| { + if (!std.mem.startsWith(u8, entry.name, "hwmon")) continue; + const dir = hwmon.openDir(entry.name) catch continue; + const wanted = name orelse return dir; + var buffer: [64]u8 = undefined; + if (std.mem.eql(u8, dir.read("name", &buffer) catch "", wanted)) return dir; + dir.close(); + } + return error.NotFound; +} + +/// A file that may hold a value: `temp2_input`, or with a label, the `temp_input` +/// whose `temp_label` is the label. +pub const Candidate = struct { + prefix: []const u8, + label: ?[]const u8 = null, + suffix: []const u8 = "input", +}; + +/// The name of the first candidate that reads as a number, copied into `arena`. +pub fn resolve(arena: std.mem.Allocator, dir: sysfs.Dir, candidates: []const Candidate) ?[]const u8 { + var buffer: [64]u8 = undefined; + for (candidates) |candidate| { + const name = fileName(dir, candidate, &buffer) orelse continue; + _ = dir.int(u64, name) catch continue; + return arena.dupe(u8, name) catch null; + } + return null; +} + +fn fileName(dir: sysfs.Dir, candidate: Candidate, buffer: []u8) ?[]const u8 { + const label = candidate.label orelse + return std.fmt.bufPrint(buffer, "{s}_{s}", .{ candidate.prefix, candidate.suffix }) catch null; + + for (1..16) |index| { + var label_name: [64]u8 = undefined; + var text: [64]u8 = undefined; + const path = std.fmt.bufPrint(&label_name, "{s}{d}_label", .{ candidate.prefix, index }) catch return null; + if (!std.ascii.eqlIgnoreCase(dir.read(path, &text) catch continue, label)) continue; + return std.fmt.bufPrint(buffer, "{s}{d}_{s}", .{ candidate.prefix, index, candidate.suffix }) catch null; + } + return null; +} + +/// A fan PWM duty, 0 to 255, as a percentage. +pub fn pwmPercent(pwm: u64) u64 { + return (@as(u64, @min(pwm, 255)) * 100 + 127) / 255; +} + +test resolve { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + try tmp.dir.createDirPath(io, "hwmon/hwmon2"); + try tmp.dir.createDirPath(io, "hwmon/hwmon3"); + try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon2/name", .data = "nvme\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon3/name", .data = "xe\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon3/temp2_label", .data = "PKG\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon3/temp2_input", .data = "58000\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "hwmon/hwmon3/power1_max", .data = "230000000\n" }); + + const device: sysfs.Dir = .{ .handle = tmp.dir, .io = io }; + try std.testing.expectError(error.NotFound, find(device, "i915")); + const dir = try find(device, "xe"); + defer dir.close(); + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const temperature = resolve(arena.allocator(), dir, &.{ .{ .prefix = "temp", .label = "pkg" }, .{ .prefix = "temp1" } }); + try std.testing.expectEqualStrings("temp2_input", temperature.?); + const power_limit = resolve(arena.allocator(), dir, &.{ .{ .prefix = "power1", .suffix = "cap" }, .{ .prefix = "power1", .suffix = "max" } }); + try std.testing.expectEqualStrings("power1_max", power_limit.?); + try std.testing.expectEqual(null, resolve(arena.allocator(), dir, &.{.{ .prefix = "fan1" }})); +} + +test pwmPercent { + try std.testing.expectEqual(0, pwmPercent(0)); + try std.testing.expectEqual(50, pwmPercent(128)); + try std.testing.expectEqual(100, pwmPercent(255)); + try std.testing.expectEqual(100, pwmPercent(300)); +} diff --git a/bin/zml-smi/smi/model.zig b/bin/zml-smi/smi/model.zig new file mode 100644 index 00000000..b6caf35f --- /dev/null +++ b/bin/zml-smi/smi/model.zig @@ -0,0 +1,178 @@ +const std = @import("std"); + +pub const Target = enum { cuda, rocm, oneapi, neuron, tpu }; + +/// What does not change while we run. Its strings live as long as the program. +pub const Device = struct { + target: Target, + /// It comes from another zml-smi, over --remotes. + remote: bool = false, + name: ?[]const u8 = null, + driver_version: ?[]const u8 = null, + /// The strings only one platform has, such as the CUDA version. + details: []const Detail = &.{}, + + pub const Detail = struct { + /// Its name in JSON, CSV and Prometheus labels: "cuda_version". + key: []const u8, + /// Its name in the TUI: "CUDA". + label: []const u8, + value: []const u8, + /// Shown beside the driver version in the card title. + in_title: bool = false, + }; + + /// Devices are referred to by index, never by pointer. + pub const Index = enum(u16) { _ }; +}; + +/// What is read every poll. Numbers only. Its tag is the device's target. +pub const Sample = union(Target) { + cuda: Gpu, + rocm: Gpu, + oneapi: Gpu, + neuron: Neuron, + tpu: Tpu, + + /// A sample of the given target with every field null. + pub fn init(target: Target) Sample { + return switch (target) { + inline else => |tag| @unionInit(Sample, @tagName(tag), .{}), + }; + } +}; + +/// A device and its latest sample: what JSON writes and what a remote host sends. +pub const Row = struct { device: Device, sample: Sample }; + +/// Every GPU target. CSV columns and JSON keys follow the field order. +pub const Gpu = struct { + util_percent: ?u64 = null, + encoder_util_percent: ?u64 = null, + decoder_util_percent: ?u64 = null, + power_mw: ?u64 = null, + power_limit_mw: ?u64 = null, + /// Degrees Celsius. + temperature: ?u64 = null, + fan_speed_percent: ?u64 = null, + clock_graphics_mhz: ?u64 = null, + clock_sm_mhz: ?u64 = null, + clock_soc_mhz: ?u64 = null, + clock_mem_mhz: ?u64 = null, + clock_graphics_max_mhz: ?u64 = null, + clock_mem_max_mhz: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + /// Bits. + mem_bus_width: ?u64 = null, + pcie_tx_kbps: ?u64 = null, + pcie_rx_kbps: ?u64 = null, + /// The line rate of the negotiated link across all its lanes. + pcie_bandwidth_mbps: ?u64 = null, + pcie_link_gen: ?u64 = null, + pcie_link_width: ?u64 = null, +}; + +/// One Neuron core. +pub const Neuron = struct { + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + // Device memory of the core by category, in bytes. + nc_tensors: ?u64 = null, + nc_constants: ?u64 = null, + nc_model_code: ?u64 = null, + nc_shared_scratchpad: ?u64 = null, + nc_nonshared_scratchpad: ?u64 = null, + nc_runtime: ?u64 = null, + nc_driver: ?u64 = null, + nc_dma_rings: ?u64 = null, + nc_collectives: ?u64 = null, + nc_notifications: ?u64 = null, + nc_uncategorized: ?u64 = null, +}; + +pub const Tpu = struct { + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, +}; + +pub const ProcessInfo = struct { + pid: u32 = 0, + /// The global index of the device, once the collector has the row. + device_idx: u16 = 0, + dev_util_percent: ?u16 = null, + dev_mem_kib: ?u64 = null, + uid: u32 = 0, + username: []const u8 = "", + comm: []const u8 = "", + /// Tenths of a percent of every CPU. + cpu_percent: u16 = 0, + rss_kib: u64 = 0, + remote: bool = false, +}; + +pub const HostData = struct { + hostname: ?[]const u8 = null, + kernel: ?[]const u8 = null, + cpu_name: ?[]const u8 = null, + cpu_cores: ?u64 = null, + mem_total_kib: ?u64 = null, + mem_available_kib: ?u64 = null, + load_1: ?f32 = null, + load_5: ?f32 = null, + load_15: ?f32 = null, + uptime_seconds: ?u64 = null, +}; + +/// A source of devices: a local platform or a remote host. +/// It is a field of its implementation, which finds itself with @fieldParentPtr. +pub const Platform = struct { + /// The devices found when the platform was opened. The list never changes afterwards. + devices: []const Device, + /// Values are derived from two polls, so a one-shot print polls twice. + two_readings: bool = false, + vtable: *const VTable, + + pub const VTable = struct { + /// Called every poll, with one empty sample per device, in the order of `devices`. + poll: *const fn (platform: *Platform, scratch: std.mem.Allocator, samples: []Sample) void, + /// Called right after `poll`, with the samples it just filled. A row's device is its position in `devices`. + processes: ?*const fn (platform: *Platform, scratch: std.mem.Allocator, samples: []const Sample, out: *ProcessList) ProcessList.Error!void = null, + deinit: ?*const fn (platform: *Platform) void = null, + }; +}; + +/// The process rows of one source for one poll, and the strings they point into. +pub const ProcessList = struct { + rows: std.ArrayList(ProcessInfo) = .empty, + /// Owns every string a row holds. + strings: std.heap.ArenaAllocator, + gpa: std.mem.Allocator, + + pub const Error = error{ OutOfMemory, Unavailable }; + + pub fn init(gpa: std.mem.Allocator) ProcessList { + return .{ .strings = .init(gpa), .gpa = gpa }; + } + + pub fn deinit(list: *ProcessList) void { + list.rows.deinit(list.gpa); + list.strings.deinit(); + } + + pub fn clear(list: *ProcessList) void { + list.rows.clearRetainingCapacity(); + _ = list.strings.reset(.retain_capacity); + } + + pub fn append(list: *ProcessList, row: ProcessInfo) error{OutOfMemory}!void { + try list.rows.append(list.gpa, row); + } + + /// Copies a string into the list, for a row to hold. + pub fn dupe(list: *ProcessList, string: []const u8) error{OutOfMemory}![]const u8 { + return list.strings.allocator().dupe(u8, string); + } +}; diff --git a/bin/zml-smi/smi/pci.zig b/bin/zml-smi/smi/pci.zig new file mode 100644 index 00000000..104eb4d5 --- /dev/null +++ b/bin/zml-smi/smi/pci.zig @@ -0,0 +1,164 @@ +const std = @import("std"); +const sysfs = @import("sysfs.zig"); + +/// The PCI devices of one vendor, from /sys/bus/pci/devices. +pub const Devices = struct { + root: sysfs.Dir, + it: std.Io.Dir.Iterator, + vendor: u16, + + pub const Entry = struct { + /// The device id. + id: u16, + /// The caller closes it. + dir: sysfs.Dir, + }; + + pub fn open(io: std.Io, vendor: u16) !Devices { + const root = try sysfs.Dir.open(io, "/sys/bus/pci/devices"); + return .{ .root = root, .it = root.iterate(), .vendor = vendor }; + } + + pub fn close(self: *Devices) void { + self.root.close(); + } + + pub fn next(self: *Devices) ?Entry { + while (self.it.next(self.root.io) catch null) |entry| { + const dir = self.root.openDir(entry.name) catch continue; + if ((dir.int(u16, "vendor") catch 0) == self.vendor) { + if (dir.int(u16, "device")) |id| return .{ .id = id, .dir = dir } else |_| {} + } + dir.close(); + } + return null; + } +}; + +pub const Link = struct { + gen: ?u64 = null, + width: ?u64 = null, + + /// The line rate across every lane, in megabits per second. + pub fn mbps(self: Link) ?u64 { + return (laneMbps(self.gen orelse return null) orelse return null) * (self.width orelse return null); + } +}; + +/// The link a device negotiated, or the widest it supports. +pub fn link(dir: sysfs.Dir, comptime which: enum { current, max }) Link { + var buffer: [32]u8 = undefined; + const speed = dir.read(@tagName(which) ++ "_link_speed", &buffer) catch null; + return .{ + .gen = generation(speed orelse ""), + .width = dir.int(u64, @tagName(which) ++ "_link_width") catch null, + }; +} + +/// The device, or the bridge above it that carries its link: the one with the widest link, +/// climbing only while the vendor is the device's own. +pub fn upstream(device: sysfs.Dir) !sysfs.Dir { + const max_depth = 8; + const vendor = try device.int(u16, "vendor"); + var path: [3 * max_depth]u8 = undefined; + var best_depth: usize = 0; + var best_mbps = link(device, .max).mbps() orelse 0; + + for (1..max_depth + 1) |depth| { + const dir = device.openDir(parents(&path, depth)) catch break; + defer dir.close(); + if ((dir.int(u16, "vendor") catch break) != vendor) break; + const mbps = link(dir, .max).mbps() orelse 0; + if (mbps > best_mbps) { + best_depth = depth; + best_mbps = mbps; + } + } + return device.openDir(parents(&path, best_depth)); +} + +fn parents(buffer: []u8, depth: usize) []const u8 { + if (depth == 0) return "."; + for (0..depth) |i| @memcpy(buffer[3 * i ..][0..3], "../"); + return buffer[0 .. 3 * depth - 1]; +} + +/// The generation of a link speed such as "32.0 GT/s PCIe". +pub fn generation(speed: []const u8) ?u64 { + const transfers = std.fmt.parseFloat(f32, std.mem.sliceTo(speed, ' ')) catch return null; + if (!(transfers > 0 and transfers < 100)) return null; + return switch (@as(u32, @intFromFloat(transfers * 10))) { + 25 => 1, + 50 => 2, + 80 => 3, + 160 => 4, + 320 => 5, + 640 => 6, + else => null, + }; +} + +/// The line rate of one lane, in megabits per second. +pub fn laneMbps(gen: u64) ?u64 { + return switch (gen) { + 1 => 2_500, + 2 => 5_000, + 3 => 8_000, + 4 => 16_000, + 5 => 32_000, + 6 => 64_000, + else => null, + }; +} + +test generation { + try std.testing.expectEqual(1, generation("2.5 GT/s PCIe")); + try std.testing.expectEqual(3, generation("8 GT/s")); + try std.testing.expectEqual(4, generation("16.0 GT/s PCIe")); + try std.testing.expectEqual(null, generation("Unknown")); +} + +test Link { + try std.testing.expectEqual(10_000, (Link{ .gen = 1, .width = 4 }).mbps()); + try std.testing.expectEqual(512_000, (Link{ .gen = 5, .width = 16 }).mbps()); + try std.testing.expectEqual(null, (Link{ .gen = 5 }).mbps()); +} + +test upstream { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + try tmp.dir.createDirPath(io, "root/bridge/gpu"); + try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/vendor", .data = "0x8086\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/max_link_speed", .data = "32.0 GT/s PCIe\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/max_link_width", .data = "16\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/gpu/vendor", .data = "0x8086\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/gpu/max_link_speed", .data = "2.5 GT/s PCIe\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "root/bridge/gpu/max_link_width", .data = "1\n" }); + + const root: sysfs.Dir = .{ .handle = tmp.dir, .io = io }; + const gpu = try root.openDir("root/bridge/gpu"); + defer gpu.close(); + const dir = try upstream(gpu); + defer dir.close(); + try std.testing.expectEqual(512_000, link(dir, .max).mbps()); +} + +test Devices { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + try tmp.dir.createDirPath(io, "0000:03:00.0"); + try tmp.dir.createDirPath(io, "0000:04:00.0"); + try tmp.dir.writeFile(io, .{ .sub_path = "0000:03:00.0/vendor", .data = "0x8086\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "0000:03:00.0/device", .data = "0xe223\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "0000:04:00.0/vendor", .data = "0x10de\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "0000:04:00.0/device", .data = "0x2b85\n" }); + + const root: sysfs.Dir = .{ .handle = tmp.dir, .io = io }; + var devices: Devices = .{ .root = root, .it = root.iterate(), .vendor = 0x8086 }; + const entry = devices.next().?; + defer entry.dir.close(); + try std.testing.expectEqual(0xe223, entry.id); + try std.testing.expectEqual(null, devices.next()); +} diff --git a/bin/zml-smi/smi/proc.zig b/bin/zml-smi/smi/proc.zig new file mode 100644 index 00000000..07042dd8 --- /dev/null +++ b/bin/zml-smi/smi/proc.zig @@ -0,0 +1,94 @@ +const std = @import("std"); + +/// The open files of every process whose path starts with one of the prefixes. +/// One walk of /proc, whatever the number of prefixes. Skips zml-smi itself, +/// which may hold device nodes through a library it opened. +pub const FdHolders = struct { + io: std.Io, + prefixes: []const []const u8, + proc: ?std.Io.Dir, + pids: std.Io.Dir.Iterator = undefined, + fds: ?std.Io.Dir = null, + fd_names: std.Io.Dir.Iterator = undefined, + pid: u32 = 0, + own_pid: u32, + link: [std.Io.Dir.max_path_bytes]u8 = undefined, + + pub const Holder = struct { + pid: u32, + /// The descriptor, for /proc//fdinfo/. + fd: u32, + /// The path after the prefix, valid until the next call: "0" for /dev/accel0. + rest: []const u8, + }; + + pub fn init(io: std.Io, prefixes: []const []const u8) FdHolders { + return initAt(io, std.Io.Dir.cwd().openDir(io, "/proc", .{ .iterate = true }) catch null, prefixes); + } + + fn initAt(io: std.Io, proc: ?std.Io.Dir, prefixes: []const []const u8) FdHolders { + var self: FdHolders = .{ .io = io, .prefixes = prefixes, .proc = proc, .own_pid = @intCast(std.c.getpid()) }; + if (proc) |dir| self.pids = dir.iterate(); + return self; + } + + pub fn next(self: *FdHolders) ?Holder { + const proc = self.proc orelse return null; + while (true) { + if (self.fds) |fds| { + while (self.fd_names.next(self.io) catch null) |entry| { + const fd = std.fmt.parseInt(u32, entry.name, 10) catch continue; + const len = fds.readLink(self.io, entry.name, &self.link) catch continue; + const target = self.link[0..len]; + for (self.prefixes) |prefix| { + if (std.mem.startsWith(u8, target, prefix)) return .{ .pid = self.pid, .fd = fd, .rest = target[prefix.len..] }; + } + } + fds.close(self.io); + self.fds = null; + } + + const entry = (self.pids.next(self.io) catch null) orelse return null; + const pid = std.fmt.parseInt(u32, entry.name, 10) catch continue; + if (pid == self.own_pid) continue; + var path: [16]u8 = undefined; + const fd_path = std.fmt.bufPrint(&path, "{d}/fd", .{pid}) catch continue; + const fds = proc.openDir(self.io, fd_path, .{ .iterate = true }) catch continue; + self.fds = fds; + self.fd_names = fds.iterate(); + self.pid = pid; + } + } + + pub fn deinit(self: *FdHolders) void { + if (self.fds) |fds| fds.close(self.io); + if (self.proc) |proc| proc.close(self.io); + } +}; + +test FdHolders { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + try tmp.dir.createDirPath(io, "100/fd"); + try tmp.dir.createDirPath(io, "200/fd"); + try tmp.dir.symLink(io, "/dev/accel3", "100/fd/5", .{}); + try tmp.dir.symLink(io, "/dev/null", "100/fd/0", .{}); + try tmp.dir.symLink(io, "/dev/vfio/1", "200/fd/9", .{}); + + var holders: FdHolders = .initAt(io, try tmp.dir.openDir(io, ".", .{ .iterate = true }), &.{ "/dev/accel", "/dev/vfio/" }); + defer holders.deinit(); + + var count: usize = 0; + while (holders.next()) |holder| : (count += 1) { + switch (holder.pid) { + 100 => { + try std.testing.expectEqual(5, holder.fd); + try std.testing.expectEqualStrings("3", holder.rest); + }, + 200 => try std.testing.expectEqualStrings("1", holder.rest), + else => return error.TestUnexpectedResult, + } + } + try std.testing.expectEqual(2, count); +} diff --git a/bin/zml-smi/smi/remote.zig b/bin/zml-smi/smi/remote.zig new file mode 100644 index 00000000..78320dd9 --- /dev/null +++ b/bin/zml-smi/smi/remote.zig @@ -0,0 +1,147 @@ +const std = @import("std"); +const smi = @import("smi.zig"); + +const log = std.log.scoped(.@"zml-smi/remote"); + +/// The devices of another zml-smi, read from its API. +pub const Remote = struct { + platform: smi.Platform, + io: std.Io, + /// "http://host:9090/metrics". + url: []const u8, + /// For each row of the peer, the position of its device here, or null when this build does not know its target. + positions: []const ?u16, + timeout: std.Io.Duration, + /// What the poll of this round fetched, in scratch, for `processes`. + fetched: ?std.json.Value = null, + warned: bool = false, + + const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes }; + + /// Fetches the devices once. `interval` bounds every later fetch, to at least a second. + pub fn open(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, host: []const u8, interval: std.Io.Duration) !Remote { + if (!std.mem.startsWith(u8, host, "http://") and !std.mem.startsWith(u8, host, "https://")) { + log.err("{s}: a remote host starts with http:// or https://", .{host}); + return error.InvalidUrl; + } + const url = try std.fmt.allocPrint(arena, "{s}/metrics", .{host}); + + var scratch: std.heap.ArenaAllocator = .init(gpa); + defer scratch.deinit(); + const rows = items(try fetch(scratch.allocator(), io, url, .fromSeconds(3)), "devices"); + + var devices: std.ArrayList(smi.Device) = try .initCapacity(arena, rows.len); + const positions = try arena.alloc(?u16, rows.len); + for (rows, positions) |item, *position| { + const row = std.json.parseFromValueLeaky(smi.Row, scratch.allocator(), item, .{ .ignore_unknown_fields = true }) catch { + position.* = null; + continue; + }; + position.* = @intCast(devices.items.len); + devices.appendAssumeCapacity(try copy(arena, row.device)); + } + + return .{ + .platform = .{ .devices = devices.items, .vtable = &vtable }, + .io = io, + .url = url, + .positions = positions, + .timeout = if (interval.nanoseconds < std.time.ns_per_s) .fromSeconds(1) else interval, + }; + } + + fn poll(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []smi.Sample) void { + const self: *Remote = @alignCast(@fieldParentPtr("platform", platform)); + self.fetched = null; + const root = fetch(scratch, self.io, self.url, self.timeout) catch return; + + const rows = items(root, "devices"); + if (rows.len != self.positions.len) return self.changed(samples); + for (rows, self.positions) |item, maybe_position| { + const position = maybe_position orelse continue; + const row = std.json.parseFromValueLeaky(smi.Row, scratch, item, .{ .ignore_unknown_fields = true }) catch return self.changed(samples); + const device = self.platform.devices[position]; + if (row.device.target != device.target or !std.mem.eql(u8, row.device.name orelse "", device.name orelse "")) { + return self.changed(samples); + } + samples[position] = row.sample; + } + self.fetched = root; + } + + /// The peer's devices are not the ones it had when it was opened: show nothing. + fn changed(self: *Remote, samples: []smi.Sample) void { + for (samples, self.platform.devices) |*sample, device| sample.* = .init(device.target); + if (!self.warned) log.warn("{s}: the devices changed since zml-smi started", .{self.url}); + self.warned = true; + } + + fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { + const self: *Remote = @alignCast(@fieldParentPtr("platform", platform)); + const root = self.fetched orelse return error.Unavailable; + for (items(root, "processes")) |item| { + var row = std.json.parseFromValueLeaky(smi.ProcessInfo, scratch, item, .{ .ignore_unknown_fields = true }) catch continue; + if (row.device_idx >= self.positions.len) continue; + row.device_idx = self.positions[row.device_idx] orelse continue; + row.remote = true; + row.username = try out.dupe(row.username); + row.comm = try out.dupe(row.comm); + try out.append(row); + } + } +}; + +fn copy(arena: std.mem.Allocator, device: smi.Device) !smi.Device { + const details = try arena.alloc(smi.Device.Detail, device.details.len); + for (details, device.details) |*detail, source| { + detail.* = .{ + .key = try arena.dupe(u8, source.key), + .label = try arena.dupe(u8, source.label), + .value = try arena.dupe(u8, source.value), + .in_title = source.in_title, + }; + } + return .{ + .target = device.target, + .remote = true, + .name = if (device.name) |name| try arena.dupe(u8, name) else null, + .driver_version = if (device.driver_version) |version| try arena.dupe(u8, version) else null, + .details = details, + }; +} + +fn items(root: std.json.Value, key: []const u8) []const std.json.Value { + if (root != .object) return &.{}; + const value = root.object.get(key) orelse return &.{}; + return if (value == .array) value.array.items else &.{}; +} + +/// GETs `url` and parses its body into `scratch`, giving up after `timeout`. +/// std.http has no timeout, so the fetch races a timer and the loser is canceled. +fn fetch(scratch: std.mem.Allocator, io: std.Io, url: []const u8, timeout: std.Io.Duration) !std.json.Value { + const Race = union(enum) { + body: @typeInfo(@TypeOf(get)).@"fn".return_type.?, + timer: std.Io.Cancelable!void, + }; + var results: [2]Race = undefined; + var race: std.Io.Select(Race) = .init(io, &results); + defer race.cancelDiscard(); + + try race.concurrent(.body, get, .{ scratch, io, url }); + try race.concurrent(.timer, std.Io.sleep, .{ io, timeout, .awake }); + const body = switch (try race.await()) { + .body => |body| try body, + .timer => return error.Timeout, + }; + return std.json.parseFromSliceLeaky(std.json.Value, scratch, body, .{}); +} + +fn get(scratch: std.mem.Allocator, io: std.Io, url: []const u8) ![]const u8 { + var client: std.http.Client = .{ .allocator = scratch, .io = io }; + defer client.deinit(); + + var body: std.Io.Writer.Allocating = .init(scratch); + const result = try client.fetch(.{ .location = .{ .url = url }, .response_writer = &body.writer }); + if (result.status != .ok) return error.HttpError; + return body.written(); +} diff --git a/bin/zml-smi/smi/sandbox.zig b/bin/zml-smi/smi/sandbox.zig new file mode 100644 index 00000000..9f796cdb --- /dev/null +++ b/bin/zml-smi/smi/sandbox.zig @@ -0,0 +1,30 @@ +const std = @import("std"); +const bazel_runfiles = @import("runfiles"); +const bazel_builtin = @import("bazel_builtin"); + +/// The directory of the libraries shipped with zml-smi, found through the runfiles of the binary. +pub fn path(io: std.Io, buffer: *[std.Io.Dir.max_path_bytes]u8) ?[]const u8 { + var exe: [std.Io.Dir.max_path_bytes]u8 = undefined; + const exe_len = std.process.executablePath(io, &exe) catch return null; + + var runfiles = (bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = io, + .argv0 = exe[0..exe_len], + .directory = env("RUNFILES_DIR"), + .manifest = env("RUNFILES_MANIFEST_FILE"), + }) catch return null) orelse return null; + defer runfiles.deinit(std.heap.c_allocator); + + return runfiles.withSourceRepo(bazel_builtin.current_repository).rlocation("zml/bin/zml-smi/sandbox", buffer) catch null; +} + +fn env(name: [*:0]const u8) ?[]const u8 { + return if (std.c.getenv(name)) |value| std.mem.span(value) else null; +} + +test path { + const io = std.testing.io; + var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + try std.Io.Dir.cwd().access(io, path(io, &buffer) orelse return error.NoSandbox, .{}); +} diff --git a/bin/zml-smi/smi/smi.zig b/bin/zml-smi/smi/smi.zig new file mode 100644 index 00000000..003d5279 --- /dev/null +++ b/bin/zml-smi/smi/smi.zig @@ -0,0 +1,50 @@ +//! The core of zml-smi: the model, the collector, the blocks platforms are made of, and the outputs. +const builtin = @import("builtin"); + +const model = @import("model.zig"); +pub const Target = model.Target; +pub const Device = model.Device; +pub const Sample = model.Sample; +pub const Row = model.Row; +pub const Gpu = model.Gpu; +pub const Neuron = model.Neuron; +pub const Tpu = model.Tpu; +pub const ProcessInfo = model.ProcessInfo; +pub const HostData = model.HostData; +pub const Platform = model.Platform; +pub const ProcessList = model.ProcessList; + +pub const Collector = @import("collector.zig").Collector; +pub const Remote = @import("remote.zig").Remote; +pub const Host = switch (builtin.os.tag) { + .linux => @import("host/linux.zig").Host, + .macos => @import("host/macos.zig").Host, + else => @compileError("zml-smi runs on Linux and macOS"), +}; + +pub const sysfs = @import("sysfs.zig"); +pub const hwmon = @import("hwmon.zig"); +pub const pci = @import("pci.zig"); +pub const fdinfo = @import("fdinfo.zig"); +pub const proc = @import("proc.zig"); +pub const dynlib = @import("dynlib.zig"); +pub const units = @import("units.zig"); +const counter = @import("counter.zig"); +pub const Counter = counter.Counter; +pub const Counters = counter.Counters; +pub const Growth = counter.Growth; +pub const DoubleBuffer = @import("double_buffer.zig").DoubleBuffer; + +test { + _ = @import("collector.zig"); + _ = counter; + _ = @import("double_buffer.zig"); + _ = dynlib; + _ = fdinfo; + _ = hwmon; + _ = pci; + _ = proc; + _ = @import("sandbox.zig"); + _ = sysfs; + _ = units; +} diff --git a/bin/zml-smi/smi/sysfs.zig b/bin/zml-smi/smi/sysfs.zig new file mode 100644 index 00000000..44c0bf75 --- /dev/null +++ b/bin/zml-smi/smi/sysfs.zig @@ -0,0 +1,148 @@ +const std = @import("std"); +const units = @import("units.zig"); + +/// A directory of /sys, /proc or /dev, kept open for the life of a device. +pub const Dir = struct { + handle: std.Io.Dir, + io: std.Io, + + pub fn open(io: std.Io, path: []const u8) !Dir { + return .{ .handle = try std.Io.Dir.cwd().openDir(io, path, .{ .iterate = true }), .io = io }; + } + + pub fn openDir(self: Dir, sub_path: []const u8) !Dir { + return .{ .handle = try self.handle.openDir(self.io, sub_path, .{ .iterate = true }), .io = self.io }; + } + + pub fn close(self: Dir) void { + self.handle.close(self.io); + } + + /// Trimmed content of a file. The slice points into `buffer`. + /// A file that does not fit is error.StreamTooLong, never a silent cut. + pub fn read(self: Dir, sub_path: []const u8, buffer: []u8) ![]const u8 { + const file = try self.handle.openFile(self.io, sub_path, .{}); + defer file.close(self.io); + + const len = try readAll(file, self.io, buffer); + var extra: [1]u8 = undefined; + if (len == buffer.len and try readAll(file, self.io, &extra) != 0) return error.StreamTooLong; + return std.mem.trim(u8, buffer[0..len], &std.ascii.whitespace); + } + + /// The first bytes of a file, cut at `buffer.len`: for /proc//cmdline. + pub fn head(self: Dir, sub_path: []const u8, buffer: []u8) ![]const u8 { + const file = try self.handle.openFile(self.io, sub_path, .{}); + defer file.close(self.io); + + return buffer[0..try readAll(file, self.io, buffer)]; + } + + /// A number, in decimal or in hex with a 0x prefix. + pub fn int(self: Dir, comptime T: type, sub_path: []const u8) !T { + var buffer: [64]u8 = undefined; + return parseInt(T, try self.read(sub_path, &buffer)); + } + + /// `int`, converted between unit prefixes. + pub fn scaled(self: Dir, sub_path: []const u8, from: units.Prefix, to: units.Prefix) !u64 { + return units.scale(try self.int(u64, sub_path), from, to); + } + + /// Value of `key` in a "key: value" file such as /proc/meminfo. + /// Reads line by line, with `buffer` as the line. + pub fn field(self: Dir, sub_path: []const u8, key: []const u8, buffer: []u8) ![]const u8 { + const file = try self.handle.openFile(self.io, sub_path, .{}); + defer file.close(self.io); + + var reader = file.readerStreaming(self.io, buffer); + while (true) { + const line = reader.interface.takeDelimiter('\n') catch |err| switch (err) { + // A line longer than the buffer is not the one we look for. + error.StreamTooLong => { + _ = reader.interface.discardDelimiterInclusive('\n') catch return error.NotFound; + continue; + }, + error.ReadFailed => return reader.err.?, + } orelse return error.NotFound; + if (fieldValue(line, key)) |value| return value; + } + } + + pub fn iterate(self: Dir) std.Io.Dir.Iterator { + return self.handle.iterate(); + } + + pub fn readLink(self: Dir, sub_path: []const u8, buffer: []u8) ![]const u8 { + return buffer[0..try self.handle.readLink(self.io, sub_path, buffer)]; + } +}; + +/// Whether a path exists: the usual body of `detect`. +pub fn exists(io: std.Io, path: []const u8) bool { + std.Io.Dir.cwd().access(io, path, .{}) catch return false; + return true; +} + +/// A number, in decimal or in hex with a 0x prefix. +pub fn parseInt(comptime T: type, text: []const u8) !T { + if (std.mem.startsWith(u8, text, "0x")) return std.fmt.parseInt(T, text[2..], 16); + return std.fmt.parseInt(T, text, 10); +} + +/// The first number of a value: "100 ns" is 100. +pub fn firstNumber(text: []const u8) ?u64 { + var words = std.mem.tokenizeAny(u8, text, " \t"); + return std.fmt.parseInt(u64, words.next() orelse return null, 10) catch null; +} + +/// The value of `key` on a "key: value" or "key=value" line. +pub fn fieldValue(line: []const u8, key: []const u8) ?[]const u8 { + if (!std.mem.startsWith(u8, line, key)) return null; + const rest = line[key.len..]; + if (rest.len > 0 and std.mem.indexOfScalar(u8, ":= \t", rest[0]) == null) return null; + return std.mem.trim(u8, rest, ":= \t"); +} + +fn readAll(file: std.Io.File, io: std.Io, buffer: []u8) !usize { + var len: usize = 0; + while (len < buffer.len) { + len += file.readStreaming(io, &.{buffer[len..]}) catch |err| switch (err) { + error.EndOfStream => break, + else => |e| return e, + }; + } + return len; +} + +test Dir { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "vendor", .data = "0x8086\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "temp1_input", .data = "58500\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "meminfo", .data = "MemTotal: 65137624 kB\nMemTotalX: 1 kB\nFlags: fpu vme de pse tsc msr pae mce cx8\nMemAvailable: 33697096 kB\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "cmdline", .data = "python\x00-m\x00llmd\x00" }); + + const dir: Dir = .{ .handle = tmp.dir, .io = io }; + var buffer: [16]u8 = undefined; + + try std.testing.expectEqualStrings("0x8086", try dir.read("vendor", &buffer)); + try std.testing.expectEqual(0x8086, try dir.int(u16, "vendor")); + try std.testing.expectEqual(59, try dir.scaled("temp1_input", .milli, .one)); + try std.testing.expectError(error.StreamTooLong, dir.read("vendor", buffer[0..4])); + try std.testing.expectEqualStrings("python\x00-m", try dir.head("cmdline", buffer[0..9])); + + var line: [64]u8 = undefined; + try std.testing.expectEqualStrings("33697096 kB", try dir.field("meminfo", "MemAvailable", &line)); + try std.testing.expectEqualStrings("65137624 kB", try dir.field("meminfo", "MemTotal", &line)); + try std.testing.expectError(error.NotFound, dir.field("meminfo", "Mem", &line)); + // A line longer than the buffer is skipped, not cut. + try std.testing.expectEqualStrings("33697096 kB", try dir.field("meminfo", "MemAvailable", line[0..30])); +} + +test fieldValue { + try std.testing.expectEqualStrings("0000:03:00.0", fieldValue("PCI_SLOT_NAME=0000:03:00.0", "PCI_SLOT_NAME").?); + try std.testing.expectEqualStrings("AMD Ryzen", fieldValue("model name\t: AMD Ryzen", "model name").?); + try std.testing.expectEqual(null, fieldValue("model names: x", "model name")); +} diff --git a/bin/zml-smi/smi/units.zig b/bin/zml-smi/smi/units.zig new file mode 100644 index 00000000..ca2b01fc --- /dev/null +++ b/bin/zml-smi/smi/units.zig @@ -0,0 +1,30 @@ +const std = @import("std"); + +/// Decimal prefixes, a factor of 1000 apart. +pub const Prefix = enum(u3) { nano, micro, milli, one, kilo, mega, giga }; + +/// Converts between prefixes of one unit, rounding to nearest: `scale(1500, .micro, .milli)` is 2. +/// A result that does not fit is the largest u64. +pub fn scale(value: u64, from: Prefix, to: Prefix) u64 { + const from_step: u8 = @backingInt(from); + const to_step: u8 = @backingInt(to); + if (from_step >= to_step) { + return std.math.mul(u64, value, factor(from_step - to_step)) catch std.math.maxInt(u64); + } + const divisor = factor(to_step - from_step); + const rounded_up: u64 = @intFromBool(value % divisor * 2 >= divisor); + return value / divisor + rounded_up; +} + +fn factor(steps: u8) u64 { + return std.math.powi(u64, 1000, steps) catch unreachable; +} + +test scale { + try std.testing.expectEqual(2, scale(1500, .micro, .milli)); + try std.testing.expectEqual(1, scale(1499, .micro, .milli)); + try std.testing.expectEqual(58, scale(58_000, .milli, .one)); + try std.testing.expectEqual(230_000, scale(230_000_000, .micro, .milli)); + try std.testing.expectEqual(3_000_000, scale(3, .one, .micro)); + try std.testing.expectEqual(std.math.maxInt(u64), scale(std.math.maxInt(u64) / 10, .giga, .nano)); +} diff --git a/bin/zml-smi/utils/BUILD.bazel b/bin/zml-smi/utils/BUILD.bazel deleted file mode 100644 index a154556c..00000000 --- a/bin/zml-smi/utils/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "double_buffer", - import_name = "zml-smi/double_buffer", - main = "double_buffer.zig", - visibility = ["//visibility:public"], -) - -zig_library( - name = "dynlib", - import_name = "zml-smi/dynlib", - main = "dynlib.zig", - visibility = ["//visibility:public"], -) - -zig_library( - name = "sandbox", - import_name = "zml-smi/sandbox", - main = "sandbox.zig", - visibility = ["//visibility:public"], - deps = ["@rules_zig//zig/runfiles"], -) - -zig_library( - name = "sysfs", - import_name = "zml-smi/sysfs", - main = "sysfs.zig", - visibility = ["//visibility:public"], -) diff --git a/bin/zml-smi/utils/double_buffer.zig b/bin/zml-smi/utils/double_buffer.zig deleted file mode 100644 index add3037b..00000000 --- a/bin/zml-smi/utils/double_buffer.zig +++ /dev/null @@ -1,32 +0,0 @@ -const std = @import("std"); - -pub fn DoubleBuffer(comptime T: type) type { - return struct { - const Self = @This(); - pub const Value = T; - - values: [2]T, - current: std.atomic.Value(u8) = std.atomic.Value(u8).init(0), - - pub fn front(self: *const Self) *const T { - return &self.values[self.current.load(.acquire)]; - } - - pub fn back(self: *Self) *T { - return &self.values[1 - self.current.load(.acquire)]; - } - - pub fn swap(self: *Self) void { - self.current.store(1 - self.current.load(.acquire), .release); - } - - pub fn jsonStringify(self: *const Self, jw: *std.json.Stringify) !void { - try jw.write(self.front().*); - } - - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { - const val = try std.json.innerParseFromValue(T, allocator, source, options); - return .{ .values = .{ val, val } }; - } - }; -} diff --git a/bin/zml-smi/utils/dynlib.zig b/bin/zml-smi/utils/dynlib.zig deleted file mode 100644 index 4a478107..00000000 --- a/bin/zml-smi/utils/dynlib.zig +++ /dev/null @@ -1,10 +0,0 @@ -const std = @import("std"); - -pub fn lookupStruct(dynlib: *std.DynLib, comptime VTable: type) !VTable { - var result: VTable = undefined; - inline for (comptime std.meta.fieldNames(VTable), comptime std.meta.fieldTypes(VTable)) |field_name, Field| { - @field(result, field_name) = dynlib.lookup(Field, field_name) orelse return error.SymbolResolutionFailed; - } - - return result; -} diff --git a/bin/zml-smi/utils/sandbox.zig b/bin/zml-smi/utils/sandbox.zig deleted file mode 100644 index d4e268bd..00000000 --- a/bin/zml-smi/utils/sandbox.zig +++ /dev/null @@ -1,44 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const bazel_runfiles = @import("runfiles"); -const bazel_builtin = @import("bazel_builtin"); - -var runfiles_global: bazel_runfiles.Runfiles = undefined; - -const init_array_section = switch (builtin.object_format) { - .macho => "__DATA,__mod_init_func", - .elf => ".init_array", - else => "", -}; - -export const _ align(@alignOf(usize)) linksection(init_array_section) = &struct { - fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { - _ = argc; - _ = argv; - _ = envp; - var threaded: std.Io.Threaded = .init_single_threaded; - const io = threaded.io(); - - var exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const exe_size = std.process.executablePath(io, &exe_path_buf) catch - std.debug.panic("Unable to get executable path", .{}); - - runfiles_global = bazel_runfiles.Runfiles.create(.{ - .allocator = std.heap.c_allocator, - .io = io, - .argv0 = exe_path_buf[0..exe_size], - .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, - .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, - }) catch { - std.debug.panic("Unable to find runfiles", .{}); - } orelse { - std.debug.panic("Unable to initialize runfiles", .{}); - }; - } -}.call; - -pub fn path(buf: *[std.Io.Dir.max_path_bytes]u8) ?[]const u8 { - const with_repo = runfiles_global.withSourceRepo(bazel_builtin.current_repository); - - return with_repo.rlocation("zml/bin/zml-smi/sandbox", buf) catch null; -} diff --git a/bin/zml-smi/utils/sysfs.zig b/bin/zml-smi/utils/sysfs.zig deleted file mode 100644 index 636883b2..00000000 --- a/bin/zml-smi/utils/sysfs.zig +++ /dev/null @@ -1,70 +0,0 @@ -const std = @import("std"); - -pub fn readInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { - const data = try readFirstLine(allocator, io, path); - return std.fmt.parseInt(u64, data, 10); -} - -pub fn readString(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { - return readFirstLine(allocator, io, path); -} - -pub fn readFieldInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) !u64 { - return nthTokenInt(try findField(allocator, io, path, key), 0); -} - -pub fn nthTokenInt(data: []const u8, n: usize) u64 { - var i: usize = 0; - - var iter = std.mem.tokenizeAny(u8, data, " \t"); - while (iter.next()) |tok| : (i += 1) { - if (i == n) { - return std.fmt.parseInt(u64, tok, 10) catch 0; - } - } - - return 0; -} - -pub fn readFieldString(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { - const value = try findField(allocator, io, path, key); - return std.mem.trimEnd(u8, value, &std.ascii.whitespace); -} - -fn findField(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { - var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); - defer file.close(io); - - var read_buf: [4096]u8 = undefined; - var reader = file.reader(io, &read_buf); - - var line_writer: std.Io.Writer.Allocating = .init(allocator); - errdefer line_writer.deinit(); - - while (true) { - line_writer.clearRetainingCapacity(); - _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch return error.NotFound; - reader.interface.toss(1); - - const line = line_writer.written(); - if (std.mem.startsWith(u8, line, key)) { - const owned = line_writer.toOwnedSlice() catch return error.NotFound; - return std.mem.trimStart(u8, owned[key.len..], " :\t"); - } - } -} - -pub fn readFirstLine(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]u8 { - var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); - defer file.close(io); - - var read_buf: [4096]u8 = undefined; - var reader = file.reader(io, &read_buf); - var writer: std.Io.Writer.Allocating = .init(allocator); - errdefer writer.deinit(); - _ = reader.interface.streamDelimiter(&writer.writer, '\n') catch |err| switch (err) { - error.EndOfStream => 0, - else => |e| return e, - }; - return writer.toOwnedSlice(); -} From 5613d5e7c9770a002400c37bd1ee79e1fe74830b Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 20:16:12 +0100 Subject: [PATCH 07/19] bin/zml-smi: write CSV, JSON and Prometheus from the collector Every output reads the device table and the latest samples by reflection, so no field is named by a string. JSON writes each device as {"device": ..., "sample": {"": ...}}, which is also what --remotes reads. CSV keeps one table per target and leaves out a device column that is empty for the whole table. The CUDA version becomes the detail cuda_version. Prometheus has one literal per struct with an entry per field, and writes metric by metric, so names, labels and family order are unchanged. PCIe throughput is kilobytes per second, now multiplied by 1000 instead of 128. The API and Prometheus servers share one loop, and write each body to memory before sending it. --- bin/zml-smi/api/BUILD.bazel | 18 --- bin/zml-smi/api/api.zig | 2 - bin/zml-smi/api/client.zig | 138 -------------------- bin/zml-smi/api/server.zig | 91 ------------- bin/zml-smi/csv/BUILD.bazel | 11 -- bin/zml-smi/csv/csv.zig | 46 ------- bin/zml-smi/json/BUILD.bazel | 11 -- bin/zml-smi/json/json.zig | 15 --- bin/zml-smi/prometheus/BUILD.bazel | 16 --- bin/zml-smi/prometheus/exposition.zig | 159 ----------------------- bin/zml-smi/prometheus/prometheus.zig | 2 - bin/zml-smi/prometheus/server.zig | 87 ------------- bin/zml-smi/smi/BUILD.bazel | 4 + bin/zml-smi/smi/api.zig | 65 ++++++++++ bin/zml-smi/smi/csv.zig | 66 ++++++++++ bin/zml-smi/smi/json.zig | 21 +++ bin/zml-smi/smi/prometheus.zig | 179 ++++++++++++++++++++++++++ bin/zml-smi/smi/smi.zig | 5 + 18 files changed, 340 insertions(+), 596 deletions(-) delete mode 100644 bin/zml-smi/api/BUILD.bazel delete mode 100644 bin/zml-smi/api/api.zig delete mode 100644 bin/zml-smi/api/client.zig delete mode 100644 bin/zml-smi/api/server.zig delete mode 100644 bin/zml-smi/csv/BUILD.bazel delete mode 100644 bin/zml-smi/csv/csv.zig delete mode 100644 bin/zml-smi/json/BUILD.bazel delete mode 100644 bin/zml-smi/json/json.zig delete mode 100644 bin/zml-smi/prometheus/BUILD.bazel delete mode 100644 bin/zml-smi/prometheus/exposition.zig delete mode 100644 bin/zml-smi/prometheus/prometheus.zig delete mode 100644 bin/zml-smi/prometheus/server.zig create mode 100644 bin/zml-smi/smi/api.zig create mode 100644 bin/zml-smi/smi/csv.zig create mode 100644 bin/zml-smi/smi/json.zig create mode 100644 bin/zml-smi/smi/prometheus.zig diff --git a/bin/zml-smi/api/BUILD.bazel b/bin/zml-smi/api/BUILD.bazel deleted file mode 100644 index 7f1f7a2d..00000000 --- a/bin/zml-smi/api/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "api", - import_name = "zml-smi/api", - main = "api.zig", - srcs = [ - "client.zig", - "server.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/json", - "//bin/zml-smi/utils:double_buffer", - ], -) diff --git a/bin/zml-smi/api/api.zig b/bin/zml-smi/api/api.zig deleted file mode 100644 index fb39ddd8..00000000 --- a/bin/zml-smi/api/api.zig +++ /dev/null @@ -1,2 +0,0 @@ -pub const Server = @import("server.zig").Server; -pub const addRemotes = @import("client.zig").addRemotes; diff --git a/bin/zml-smi/api/client.zig b/bin/zml-smi/api/client.zig deleted file mode 100644 index ddec6d2d..00000000 --- a/bin/zml-smi/api/client.zig +++ /dev/null @@ -1,138 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const pi = smi_info.process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; - -fn setRemote(device: *DeviceInfo) void { - inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { - if (device.* == @field(smi_info.device_info.Target, field_name)) { - @field(device, field_name).values[0].remote = true; - @field(device, field_name).values[1].remote = true; - } - } -} - -pub fn addRemotes(collector: *Collector, hosts: []const u8) !void { - var it = std.mem.splitScalar(u8, hosts, ','); - while (it.next()) |host| { - addHost(collector, host) catch |err| { - std.log.err("{s}: {s}", .{ host, @errorName(err) }); - }; - } -} - -fn addHost(collector: *Collector, host: []const u8) !void { - const body = try httpGet(collector.gpa, collector.io, host); - defer collector.gpa.free(body); - - const parsed = try std.json.parseFromSlice(std.json.Value, collector.gpa, body, .{}); - defer parsed.deinit(); - - const root = parsed.value.object; - const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; - - const dev_offset: u16 = @intCast(collector.device_infos.items.len); - - var devices: std.ArrayList(*DeviceInfo) = .empty; - for (json_devices) |item| { - var device = std.json.parseFromValueLeaky(DeviceInfo, collector.arena, item, .{ .ignore_unknown_fields = true }) catch continue; - setRemote(&device); - const info = try collector.addDevice(device); - try devices.append(collector.arena, info); - } - - if (devices.items.len == 0) { - return; - } - - const processes = try collector.createProcessList(); - const str_arenas: [2]*std.heap.ArenaAllocator = .{ try collector.createPollArena(), try collector.createPollArena() }; - - const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; - const back_idx: usize = 1 - processes.current.load(.acquire); - const back = processes.back(); - for (json_processes) |item| { - var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, str_arenas[back_idx].allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; - proc.device_idx += dev_offset; - proc.remote = true; - back.append(collector.gpa, proc) catch continue; - } - processes.swap(); - - const url = try collector.arena.dupe(u8, host); - const poll_arena = try collector.createPollArena(); - try collector.spawnPoll(pollOnce, .{ poll_arena, collector.gpa, collector.io, url, devices.items, processes, dev_offset, str_arenas }); -} - -fn pollOnce(poll_arena: *std.heap.ArenaAllocator, gpa: std.mem.Allocator, io: std.Io, host: []const u8, devices: []const *DeviceInfo, processes: *ProcessDoubleBuffer, dev_offset: u16, str_arenas: [2]*std.heap.ArenaAllocator) void { - _ = poll_arena.reset(.retain_capacity); - - const body = httpGet(gpa, io, host) catch return; - defer gpa.free(body); - - const parsed = std.json.parseFromSlice(std.json.Value, gpa, body, .{}) catch return; - defer parsed.deinit(); - - const root = parsed.value.object; - const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; - - for (json_devices, 0..) |item, i| { - if (i >= devices.len) { - std.log.warn("{s}: remote has more devices than expected ({d} > {d})", .{ host, json_devices.len, devices.len }); - break; - } - var device = std.json.parseFromValueLeaky(DeviceInfo, poll_arena.allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; - setRemote(&device); - - inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { - if (devices[i].* == @field(smi_info.device_info.Target, field_name)) { - @field(devices[i], field_name).back().* = @field(device, field_name).front().*; - @field(devices[i], field_name).swap(); - } - } - } - - const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; - const back_idx: usize = 1 - processes.current.load(.acquire); - _ = str_arenas[back_idx].reset(.retain_capacity); - const alloc = str_arenas[back_idx].allocator(); - const back = processes.back(); - back.clearRetainingCapacity(); - for (json_processes) |item| { - var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, alloc, item, .{ .ignore_unknown_fields = true }) catch continue; - proc.device_idx += dev_offset; - proc.remote = true; - back.append(gpa, proc) catch continue; - } - processes.swap(); -} - -fn httpGet(allocator: std.mem.Allocator, io: std.Io, host: []const u8) ![]const u8 { - if (!std.mem.startsWith(u8, host, "http://") and !std.mem.startsWith(u8, host, "https://")) { - std.log.err("remote host must start with http:// or https://: {s}", .{host}); - return error.InvalidUrl; - } - - const url = try std.fmt.allocPrint(allocator, "{s}/metrics", .{host}); - defer allocator.free(url); - - var client: std.http.Client = .{ .allocator = allocator, .io = io }; - defer client.deinit(); - - var aw: std.Io.Writer.Allocating = .init(allocator); - errdefer aw.deinit(); - - const result = try client.fetch(.{ - .location = .{ .url = url }, - .response_writer = &aw.writer, - }); - - if (result.status != .ok) { - aw.deinit(); - return error.HttpError; - } - - return try aw.toOwnedSlice(); -} diff --git a/bin/zml-smi/api/server.zig b/bin/zml-smi/api/server.zig deleted file mode 100644 index 517d0cd9..00000000 --- a/bin/zml-smi/api/server.zig +++ /dev/null @@ -1,91 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const ProcessInfo = smi_info.process_info.ProcessInfo; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(ProcessInfo)); -const json = @import("zml-smi/json"); - -pub const Server = struct { - allocator: std.mem.Allocator, - io: std.Io, - devices: []const *DeviceInfo, - processes: *const ProcessDoubleBuffer, - tcp: std.Io.net.Server, - connection_group: std.Io.Group = .init, - - pub fn init(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, processes: *const ProcessDoubleBuffer) !Server { - const address: std.Io.net.IpAddress = .{ .ip4 = .{ - .bytes = .{ 0, 0, 0, 0 }, - .port = port, - } }; - var tcp = try address.listen(io, .{ .reuse_address = true }); - errdefer tcp.deinit(io); - - return .{ - .allocator = allocator, - .io = io, - .devices = devices, - .processes = processes, - .tcp = tcp, - }; - } - - pub fn run(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, processes: *const ProcessDoubleBuffer) void { - var self = init(allocator, io, port, devices, processes) catch |err| { - std.log.err("api server failed to start: {s}", .{@errorName(err)}); - return; - }; - defer self.deinit(); - self.serve(); - } - - pub fn deinit(self: *Server) void { - self.connection_group.await(self.io) catch {}; - self.tcp.deinit(self.io); - } - - pub fn serve(self: *Server) void { - while (true) { - const stream = self.tcp.accept(self.io) catch break; - self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; - } - } - - fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { - defer stream.close(self.io); - self.handleConn(stream) catch {}; - } - - fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { - var read_buf: [8192]u8 = undefined; - var reader = stream.reader(self.io, &read_buf); - - var write_buf: [4096]u8 = undefined; - var writer = stream.writer(self.io, &write_buf); - - var http_server: std.http.Server = .init(&reader.interface, &writer.interface); - - var body: std.Io.Writer.Allocating = .init(self.allocator); - defer body.deinit(); - - while (true) { - var request = http_server.receiveHead() catch |err| switch (err) { - error.HttpConnectionClosing => return, - else => return err, - }; - - // The processes are only valid until they are polled again, so the body - // is written out before it is sent: a slow client cannot outlive them. - body.clearRetainingCapacity(); - try json.write(&body.writer, self.devices, self.processes.front().items); - - try request.respond(body.written(), .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - .{ .name = "access-control-allow-origin", .value = "*" }, - }, - }); - } - } -}; diff --git a/bin/zml-smi/csv/BUILD.bazel b/bin/zml-smi/csv/BUILD.bazel deleted file mode 100644 index a0cb3422..00000000 --- a/bin/zml-smi/csv/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "csv", - import_name = "zml-smi/csv", - main = "csv.zig", - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/info", - ], -) diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig deleted file mode 100644 index dbd80dd9..00000000 --- a/bin/zml-smi/csv/csv.zig +++ /dev/null @@ -1,46 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; - -pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { - const info = @typeInfo(DeviceInfo).@"union"; - inline for (info.field_names, info.field_types) |type_name, DeviceType| { - const tag = @field(smi_info.Target, type_name); - var header_printed = false; - - for (devices, 0..) |dev, i| { - switch (dev.*) { - tag => |*db| { - if (!header_printed) { - try writer.writeAll("index,type"); - inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { - try writer.writeAll("," ++ field_name); - } - try writer.writeAll("\n"); - - header_printed = true; - } - - const val = db.front().*; - - try writer.print("{d},{s}", .{ i, type_name }); - inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { - try writer.writeAll(","); - try writeValue(writer, @field(val, field_name)); - } - try writer.writeAll("\n"); - }, - else => {}, - } - } - } -} - -fn writeValue(writer: *std.Io.Writer, field: anytype) !void { - if (@typeInfo(@TypeOf(field)) == .optional) { - if (field) |v| return writeValue(writer, v); - } else switch (@typeInfo(@TypeOf(field))) { - .pointer => return writer.writeAll(field), - else => return writer.print("{any}", .{field}), - } -} diff --git a/bin/zml-smi/json/BUILD.bazel b/bin/zml-smi/json/BUILD.bazel deleted file mode 100644 index decc8ca1..00000000 --- a/bin/zml-smi/json/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "json", - import_name = "zml-smi/json", - main = "json.zig", - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/info", - ], -) diff --git a/bin/zml-smi/json/json.zig b/bin/zml-smi/json/json.zig deleted file mode 100644 index c32a16cf..00000000 --- a/bin/zml-smi/json/json.zig +++ /dev/null @@ -1,15 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const ProcessInfo = smi_info.process_info.ProcessInfo; - -pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, processes: []const ProcessInfo) !void { - var jw: std.json.Stringify = .{ .writer = writer }; - try jw.write(Response{ .devices = devices, .processes = processes }); - try writer.writeAll("\n"); -} - -const Response = struct { - devices: []const *DeviceInfo, - processes: []const ProcessInfo, -}; diff --git a/bin/zml-smi/prometheus/BUILD.bazel b/bin/zml-smi/prometheus/BUILD.bazel deleted file mode 100644 index c00a26b5..00000000 --- a/bin/zml-smi/prometheus/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "prometheus", - import_name = "zml-smi/prometheus", - main = "prometheus.zig", - srcs = [ - "server.zig", - "exposition.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - ], -) diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig deleted file mode 100644 index 54d48cb6..00000000 --- a/bin/zml-smi/prometheus/exposition.zig +++ /dev/null @@ -1,159 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const HostInfo = smi_info.host_info.HostInfo; - -const device_metrics = .{ - // Shared across all device types - .{ .field = "util_percent", .metric = "zml_device_utilization_percent", .help = "Device utilization percentage" }, - .{ .field = "mem_used_bytes", .metric = "zml_device_memory_used_bytes", .help = "Device memory used in bytes" }, - .{ .field = "mem_total_bytes", .metric = "zml_device_memory_total_bytes", .help = "Device memory total in bytes" }, - // GPU (CUDA/ROCm/oneAPI) - .{ .field = "encoder_util_percent", .metric = "zml_device_encoder_utilization_percent", .help = "Encoder utilization percentage" }, - .{ .field = "decoder_util_percent", .metric = "zml_device_decoder_utilization_percent", .help = "Decoder utilization percentage" }, - .{ .field = "power_mw", .metric = "zml_device_power_watts", .help = "Power draw in watts", .fmt = fmtMilliwattsAsWatts }, - .{ .field = "power_limit_mw", .metric = "zml_device_power_limit_watts", .help = "Power limit in watts", .fmt = fmtMilliwattsAsWatts }, - .{ .field = "temperature", .metric = "zml_device_temperature_celsius", .help = "Device temperature in celsius" }, - .{ .field = "fan_speed_percent", .metric = "zml_device_fan_speed_percent", .help = "Fan speed percentage" }, - .{ .field = "clock_graphics_mhz", .metric = "zml_device_clock_graphics_mhz", .help = "Graphics clock frequency in MHz" }, - .{ .field = "clock_sm_mhz", .metric = "zml_device_clock_sm_mhz", .help = "SM clock frequency in MHz" }, - .{ .field = "clock_mem_mhz", .metric = "zml_device_clock_memory_mhz", .help = "Memory clock frequency in MHz" }, - .{ .field = "pcie_tx_kbps", .metric = "zml_device_pcie_tx_bytes_per_second", .help = "PCIe TX throughput in bytes per second", .fmt = fmtScaled(128) }, - .{ .field = "pcie_rx_kbps", .metric = "zml_device_pcie_rx_bytes_per_second", .help = "PCIe RX throughput in bytes per second", .fmt = fmtScaled(128) }, - // Neuron - .{ .field = "nc_tensors", .metric = "zml_device_neuron_tensors_bytes", .help = "Neuron core tensor memory in bytes" }, - .{ .field = "nc_constants", .metric = "zml_device_neuron_constants_bytes", .help = "Neuron core constants memory in bytes" }, - .{ .field = "nc_model_code", .metric = "zml_device_neuron_model_code_bytes", .help = "Neuron core model code memory in bytes" }, - .{ .field = "nc_shared_scratchpad", .metric = "zml_device_neuron_shared_scratchpad_bytes", .help = "Neuron core shared scratchpad memory in bytes" }, - .{ .field = "nc_nonshared_scratchpad", .metric = "zml_device_neuron_nonshared_scratchpad_bytes", .help = "Neuron core non-shared scratchpad memory in bytes" }, - .{ .field = "nc_runtime", .metric = "zml_device_neuron_runtime_bytes", .help = "Neuron core runtime memory in bytes" }, - .{ .field = "nc_driver", .metric = "zml_device_neuron_driver_bytes", .help = "Neuron core driver memory in bytes" }, - .{ .field = "nc_dma_rings", .metric = "zml_device_neuron_dma_rings_bytes", .help = "Neuron core DMA rings memory in bytes" }, - .{ .field = "nc_collectives", .metric = "zml_device_neuron_collectives_bytes", .help = "Neuron core collectives memory in bytes" }, - .{ .field = "nc_notifications", .metric = "zml_device_neuron_notifications_bytes", .help = "Neuron core notifications memory in bytes" }, - .{ .field = "nc_uncategorized", .metric = "zml_device_neuron_uncategorized_bytes", .help = "Neuron core uncategorized memory in bytes" }, -}; - -const host_metrics = .{ - .{ .field = "cpu_cores", .metric = "zml_host_cpu_cores", .help = "Number of CPU cores" }, - .{ .field = "mem_total_kib", .metric = "zml_host_memory_total_bytes", .help = "Total host memory in bytes", .fmt = fmtScaled(1024) }, - .{ .field = "mem_available_kib", .metric = "zml_host_memory_available_bytes", .help = "Available host memory in bytes", .fmt = fmtScaled(1024) }, - .{ .field = "load_1", .metric = "zml_host_load_1", .help = "1-minute load average" }, - .{ .field = "load_5", .metric = "zml_host_load_5", .help = "5-minute load average" }, - .{ .field = "load_15", .metric = "zml_host_load_15", .help = "15-minute load average" }, - .{ .field = "uptime_seconds", .metric = "zml_host_uptime_seconds", .help = "System uptime in seconds" }, -}; - -pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, host: *const HostInfo) !void { - try writer.writeAll("# HELP zml_device_info Device metadata\n"); - try writer.writeAll("# TYPE zml_device_info gauge\n"); - for (devices, 0..) |dev, i| { - inline for (@typeInfo(DeviceInfo).@"union".field_names) |type_name| { - const tag = @field(smi_info.device_info.Target, type_name); - - switch (dev.*) { - tag => |*db| { - const val = db.front().*; - - try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, type_name }); - - inline for (.{ - .{ "name", "name" }, - .{ "driver_version", "driver" }, - .{ "cuda_driver_version", "cuda_version" }, - }) |label| { - if (@hasField(@TypeOf(val), label[0])) { - if (@field(val, label[0])) |v| { - try writer.print(",{s}=\"", .{label[1]}); - try std.zig.stringEscape(v, writer); - try writer.writeAll("\""); - } - } - } - - try writer.writeAll("} 1\n"); - }, - else => {}, - } - } - } - try writer.writeAll("\n"); - - try writeAllDeviceMetrics(writer, device_metrics, devices); - try writeHostMetrics(writer, host.front().*); -} - -fn writeAllDeviceMetrics( - writer: *std.Io.Writer, - comptime metrics: anytype, - devices: []const *DeviceInfo, -) !void { - inline for (metrics) |m| { - var header_written = false; - - for (devices, 0..) |dev, i| { - switch (dev.*) { - inline else => |*db| { - const val = db.front().*; - if (@hasField(@TypeOf(val), m.field)) { - if (@field(val, m.field)) |raw| { - if (!header_written) { - try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n", .{ m.metric, m.help, m.metric }); - header_written = true; - } - - const device_type = @tagName(std.meta.activeTag(dev.*)); - try writer.print("{s}{{device=\"{d}\",type=\"{s}\",name=\"", .{ m.metric, i, device_type }); - try std.zig.stringEscape(val.name orelse "", writer); - try writer.writeAll("\"} "); - try fmtValue(m, writer, raw); - try writer.writeAll("\n"); - } - } - }, - } - } - - if (header_written) { - try writer.writeAll("\n"); - } - } -} - -fn writeHostMetrics(writer: *std.Io.Writer, val: anytype) !void { - inline for (host_metrics) |m| { - if (@field(val, m.field)) |raw| { - try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n{s} ", .{ m.metric, m.help, m.metric, m.metric }); - try fmtValue(m, writer, raw); - try writer.writeAll("\n\n"); - } - } -} - -fn fmtValue(comptime m: anytype, writer: *std.Io.Writer, raw: anytype) !void { - if (@hasField(@TypeOf(m), "fmt")) { - try m.fmt(writer, raw); - } else { - try writeValue(writer, raw); - } -} - -fn writeValue(writer: *std.Io.Writer, val: anytype) std.Io.Writer.Error!void { - switch (@typeInfo(@TypeOf(val))) { - .float, .comptime_float => try writer.print("{d:.2}", .{val}), - .pointer => try writer.writeAll(val), - else => try writer.print("{d}", .{val}), - } -} - -fn fmtMilliwattsAsWatts(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { - try writer.print("{d}.{d:0>3}", .{ val / 1000, val % 1000 }); -} - -fn fmtScaled(comptime scale: u64) fn (*std.Io.Writer, u64) std.Io.Writer.Error!void { - return struct { - fn f(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { - try writer.print("{d}", .{val * scale}); - } - }.f; -} diff --git a/bin/zml-smi/prometheus/prometheus.zig b/bin/zml-smi/prometheus/prometheus.zig deleted file mode 100644 index cfa2fc28..00000000 --- a/bin/zml-smi/prometheus/prometheus.zig +++ /dev/null @@ -1,2 +0,0 @@ -pub const Server = @import("server.zig").Server; -pub const exposition = @import("exposition.zig"); diff --git a/bin/zml-smi/prometheus/server.zig b/bin/zml-smi/prometheus/server.zig deleted file mode 100644 index 9517ecac..00000000 --- a/bin/zml-smi/prometheus/server.zig +++ /dev/null @@ -1,87 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const HostInfo = smi_info.host_info.HostInfo; -const exposition = @import("exposition.zig"); - -pub const Server = struct { - io: std.Io, - devices: []const *DeviceInfo, - host: *const HostInfo, - tcp: std.Io.net.Server, - connection_group: std.Io.Group = .init, - - pub fn init(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host: *const HostInfo) !Server { - const address = try std.Io.net.IpAddress.parseLiteral(listen); - var tcp = try address.listen(io, .{ .reuse_address = true }); - errdefer tcp.deinit(io); - - return .{ - .io = io, - .devices = devices, - .host = host, - .tcp = tcp, - }; - } - - pub fn run(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host_info: *const HostInfo) void { - var self = init(io, listen, devices, host_info) catch |err| { - std.log.err("prometheus server failed to start: {s}", .{@errorName(err)}); - return; - }; - defer self.deinit(); - self.serve(); - } - - pub fn deinit(self: *Server) void { - self.connection_group.await(self.io) catch {}; - self.tcp.deinit(self.io); - } - - pub fn serve(self: *Server) void { - while (true) { - const stream = self.tcp.accept(self.io) catch break; - self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; - } - } - - fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { - defer stream.close(self.io); - self.handleConn(stream) catch {}; - } - - fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { - var read_buf: [8192]u8 = undefined; - var reader = stream.reader(self.io, &read_buf); - - var write_buf: [4096]u8 = undefined; - var writer = stream.writer(self.io, &write_buf); - - var http_server: std.http.Server = .init(&reader.interface, &writer.interface); - - while (true) { - var request = http_server.receiveHead() catch |err| switch (err) { - error.HttpConnectionClosing => return, - else => return err, - }; - - if (request.head.method != .GET) { - try request.respond("Method Not Allowed\n", .{ .status = .method_not_allowed }); - continue; - } - - var body_buf: [4096]u8 = undefined; - var body_writer = try request.respondStreaming(&body_buf, .{ - .respond_options = .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "text/plain" }, - }, - }, - }); - - try exposition.write(&body_writer.writer, self.devices, self.host); - try body_writer.end(); - } - } -}; diff --git a/bin/zml-smi/smi/BUILD.bazel b/bin/zml-smi/smi/BUILD.bazel index fd3bf901..00614864 100644 --- a/bin/zml-smi/smi/BUILD.bazel +++ b/bin/zml-smi/smi/BUILD.bazel @@ -3,17 +3,21 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "smi", srcs = [ + "api.zig", "collector.zig", "counter.zig", + "csv.zig", "double_buffer.zig", "dynlib.zig", "fdinfo.zig", "host/linux.zig", "host/macos.zig", "hwmon.zig", + "json.zig", "model.zig", "pci.zig", "proc.zig", + "prometheus.zig", "remote.zig", "sandbox.zig", "sysfs.zig", diff --git a/bin/zml-smi/smi/api.zig b/bin/zml-smi/smi/api.zig new file mode 100644 index 00000000..16cf37e9 --- /dev/null +++ b/bin/zml-smi/smi/api.zig @@ -0,0 +1,65 @@ +const std = @import("std"); +const smi = @import("smi.zig"); + +const log = std.log.scoped(.@"zml-smi/api"); + +pub const Format = enum { + /// The API: what --json prints, which --remotes reads. + json, + prometheus, +}; + +/// Answers every request on `address` with the latest values, until canceled. +pub fn serve(gpa: std.mem.Allocator, io: std.Io, address: std.Io.net.IpAddress, collector: *smi.Collector, format: Format) void { + var server = address.listen(io, .{ .reuse_address = true }) catch |err| { + log.err("{t} server cannot listen on {f}: {t}", .{ format, address, err }); + return; + }; + defer server.deinit(io); + + var connections: std.Io.Group = .init; + defer connections.cancel(io); + while (true) { + const stream = server.accept(io) catch return; + connections.concurrent(io, answer, .{ gpa, io, stream, collector, format }) catch { + stream.close(io); + return; + }; + } +} + +fn answer(gpa: std.mem.Allocator, io: std.Io, stream: std.Io.net.Stream, collector: *smi.Collector, format: Format) void { + defer stream.close(io); + var read_buffer: [8192]u8 = undefined; + var write_buffer: [4096]u8 = undefined; + var reader = stream.reader(io, &read_buffer); + var writer = stream.writer(io, &write_buffer); + var http: std.http.Server = .init(&reader.interface, &writer.interface); + + // The body is written to memory first, so a slow client never holds a lock. + var body: std.Io.Writer.Allocating = .init(gpa); + defer body.deinit(); + + while (true) { + var request = http.receiveHead() catch return; + if (format == .prometheus and request.head.method != .GET) { + request.respond("Method Not Allowed\n", .{ .status = .method_not_allowed }) catch return; + continue; + } + + body.clearRetainingCapacity(); + const written = switch (format) { + .json => smi.json.write(&body.writer, collector), + .prometheus => smi.prometheus.write(&body.writer, collector), + }; + written catch return; + + request.respond(body.written(), .{ .extra_headers = switch (format) { + .json => &.{ + .{ .name = "content-type", .value = "application/json" }, + .{ .name = "access-control-allow-origin", .value = "*" }, + }, + .prometheus => &.{.{ .name = "content-type", .value = "text/plain" }}, + } }) catch return; + } +} diff --git a/bin/zml-smi/smi/csv.zig b/bin/zml-smi/smi/csv.zig new file mode 100644 index 00000000..f757362f --- /dev/null +++ b/bin/zml-smi/smi/csv.zig @@ -0,0 +1,66 @@ +const std = @import("std"); +const smi = @import("smi.zig"); + +/// One table per target: index, type, remote, the strings of the device, then the fields of its sample. +/// A device column that is empty for the whole table is left out. +pub fn write(writer: *std.Io.Writer, collector: *smi.Collector) !void { + inline for (@typeInfo(smi.Target).@"enum".field_names) |name| { + try writeTable(writer, collector, @field(smi.Target, name)); + } +} + +fn writeTable(writer: *std.Io.Writer, collector: *smi.Collector, comptime target: smi.Target) !void { + const fields = @typeInfo(@FieldType(smi.Sample, @tagName(target))).@"struct".field_names; + + var rows: usize = 0; + var names = false; + var drivers = false; + var key_buffer: [16][]const u8 = undefined; + var keys: std.ArrayList([]const u8) = .initBuffer(&key_buffer); + for (collector.devices.items) |device| { + if (device.target != target) continue; + rows += 1; + names = names or device.name != null; + drivers = drivers or device.driver_version != null; + for (device.details) |detail| { + if (!contains(keys.items, detail.key)) keys.appendBounded(detail.key) catch {}; + } + } + if (rows == 0) return; + + try writer.writeAll("index,type,remote"); + if (names) try writer.writeAll(",name"); + if (drivers) try writer.writeAll(",driver_version"); + for (keys.items) |key| try writer.print(",{s}", .{key}); + inline for (fields) |field| try writer.writeAll("," ++ field); + try writer.writeAll("\n"); + + for (collector.devices.items, 0..) |device, i| { + if (device.target != target) continue; + try writer.print("{d},{t},{any}", .{ i, target, device.remote }); + if (names) try writer.print(",{s}", .{device.name orelse ""}); + if (drivers) try writer.print(",{s}", .{device.driver_version orelse ""}); + for (keys.items) |key| try writer.print(",{s}", .{detailValue(device, key) orelse ""}); + + const sample = @field(collector.sample(@fromBackingInt(@intCast(i))), @tagName(target)); + inline for (fields) |field| { + try writer.writeAll(","); + if (@field(sample, field)) |value| try writer.print("{d}", .{value}); + } + try writer.writeAll("\n"); + } +} + +fn contains(keys: []const []const u8, key: []const u8) bool { + for (keys) |other| { + if (std.mem.eql(u8, other, key)) return true; + } + return false; +} + +fn detailValue(device: smi.Device, key: []const u8) ?[]const u8 { + for (device.details) |item| { + if (std.mem.eql(u8, item.key, key)) return item.value; + } + return null; +} diff --git a/bin/zml-smi/smi/json.zig b/bin/zml-smi/smi/json.zig new file mode 100644 index 00000000..43d53d72 --- /dev/null +++ b/bin/zml-smi/smi/json.zig @@ -0,0 +1,21 @@ +const std = @import("std"); +const smi = @import("smi.zig"); + +/// Every device with its latest sample, then the merged processes. +pub fn write(writer: *std.Io.Writer, collector: *smi.Collector) !void { + var json: std.json.Stringify = .{ .writer = writer }; + try json.beginObject(); + try json.objectField("devices"); + try json.beginArray(); + for (collector.devices.items, 0..) |device, i| { + try json.write(smi.Row{ .device = device, .sample = collector.sample(@fromBackingInt(@intCast(i))) }); + } + try json.endArray(); + + try json.objectField("processes"); + const processes = collector.processes.lock(collector.io); + defer collector.processes.unlock(collector.io); + try json.write(processes.rows.items); + try json.endObject(); + try writer.writeAll("\n"); +} diff --git a/bin/zml-smi/smi/prometheus.zig b/bin/zml-smi/smi/prometheus.zig new file mode 100644 index 00000000..6cff0cfd --- /dev/null +++ b/bin/zml-smi/smi/prometheus.zig @@ -0,0 +1,179 @@ +const std = @import("std"); +const smi = @import("smi.zig"); + +/// A metric family. `decimals` moves the decimal point and `factor` multiplies, so conversions are exact. +const Metric = struct { name: []const u8, help: []const u8, decimals: u8 = 0, factor: u64 = 1 }; + +/// One entry per field of `T`. A missing or misspelled field does not compile. +fn Metrics(comptime T: type) type { + return std.enums.EnumFieldStruct(std.meta.FieldEnum(T), ?Metric, null); +} + +const utilization: Metric = .{ .name = "zml_device_utilization_percent", .help = "Device utilization percentage" }; +const memory_used: Metric = .{ .name = "zml_device_memory_used_bytes", .help = "Device memory used in bytes" }; +const memory_total: Metric = .{ .name = "zml_device_memory_total_bytes", .help = "Device memory total in bytes" }; + +const gpu: Metrics(smi.Gpu) = .{ + .util_percent = utilization, + .encoder_util_percent = .{ .name = "zml_device_encoder_utilization_percent", .help = "Encoder utilization percentage" }, + .decoder_util_percent = .{ .name = "zml_device_decoder_utilization_percent", .help = "Decoder utilization percentage" }, + // Milliwatts, written as watts. + .power_mw = .{ .name = "zml_device_power_watts", .help = "Power draw in watts", .decimals = 3 }, + .power_limit_mw = .{ .name = "zml_device_power_limit_watts", .help = "Power limit in watts", .decimals = 3 }, + .temperature = .{ .name = "zml_device_temperature_celsius", .help = "Device temperature in celsius" }, + .fan_speed_percent = .{ .name = "zml_device_fan_speed_percent", .help = "Fan speed percentage" }, + .clock_graphics_mhz = .{ .name = "zml_device_clock_graphics_mhz", .help = "Graphics clock frequency in MHz" }, + .clock_sm_mhz = .{ .name = "zml_device_clock_sm_mhz", .help = "SM clock frequency in MHz" }, + .clock_soc_mhz = null, + .clock_mem_mhz = .{ .name = "zml_device_clock_memory_mhz", .help = "Memory clock frequency in MHz" }, + .clock_graphics_max_mhz = null, + .clock_mem_max_mhz = null, + .mem_used_bytes = memory_used, + .mem_total_bytes = memory_total, + .mem_bus_width = null, + // Kilobytes per second, written as bytes per second. + .pcie_tx_kbps = .{ .name = "zml_device_pcie_tx_bytes_per_second", .help = "PCIe TX throughput in bytes per second", .factor = 1000 }, + .pcie_rx_kbps = .{ .name = "zml_device_pcie_rx_bytes_per_second", .help = "PCIe RX throughput in bytes per second", .factor = 1000 }, + .pcie_bandwidth_mbps = null, + .pcie_link_gen = null, + .pcie_link_width = null, +}; + +const neuron: Metrics(smi.Neuron) = .{ + .util_percent = utilization, + .mem_used_bytes = memory_used, + .mem_total_bytes = memory_total, + .nc_tensors = .{ .name = "zml_device_neuron_tensors_bytes", .help = "Neuron core tensor memory in bytes" }, + .nc_constants = .{ .name = "zml_device_neuron_constants_bytes", .help = "Neuron core constants memory in bytes" }, + .nc_model_code = .{ .name = "zml_device_neuron_model_code_bytes", .help = "Neuron core model code memory in bytes" }, + .nc_shared_scratchpad = .{ .name = "zml_device_neuron_shared_scratchpad_bytes", .help = "Neuron core shared scratchpad memory in bytes" }, + .nc_nonshared_scratchpad = .{ .name = "zml_device_neuron_nonshared_scratchpad_bytes", .help = "Neuron core non-shared scratchpad memory in bytes" }, + .nc_runtime = .{ .name = "zml_device_neuron_runtime_bytes", .help = "Neuron core runtime memory in bytes" }, + .nc_driver = .{ .name = "zml_device_neuron_driver_bytes", .help = "Neuron core driver memory in bytes" }, + .nc_dma_rings = .{ .name = "zml_device_neuron_dma_rings_bytes", .help = "Neuron core DMA rings memory in bytes" }, + .nc_collectives = .{ .name = "zml_device_neuron_collectives_bytes", .help = "Neuron core collectives memory in bytes" }, + .nc_notifications = .{ .name = "zml_device_neuron_notifications_bytes", .help = "Neuron core notifications memory in bytes" }, + .nc_uncategorized = .{ .name = "zml_device_neuron_uncategorized_bytes", .help = "Neuron core uncategorized memory in bytes" }, +}; + +const tpu: Metrics(smi.Tpu) = .{ + .util_percent = utilization, + .mem_used_bytes = memory_used, + .mem_total_bytes = memory_total, +}; + +const host: Metrics(smi.HostData) = .{ + .hostname = null, + .kernel = null, + .cpu_name = null, + .cpu_cores = .{ .name = "zml_host_cpu_cores", .help = "Number of CPU cores" }, + .mem_total_kib = .{ .name = "zml_host_memory_total_bytes", .help = "Total host memory in bytes", .factor = 1024 }, + .mem_available_kib = .{ .name = "zml_host_memory_available_bytes", .help = "Available host memory in bytes", .factor = 1024 }, + .load_1 = .{ .name = "zml_host_load_1", .help = "1-minute load average", .decimals = 2 }, + .load_5 = .{ .name = "zml_host_load_5", .help = "5-minute load average", .decimals = 2 }, + .load_15 = .{ .name = "zml_host_load_15", .help = "15-minute load average", .decimals = 2 }, + .uptime_seconds = .{ .name = "zml_host_uptime_seconds", .help = "System uptime in seconds" }, +}; + +fn metricsOf(comptime T: type) Metrics(T) { + return switch (T) { + smi.Gpu => gpu, + smi.Neuron => neuron, + smi.Tpu => tpu, + else => @compileError("no metrics for " ++ @typeName(T)), + }; +} + +/// The distinct families of every struct, in the order the structs give them. +/// TPU first keeps the order of the families: utilization and memory, then GPU, then Neuron. +const families: []const Metric = families: { + @setEvalBranchQuota(10_000); + var list: []const Metric = &.{}; + for (.{ smi.Tpu, smi.Gpu, smi.Neuron }) |T| { + const metrics = metricsOf(T); + next: for (@typeInfo(T).@"struct".field_names) |field| { + const metric = @field(metrics, field) orelse continue; + for (list) |seen| { + if (!std.mem.eql(u8, seen.name, metric.name)) continue; + if (!std.mem.eql(u8, seen.help, metric.help) or seen.decimals != metric.decimals or seen.factor != metric.factor) { + @compileError(metric.name ++ " has two definitions"); + } + continue :next; + } + list = list ++ .{metric}; + } + } + break :families list; +}; + +/// The field of `T` written as `family`, or null. +fn fieldOf(comptime T: type, comptime family: Metric) ?[]const u8 { + @setEvalBranchQuota(10_000); + const metrics = metricsOf(T); + for (@typeInfo(T).@"struct".field_names) |field| { + const metric = @field(metrics, field) orelse continue; + if (std.mem.eql(u8, metric.name, family.name)) return field; + } + return null; +} + +pub fn write(writer: *std.Io.Writer, collector: *smi.Collector) !void { + const devices = collector.devices.items; + + try writer.writeAll("# HELP zml_device_info Device metadata\n# TYPE zml_device_info gauge\n"); + for (devices, 0..) |device, i| { + try writer.print("zml_device_info{{device=\"{d}\",type=\"{t}\"", .{ i, device.target }); + if (device.name) |name| try label(writer, "name", name); + if (device.driver_version) |driver| try label(writer, "driver", driver); + for (device.details) |detail| try label(writer, detail.key, detail.value); + try writer.writeAll("} 1\n"); + } + try writer.writeAll("\n"); + + inline for (families) |family| { + var written = false; + for (devices, 0..) |device, i| { + const value: u64 = switch (collector.sample(@fromBackingInt(@intCast(i)))) { + inline else => |fields| if (comptime fieldOf(@TypeOf(fields), family)) |field| @field(fields, field) else null, + } orelse continue; + if (!written) try header(writer, family); + written = true; + try writer.print("{s}{{device=\"{d}\",type=\"{t}\",name=\"", .{ family.name, i, device.target }); + try std.zig.stringEscape(device.name orelse "", writer); + try writer.writeAll("\"} "); + try number(writer, family, value); + try writer.writeAll("\n"); + } + if (written) try writer.writeAll("\n"); + } + + const data = collector.host.read(collector.io); + inline for (@typeInfo(smi.HostData).@"struct".field_names) |field| { + if (comptime @field(host, field)) |family| { + if (@field(data, field)) |value| { + try header(writer, family); + try writer.print("{s} ", .{family.name}); + try number(writer, family, value); + try writer.writeAll("\n\n"); + } + } + } +} + +fn header(writer: *std.Io.Writer, comptime family: Metric) !void { + try writer.writeAll("# HELP " ++ family.name ++ " " ++ family.help ++ "\n# TYPE " ++ family.name ++ " gauge\n"); +} + +fn label(writer: *std.Io.Writer, name: []const u8, value: []const u8) !void { + try writer.print(",{s}=\"", .{name}); + try std.zig.stringEscape(value, writer); + try writer.writeAll("\""); +} + +fn number(writer: *std.Io.Writer, comptime family: Metric, value: anytype) !void { + if (@TypeOf(value) == f32) return writer.print(std.fmt.comptimePrint("{{d:.{d}}}", .{family.decimals}), .{value}); + const scaled = value *| family.factor; + if (family.decimals == 0) return writer.print("{d}", .{scaled}); + const one = comptime std.math.pow(u64, 10, family.decimals); + try writer.print(std.fmt.comptimePrint("{{d}}.{{d:0>{d}}}", .{family.decimals}), .{ scaled / one, scaled % one }); +} diff --git a/bin/zml-smi/smi/smi.zig b/bin/zml-smi/smi/smi.zig index 003d5279..d800803a 100644 --- a/bin/zml-smi/smi/smi.zig +++ b/bin/zml-smi/smi/smi.zig @@ -35,6 +35,11 @@ pub const Counters = counter.Counters; pub const Growth = counter.Growth; pub const DoubleBuffer = @import("double_buffer.zig").DoubleBuffer; +pub const csv = @import("csv.zig"); +pub const json = @import("json.zig"); +pub const prometheus = @import("prometheus.zig"); +pub const api = @import("api.zig"); + test { _ = @import("collector.zig"); _ = counter; From aa123c11f5e077c575d2b7c28346dc1e957c801a Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 20:16:22 +0100 Subject: [PATCH 08/19] bin/zml-smi/tui: read devices and samples from the collector The card, the history and the detail page switch on the sample, with one prong for the GPU targets, so a new target that is not added to them does not compile. Titles show the driver version and the details, the card only those marked for its title. Logos are kept by target. The process table copies the merged rows, and their strings, into its frame while it holds their lock. --- bin/zml-smi/tui/BUILD.bazel | 3 +- bin/zml-smi/tui/data.zig | 74 +++++------ bin/zml-smi/tui/image_cache.zig | 42 +++--- bin/zml-smi/tui/lib/utils.zig | 15 +++ bin/zml-smi/tui/pages/detail.zig | 10 +- .../tui/pages/detail/detail_common.zig | 6 +- bin/zml-smi/tui/pages/detail/gpu_detail.zig | 18 +-- .../tui/pages/detail/neuron_detail.zig | 10 +- bin/zml-smi/tui/pages/detail/tpu_detail.zig | 6 +- bin/zml-smi/tui/pages/overview.zig | 4 +- bin/zml-smi/tui/print.zig | 2 +- bin/zml-smi/tui/top.zig | 4 +- bin/zml-smi/tui/widgets/device_card.zig | 120 ++++++++---------- bin/zml-smi/tui/widgets/info_lines.zig | 2 +- bin/zml-smi/tui/widgets/process_table.zig | 15 ++- 15 files changed, 162 insertions(+), 169 deletions(-) diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index e4a41e3c..8d16df45 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -43,8 +43,7 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//bazel/stamp:zig", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/smi", "//zml:logo", "@libvaxis//:vaxis", ], diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index 553b1023..c90838b5 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -1,17 +1,5 @@ const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const di = smi_info.device_info; -pub const DeviceInfo = di.DeviceInfo; -pub const GpuInfo = di.GpuInfo; -pub const NeuronInfo = di.NeuronInfo; -pub const TpuInfo = di.TpuInfo; -pub const Target = di.Target; -pub const Targets = smi_info.Targets; -const hi = smi_info.host_info; -pub const HostInfo = hi.HostInfo; -pub const HostData = hi.HostData; -pub const pi = smi_info.process_info; -pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const smi = @import("zml-smi"); /// The samples of one metric, oldest first. pub const History = std.ArrayList(u64); @@ -47,25 +35,19 @@ pub const HistoryBuffers = struct { }; pub const SystemState = struct { - devices: []*DeviceInfo, - host: *HostInfo, + collector: *smi.Collector, history: HistoryBuffers = .{}, /// Charts plot one sample per braille dot, two per cell: the history /// keeps enough of them to fill the widest terminal seen. history_len: usize = 0, - targets: Targets, - processes: *const ProcessDoubleBuffer, gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, tui_refresh_rate: u16, pub const Config = struct { - devices: []*DeviceInfo, - host: *HostInfo, - targets: Targets, + collector: *smi.Collector, tui_refresh_rate: u16, - processes: *const ProcessDoubleBuffer, gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, @@ -73,12 +55,9 @@ pub const SystemState = struct { pub fn init(cfg: Config) !SystemState { return .{ - .devices = cfg.devices, - .host = cfg.host, - .history = try HistoryBuffers.init(cfg.arena, cfg.devices.len), - .targets = cfg.targets, + .collector = cfg.collector, + .history = try HistoryBuffers.init(cfg.arena, cfg.collector.devices.items.len), .tui_refresh_rate = cfg.tui_refresh_rate, - .processes = cfg.processes, .gpa = cfg.gpa, .arena = cfg.arena, .io = cfg.io, @@ -90,27 +69,36 @@ pub const SystemState = struct { } pub fn deviceCount(self: *const SystemState) usize { - return self.devices.len; + return self.collector.devices.items.len; + } + + pub fn device(self: *const SystemState, id: u16) smi.Device { + return self.collector.devices.items[id]; + } + + pub fn sample(self: *const SystemState, id: u16) smi.Sample { + return self.collector.sample(@fromBackingInt(id)); + } + + pub fn host(self: *const SystemState) smi.HostData { + return self.collector.host.read(self.io); } pub fn recordHistory(self: *SystemState) void { for (0..self.deviceCount()) |i| { - switch (self.devices[i].*) { - .cuda, .rocm, .oneapi => |*sv| { - const gpu = sv.front().*; - self.record(&self.history.util[i], gpu.util_percent orelse 0); - const used = gpu.mem_used_bytes orelse 0; - const total = gpu.mem_total_bytes orelse 0; - self.record(&self.history.mem_util[i], if (total > 0) used * 100 / total else 0); + switch (self.sample(@intCast(i))) { + .cuda, .rocm, .oneapi => |gpu| { + self.recordUsage(i, gpu.util_percent, gpu.mem_used_bytes, gpu.mem_total_bytes); self.record(&self.history.temp[i], gpu.temperature orelse 0); self.record(&self.history.power[i], gpu.power_mw orelse 0); }, - inline else => |*sv| { - const info = sv.front().*; - self.record(&self.history.util[i], info.util_percent orelse 0); - const used = info.mem_used_bytes orelse 0; - const total = info.mem_total_bytes orelse 0; - self.record(&self.history.mem_util[i], if (total > 0) used * 100 / total else 0); + .neuron => |core| { + self.recordUsage(i, core.util_percent, core.mem_used_bytes, core.mem_total_bytes); + self.record(&self.history.temp[i], 0); + self.record(&self.history.power[i], 0); + }, + .tpu => |chip| { + self.recordUsage(i, chip.util_percent, chip.mem_used_bytes, chip.mem_total_bytes); self.record(&self.history.temp[i], 0); self.record(&self.history.power[i], 0); }, @@ -118,6 +106,12 @@ pub const SystemState = struct { } } + fn recordUsage(self: *SystemState, i: usize, util: ?u64, used: ?u64, total: ?u64) void { + self.record(&self.history.util[i], util orelse 0); + const total_bytes = total orelse 0; + self.record(&self.history.mem_util[i], if (total_bytes > 0) (used orelse 0) * 100 / total_bytes else 0); + } + fn record(self: *SystemState, history: *History, value: u64) void { if (history.items.len > 0 and history.items.len >= self.history_len) { _ = history.orderedRemove(0); diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig index 1aea4778..f04be14d 100644 --- a/bin/zml-smi/tui/image_cache.zig +++ b/bin/zml-smi/tui/image_cache.zig @@ -1,37 +1,35 @@ const std = @import("std"); const vaxis = @import("vaxis"); +const smi = @import("zml-smi"); const ImageCache = @This(); pub var global: ImageCache = .{}; -map: std.StringHashMapUnmanaged(vaxis.Image) = .empty, +/// A target without a logo shows none. +const logos: std.EnumMap(smi.Target, []const u8) = .init(.{ + .cuda = @embedFile("assets/nvidia.png"), + .rocm = @embedFile("assets/amd.png"), + .oneapi = @embedFile("assets/intel.png"), + .neuron = @embedFile("assets/neuron.png"), + .tpu = @embedFile("assets/tpu.png"), +}); -pub fn deinit(self: *ImageCache, allocator: std.mem.Allocator) void { - self.map.deinit(allocator); -} +zml: ?vaxis.Image = null, +targets: std.EnumMap(smi.Target, vaxis.Image) = .{}, -pub fn load( - self: *ImageCache, - vx: *vaxis.Vaxis, - allocator: std.mem.Allocator, - writer: *std.Io.Writer, - key: []const u8, - data: []const u8, -) void { - const image = vx.loadImage(allocator, writer, .{ .mem = data }) catch return; - self.map.put(allocator, key, image) catch return; +pub fn deinit(self: *ImageCache) void { + self.* = .{}; } pub fn loadAll(self: *ImageCache, vx: *vaxis.Vaxis, allocator: std.mem.Allocator, writer: *std.Io.Writer) void { - self.load(vx, allocator, writer, "logo", @embedFile("assets/logo.png")); - self.load(vx, allocator, writer, "gpu_cuda", @embedFile("assets/nvidia.png")); - self.load(vx, allocator, writer, "gpu_rocm", @embedFile("assets/amd.png")); - self.load(vx, allocator, writer, "gpu_oneapi", @embedFile("assets/intel.png")); - self.load(vx, allocator, writer, "gpu_neuron", @embedFile("assets/neuron.png")); - self.load(vx, allocator, writer, "gpu_tpu", @embedFile("assets/tpu.png")); + self.zml = vx.loadImage(allocator, writer, .{ .mem = @embedFile("assets/logo.png") }) catch null; + for (std.enums.values(smi.Target)) |target| { + const png = logos.get(target) orelse continue; + self.targets.put(target, vx.loadImage(allocator, writer, .{ .mem = png }) catch continue); + } } -pub fn get(self: *const ImageCache, key: []const u8) ?vaxis.Image { - return self.map.get(key); +pub fn get(self: *const ImageCache, target: smi.Target) ?vaxis.Image { + return self.targets.get(target); } diff --git a/bin/zml-smi/tui/lib/utils.zig b/bin/zml-smi/tui/lib/utils.zig index 993703c9..4f9a05c8 100644 --- a/bin/zml-smi/tui/lib/utils.zig +++ b/bin/zml-smi/tui/lib/utils.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const smi = @import("zml-smi"); pub fn bytesToMb(val: ?u64) u64 { return if (val) |v| v / (1024 * 1024) else 0; @@ -28,3 +29,17 @@ pub fn normalizeRange(arena: std.mem.Allocator, raw: []const u64, min_val: u64, } return result; } + +/// " (driver, CUDA 13.4)": the driver version and the details, all of them or those shown in titles. +pub fn versions(arena: std.mem.Allocator, device: smi.Device, comptime details: enum { in_title, all }) std.mem.Allocator.Error![]const u8 { + var text: std.Io.Writer.Allocating = .init(arena); + const writer = &text.writer; + if (device.driver_version) |driver| writer.writeAll(driver) catch return error.OutOfMemory; + for (device.details) |detail| { + if (details == .in_title and !detail.in_title) continue; + if (text.written().len > 0) writer.writeAll(", ") catch return error.OutOfMemory; + writer.print("{s} {s}", .{ detail.label, detail.value }) catch return error.OutOfMemory; + } + if (text.written().len == 0) return ""; + return std.fmt.allocPrint(arena, " ({s})", .{text.written()}); +} diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig index 86d4b7a3..f5d50c67 100644 --- a/bin/zml-smi/tui/pages/detail.zig +++ b/bin/zml-smi/tui/pages/detail.zig @@ -17,12 +17,12 @@ process_table: *ProcessTable = undefined, pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const w = ctx.max.width orelse 80; const id = self.device_id; - const dev = self.state.devices[id]; + const device = self.state.device(id); const wgt = ui.widget(self); - return switch (dev.*) { - .cuda, .rocm, .oneapi => |*sv| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), - .neuron => |*sv| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), - .tpu => |*sv| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + return switch (self.state.sample(id)) { + .cuda, .rocm, .oneapi => |gpu| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, device, gpu, wgt), + .neuron => |core| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, device, core, wgt), + .tpu => |chip| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, device, chip, wgt), }; } diff --git a/bin/zml-smi/tui/pages/detail/detail_common.zig b/bin/zml-smi/tui/pages/detail/detail_common.zig index 8720c6fc..52a68fda 100644 --- a/bin/zml-smi/tui/pages/detail/detail_common.zig +++ b/bin/zml-smi/tui/pages/detail/detail_common.zig @@ -1,6 +1,7 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; +const smi = @import("zml-smi"); const data = @import("../../data.zig"); const theme = @import("../../theme.zig"); const utils = @import("../../lib/utils.zig"); @@ -9,9 +10,10 @@ const compose = @import("../../lib/compose.zig"); const Chart = @import("../../widgets/chart.zig"); const ColumnLayout = @import("../../widgets/column_layout.zig"); -pub fn headerText(arena: std.mem.Allocator, id: u16, name: []const u8) std.mem.Allocator.Error!vxfw.Text { +/// "Device 0: NVIDIA GeForce RTX 5090 (615.71.09, CUDA 13.4)", with every detail. +pub fn headerText(arena: std.mem.Allocator, id: u16, device: smi.Device) std.mem.Allocator.Error!vxfw.Text { return .{ - .text = try std.fmt.allocPrint(arena, "Device {d}: {s}", .{ id, name }), + .text = try std.fmt.allocPrint(arena, "Device {d}: {s}{s}", .{ id, device.name orelse "Unknown", try utils.versions(arena, device, .all) }), .style = theme.header_style, .softwrap = false, .overflow = .clip, diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig index ef889acd..7cd21e42 100644 --- a/bin/zml-smi/tui/pages/detail/gpu_detail.zig +++ b/bin/zml-smi/tui/pages/detail/gpu_detail.zig @@ -1,6 +1,7 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; +const smi = @import("zml-smi"); const data = @import("../../data.zig"); const utils = @import("../../lib/utils.zig"); const ui = @import("../../lib/ui.zig"); @@ -16,23 +17,12 @@ pub fn draw( ctx: vxfw.DrawContext, w: u16, id: u16, - gpu: data.GpuInfo, + device: smi.Device, + gpu: smi.Gpu, parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { // ── Header ────────────────────────────────────────────── - const name_suffix: []const u8 = blk: { - const drv = gpu.driver_version orelse ""; - const cuda = gpu.cuda_driver_version orelse ""; - break :blk if (drv.len > 0 and cuda.len > 0) - try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) - else if (drv.len > 0) - try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) - else if (cuda.len > 0) - try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) - else - @as([]const u8, ""); - }; - const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ gpu.name orelse "Unknown", name_suffix })); + const header = try common.headerText(ctx.arena, id, device); // ── GPU Utilization + Memory charts ───────────────────── const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ diff --git a/bin/zml-smi/tui/pages/detail/neuron_detail.zig b/bin/zml-smi/tui/pages/detail/neuron_detail.zig index e0dc3bcd..594d5fff 100644 --- a/bin/zml-smi/tui/pages/detail/neuron_detail.zig +++ b/bin/zml-smi/tui/pages/detail/neuron_detail.zig @@ -1,6 +1,7 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; +const smi = @import("zml-smi"); const data = @import("../../data.zig"); const utils = @import("../../lib/utils.zig"); const ui = @import("../../lib/ui.zig"); @@ -15,15 +16,12 @@ pub fn draw( ctx: vxfw.DrawContext, w: u16, id: u16, - nc: data.NeuronInfo, + device: smi.Device, + nc: smi.Neuron, parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { // ── Header ────────────────────────────────────────────── - const name_suffix: []const u8 = if (nc.driver_version) |v| - try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) - else - ""; - const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ nc.name orelse "Unknown", name_suffix })); + const header = try common.headerText(ctx.arena, id, device); // ── Utilization + Memory charts ───────────────────────── const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ diff --git a/bin/zml-smi/tui/pages/detail/tpu_detail.zig b/bin/zml-smi/tui/pages/detail/tpu_detail.zig index ca6d3b66..953a894e 100644 --- a/bin/zml-smi/tui/pages/detail/tpu_detail.zig +++ b/bin/zml-smi/tui/pages/detail/tpu_detail.zig @@ -1,6 +1,7 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; +const smi = @import("zml-smi"); const data = @import("../../data.zig"); const utils = @import("../../lib/utils.zig"); const ui = @import("../../lib/ui.zig"); @@ -14,11 +15,12 @@ pub fn draw( ctx: vxfw.DrawContext, w: u16, id: u16, - tp: data.TpuInfo, + device: smi.Device, + tp: smi.Tpu, parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { // ── Header ────────────────────────────────────────────── - const header = try common.headerText(ctx.arena, id, tp.name orelse "Unknown"); + const header = try common.headerText(ctx.arena, id, device); // ── Utilization + Memory charts ───────────────────────── const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ diff --git a/bin/zml-smi/tui/pages/overview.zig b/bin/zml-smi/tui/pages/overview.zig index e91253bd..2407f3d6 100644 --- a/bin/zml-smi/tui/pages/overview.zig +++ b/bin/zml-smi/tui/pages/overview.zig @@ -48,7 +48,7 @@ pub fn deinit(self: *Overview, allocator: std.mem.Allocator) void { } fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { - const logo: Logo = .{ .image = image_cache.global.get("logo") }; + const logo: Logo = .{ .image = image_cache.global.zml }; const info_lines: InfoLines = .{ .state = self.state }; const children = [2]vxfw.Widget{ @@ -60,7 +60,7 @@ fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16 } fn drawWideBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { - const logo: Logo = .{ .image = image_cache.global.get("logo") }; + const logo: Logo = .{ .image = image_cache.global.zml }; const info_lines: InfoLines = .{ .state = self.state }; const logo_box_w = Logo.logo_width + 6; // +4 centering + 2 absorbed page margin diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig index 238534cd..d1e33507 100644 --- a/bin/zml-smi/tui/print.zig +++ b/bin/zml-smi/tui/print.zig @@ -48,7 +48,7 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Envir } image_cache.global.loadAll(&vx, allocator, tty.writer()); - defer image_cache.global.deinit(allocator); + defer image_cache.global.deinit(); vxfw.DrawContext.init(vx.screen.width_method); diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig index 96438028..9fe0e336 100644 --- a/bin/zml-smi/tui/top.zig +++ b/bin/zml-smi/tui/top.zig @@ -51,7 +51,7 @@ const Model = struct { fn deinit(self: *Model) void { self.process_table.deinit(self.allocator); self.overview.deinit(self.allocator); - image_cache.global.deinit(self.allocator); + image_cache.global.deinit(); } pub fn handleEvent(self: *Model, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { @@ -119,7 +119,7 @@ const Model = struct { self.prev_viewing_device = self.viewing_device; } - self.process_table.prepare(self.state, self.viewing_device); + self.process_table.prepare(self.state, self.viewing_device, ctx.arena); var content_surf: vxfw.Surface = undefined; diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig index 2c8c3894..ffd81565 100644 --- a/bin/zml-smi/tui/widgets/device_card.zig +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -1,6 +1,7 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; +const smi = @import("zml-smi"); const data = @import("../data.zig"); const theme = @import("../theme.zig"); const utils = @import("../lib/utils.zig"); @@ -39,67 +40,54 @@ pub fn handleEvent(self: *DeviceCard, ctx: *vxfw.EventContext, event: vxfw.Event } } +/// What a card shows of a device: utilization and memory, with a suffix after utilization. +const Usage = struct { + label: []const u8, + util_label: []const u8, + util_percent: ?u64, + mem_used: ?u64, + mem_total: ?u64, + suffix: []const u8 = "", +}; + pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const i = self.device_id; - const dev = self.state.devices[i]; - const target: data.Target = std.meta.activeTag(dev.*); - const cf = switch (dev.*) { - inline else => |*sv| blk: { - const f = sv.front().*; - break :blk .{ - .name = f.name orelse "Unknown", - .util_percent = @as(u8, @intCast(@min(f.util_percent orelse 0, 100))), - .mem_used = f.mem_used_bytes orelse 0, - .mem_total = f.mem_total_bytes orelse 0, - }; + const device = self.state.device(i); + const usage: Usage = switch (self.state.sample(i)) { + .cuda, .rocm, .oneapi => |gpu| .{ + .label = "GPU", + .util_label = "GPU", + .util_percent = gpu.util_percent, + .mem_used = gpu.mem_used_bytes, + .mem_total = gpu.mem_total_bytes, + .suffix = try gpuSuffix(ctx.arena, gpu), + }, + .neuron => |core| .{ + .label = "NC", + .util_label = "Core", + .util_percent = core.util_percent, + .mem_used = core.mem_used_bytes, + .mem_total = core.mem_total_bytes, + }, + .tpu => |chip| .{ + .label = "TPU", + .util_label = "Duty", + .util_percent = chip.util_percent, + .mem_used = chip.mem_used_bytes, + .mem_total = chip.mem_total_bytes, }, }; - const mem_pct: u8 = if (cf.mem_total > 0) @intCast(@min(cf.mem_used * 100 / cf.mem_total, 100)) else 0; + const util_percent: u8 = @intCast(@min(usage.util_percent orelse 0, 100)); + const mem_used = usage.mem_used orelse 0; + const mem_total = usage.mem_total orelse 0; + const mem_pct: u8 = if (mem_total > 0) @intCast(@min(mem_used * 100 / mem_total, 100)) else 0; - const device_label, const util_label = switch (target) { - .cuda, .rocm, .oneapi => .{ "GPU", "GPU" }, - .neuron => .{ "NC", "Core" }, - .tpu => .{ "TPU", "Duty" }, - }; - const driver_ver: []const u8 = switch (dev.*) { - .cuda => |*sv| blk: { - const gpu = sv.front(); - const drv = gpu.driver_version orelse ""; - const cuda = gpu.cuda_driver_version orelse ""; - break :blk if (drv.len > 0 and cuda.len > 0) - try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) - else if (drv.len > 0) - try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) - else if (cuda.len > 0) - try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) - else - @as([]const u8, ""); - }, - .rocm, .oneapi => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", - .neuron => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", - else => "", - }; - const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}{s}", .{ device_label, i, cf.name, driver_ver }); + const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}{s}", .{ + usage.label, i, device.name orelse "Unknown", try utils.versions(ctx.arena, device, .in_title), + }); - const dev_suffix = switch (dev.*) { - .cuda, .rocm, .oneapi => |*sv| blk: { - const gpu = sv.front().*; - const power = (gpu.power_mw orelse 0) / 1000; - const power_limit = (gpu.power_limit_mw orelse 0) / 1000; - - const power_limit_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power_limit}); - const power_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power}); - - const pad = try utils.repeatStr(ctx.arena, " ", @intCast(power_limit_str.len -| power_str.len)); - break :blk try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C | {s}{s} W", .{ - gpu.temperature orelse 0, pad, power_str, - }); - }, - else => "", - }; - - const mem_used_mb = utils.bytesToMb(cf.mem_used); - const mem_total_mb = utils.bytesToMb(cf.mem_total); + const mem_used_mb = utils.bytesToMb(mem_used); + const mem_total_mb = utils.bytesToMb(mem_total); const mem_total_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_total_mb}); const mem_used_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_used_mb}); @@ -110,14 +98,8 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx var card: MetricCard = .{ .title = card_title, - .title_image = image_cache.global.get(switch (dev.*) { - .cuda => "gpu_cuda", - .rocm => "gpu_rocm", - .oneapi => "gpu_oneapi", - .neuron => "gpu_neuron", - .tpu => "gpu_tpu", - }), - .title_right = if (self.state.devices[i].isRemote()) "remote" else null, + .title_image = image_cache.global.get(device.target), + .title_right = if (device.remote) "remote" else null, .cell_size = ctx.cell_size, .highlighted = self.highlighted, }; @@ -126,11 +108,11 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx const mem_chart_data = try utils.normalizeRange(ctx.arena, self.state.history.mem_util[i].items, 0, 100); const chart_entries = [2]MetricCard.ChartEntry{ - .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix, .data = util_chart_data }, + .{ .label = usage.util_label, .value = util_percent, .suffix = usage.suffix, .data = util_chart_data }, .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix, .data = mem_chart_data }, }; const gauge_entries = [2]MetricCard.GaugeEntry{ - .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix }, + .{ .label = usage.util_label, .value = util_percent, .suffix = usage.suffix }, .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix }, }; @@ -144,3 +126,11 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx card_surf.widget = ui.widget(self); return card_surf; } + +/// "41°C | 10 W", the power padded to the width of the limit. +fn gpuSuffix(arena: std.mem.Allocator, gpu: smi.Gpu) std.mem.Allocator.Error![]const u8 { + const power_str = try std.fmt.allocPrint(arena, "{d}", .{(gpu.power_mw orelse 0) / 1000}); + const power_limit_str = try std.fmt.allocPrint(arena, "{d}", .{(gpu.power_limit_mw orelse 0) / 1000}); + const pad = try utils.repeatStr(arena, " ", @intCast(power_limit_str.len -| power_str.len)); + return std.fmt.allocPrint(arena, "{d}\u{00b0}C | {s}{s} W", .{ gpu.temperature orelse 0, pad, power_str }); +} diff --git a/bin/zml-smi/tui/widgets/info_lines.zig b/bin/zml-smi/tui/widgets/info_lines.zig index fcf1ae0d..00ba8f5a 100644 --- a/bin/zml-smi/tui/widgets/info_lines.zig +++ b/bin/zml-smi/tui/widgets/info_lines.zig @@ -34,7 +34,7 @@ const Entry = struct { pub fn draw(self: *const InfoLines, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { const w = ctx.max.width orelse 80; - const host = self.state.host.front().*; + const host = self.state.host(); const hostname_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.hostname orelse "Unknown"}); const kernel_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.kernel orelse "Unknown"}); diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig index af740f3d..9ac7e09a 100644 --- a/bin/zml-smi/tui/widgets/process_table.zig +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -7,8 +7,7 @@ const ui = @import("../lib/ui.zig"); const compose = @import("../lib/compose.zig"); const TitledBorder = @import("titled_border.zig"); const ColumnLayout = @import("column_layout.zig"); -const pi = @import("zml-smi/info").process_info; -const ProcessInfo = pi.ProcessInfo; +const ProcessInfo = @import("zml-smi").ProcessInfo; const RichText = vxfw.RichText; const Segment = vaxis.Cell.Segment; @@ -30,17 +29,23 @@ pub fn deinit(self: *ProcessTable, allocator: std.mem.Allocator) void { self.processes.deinit(allocator); } -pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16) void { +/// Copies the rows to draw, and their strings into `frame`, the arena of the frame. +pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16, frame: std.mem.Allocator) void { self.scroll_bars.scroll_view.children.builder.userdata = self; self.processes.clearRetainingCapacity(); - for (state.processes.front().items) |entry| { + const merged = state.collector.processes.lock(state.io); + defer state.collector.processes.unlock(state.io); + for (merged.rows.items) |entry| { if (device_id) |did| { if (entry.device_idx != did) { continue; } } - self.processes.append(state.gpa, entry) catch break; + var row = entry; + row.username = frame.dupe(u8, entry.username) catch ""; + row.comm = frame.dupe(u8, entry.comm) catch ""; + self.processes.append(state.gpa, row) catch break; } // Sort by dev_mem descending From 959592ca2556f6a93456161359016e03096ad980 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 20:16:22 +0100 Subject: [PATCH 09/19] bin/zml-smi: port NVML One file, Nvml.zig, replaces nvml.zig, metrics.zig and process.zig: init finds the GPUs, poll assigns each field from the call that reads it, and processes reads the running processes and their utilization. Functions that return a status are Checked, with two errors instead of thirty, and size queries go through the raw status. PCIe throughput is still read every fourth poll. pcie_bandwidth_mbps is now the lane speed times the link width, as on every platform. --- bin/zml-smi/platforms/nvml/BUILD.bazel | 22 ++- bin/zml-smi/platforms/nvml/Nvml.zig | 216 +++++++++++++++++++++++++ 2 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 bin/zml-smi/platforms/nvml/Nvml.zig diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel index 0ab0fae1..54f68d15 100644 --- a/bin/zml-smi/platforms/nvml/BUILD.bazel +++ b/bin/zml-smi/platforms/nvml/BUILD.bazel @@ -1,22 +1,18 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "nvml", - srcs = [ - "nvml.zig", - "process.zig", - ], import_name = "zml-smi/platforms/nvml", - main = "metrics.zig", + main = "Nvml.zig", target_compatible_with = ["@platforms//os:linux"], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:dynlib", - ] + select({ + visibility = ["//bin/zml-smi:__subpackages__"], + deps = ["//bin/zml-smi/smi"] + select({ "@llvm//platforms/config:linux_x86_64": ["@cuda_nvml_dev_linux_x86_64//:nvml"], "@llvm//platforms/config:linux_aarch64": ["@cuda_nvml_dev_linux_sbsa//:nvml"], }), ) + +zig_test( + name = "test", + deps = [":nvml"], +) diff --git a/bin/zml-smi/platforms/nvml/Nvml.zig b/bin/zml-smi/platforms/nvml/Nvml.zig new file mode 100644 index 00000000..bc971f70 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/Nvml.zig @@ -0,0 +1,216 @@ +//! NVIDIA GPUs, through libnvidia-ml. +const std = @import("std"); +const c = @import("c"); +const smi = @import("zml-smi"); + +const Nvml = @This(); +const log = std.log.scoped(.@"zml-smi/nvml"); + +platform: smi.Platform, +api: Api, +gpus: []Gpu, +tick: u8 = 0, + +const Gpu = struct { + handle: c.nvmlDevice_t, + /// PCIe throughput is read every fourth poll and kept in between. + pcie_tx_kbps: ?u64 = null, + pcie_rx_kbps: ?u64 = null, + /// The newest process utilization the GPU reported, so that a poll reads only newer ones. + last_seen: u64 = 0, +}; + +const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes }; + +const Api = struct { + nvmlInit_v2: Checked(c.nvmlInit_v2), + nvmlErrorString: *const @TypeOf(c.nvmlErrorString), + nvmlSystemGetDriverVersion: Checked(c.nvmlSystemGetDriverVersion), + nvmlSystemGetCudaDriverVersion_v2: Checked(c.nvmlSystemGetCudaDriverVersion_v2), + nvmlDeviceGetCount_v2: Checked(c.nvmlDeviceGetCount_v2), + nvmlDeviceGetHandleByIndex_v2: Checked(c.nvmlDeviceGetHandleByIndex_v2), + nvmlDeviceGetName: Checked(c.nvmlDeviceGetName), + nvmlDeviceGetPowerUsage: Checked(c.nvmlDeviceGetPowerUsage), + nvmlDeviceGetEnforcedPowerLimit: Checked(c.nvmlDeviceGetEnforcedPowerLimit), + nvmlDeviceGetTemperature: Checked(c.nvmlDeviceGetTemperature), + nvmlDeviceGetFanSpeed: Checked(c.nvmlDeviceGetFanSpeed), + nvmlDeviceGetUtilizationRates: Checked(c.nvmlDeviceGetUtilizationRates), + nvmlDeviceGetEncoderUtilization: Checked(c.nvmlDeviceGetEncoderUtilization), + nvmlDeviceGetDecoderUtilization: Checked(c.nvmlDeviceGetDecoderUtilization), + nvmlDeviceGetClockInfo: Checked(c.nvmlDeviceGetClockInfo), + nvmlDeviceGetMaxClockInfo: Checked(c.nvmlDeviceGetMaxClockInfo), + nvmlDeviceGetMemoryInfo: Checked(c.nvmlDeviceGetMemoryInfo), + nvmlDeviceGetMemoryBusWidth: Checked(c.nvmlDeviceGetMemoryBusWidth), + nvmlDeviceGetPcieThroughput: Checked(c.nvmlDeviceGetPcieThroughput), + nvmlDeviceGetPcieSpeed: Checked(c.nvmlDeviceGetPcieSpeed), + nvmlDeviceGetCurrPcieLinkGeneration: Checked(c.nvmlDeviceGetCurrPcieLinkGeneration), + nvmlDeviceGetCurrPcieLinkWidth: Checked(c.nvmlDeviceGetCurrPcieLinkWidth), + nvmlDeviceGetComputeRunningProcesses_v3: Checked(c.nvmlDeviceGetComputeRunningProcesses_v3), + nvmlDeviceGetGraphicsRunningProcesses_v3: Checked(c.nvmlDeviceGetGraphicsRunningProcesses_v3), + nvmlDeviceGetProcessUtilization: Checked(c.nvmlDeviceGetProcessUtilization), + + fn Checked(comptime function: anytype) type { + return smi.dynlib.Checked(function, check); + } + + fn check(status: c.nvmlReturn_t) error{ Unsupported, NvmlError }!void { + return switch (status) { + c.NVML_SUCCESS => {}, + c.NVML_ERROR_NOT_SUPPORTED, c.NVML_ERROR_NO_DATA => error.Unsupported, + else => error.NvmlError, + }; + } +}; + +pub fn detect(io: std.Io) bool { + return smi.sysfs.exists(io, "/dev/nvidiactl"); +} + +pub fn init(arena: std.mem.Allocator, _: std.mem.Allocator, _: std.Io) !Nvml { + const api = try smi.dynlib.open(Api, &.{ "libnvidia-ml.so.1", "libnvidia-ml.so" }); + + // The first call says why the library cannot be used. + const status = api.nvmlInit_v2.function(); + Api.check(status) catch |err| { + log.warn("nvmlInit: {s}", .{std.mem.span(api.nvmlErrorString(status))}); + return err; + }; + + var version: [c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE]u8 = undefined; + const driver_version = if (api.nvmlSystemGetDriverVersion.call(.{ &version, version.len })) |_| + try arena.dupe(u8, std.mem.sliceTo(&version, 0)) + else |_| + null; + const details: []const smi.Device.Detail = if (api.nvmlSystemGetCudaDriverVersion_v2.get(.{})) |cuda| try arena.dupe(smi.Device.Detail, &.{.{ + .key = "cuda_version", + .label = "CUDA", + .value = try std.fmt.allocPrint(arena, "{d}.{d}", .{ @divTrunc(cuda, 1000), @divTrunc(@mod(cuda, 1000), 10) }), + .in_title = true, + }}) else |_| &.{}; + + const count = try api.nvmlDeviceGetCount_v2.get(.{}); + var gpus: std.ArrayList(Gpu) = try .initCapacity(arena, count); + var devices: std.ArrayList(smi.Device) = try .initCapacity(arena, count); + for (0..count) |index| { + // A GPU that fell off the bus is skipped; the others still show. + const handle = api.nvmlDeviceGetHandleByIndex_v2.get(.{@intCast(index)}) catch continue; + var name: [c.NVML_DEVICE_NAME_V2_BUFFER_SIZE]u8 = undefined; + gpus.appendAssumeCapacity(.{ .handle = handle }); + devices.appendAssumeCapacity(.{ + .target = .cuda, + .name = if (api.nvmlDeviceGetName.call(.{ handle, &name, name.len })) |_| try arena.dupe(u8, std.mem.sliceTo(&name, 0)) else |_| null, + .driver_version = driver_version, + .details = details, + }); + } + + return .{ + .platform = .{ .devices = devices.items, .vtable = &vtable }, + .api = api, + .gpus = gpus.items, + }; +} + +fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { + const self: *Nvml = @alignCast(@fieldParentPtr("platform", platform)); + const api = &self.api; + // PCIe throughput blocks while it measures (#766). + const read_pcie = self.tick % 4 == 0; + self.tick +%= 1; + + for (self.gpus, samples) |*gpu, *sample| { + const fields = &sample.cuda; + const handle = gpu.handle; + + fields.power_mw = api.nvmlDeviceGetPowerUsage.get(.{handle}) catch null; + fields.power_limit_mw = api.nvmlDeviceGetEnforcedPowerLimit.get(.{handle}) catch null; + fields.temperature = api.nvmlDeviceGetTemperature.get(.{ handle, c.NVML_TEMPERATURE_GPU }) catch null; + fields.fan_speed_percent = api.nvmlDeviceGetFanSpeed.get(.{handle}) catch null; + fields.util_percent = if (api.nvmlDeviceGetUtilizationRates.get(.{handle})) |rates| rates.gpu else |_| null; + + // Utilization and its sampling period. + var util: c_uint, var period: c_uint = .{ 0, 0 }; + fields.encoder_util_percent = if (api.nvmlDeviceGetEncoderUtilization.call(.{ handle, &util, &period })) |_| util else |_| null; + fields.decoder_util_percent = if (api.nvmlDeviceGetDecoderUtilization.call(.{ handle, &util, &period })) |_| util else |_| null; + + fields.clock_graphics_mhz = api.nvmlDeviceGetClockInfo.get(.{ handle, c.NVML_CLOCK_GRAPHICS }) catch null; + fields.clock_sm_mhz = api.nvmlDeviceGetClockInfo.get(.{ handle, c.NVML_CLOCK_SM }) catch null; + fields.clock_mem_mhz = api.nvmlDeviceGetClockInfo.get(.{ handle, c.NVML_CLOCK_MEM }) catch null; + fields.clock_graphics_max_mhz = api.nvmlDeviceGetMaxClockInfo.get(.{ handle, c.NVML_CLOCK_GRAPHICS }) catch null; + fields.clock_mem_max_mhz = api.nvmlDeviceGetMaxClockInfo.get(.{ handle, c.NVML_CLOCK_MEM }) catch null; + + // One call fills two fields. On failure both stay null. + if (api.nvmlDeviceGetMemoryInfo.get(.{handle})) |memory| { + fields.mem_used_bytes = memory.used; + fields.mem_total_bytes = memory.total; + } else |_| {} + fields.mem_bus_width = api.nvmlDeviceGetMemoryBusWidth.get(.{handle}) catch null; + + fields.pcie_link_gen = api.nvmlDeviceGetCurrPcieLinkGeneration.get(.{handle}) catch null; + fields.pcie_link_width = api.nvmlDeviceGetCurrPcieLinkWidth.get(.{handle}) catch null; + // NVML gives the speed of one lane. + if (api.nvmlDeviceGetPcieSpeed.get(.{handle})) |lane_mbps| { + if (fields.pcie_link_width) |width| fields.pcie_bandwidth_mbps = lane_mbps * width; + } else |_| {} + + if (read_pcie) { + gpu.pcie_tx_kbps = api.nvmlDeviceGetPcieThroughput.get(.{ handle, c.NVML_PCIE_UTIL_TX_BYTES }) catch null; + gpu.pcie_rx_kbps = api.nvmlDeviceGetPcieThroughput.get(.{ handle, c.NVML_PCIE_UTIL_RX_BYTES }) catch null; + } + fields.pcie_tx_kbps = gpu.pcie_tx_kbps; + fields.pcie_rx_kbps = gpu.pcie_rx_kbps; + } +} + +fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { + const self: *Nvml = @alignCast(@fieldParentPtr("platform", platform)); + var read_any = self.gpus.len == 0; + + for (self.gpus, 0..) |*gpu, device| { + const handle = gpu.handle; + const first_row = out.rows.items.len; + inline for (.{ self.api.nvmlDeviceGetComputeRunningProcesses_v3, self.api.nvmlDeviceGetGraphicsRunningProcesses_v3 }) |query| { + if (running(scratch, query, handle)) |list| { + read_any = true; + for (list) |process| { + try out.append(.{ .pid = process.pid, .device_idx = @intCast(device), .dev_mem_kib = process.usedGpuMemory / 1024 }); + } + } else |_| {} + } + + const since = gpu.last_seen; + const utilization = self.processUtilization(scratch, handle, since) catch continue; + for (utilization) |util| { + if (util.smUtil > 100 or util.timeStamp <= since) continue; + gpu.last_seen = @max(gpu.last_seen, util.timeStamp); + for (out.rows.items[first_row..]) |*row| { + if (row.pid == util.pid) row.dev_util_percent = @intCast(util.smUtil); + } + } + } + if (!read_any) return error.Unavailable; +} + +/// The processes of a GPU, from a query sized by a first call. +fn running(scratch: std.mem.Allocator, query: anytype, handle: c.nvmlDevice_t) ![]const c.nvmlProcessInfo_t { + var count: c_uint = 0; + const status = query.function(handle, &count, null); + if (status == c.NVML_SUCCESS) return &.{}; + if (status != c.NVML_ERROR_INSUFFICIENT_SIZE) try Api.check(status); + + const list = try scratch.alloc(c.nvmlProcessInfo_t, count); + try query.call(.{ handle, &count, list.ptr }); + return list[0..count]; +} + +/// The utilization samples newer than `since`, from a query sized by a first call. +fn processUtilization(self: *Nvml, scratch: std.mem.Allocator, handle: c.nvmlDevice_t, since: u64) ![]const c.nvmlProcessUtilizationSample_t { + var count: c_uint = 0; + const status = self.api.nvmlDeviceGetProcessUtilization.function(handle, null, &count, since); + if (status == c.NVML_SUCCESS) return &.{}; + if (status != c.NVML_ERROR_INSUFFICIENT_SIZE) try Api.check(status); + + const samples = try scratch.alloc(c.nvmlProcessUtilizationSample_t, count); + try self.api.nvmlDeviceGetProcessUtilization.call(.{ handle, samples.ptr, &count, since }); + return samples[0..count]; +} From c357487b8e746d4f6619b846c7b72a477e035182 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 20:16:22 +0100 Subject: [PATCH 10/19] bin/zml-smi: port Intel oneAPI One file, OneApi.zig, replaces monitor.zig, oneapi.zig, metrics.zig and process.zig, on top of the sysfs, hwmon, pci, fdinfo and FdHolders blocks. hwmon file names are chosen once in init. Each round scans the DRM clients once, where every metric used to scan /proc on its own, and reads only the fdinfo of descriptors that point into /dev/dri. A file that vanished between its open and its read is skipped. Utilization keeps one counter per process instead of summing running totals that drop when a process exits: busy time per engine class for i915, busy cycles over total cycles for xe. pcie_bandwidth_mbps is now the line rate of the link in megabits per second, 512000 on a gen 5 x16 link, where it was megabytes per second after encoding. --- bin/zml-smi/platforms/oneapi/BUILD.bazel | 16 +- bin/zml-smi/platforms/oneapi/OneApi.zig | 481 +++++++++++++++++++++++ 2 files changed, 484 insertions(+), 13 deletions(-) create mode 100644 bin/zml-smi/platforms/oneapi/OneApi.zig diff --git a/bin/zml-smi/platforms/oneapi/BUILD.bazel b/bin/zml-smi/platforms/oneapi/BUILD.bazel index c0b27d84..c9096766 100644 --- a/bin/zml-smi/platforms/oneapi/BUILD.bazel +++ b/bin/zml-smi/platforms/oneapi/BUILD.bazel @@ -3,19 +3,9 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "oneapi", import_name = "zml-smi/platforms/oneapi", - main = "metrics.zig", - srcs = [ - "monitor.zig", - "oneapi.zig", - "process.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:sysfs", - ], + main = "OneApi.zig", + visibility = ["//bin/zml-smi:__subpackages__"], + deps = ["//bin/zml-smi/smi"], ) zig_test( diff --git a/bin/zml-smi/platforms/oneapi/OneApi.zig b/bin/zml-smi/platforms/oneapi/OneApi.zig new file mode 100644 index 00000000..af8a1601 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/OneApi.zig @@ -0,0 +1,481 @@ +//! Intel GPUs, through sysfs: drm, hwmon, and the fdinfo of every DRM client. +const std = @import("std"); +const smi = @import("zml-smi"); + +const OneApi = @This(); + +platform: smi.Platform, +io: std.Io, +gpus: []Gpu, +proc: ?smi.sysfs.Dir, +/// Busy nanoseconds of each process, device and engine class (i915), or busy cycles of each process and device (xe). +busy: smi.Counters(EngineKey), +/// Total cycles of each process and device (xe). +cycles: smi.Counters(EngineKey), +/// The rows of this round's scan, in scratch. Null when the scan failed. +rows: ?[]const smi.ProcessInfo = null, + +const Gpu = struct { + /// The PCI device, /sys/class/drm/renderD/device. + dir: smi.sysfs.Dir, + /// The device or the bridge above it that carries its link. + link: smi.sysfs.Dir, + address: [12]u8, + hwmon: ?Hwmon, + energy: smi.Counter = .{}, + mem_total_bytes: ?u64, +}; + +/// The files chosen in init, read every poll. +const Hwmon = struct { + dir: smi.sysfs.Dir, + temperature: ?[]const u8, + energy: ?[]const u8, + power_limit: ?[]const u8, + fan: ?Fan, +}; + +/// A fan PWM, else a fan speed and its maximum. +const Fan = union(enum) { + pwm: []const u8, + speed: struct { input: []const u8, max: []const u8 }, + + /// The first PWM that reads, else the first speed that reads with a maximum above 0. + fn find(arena: std.mem.Allocator, dir: smi.sysfs.Dir) ?Fan { + var input: [16]u8 = undefined; + var max: [16]u8 = undefined; + for (1..16) |index| { + const pwm = std.fmt.bufPrint(&input, "pwm{d}", .{index}) catch unreachable; + _ = dir.int(u64, pwm) catch continue; + return .{ .pwm = arena.dupe(u8, pwm) catch return null }; + } + for (1..16) |index| { + const speed = std.fmt.bufPrint(&input, "fan{d}_input", .{index}) catch unreachable; + const limit = std.fmt.bufPrint(&max, "fan{d}_max", .{index}) catch unreachable; + _ = dir.int(u64, speed) catch continue; + if ((dir.int(u64, limit) catch 0) == 0) continue; + return .{ .speed = .{ .input = arena.dupe(u8, speed) catch return null, .max = arena.dupe(u8, limit) catch return null } }; + } + return null; + } + + fn percent(fan: Fan, dir: smi.sysfs.Dir) ?u64 { + switch (fan) { + .pwm => |file| return smi.hwmon.pwmPercent(dir.int(u64, file) catch return null), + .speed => |files| { + const input = dir.int(u64, files.input) catch return null; + const max = dir.int(u64, files.max) catch return null; + return if (max > 0) @min(input * 100 / max, 100) else null; + }, + } + } +}; + +const EngineKey = struct { pid: u32, device: u16, class: u64 }; + +const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit }; + +const intel = 0x8086; + +pub fn detect(io: std.Io) bool { + const drm = smi.sysfs.Dir.open(io, "/sys/class/drm") catch return false; + defer drm.close(); + var it = drm.iterate(); + while (it.next(io) catch null) |entry| { + const node = renderNode(entry.name) orelse continue; + const dir = openDevice(drm, node) catch continue; + defer dir.close(); + if ((dir.int(u16, "vendor") catch 0) == intel) return true; + } + return false; +} + +pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io) !OneApi { + const drm = try smi.sysfs.Dir.open(io, "/sys/class/drm"); + defer drm.close(); + + var nodes: std.ArrayList(u32) = .empty; + var it = drm.iterate(); + while (try it.next(io)) |entry| { + if (renderNode(entry.name)) |node| try nodes.append(arena, node); + } + std.mem.sort(u32, nodes.items, {}, std.sort.asc(u32)); + + const driver_version = driverVersion(arena, io); + var gpus: std.ArrayList(Gpu) = .empty; + var devices: std.ArrayList(smi.Device) = .empty; + for (nodes.items) |node| { + const dir = openDevice(drm, node) catch continue; + const gpu = openGpu(arena, dir) catch { + dir.close(); + continue; + }; + try gpus.append(arena, gpu); + try devices.append(arena, .{ .target = .oneapi, .name = try name(arena, io, gpu), .driver_version = driver_version }); + } + + return .{ + .platform = .{ .devices = devices.items, .two_readings = true, .vtable = &vtable }, + .io = io, + .gpus = gpus.items, + .proc = smi.sysfs.Dir.open(io, "/proc") catch null, + .busy = .init(gpa), + .cycles = .init(gpa), + }; +} + +fn renderNode(entry: []const u8) ?u32 { + if (!std.mem.startsWith(u8, entry, "renderD")) return null; + return std.fmt.parseInt(u32, entry["renderD".len..], 10) catch null; +} + +fn openDevice(drm: smi.sysfs.Dir, node: u32) !smi.sysfs.Dir { + var path: [32]u8 = undefined; + return drm.openDir(try std.fmt.bufPrint(&path, "renderD{d}/device", .{node})); +} + +fn openGpu(arena: std.mem.Allocator, dir: smi.sysfs.Dir) !Gpu { + if (try dir.int(u16, "vendor") != intel) return error.NotIntel; + + var buffer: [2048]u8 = undefined; + const slot = try dir.field("uevent", "PCI_SLOT_NAME", &buffer); + var address: [12]u8 = undefined; + if (slot.len != address.len) return error.NotFound; + @memcpy(&address, slot); + + return .{ + .dir = dir, + .link = try smi.pci.upstream(dir), + .address = address, + .hwmon = openHwmon(arena, dir), + .mem_total_bytes = largestBar(dir.read("resource", &buffer) catch ""), + }; +} + +fn openHwmon(arena: std.mem.Allocator, device: smi.sysfs.Dir) ?Hwmon { + var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const driver = device.readLink("driver", &buffer) catch null; + const dir = smi.hwmon.find(device, if (driver) |path| std.fs.path.basename(path) else null) catch return null; + return .{ + .dir = dir, + .temperature = smi.hwmon.resolve(arena, dir, &.{ + .{ .prefix = "temp", .label = "pkg" }, + .{ .prefix = "temp", .label = "gpu" }, + .{ .prefix = "temp2" }, + .{ .prefix = "temp1" }, + }), + .energy = smi.hwmon.resolve(arena, dir, &.{ + .{ .prefix = "energy", .label = "card" }, + .{ .prefix = "energy", .label = "pkg" }, + .{ .prefix = "energy1" }, + }), + .power_limit = smi.hwmon.resolve(arena, dir, &.{ .{ .prefix = "power1", .suffix = "cap" }, .{ .prefix = "power1", .suffix = "max" } }), + .fan = .find(arena, dir), + }; +} + +fn deinit(platform: *smi.Platform) void { + const self: *OneApi = @alignCast(@fieldParentPtr("platform", platform)); + for (self.gpus) |gpu| { + gpu.dir.close(); + gpu.link.close(); + if (gpu.hwmon) |hwmon| hwmon.dir.close(); + } + if (self.proc) |proc| proc.close(); + self.busy.deinit(); + self.cycles.deinit(); +} + +fn poll(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []smi.Sample) void { + const self: *OneApi = @alignCast(@fieldParentPtr("platform", platform)); + + for (self.gpus, samples) |*gpu, *sample| { + const fields = &sample.oneapi; + + fields.clock_graphics_mhz = gpu.dir.int(u64, "tile0/gt0/freq0/act_freq") catch gpu.dir.int(u64, "tile0/gt0/freq0/cur_freq") catch null; + fields.clock_graphics_max_mhz = gpu.dir.int(u64, "tile0/gt0/freq0/max_freq") catch null; + fields.mem_total_bytes = gpu.mem_total_bytes; + + const link = smi.pci.link(gpu.link, .current); + fields.pcie_link_gen = link.gen; + fields.pcie_link_width = link.width; + fields.pcie_bandwidth_mbps = link.mbps(); + + const hwmon = gpu.hwmon orelse continue; + // hwmon reports millidegrees, microwatts and microjoules. + if (hwmon.temperature) |file| fields.temperature = hwmon.dir.scaled(file, .milli, .one) catch null; + if (hwmon.power_limit) |file| fields.power_limit_mw = hwmon.dir.scaled(file, .micro, .milli) catch null; + if (hwmon.energy) |file| { + // Energy is a running total: its growth per second is microwatts. + if (hwmon.dir.int(u64, file)) |microjoules| { + if (gpu.energy.advance(self.io, microjoules)) |growth| fields.power_mw = smi.units.scale(growth.perSecond(), .micro, .milli); + } else |_| {} + } + if (hwmon.fan) |fan| fields.fan_speed_percent = fan.percent(hwmon.dir); + } + + self.scan(scratch, samples); +} + +fn processes(platform: *smi.Platform, _: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { + const self: *OneApi = @alignCast(@fieldParentPtr("platform", platform)); + for (self.rows orelse return error.Unavailable) |row| try out.append(row); +} + +/// One process on one device: its memory, and the growth of its busy time this round. +const Usage = struct { + row: smi.ProcessInfo, + busy: ?smi.Growth = null, + /// xe: the growth of the total cycles the busy cycles are a share of. + cycles: ?smi.Growth = null, +}; + +/// The busy time of every process on one device. +const DeviceUsage = struct { + busy: ?smi.Growth = null, + cycles: ?smi.Growth = null, + encoder: ?smi.Growth = null, + decoder: ?smi.Growth = null, + has_encoder: bool = false, + has_decoder: bool = false, +}; + +/// Reads every DRM client once: memory in use, and utilization through one counter per process. +fn scan(self: *OneApi, scratch: std.mem.Allocator, samples: []smi.Sample) void { + defer self.busy.sweep(); + defer self.cycles.sweep(); + self.rows = null; + const proc = self.proc orelse return; + + var usages: std.AutoArrayHashMapUnmanaged(struct { u32, u16 }, Usage) = .empty; + var engines: std.AutoArrayHashMapUnmanaged(EngineKey, Engine) = .empty; + // A client shared through dup() or fork() is counted once. + var clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .empty; + + var holders: smi.proc.FdHolders = .init(self.io, &.{"/dev/dri/"}); + defer holders.deinit(); + var buffer: [8192]u8 = undefined; + while (holders.next()) |holder| { + var path: [48]u8 = undefined; + const fdinfo_path = std.fmt.bufPrint(&path, "{d}/fdinfo/{d}", .{ holder.pid, holder.fd }) catch continue; + // The process may have closed it since: such a file is skipped. + const info = smi.fdinfo.read(proc, fdinfo_path, &buffer) catch continue; + const device = self.deviceOf(info.pdev orelse continue) orelse continue; + if (info.client_id) |id| { + if ((clients.getOrPut(scratch, .{ device, id }) catch continue).found_existing) continue; + } + + const usage = usages.getOrPutValue(scratch, .{ holder.pid, device }, .{ .row = .{ .pid = holder.pid, .device_idx = device } }) catch continue; + if (memoryBytes(&info)) |bytes| usage.value_ptr.row.dev_mem_kib = (usage.value_ptr.row.dev_mem_kib orelse 0) + bytes / 1024; + for (info.engines()) |engine| { + const xe = engine.total_cycles != null; + const key: EngineKey = .{ .pid = holder.pid, .device = device, .class = if (xe) 0 else std.hash.Wyhash.hash(0, engine.class) }; + const totals = (engines.getOrPutValue(scratch, key, .{ .kind = .of(engine.class) }) catch continue).value_ptr; + if (engine.total_cycles) |total| { + totals.busy += engine.cycles orelse 0; + totals.total = @max(totals.total orelse 0, total); + } else { + totals.busy += engine.busy_ns orelse 0; + } + } + } + + const devices = scratch.alloc(DeviceUsage, self.gpus.len) catch return; + @memset(devices, .{}); + var it = engines.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const usage = usages.getPtr(.{ key.pid, key.device }).?; + const device = &devices[key.device]; + device.has_encoder = device.has_encoder or entry.value_ptr.kind == .encoder; + device.has_decoder = device.has_decoder or entry.value_ptr.kind == .decoder; + + // Both counters advance at every reading, even the first. + const busy = self.busy.advance(key, self.io, entry.value_ptr.busy) catch continue; + const cycles = if (entry.value_ptr.total) |total| self.cycles.advance(key, self.io, total) catch continue else null; + const growth = busy orelse continue; + if (entry.value_ptr.total != null and cycles == null) continue; + smi.Growth.add(&usage.busy, growth); + smi.Growth.add(&device.busy, growth); + switch (entry.value_ptr.kind) { + .encoder => smi.Growth.add(&device.encoder, growth), + .decoder => smi.Growth.add(&device.decoder, growth), + .other => {}, + } + if (cycles) |total| { + usage.cycles = total; + if (device.cycles == null or total.amount > device.cycles.?.amount) device.cycles = total; + } + } + + const rows = scratch.alloc(smi.ProcessInfo, usages.count()) catch return; + for (rows, usages.values()) |*row, usage| { + row.* = usage.row; + row.dev_util_percent = percent(usage.busy, usage.cycles); + } + self.rows = rows; + + for (samples, devices, 0..) |*sample, device, index| { + const fields = &sample.oneapi; + fields.util_percent = percent(device.busy, device.cycles) orelse 0; + fields.encoder_util_percent = if (device.has_encoder) percent(device.encoder, null) orelse 0 else null; + fields.decoder_util_percent = if (device.has_decoder) percent(device.decoder, null) orelse 0 else null; + var mem_kib: u64 = 0; + for (rows) |row| { + if (row.device_idx == index) mem_kib += row.dev_mem_kib orelse 0; + } + fields.mem_used_bytes = mem_kib * 1024; + } +} + +/// The busy time of one engine class, summed over the clients of a process. +const Engine = struct { + busy: u64 = 0, + /// xe: the total cycles the busy cycles are counted against. + total: ?u64 = null, + kind: Kind, + + /// Encoder and decoder utilization count only the classes named so. + const Kind = enum { + encoder, + decoder, + other, + + fn of(class: []const u8) Kind { + if (std.ascii.findIgnoreCase(class, "encode") != null) return .encoder; + if (std.ascii.findIgnoreCase(class, "decode") != null) return .decoder; + return .other; + } + }; +}; + +/// Busy nanoseconds over the elapsed time (i915), or busy cycles over total cycles (xe). +fn percent(busy: ?smi.Growth, cycles: ?smi.Growth) ?u16 { + const growth = busy orelse return null; + return if (cycles) |total| growth.percentOf(total) else growth.percentOfTime(); +} + +fn deviceOf(self: *const OneApi, pdev: []const u8) ?u16 { + for (self.gpus, 0..) |gpu, index| { + if (std.mem.eql(u8, &gpu.address, pdev)) return @intCast(index); + } + return null; +} + +/// The memory of one client: its resident VRAM, else its VRAM, else what it holds resident, else in total. +fn memoryBytes(info: *const smi.fdinfo.Fdinfo) ?u64 { + var sums: [4]u64 = @splat(0); + for (info.regions()) |region| { + if (std.mem.startsWith(u8, region.name, "vram")) { + sums[0] += region.resident orelse 0; + sums[1] += region.total orelse 0; + } + sums[2] += region.resident orelse 0; + sums[3] += region.total orelse 0; + } + for (sums) |sum| { + if (sum > 0) return sum; + } + return null; +} + +/// Device memory: the largest BAR of the PCI resource file, if it is at least 256 MiB. +fn largestBar(resource: []const u8) ?u64 { + var largest: u64 = 0; + var lines = std.mem.tokenizeScalar(u8, resource, '\n'); + while (lines.next()) |line| { + var words = std.mem.tokenizeAny(u8, line, " \t"); + const start = smi.sysfs.parseInt(u64, words.next() orelse continue) catch continue; + const end = smi.sysfs.parseInt(u64, words.next() orelse continue) catch continue; + if (end > start) largest = @max(largest, end - start + 1); + } + return if (largest >= 256 * 1024 * 1024) largest else null; +} + +fn driverVersion(arena: std.mem.Allocator, io: std.Io) ?[]const u8 { + const module = smi.sysfs.Dir.open(io, "/sys/module") catch return null; + defer module.close(); + var buffer: [128]u8 = undefined; + for ([_][]const u8{ "xe/version", "xe/srcversion", "i915/version", "i915/srcversion" }) |file| { + const version = module.read(file, &buffer) catch continue; + if (version.len > 0) return arena.dupe(u8, version) catch null; + } + return null; +} + +/// The name lspci gives, else the device id. +fn name(arena: std.mem.Allocator, io: std.Io, gpu: Gpu) ![]const u8 { + if (std.process.run(arena, io, .{ + .argv = &.{ "lspci", "-Dnn", "-s", &gpu.address }, + .stdout_limit = .limited(4096), + .stderr_limit = .limited(1024), + })) |result| { + if (result.term == .exited and result.term.exited == 0) { + if (parseLspciName(result.stdout)) |lspci| return arena.dupe(u8, lspci); + } + } else |_| {} + + var buffer: [16]u8 = undefined; + return std.fmt.allocPrint(arena, "Intel GPU {s}", .{gpu.dir.read("device", &buffer) catch "unknown"}); +} + +fn parseLspciName(output: []const u8) ?[]const u8 { + var lines = std.mem.tokenizeScalar(u8, output, '\n'); + while (lines.next()) |line| { + const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace); + const header_end = std.mem.indexOf(u8, trimmed, "]: ") orelse continue; + const header = trimmed[0 .. header_end + 1]; + if (std.ascii.findIgnoreCase(header, "VGA") == null and std.ascii.findIgnoreCase(header, "Display") == null) continue; + + const rest = trimmed[header_end + "]: ".len ..]; + const end = std.mem.lastIndexOf(u8, rest, " [8086:") orelse std.mem.indexOf(u8, rest, " (rev ") orelse rest.len; + const device_name = std.mem.trim(u8, rest[0..end], &std.ascii.whitespace); + if (device_name.len == 0 or std.mem.startsWith(u8, device_name, "Intel Corporation Device ")) return null; + return device_name; + } + return null; +} + +test parseLspciName { + try std.testing.expectEqualStrings( + "Intel Corporation Battlemage G31 [Arc Pro B70]", + parseLspciName("0000:84:00.0 VGA compatible controller [0300]: Intel Corporation Battlemage G31 [Arc Pro B70] [8086:e223] (rev 04)\n").?, + ); + // A generic name is no better than the device id. + try std.testing.expectEqual(null, parseLspciName("0000:84:00.0 Display controller [0380]: Intel Corporation Device e223 [8086:e223] (rev 04)\n")); +} + +test largestBar { + try std.testing.expectEqual(34_359_738_368, largestBar( + \\0x0000048800000000 0x0000048fffffffff 0x000000000014220c + \\0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200 + \\0x0000000000000000 0x0000000000000000 0x0000000000000000 + )); + try std.testing.expectEqual(null, largestBar("0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200\n")); +} + +test memoryBytes { + const xe = smi.fdinfo.parse("drm-total-system:\t184 MiB\ndrm-total-vram0:\t4096 KiB\ndrm-resident-vram0:\t3072 KiB\n"); + try std.testing.expectEqual(3072 * 1024, memoryBytes(&xe)); + const system = smi.fdinfo.parse("drm-total-system0:\t2 MiB\n"); + try std.testing.expectEqual(2 * 1024 * 1024, memoryBytes(&system)); +} + +test Fan { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const dir: smi.sysfs.Dir = .{ .handle = tmp.dir, .io = io }; + + // fan1 has no maximum, so the speed is fan2's. + try tmp.dir.writeFile(io, .{ .sub_path = "fan1_input", .data = "900\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "fan2_input", .data = "1500\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "fan2_max", .data = "3000\n" }); + try std.testing.expectEqual(50, Fan.find(arena.allocator(), dir).?.percent(dir)); + + // A PWM comes first, whatever its index. + try tmp.dir.writeFile(io, .{ .sub_path = "pwm3", .data = "255\n" }); + try std.testing.expectEqual(100, Fan.find(arena.allocator(), dir).?.percent(dir)); +} From b291cf59a20feb9f1c3f599c59c8938df494dc75 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 20:16:29 +0100 Subject: [PATCH 11/19] bin/zml-smi: register the platforms platforms.zig lists the platform files of each OS and architecture. main opens the platforms whose detect is true and the remote hosts, skips those that fail to open, and runs one output. AMD, Neuron and TPU have their directory, BUILD target, registry line and select entry, with a platform whose detect is false until they are ported. std's unexpected errno traces are off: a /proc file can vanish between its open and its read, which printed stack traces in debug builds. --- bin/zml-smi/BUILD.bazel | 21 +-- bin/zml-smi/main.zig | 228 +++++++---------------- bin/zml-smi/platforms.zig | 19 ++ bin/zml-smi/platforms/amdsmi/AmdSmi.zig | 15 ++ bin/zml-smi/platforms/amdsmi/BUILD.bazel | 15 +- bin/zml-smi/platforms/neuron/BUILD.bazel | 24 +-- bin/zml-smi/platforms/neuron/Neuron.zig | 15 ++ bin/zml-smi/platforms/tpu/BUILD.bazel | 22 +-- bin/zml-smi/platforms/tpu/Tpu.zig | 15 ++ 9 files changed, 154 insertions(+), 220 deletions(-) create mode 100644 bin/zml-smi/platforms.zig create mode 100644 bin/zml-smi/platforms/amdsmi/AmdSmi.zig create mode 100644 bin/zml-smi/platforms/neuron/Neuron.zig create mode 100644 bin/zml-smi/platforms/tpu/Tpu.zig diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index cfefedf2..ddc18b9b 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -4,14 +4,10 @@ load("@bazel_lib//tools/release:hashes.bzl", "hashes") load("@bazel_skylib//rules:run_binary.bzl", "run_binary") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_binary") load("@tar.bzl//tar:mtree.bzl", "mtree_spec") load("@tar.bzl//tar:tar.bzl", "tar") -# zig libraries - -# smi - run_binary( name = "libnrt.stripped", srcs = ["@aws-neuronx-runtime-lib//:libnrt.patchelf"], @@ -79,32 +75,21 @@ run_binary( zig_binary( name = "zml-smi", + srcs = ["platforms.zig"], data = [":sandbox"], main = "main.zig", visibility = ["//visibility:public"], deps = [ - "//bin/zml-smi/api", - "//bin/zml-smi/collector", - "//bin/zml-smi/csv", - "//bin/zml-smi/info", - "//bin/zml-smi/json", - "//bin/zml-smi/prometheus", + "//bin/zml-smi/smi", "//bin/zml-smi/tui", - "//bin/zml-smi/utils:sysfs", "//stdx", - "@libvaxis//:vaxis", - "@rules_zig//zig/runfiles", ] + select({ - "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], - "@platforms//os:linux": ["//bin/zml-smi/platforms/linux"], - }) + select({ "@llvm//platforms/config:linux_x86_64": [ "//bin/zml-smi/platforms/amdsmi", "//bin/zml-smi/platforms/neuron", "//bin/zml-smi/platforms/nvml", "//bin/zml-smi/platforms/oneapi", "//bin/zml-smi/platforms/tpu", - "//platforms/neuron:libnrt_headers", ], "@llvm//platforms/config:linux_aarch64": [ "//bin/zml-smi/platforms/nvml", diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index 921f07b0..7de518ca 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -1,32 +1,21 @@ const std = @import("std"); -const builtin = @import("builtin"); const stdx = @import("stdx"); +const smi = @import("zml-smi"); +const tui = @import("zml-smi/tui"); + +const platforms = @import("platforms.zig"); pub const std_options: std.Options = .{ .log_scope_levels = &.{ .{ .scope = .vaxis, .level = .warn }, }, + // A /proc file can vanish between its open and its read, which std reports + // as an unexpected errno, with a stack trace in debug builds. + .unexpected_error_tracing = false, }; -const Collector = @import("zml-smi/collector").Collector; -const smi_info = @import("zml-smi/info"); -const HostInfo = smi_info.host_info.HostInfo; -const platform = if (builtin.os.tag == .macos) - @import("zml-smi/platforms/macos") -else - @import("zml-smi/platforms/linux"); -const host = platform.metrics; -const c = @import("c"); -const csv = @import("zml-smi/csv"); -const json = @import("zml-smi/json"); -const api = @import("zml-smi/api"); -const prometheus = @import("zml-smi/prometheus"); -const sysfs = @import("zml-smi/sysfs"); -const smi_tui = @import("zml-smi/tui"); -const tui = smi_tui.top; -const static_print = smi_tui.print; -const data = smi_tui.data; +const log = std.log; const CliArgs = struct { top: bool = false, @@ -58,174 +47,83 @@ const CliArgs = struct { // \\ --remotes Add devices from remote hosts (comma-separated URLs) }; -const device_backends = switch (builtin.os.tag) { - .macos => .{}, - .linux => switch (builtin.cpu.arch) { - .x86_64 => .{ - .{ .cuda, @import("zml-smi/platforms/nvml") }, - .{ .rocm, @import("zml-smi/platforms/amdsmi") }, - .{ .oneapi, @import("zml-smi/platforms/oneapi") }, - .{ .neuron, @import("zml-smi/platforms/neuron") }, - .{ .tpu, @import("zml-smi/platforms/tpu") }, - }, - .aarch64 => .{ - .{ .cuda, @import("zml-smi/platforms/nvml") }, - }, - else => unreachable, - }, - else => unreachable, -}; - pub fn main(init: std.process.Init) !void { const gpa = init.gpa; const arena = init.arena.allocator(); const io = init.io; const args = stdx.flags.parse(init.minimal.args, CliArgs); + const interval: std.Io.Duration = .fromMilliseconds(args.poll_interval); - const targets = detect(gpa, io); - - var collector: Collector = .init(arena, gpa, io, .{ - .poll_interval_ms = args.poll_interval, - .poll_only = !args.top and args.prometheus_listen == null and !args.api, - }); + var collector: smi.Collector = .init(arena, gpa, io, .{ .poll_interval_ms = args.poll_interval }); defer collector.deinit(); - var host_info: HostInfo = .{ .values = .{ .{}, .{} } }; - try host.init(&collector, &host_info); - - if (args.api) { - std.log.info("serving api metrics on port {d}", .{args.api_port}); - } - - if (args.prometheus_listen) |listen| { - std.log.info("serving prometheus metrics on {s}", .{listen}); - } - - inline for (device_backends) |entry| { - const target, const backend = entry; - if (targets.contains(target)) { - backend.start(&collector) catch |err| { - std.log.err("{s} skipped: {s}", .{ @tagName(target), @errorName(err) }); - }; + inline for (platforms.all) |Platform| { + if (Platform.detect(io)) { + const platform = try arena.create(Platform); + if (Platform.init(arena, gpa, io)) |value| { + platform.* = value; + try collector.add(&platform.platform); + } else |err| log.warn("{s} skipped: {t}", .{ @typeName(Platform), err }); } } - var api_group: std.Io.Group = .init; - if (args.remotes) |hosts| { - try api.addRemotes(&collector, hosts); + var it = std.mem.splitScalar(u8, hosts, ','); + while (it.next()) |host| { + const remote = try arena.create(smi.Remote); + if (smi.Remote.open(arena, gpa, io, host, interval)) |value| { + remote.* = value; + try collector.add(&remote.platform); + } else |err| log.warn("{s} skipped: {t}", .{ host, err }); + } } - try platform.process.start(&collector); + const serving = args.prometheus_listen != null or args.api; + if (serving or args.top) { + try collector.start(); + } else { + collector.round(); + if (collector.two_readings) { + try io.sleep(interval, .awake); + collector.round(); + } + } - var state = try data.SystemState.init(.{ - .devices = collector.device_infos.items, - .host = &host_info, - .targets = targets, + if (serving) { + var servers: std.Io.Group = .init; + defer servers.cancel(io); + if (args.prometheus_listen) |listen| { + log.info("serving prometheus metrics on {s}", .{listen}); + try servers.concurrent(io, smi.api.serve, .{ gpa, io, try .parseLiteral(listen), &collector, .prometheus }); + } + if (args.api) { + log.info("serving api metrics on port {d}", .{args.api_port}); + const address: std.Io.net.IpAddress = .{ .ip4 = .{ .bytes = .{ 0, 0, 0, 0 }, .port = args.api_port } }; + try servers.concurrent(io, smi.api.serve, .{ gpa, io, address, &collector, .json }); + } + try servers.await(io); + return; + } + + if (args.csv or args.json) { + var buffer: [4096]u8 = undefined; + var stdout = std.Io.File.stdout().writer(io, &buffer); + try if (args.csv) smi.csv.write(&stdout.interface, &collector) else smi.json.write(&stdout.interface, &collector); + return stdout.interface.flush(); + } + + var state: tui.data.SystemState = try .init(.{ + .collector = &collector, .tui_refresh_rate = args.tui_refresh_rate, - .processes = &collector.processes, .gpa = gpa, .arena = arena, .io = io, }); defer state.deinit(arena); - if (args.prometheus_listen) |listen| { - try api_group.concurrent(io, prometheus.Server.run, .{ io, listen, collector.device_infos.items, &host_info }); - } - - if (args.api) { - try api_group.concurrent(io, api.Server.run, .{ gpa, io, args.api_port, collector.device_infos.items, &collector.processes }); - } - - if (args.prometheus_listen != null or args.api) { - try api_group.await(io); - } else if (args.csv) { - var csv_buf: [4096]u8 = undefined; - var csv_writer = std.Io.File.stdout().writer(io, &csv_buf); - - try csv.write(&csv_writer.interface, collector.device_infos.items); - try csv_writer.flush(); - } else if (args.json) { - var json_buf: [4096]u8 = undefined; - var json_writer = std.Io.File.stdout().writer(io, &json_buf); - - try json.write(&json_writer.interface, collector.device_infos.items, collector.processes.front().items); - try json_writer.flush(); - } else if (args.top) { - try tui.run(gpa, io, init.environ_map, &state); + if (args.top) { + try tui.top.run(gpa, io, init.environ_map, &state); } else { - try static_print.run(arena, io, init.environ_map, &state); + try tui.print.run(arena, io, init.environ_map, &state); } } - -fn detect(allocator: std.mem.Allocator, io: std.Io) smi_info.Targets { - if (comptime builtin.os.tag == .macos) { - return .{}; - } - - var targets: smi_info.Targets = .{}; - - switch (builtin.os.tag) { - .macos => {}, - .linux => { - switch (builtin.cpu.arch) { - .x86_64 => { - if (hasDevice(io, "/dev/nvidiactl")) { - targets.insert(.cuda); - } - - if (hasDevice(io, "/dev/kfd")) { - targets.insert(.rocm); - } - - if (hasDrmRenderDevice(allocator, io, "0x8086")) { - targets.insert(.oneapi); - } - - if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { - targets.insert(.neuron); - } - - if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { - targets.insert(.tpu); - } - }, - .aarch64 => { - if (hasDevice(io, "/dev/nvidiactl")) { - targets.insert(.cuda); - } - }, - else => unreachable, - } - }, - else => unreachable, - } - - return targets; -} - -fn hasDevice(io: std.Io, path: []const u8) bool { - std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; - return true; -} - -fn hasDrmRenderDevice(allocator: std.mem.Allocator, io: std.Io, comptime pci_vendor_id: []const u8) bool { - var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; - defer dir.close(io); - - var it = dir.iterate(); - while (it.next(io) catch null) |entry| { - if (!std.mem.startsWith(u8, entry.name, "renderD")) continue; - - var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const vendor_path = std.fmt.bufPrint(&vendor_path_buf, "/sys/class/drm/{s}/device/vendor", .{entry.name}) catch continue; - const vendor = sysfs.readString(allocator, io, vendor_path) catch continue; - defer allocator.free(vendor); - if (std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), pci_vendor_id)) { - return true; - } - } - - return false; -} diff --git a/bin/zml-smi/platforms.zig b/bin/zml-smi/platforms.zig new file mode 100644 index 00000000..d1e2e23a --- /dev/null +++ b/bin/zml-smi/platforms.zig @@ -0,0 +1,19 @@ +//! The platforms of this OS and architecture. Each one is a file whose struct holds an `smi.Platform`. +const builtin = @import("builtin"); + +pub const all = switch (builtin.os.tag) { + .linux => switch (builtin.cpu.arch) { + .x86_64 => .{ + @import("zml-smi/platforms/nvml"), + @import("zml-smi/platforms/amdsmi"), + @import("zml-smi/platforms/oneapi"), + @import("zml-smi/platforms/neuron"), + @import("zml-smi/platforms/tpu"), + }, + .aarch64 => .{ + @import("zml-smi/platforms/nvml"), + }, + else => .{}, + }, + else => .{}, +}; diff --git a/bin/zml-smi/platforms/amdsmi/AmdSmi.zig b/bin/zml-smi/platforms/amdsmi/AmdSmi.zig new file mode 100644 index 00000000..75f75c5c --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/AmdSmi.zig @@ -0,0 +1,15 @@ +//! AMD GPUs, through the bundled libamd_smi. +const std = @import("std"); +const smi = @import("zml-smi"); + +const AmdSmi = @This(); + +platform: smi.Platform, + +pub fn detect(_: std.Io) bool { + return false; +} + +pub fn init(_: std.mem.Allocator, _: std.mem.Allocator, _: std.Io) !AmdSmi { + return error.Unsupported; +} diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel index 4fde01d2..a285ca7a 100644 --- a/bin/zml-smi/platforms/amdsmi/BUILD.bazel +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -2,21 +2,12 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "amdsmi", - srcs = [ - "amdsmi.zig", - "process.zig", - ], import_name = "zml-smi/platforms/amdsmi", - main = "metrics.zig", + main = "AmdSmi.zig", target_compatible_with = ["@platforms//os:linux"], - visibility = ["//visibility:public"], + visibility = ["//bin/zml-smi:__subpackages__"], deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:dynlib", - "//bin/zml-smi/utils:sandbox", - "//stdx", + "//bin/zml-smi/smi", "@amdrocm-amdsmi//:amdsmi", ], ) diff --git a/bin/zml-smi/platforms/neuron/BUILD.bazel b/bin/zml-smi/platforms/neuron/BUILD.bazel index c7129ac7..0057edd0 100644 --- a/bin/zml-smi/platforms/neuron/BUILD.bazel +++ b/bin/zml-smi/platforms/neuron/BUILD.bazel @@ -1,22 +1,18 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "neuron", import_name = "zml-smi/platforms/neuron", - main = "metrics.zig", - srcs = [ - "nrt.zig", - "process.zig", - ], - visibility = ["//visibility:public"], + main = "Neuron.zig", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//bin/zml-smi:__subpackages__"], deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:dynlib", - "//bin/zml-smi/utils:sandbox", - "//bin/zml-smi/utils:sysfs", + "//bin/zml-smi/smi", "//platforms/neuron:libnrt_headers", - "//stdx", ], ) + +zig_test( + name = "test", + deps = [":neuron"], +) diff --git a/bin/zml-smi/platforms/neuron/Neuron.zig b/bin/zml-smi/platforms/neuron/Neuron.zig new file mode 100644 index 00000000..07bf460c --- /dev/null +++ b/bin/zml-smi/platforms/neuron/Neuron.zig @@ -0,0 +1,15 @@ +//! AWS Neuron cores, through the bundled libnrt and sysfs. +const std = @import("std"); +const smi = @import("zml-smi"); + +const Neuron = @This(); + +platform: smi.Platform, + +pub fn detect(_: std.Io) bool { + return false; +} + +pub fn init(_: std.mem.Allocator, _: std.mem.Allocator, _: std.Io) !Neuron { + return error.Unsupported; +} diff --git a/bin/zml-smi/platforms/tpu/BUILD.bazel b/bin/zml-smi/platforms/tpu/BUILD.bazel index a65377b3..4f3f2503 100644 --- a/bin/zml-smi/platforms/tpu/BUILD.bazel +++ b/bin/zml-smi/platforms/tpu/BUILD.bazel @@ -1,10 +1,11 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( name = "tpuinfo", srcs = ["tpuinfo.cc"], hdrs = ["tpuinfo.h"], + target_compatible_with = ["@platforms//os:linux"], deps = [ "@cloud_accelerator_diagnostics//:tpu_metric_service_cc_grpc", "@grpc//:grpc++_unsecure", @@ -14,17 +15,16 @@ cc_library( zig_library( name = "tpu", import_name = "zml-smi/platforms/tpu", - main = "metrics.zig", - srcs = [ - "process.zig", - "tpuinfo.zig", - ], - visibility = ["//visibility:public"], + main = "Tpu.zig", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//bin/zml-smi:__subpackages__"], deps = [ ":tpuinfo", - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:sysfs", + "//bin/zml-smi/smi", ], ) + +zig_test( + name = "test", + deps = [":tpu"], +) diff --git a/bin/zml-smi/platforms/tpu/Tpu.zig b/bin/zml-smi/platforms/tpu/Tpu.zig new file mode 100644 index 00000000..5b95f02b --- /dev/null +++ b/bin/zml-smi/platforms/tpu/Tpu.zig @@ -0,0 +1,15 @@ +//! Google TPUs, through a PCI scan and the TPU runtime metrics service. +const std = @import("std"); +const smi = @import("zml-smi"); + +const Tpu = @This(); + +platform: smi.Platform, + +pub fn detect(_: std.Io) bool { + return false; +} + +pub fn init(_: std.mem.Allocator, _: std.mem.Allocator, _: std.Io) !Tpu { + return error.Unsupported; +} From b0f46ceed6a4ec26abb97f168b0b89f85b4c08a3 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Sat, 14 Nov 2026 10:19:12 +0100 Subject: [PATCH 12/19] bin/zml-smi: port amdsmi AMD GPUs through the bundled libamd_smi: sockets and processors enumerated in init, sentinel values read as null, the process list called with stderr pointed at the null device. Memory per process is the VRAM of its fdinfo files on the GPU; utilization per process is one counter per process and GPU over graphics plus encode time. pcie_bandwidth_mbps is the line rate of the PCIe generation times the width, since the current speed amdsmi reports stays at 2.5 GT/s on an R9700 whose link runs at 32 GT/s. --- bin/zml-smi/platforms/amdsmi/AmdSmi.zig | 247 +++++++++++++++++++++++- 1 file changed, 242 insertions(+), 5 deletions(-) diff --git a/bin/zml-smi/platforms/amdsmi/AmdSmi.zig b/bin/zml-smi/platforms/amdsmi/AmdSmi.zig index 75f75c5c..b809005e 100644 --- a/bin/zml-smi/platforms/amdsmi/AmdSmi.zig +++ b/bin/zml-smi/platforms/amdsmi/AmdSmi.zig @@ -1,15 +1,252 @@ -//! AMD GPUs, through the bundled libamd_smi. +//! AMD GPUs, through the bundled libamd_smi, with the memory of each process from its fdinfo files. const std = @import("std"); +const c = @import("c"); const smi = @import("zml-smi"); const AmdSmi = @This(); +const log = std.log.scoped(.@"zml-smi/amdsmi"); platform: smi.Platform, +io: std.Io, +api: Api, +gpus: []const Gpu, +proc: ?smi.sysfs.Dir, +/// Graphics plus encode nanoseconds of each process on each GPU. +busy: smi.Counters(struct { pid: u32, device: u16 }), -pub fn detect(_: std.Io) bool { - return false; +const Gpu = struct { + handle: c.amdsmi_processor_handle, + /// The PCI address fdinfo files name the GPU by: "0000:03:00.0". + address: ?[12]u8, +}; + +const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit }; + +const Api = struct { + amdsmi_init: Checked(c.amdsmi_init), + amdsmi_status_code_to_string: Checked(c.amdsmi_status_code_to_string), + amdsmi_get_socket_handles: Checked(c.amdsmi_get_socket_handles), + amdsmi_get_processor_handles: Checked(c.amdsmi_get_processor_handles), + amdsmi_get_gpu_asic_info: Checked(c.amdsmi_get_gpu_asic_info), + amdsmi_get_gpu_driver_info: Checked(c.amdsmi_get_gpu_driver_info), + amdsmi_get_gpu_bdf_id: Checked(c.amdsmi_get_gpu_bdf_id), + amdsmi_get_gpu_metrics_info: Checked(c.amdsmi_get_gpu_metrics_info), + amdsmi_get_power_info: Checked(c.amdsmi_get_power_info), + amdsmi_get_temp_metric: Checked(c.amdsmi_get_temp_metric), + amdsmi_get_gpu_fan_speed: Checked(c.amdsmi_get_gpu_fan_speed), + amdsmi_get_gpu_activity: Checked(c.amdsmi_get_gpu_activity), + amdsmi_get_clock_info: Checked(c.amdsmi_get_clock_info), + amdsmi_get_gpu_memory_total: Checked(c.amdsmi_get_gpu_memory_total), + amdsmi_get_gpu_memory_usage: Checked(c.amdsmi_get_gpu_memory_usage), + amdsmi_get_pcie_info: Checked(c.amdsmi_get_pcie_info), + amdsmi_get_gpu_process_list: Checked(c.amdsmi_get_gpu_process_list), + + fn Checked(comptime function: anytype) type { + return smi.dynlib.Checked(function, check); + } + + fn check(status: c.amdsmi_status_t) error{ Unsupported, AmdSmiError }!void { + return switch (status) { + c.AMDSMI_STATUS_SUCCESS => {}, + c.AMDSMI_STATUS_NOT_SUPPORTED, c.AMDSMI_STATUS_NOT_YET_IMPLEMENTED, c.AMDSMI_STATUS_NO_DATA => error.Unsupported, + else => error.AmdSmiError, + }; + } +}; + +pub fn detect(io: std.Io) bool { + return smi.sysfs.exists(io, "/dev/kfd"); } -pub fn init(_: std.mem.Allocator, _: std.mem.Allocator, _: std.Io) !AmdSmi { - return error.Unsupported; +pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io) !AmdSmi { + const api = try smi.dynlib.openBundled(Api, io, "libamd_smi.so.27"); + + // The first call says why the library cannot be used. + const status = api.amdsmi_init.function(c.AMDSMI_INIT_AMD_GPUS); + Api.check(status) catch |err| { + const message: [*c]const u8 = api.amdsmi_status_code_to_string.get(.{status}) catch "unknown status"; + log.warn("amdsmi_init: {s}", .{std.mem.span(message)}); + return err; + }; + + var socket_count: u32 = 0; + try api.amdsmi_get_socket_handles.call(.{ &socket_count, null }); + const sockets = try gpa.alloc(c.amdsmi_socket_handle, socket_count); + defer gpa.free(sockets); + try api.amdsmi_get_socket_handles.call(.{ &socket_count, sockets.ptr }); + + var gpus: std.ArrayList(Gpu) = .empty; + var devices: std.ArrayList(smi.Device) = .empty; + for (sockets[0..socket_count]) |socket| { + var count: u32 = 0; + try api.amdsmi_get_processor_handles.call(.{ socket, &count, null }); + const handles = try gpa.alloc(c.amdsmi_processor_handle, count); + defer gpa.free(handles); + try api.amdsmi_get_processor_handles.call(.{ socket, &count, handles.ptr }); + + for (handles[0..count]) |handle| { + try gpus.append(arena, .{ .handle = handle, .address = if (api.amdsmi_get_gpu_bdf_id.get(.{handle})) |bdf| pciAddress(bdf) else |_| null }); + try devices.append(arena, .{ + .target = .rocm, + .name = if (api.amdsmi_get_gpu_asic_info.get(.{handle})) |info| try arena.dupe(u8, std.mem.sliceTo(&info.market_name, 0)) else |_| null, + .driver_version = if (api.amdsmi_get_gpu_driver_info.get(.{handle})) |info| try arena.dupe(u8, std.mem.sliceTo(&info.driver_version, 0)) else |_| null, + }); + } + } + + return .{ + .platform = .{ .devices = devices.items, .two_readings = true, .vtable = &vtable }, + .io = io, + .api = api, + .gpus = gpus.items, + .proc = smi.sysfs.Dir.open(io, "/proc") catch null, + .busy = .init(gpa), + }; +} + +fn deinit(platform: *smi.Platform) void { + const self: *AmdSmi = @alignCast(@fieldParentPtr("platform", platform)); + if (self.proc) |proc| proc.close(); + self.busy.deinit(); +} + +fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { + const self: *AmdSmi = @alignCast(@fieldParentPtr("platform", platform)); + const api = &self.api; + + for (self.gpus, samples) |gpu, *sample| { + const fields = &sample.rocm; + const handle = gpu.handle; + + // Watts: the current power where the GPU has it, else the average. + if (api.amdsmi_get_gpu_metrics_info.get(.{handle})) |metrics| { + if (known(metrics.current_socket_power) orelse known(metrics.average_socket_power)) |watts| fields.power_mw = watts * 1000; + } else |_| {} + if (api.amdsmi_get_power_info.get(.{handle})) |info| { + if (known(info.power_limit)) |microwatts| fields.power_limit_mw = smi.units.scale(microwatts, .micro, .milli); + } else |_| {} + if (api.amdsmi_get_temp_metric.get(.{ handle, c.AMDSMI_TEMPERATURE_TYPE_EDGE, c.AMDSMI_TEMP_CURRENT })) |celsius| { + fields.temperature = std.math.cast(u64, celsius); + } else |_| {} + if (api.amdsmi_get_gpu_fan_speed.get(.{ handle, 0 })) |speed| { + if (std.math.cast(u64, speed)) |pwm| fields.fan_speed_percent = smi.hwmon.pwmPercent(pwm); + } else |_| {} + if (api.amdsmi_get_gpu_activity.get(.{handle})) |activity| { + // Not a percentage when unsupported: 0xFFFF. + if (activity.gfx_activity <= 100) fields.util_percent = activity.gfx_activity; + } else |_| {} + + if (api.amdsmi_get_clock_info.get(.{ handle, c.AMDSMI_CLK_TYPE_SYS })) |clock| { + fields.clock_graphics_mhz = known(clock.clk); + fields.clock_graphics_max_mhz = known(clock.max_clk); + } else |_| {} + if (api.amdsmi_get_clock_info.get(.{ handle, c.AMDSMI_CLK_TYPE_SOC })) |clock| { + fields.clock_soc_mhz = known(clock.clk); + } else |_| {} + if (api.amdsmi_get_clock_info.get(.{ handle, c.AMDSMI_CLK_TYPE_MEM })) |clock| { + fields.clock_mem_mhz = known(clock.clk); + fields.clock_mem_max_mhz = known(clock.max_clk); + } else |_| {} + + fields.mem_used_bytes = api.amdsmi_get_gpu_memory_usage.get(.{ handle, c.AMDSMI_MEM_TYPE_VRAM }) catch null; + fields.mem_total_bytes = api.amdsmi_get_gpu_memory_total.get(.{ handle, c.AMDSMI_MEM_TYPE_VRAM }) catch null; + + if (api.amdsmi_get_pcie_info.get(.{handle})) |pcie| { + const link: smi.pci.Link = .{ .gen = known(pcie.pcie_static.pcie_interface_version), .width = known(pcie.pcie_metric.pcie_width) }; + fields.pcie_link_gen = link.gen; + fields.pcie_link_width = link.width; + // The rate comes from the generation: the speed amdsmi reports stays at 2.5 GT/s on an R9700 whose link runs at 32. + fields.pcie_bandwidth_mbps = link.mbps(); + } else |_| {} + } +} + +fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, _: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { + const self: *AmdSmi = @alignCast(@fieldParentPtr("platform", platform)); + defer self.busy.sweep(); + var read_any = self.gpus.len == 0; + + for (self.gpus, 0..) |gpu, index| { + const device: u16 = @intCast(index); + const list = self.processList(scratch, gpu.handle) catch continue; + read_any = true; + for (list) |process| { + const busy = try self.busy.advance(.{ .pid = process.pid, .device = device }, self.io, process.engine_usage.gfx +| process.engine_usage.enc); + try out.append(.{ + .pid = process.pid, + .device_idx = device, + .dev_util_percent = if (busy) |growth| growth.percentOfTime() else null, + .dev_mem_kib = if (gpu.address) |address| self.vramKib(process.pid, &address) else null, + }); + } + } + if (!read_any) return error.Unavailable; +} + +/// The processes on a GPU, from a query sized by a first call. +fn processList(self: *AmdSmi, scratch: std.mem.Allocator, handle: c.amdsmi_processor_handle) ![]const c.amdsmi_proc_info_t { + var count: u32 = 0; + try Api.check(self.listQuietly(handle, &count, null)); + if (count == 0) return &.{}; + + const list = try scratch.alloc(c.amdsmi_proc_info_t, count); + const status = self.listQuietly(handle, &count, list.ptr); + // The list is full, and the processes that started since are left out. + if (status != c.AMDSMI_STATUS_OUT_OF_RESOURCES) try Api.check(status); + return list[0..@min(count, list.len)]; +} + +/// amdsmi prints while it lists processes, so stderr points at the null device for the length of the call. +fn listQuietly(self: *AmdSmi, handle: c.amdsmi_processor_handle, count: *u32, list: [*c]c.amdsmi_proc_info_t) c.amdsmi_status_t { + const saved = std.c.dup(std.c.STDERR_FILENO); + if (saved >= 0) { + const null_device = std.c.open("/dev/null", .{ .ACCMODE = .WRONLY }); + if (null_device >= 0) { + _ = std.c.dup2(null_device, std.c.STDERR_FILENO); + _ = std.c.close(null_device); + } + } + defer if (saved >= 0) { + _ = std.c.dup2(saved, std.c.STDERR_FILENO); + _ = std.c.close(saved); + }; + return self.api.amdsmi_get_gpu_process_list.function(handle, count, list); +} + +/// The VRAM a process holds on a GPU, summed over its fdinfo files that name the GPU. +fn vramKib(self: *AmdSmi, pid: u32, address: []const u8) ?u64 { + const proc = self.proc orelse return null; + var path: [24]u8 = undefined; + const files = proc.openDir(std.fmt.bufPrint(&path, "{d}/fdinfo", .{pid}) catch return null) catch return null; + defer files.close(); + + var bytes: u64 = 0; + var buffer: [8192]u8 = undefined; + var it = files.iterate(); + while (it.next(self.io) catch null) |entry| { + const info = smi.fdinfo.read(files, entry.name, &buffer) catch continue; + if (!std.mem.eql(u8, info.pdev orelse continue, address)) continue; + for (info.regions()) |region| { + if (std.mem.eql(u8, region.name, "vram")) bytes += region.resident orelse 0; + } + } + return bytes / 1024; +} + +/// amdsmi reports a value it cannot read as the largest of its type. +fn known(value: anytype) ?u64 { + return if (value == std.math.maxInt(@TypeOf(value))) null else value; +} + +/// The PCI address of an amdsmi BDF id, whose domain is in the upper 32 bits. +fn pciAddress(bdf: u64) ?[12]u8 { + var address: [12]u8 = undefined; + _ = std.fmt.bufPrint(&address, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ bdf >> 32, (bdf >> 8) & 0xff, (bdf >> 3) & 0x1f, bdf & 0x7 }) catch return null; + return address; +} + +test pciAddress { + try std.testing.expectEqualStrings("0000:03:00.0", &pciAddress(0x0300).?); + // Bits 28 to 31 hold a partition id, which is not part of the address. + try std.testing.expectEqualStrings("0001:c1:00.1", &pciAddress(0x1_1000_c101).?); } From fa7f176ff8bb967b8096958c2a2d577449c49bfa Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Sat, 14 Nov 2026 10:17:16 +0100 Subject: [PATCH 13/19] bin/zml-smi: port Neuron One file, Neuron.zig, replaces nrt.zig, metrics.zig and process.zig. A device is a core. libnrt is opened from the sandbox with openBundled, and its private ndl and nds functions are found in its ELF symbol table from the address of an exported one. Each round lists the apps of each chip once: utilization sums the growth of one busy-time counter per process and core, where it used to take the growth of a sum that drops when a process exits, and the same scan gives the process rows. Memory in use of a core is read once per chip, the sum over its cores, instead of once per core. An idle chip now shows 0, and a chip whose apps cannot be read shows no utilization instead of its last value. --- bin/zml-smi/platforms/neuron/BUILD.bazel | 3 + bin/zml-smi/platforms/neuron/Neuron.zig | 342 ++++++++++++++++++++++- 2 files changed, 341 insertions(+), 4 deletions(-) diff --git a/bin/zml-smi/platforms/neuron/BUILD.bazel b/bin/zml-smi/platforms/neuron/BUILD.bazel index 0057edd0..845caeed 100644 --- a/bin/zml-smi/platforms/neuron/BUILD.bazel +++ b/bin/zml-smi/platforms/neuron/BUILD.bazel @@ -14,5 +14,8 @@ zig_library( zig_test( name = "test", + # The test opens the libnrt of the sandbox, which only linux-amd64 ships. + data = ["//bin/zml-smi:sandbox"], + target_compatible_with = ["@platforms//cpu:x86_64"], deps = [":neuron"], ) diff --git a/bin/zml-smi/platforms/neuron/Neuron.zig b/bin/zml-smi/platforms/neuron/Neuron.zig index 07bf460c..845bbf6e 100644 --- a/bin/zml-smi/platforms/neuron/Neuron.zig +++ b/bin/zml-smi/platforms/neuron/Neuron.zig @@ -1,15 +1,349 @@ //! AWS Neuron cores, through the bundled libnrt and sysfs. const std = @import("std"); +const c = @import("c"); const smi = @import("zml-smi"); const Neuron = @This(); +const log = std.log.scoped(.@"zml-smi/neuron"); platform: smi.Platform, +io: std.Io, +nrt: Private, +chips: []const Chip, +/// Busy nanoseconds of each process on each core. +busy: smi.Counters(Key), +/// The rows of this round's scan, in scratch. Null when no chip could be read. +rows: ?[]const smi.ProcessInfo = null, -pub fn detect(_: std.Io) bool { - return false; +/// A Neuron device. Each of its cores is a device of the platform. +const Chip = struct { + handle: *c.ndl_device_t, + name: ?[]const u8 = null, + mem_total_bytes: ?u64, + /// stats/memory_usage/device_mem of each core. + memory: []const ?smi.sysfs.Dir, +}; + +/// A process and the position of a core in `platform.devices`. +const Key = struct { pid: u32, core: u16 }; + +const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit }; + +const Api = struct { + nrt_get_total_nc_count: Checked(c.nrt_get_total_nc_count), + nrt_get_version: Checked(c.nrt_get_version), +}; + +/// The functions libnrt does not export, found in its symbol table. +const Private = struct { + ndl_available_devices: *const @TypeOf(c.ndl_available_devices), + ndl_open_device: Checked(c.ndl_open_device), + ndl_get_all_apps_info: Checked(c.ndl_get_all_apps_info), + nds_open: Checked(c.nds_open), + nds_close: *const @TypeOf(c.nds_close), + nds_get_nc_counter: Checked(c.nds_get_nc_counter), +}; + +fn Checked(comptime function: anytype) type { + return smi.dynlib.Checked(function, check); } -pub fn init(_: std.mem.Allocator, _: std.mem.Allocator, _: std.Io) !Neuron { - return error.Unsupported; +/// The libnrt functions that report a status, exported or not, return 0 on success. +fn check(status: i64) error{NrtError}!void { + if (status != 0) return error.NrtError; +} + +pub fn detect(io: std.Io) bool { + return smi.sysfs.exists(io, c.NEURON_DEVICE_PREFIX ++ "0"); +} + +pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io) !Neuron { + const api = try smi.dynlib.openBundled(Api, io, "libnrt.so.1"); + const nrt = try findPrivate(io, api.nrt_get_version.function); + const core_count = try api.nrt_get_total_nc_count.get(.{}); + + var version = std.mem.zeroes(c.nrt_version_t); + const driver_version = if (api.nrt_get_version.call(.{ &version, @sizeOf(c.nrt_version_t) })) |_| + try arena.dupe(u8, std.mem.sliceTo(&version.rt_detail, 0)) + else |_| + null; + + var indices: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; + const count: usize = @intCast(std.math.clamp(nrt.ndl_available_devices(&indices, indices.len), 0, indices.len)); + var opened: std.ArrayList(struct { c_int, *c.ndl_device_t }) = try .initCapacity(arena, count); + for (indices[0..count]) |index| { + var params: c.ndl_device_init_param_t = .{ .initialize_device = false, .num_dram_regions = 0, .map_hbm = false, .skip_copy_bufs = true }; + // A device that cannot be opened is skipped. + const handle: ?*c.ndl_device_t = nrt.ndl_open_device.get(.{ index, ¶ms }) catch continue; + opened.appendAssumeCapacity(.{ index, handle orelse continue }); + } + + // The devices share the cores evenly. + const cores_per_chip = core_count / @max(1, opened.items.len); + const chips = try arena.alloc(Chip, opened.items.len); + var devices: std.ArrayList(smi.Device) = try .initCapacity(arena, chips.len * cores_per_chip); + for (chips, opened.items) |*chip, entry| { + const index, const handle = entry; + chip.* = try openChip(arena, io, index, handle, cores_per_chip); + for (chip.memory) |_| devices.appendAssumeCapacity(.{ .target = .neuron, .name = chip.name, .driver_version = driver_version }); + } + + return .{ + .platform = .{ .devices = devices.items, .two_readings = true, .vtable = &vtable }, + .io = io, + .nrt = nrt, + .chips = chips, + .busy = .init(gpa), + }; +} + +/// A chip's memory total from its device type, and its name and the memory directory of each core from sysfs. +fn openChip(arena: std.mem.Allocator, io: std.Io, index: c_int, handle: *c.ndl_device_t, cores: usize) !Chip { + const memory = try arena.alloc(?smi.sysfs.Dir, cores); + @memset(memory, null); + var chip: Chip = .{ .handle = handle, .mem_total_bytes = memoryTotal(handle.device_type), .memory = memory }; + + var path: [64]u8 = undefined; + const dir = smi.sysfs.Dir.open(io, try std.fmt.bufPrint(&path, "/sys/devices/virtual/neuron_device/neuron{d}", .{index})) catch return chip; + defer dir.close(); + for (memory, 0..) |*core, core_index| { + core.* = dir.openDir(try std.fmt.bufPrint(&path, "neuron_core{d}/stats/memory_usage/device_mem", .{core_index})) catch null; + } + var buffer: [128]u8 = undefined; + chip.name = try arena.dupe(u8, dir.read("info/architecture/device_name", &buffer) catch return chip); + return chip; +} + +/// Device memory by device type: Inferentia, then Inferentia2 and Trainium, Trainium2, Trainium3. +fn memoryTotal(device_type: u8) ?u64 { + const gib: u64 = 1 << 30; + return switch (device_type) { + 1 => 8 * gib, + 2 => 32 * gib, + 3 => 96 * gib, + 4 => 144 * gib, + else => null, + }; +} + +fn deinit(platform: *smi.Platform) void { + const self: *Neuron = @alignCast(@fieldParentPtr("platform", platform)); + for (self.chips) |chip| { + for (chip.memory) |memory| if (memory) |dir| dir.close(); + } + self.busy.deinit(); +} + +fn poll(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []smi.Sample) void { + const self: *Neuron = @alignCast(@fieldParentPtr("platform", platform)); + defer self.busy.sweep(); + var rows: std.ArrayList(smi.ProcessInfo) = .empty; + var read_any = false; + + var first: usize = 0; + for (self.chips) |chip| { + const cores = samples[first..][0..chip.memory.len]; + defer first += cores.len; + + // The cores of a chip share its memory: each shows what all of them use. + const mem_used = memoryInUse(chip.memory); + for (cores, chip.memory) |*sample, memory| { + const core = &sample.neuron; + core.mem_used_bytes = mem_used; + core.mem_total_bytes = chip.mem_total_bytes; + const dir = memory orelse continue; + core.nc_tensors = dir.int(u64, "tensors/total") catch null; + core.nc_constants = dir.int(u64, "constants/total") catch null; + core.nc_model_code = dir.int(u64, "model_code/total") catch null; + core.nc_shared_scratchpad = dir.int(u64, "model_shared_scratchpad/total") catch null; + core.nc_nonshared_scratchpad = dir.int(u64, "nonshared_scratchpad/total") catch null; + core.nc_runtime = dir.int(u64, "runtime_memory/total") catch null; + core.nc_driver = dir.int(u64, "driver_memory/total") catch null; + core.nc_dma_rings = dir.int(u64, "dma_rings/total") catch null; + core.nc_collectives = dir.int(u64, "collectives/total") catch null; + core.nc_notifications = dir.int(u64, "notifications/total") catch null; + core.nc_uncategorized = dir.int(u64, "uncategorized/total") catch null; + } + + self.scan(scratch, chip.handle, @intCast(first), cores, &rows) catch continue; + read_any = true; + } + self.rows = if (read_any) rows.items else null; +} + +fn processes(platform: *smi.Platform, _: std.mem.Allocator, samples: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { + const self: *Neuron = @alignCast(@fieldParentPtr("platform", platform)); + for (self.rows orelse return error.Unavailable) |row| { + var copy = row; + if (samples[row.device_idx].neuron.util_percent) |util| copy.dev_util_percent = @intCast(util); + try out.append(copy); + } +} + +fn memoryInUse(memory: []const ?smi.sysfs.Dir) ?u64 { + var total: u64 = 0; + for (memory) |dir| total += (dir orelse return null).int(u64, "total") catch return null; + return total; +} + +/// Reads the busy time of every process on one chip: the utilization of each core, +/// and a row per process and core it has used, if it holds device memory. +fn scan(self: *Neuron, scratch: std.mem.Allocator, handle: *c.ndl_device_t, first: u16, cores: []smi.Sample, rows: *std.ArrayList(smi.ProcessInfo)) !void { + var list: ?[*]c.struct_neuron_app_info = null; + var count: usize = 0; + try self.nrt.ndl_get_all_apps_info.call(.{ handle, &list, &count, c.APP_INFO_ALL }); + defer std.c.free(list); + const apps: []const c.struct_neuron_app_info = if (list) |items| items[0..count] else &.{}; + + const busy = try scratch.alloc(?smi.Growth, cores.len); + @memset(busy, null); + for (apps) |app| { + if (app.pid <= 0) continue; + const nds = (self.nrt.nds_open.get(.{ handle, app.pid }) catch continue) orelse continue; + defer _ = self.nrt.nds_close(nds); + + for (busy, 0..) |*sum, core| { + const micros = self.nrt.nds_get_nc_counter.get(.{ nds, @intCast(core), c.NDS_NC_COUNTER_TIME_IN_USE }) catch continue; + const key: Key = .{ .pid = @intCast(app.pid), .core = first + @as(u16, @intCast(core)) }; + if (try self.busy.advance(key, self.io, smi.units.scale(micros, .micro, .nano))) |growth| smi.Growth.add(sum, growth); + + const mem_kib = app.device_mem_size / 1024; + if (micros > 0 and mem_kib > 0) try rows.append(scratch, .{ .pid = key.pid, .device_idx = key.core, .dev_mem_kib = mem_kib }); + } + } + for (cores, busy) |*sample, sum| sample.neuron.util_percent = if (sum) |growth| growth.percentOfTime() else 0; +} + +/// Finds the functions of `Private` in the symbol table of the library that holds `known`. +fn findPrivate(io: std.Io, known: *const anyopaque) !Private { + var library: DlInfo = undefined; + if (dladdr(known, &library) == 0) return error.LibraryNotFound; + const file = try std.Io.Dir.cwd().openFile(io, std.mem.span(library.fname), .{}); + defer file.close(io); + var map = try file.createMemoryMap(io, .{ .len = @intCast((try file.stat(io)).size), .protection = .{ .read = true }, .populate = false }); + defer map.destroy(io); + + const elf = std.elf.ElfN; + const image = map.memory; + const header = std.mem.bytesAsValue(elf.Ehdr, image[0..@sizeOf(elf.Ehdr)]); + const sections = std.mem.bytesAsSlice(elf.Shdr, image[header.shoff..][0 .. @as(usize, header.shnum) * @sizeOf(elf.Shdr)]); + + const info = @typeInfo(Private).@"struct"; + var private: Private = undefined; + var found: [info.field_names.len]bool = @splat(false); + for (sections) |section| { + if (section.type != .SYMTAB) continue; + const strings = image[sections[section.link].offset..][0..sections[section.link].size]; + for (std.mem.bytesAsSlice(elf.Sym, image[section.offset..][0..section.size])) |symbol| { + if (symbol.value == 0) continue; + const name = std.mem.sliceTo(strings[symbol.name..], 0); + inline for (info.field_names, info.field_types, &found) |field, Field, *seen| { + if (!seen.* and std.mem.eql(u8, name, field)) { + seen.* = true; + const address = @intFromPtr(library.fbase) + symbol.value; + @field(private, field) = if (@typeInfo(Field) == .@"struct") .{ .function = @ptrFromInt(address) } else @ptrFromInt(address); + } + } + } + } + inline for (info.field_names, found) |field, seen| { + if (!seen) { + log.warn("libnrt has no symbol {s}", .{field}); + return error.MissingSymbol; + } + } + return private; +} + +const DlInfo = extern struct { + fname: [*:0]const u8, + fbase: *anyopaque, + sname: ?[*:0]const u8, + saddr: ?*anyopaque, +}; + +extern fn dladdr(address: *const anyopaque, info: *DlInfo) c_int; + +test findPrivate { + const io = std.testing.io; + const api = try smi.dynlib.openBundled(Api, io, "libnrt.so.1"); + const nrt = try findPrivate(io, api.nrt_get_version.function); + // Listing devices works without any, so the call checks the address. + var indices: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; + try std.testing.expect(nrt.ndl_available_devices(&indices, indices.len) >= 0); +} + +test "utilization and process rows" { + const fake = struct { + var busy_us: u64 = 1000; + + fn apps(device: [*c]c.ndl_device_t, info: [*c][*c]c.struct_neuron_app_info, count: [*c]usize, _: u16) callconv(.c) c_int { + if (@intFromPtr(device) == 0x1000) return -1; + const list: [*]c.struct_neuron_app_info = @ptrCast(@alignCast(std.c.malloc(3 * @sizeOf(c.struct_neuron_app_info)).?)); + // pid 0 is skipped, and pid 8 holds less than 1 KiB. + for (list[0..3], [_]i32{ 0, 7, 8 }, [_]usize{ 1 << 20, 1 << 20, 512 }) |*app, pid, bytes| { + app.* = std.mem.zeroes(c.struct_neuron_app_info); + app.pid = pid; + app.device_mem_size = bytes; + } + info.* = list; + count.* = 3; + return 0; + } + + fn open(_: [*c]c.ndl_device_t, pid: c.pid_t, instance: [*c]?*c.nds_instance_t) callconv(.c) c_int { + instance.* = @ptrFromInt(@as(usize, @intCast(pid))); + return 0; + } + + fn close(_: ?*c.nds_instance_t) callconv(.c) c_int { + return 0; + } + + /// Every process is busy on the first core of a chip. + fn counter(_: ?*c.nds_instance_t, core: c_int, _: u32, value: [*c]u64) callconv(.c) c_int { + value.* = if (core == 0) busy_us else 0; + return 0; + } + }; + + const devices: [4]smi.Device = @splat(.{ .target = .neuron }); + var neuron: Neuron = .{ + .platform = .{ .devices = &devices, .vtable = &vtable }, + .io = std.testing.io, + .nrt = .{ + .ndl_available_devices = undefined, + .ndl_open_device = undefined, + .ndl_get_all_apps_info = .{ .function = &fake.apps }, + .nds_open = .{ .function = &fake.open }, + .nds_close = &fake.close, + .nds_get_nc_counter = .{ .function = &fake.counter }, + }, + // The first chip cannot be read. + .chips = &.{ + .{ .handle = @ptrFromInt(0x1000), .mem_total_bytes = null, .memory = &.{ null, null } }, + .{ .handle = @ptrFromInt(0x2000), .mem_total_bytes = null, .memory = &.{ null, null } }, + }, + .busy = .init(std.testing.allocator), + }; + defer neuron.busy.deinit(); + var scratch: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer scratch.deinit(); + var list: smi.ProcessList = .init(std.testing.allocator); + defer list.deinit(); + + // The first reading has no growth; the second grows by far more than the time between them. + for ([_]u8{ 0, 100 }) |util| { + var samples: [4]smi.Sample = @splat(.init(.neuron)); + list.clear(); + poll(&neuron.platform, scratch.allocator(), &samples); + try processes(&neuron.platform, scratch.allocator(), &samples, &list); + + try std.testing.expectEqual(null, samples[0].neuron.util_percent); + try std.testing.expectEqual(util, samples[2].neuron.util_percent); + try std.testing.expectEqual(0, samples[3].neuron.util_percent); + const row: smi.ProcessInfo = .{ .pid = 7, .device_idx = 2, .dev_util_percent = util, .dev_mem_kib = 1024 }; + try std.testing.expectEqualDeep(@as([]const smi.ProcessInfo, &.{row}), list.rows.items); + fake.busy_us += 1_000_000_000; + } } From d21f11c91944e72072507ef2570124ab7957544b Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Sat, 14 Nov 2026 10:14:39 +0100 Subject: [PATCH 14/19] bin/zml-smi: port TPU --- bin/zml-smi/platforms/tpu/Tpu.zig | 157 +++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 5 deletions(-) diff --git a/bin/zml-smi/platforms/tpu/Tpu.zig b/bin/zml-smi/platforms/tpu/Tpu.zig index 5b95f02b..536db90c 100644 --- a/bin/zml-smi/platforms/tpu/Tpu.zig +++ b/bin/zml-smi/platforms/tpu/Tpu.zig @@ -1,15 +1,162 @@ -//! Google TPUs, through a PCI scan and the TPU runtime metrics service. +//! Google TPUs, through a PCI scan and the runtime metrics service of libtpu. const std = @import("std"); +const c = @import("c"); const smi = @import("zml-smi"); const Tpu = @This(); platform: smi.Platform, +io: std.Io, +/// The service numbers the devices chip after chip. +devices_per_chip: u32, -pub fn detect(_: std.Io) bool { - return false; +const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes }; + +/// Each query waits at most 1 s, so a stalled service delays a poll by up to 3 s. +const address = "localhost:8431"; +const max_devices: usize = c.TPUINFO_MAX_DEVICES; +const google = 0x1ae0; + +const Chip = struct { device: u16, subsystem: ?u16 = null, name: []const u8, devices_per_chip: u32 }; + +/// From tpu_info/device.py in cloud-accelerator-diagnostics (c94f463). +const chips = [_]Chip{ + .{ .device = 0x0027, .subsystem = 0x004e, .name = "TPU v2", .devices_per_chip = 2 }, + .{ .device = 0x0027, .subsystem = 0x004f, .name = "TPU v3", .devices_per_chip = 2 }, + .{ .device = 0x005e, .name = "TPU v4", .devices_per_chip = 1 }, + .{ .device = 0x0063, .name = "TPU v5e", .devices_per_chip = 1 }, + .{ .device = 0x0062, .name = "TPU v5p", .devices_per_chip = 1 }, + .{ .device = 0x006f, .name = "TPU v6e", .devices_per_chip = 1 }, + .{ .device = 0x0076, .name = "TPU7x", .devices_per_chip = 2 }, +}; + +pub fn detect(io: std.Io) bool { + return smi.sysfs.exists(io, "/dev/accel0") or smi.sysfs.exists(io, "/dev/vfio/0"); } -pub fn init(_: std.mem.Allocator, _: std.mem.Allocator, _: std.Io) !Tpu { - return error.Unsupported; +pub fn init(arena: std.mem.Allocator, _: std.mem.Allocator, io: std.Io) !Tpu { + var scan: smi.pci.Devices = try .open(io, google); + defer scan.close(); + var found: ?Chip = null; + var count: u32 = 0; + while (scan.next()) |entry| { + defer entry.dir.close(); + found = chipOf(entry.id, entry.dir.int(u16, "subsystem_device") catch null) orelse continue; + count += 1; + } + const chip = found orelse return error.TpuUnavailable; + + const devices = try arena.alloc(smi.Device, count * chip.devices_per_chip); + @memset(devices, .{ .target = .tpu, .name = chip.name }); + return .{ + .platform = .{ .devices = devices, .vtable = &vtable }, + .io = io, + .devices_per_chip = chip.devices_per_chip, + }; +} + +fn chipOf(device: u16, subsystem: ?u16) ?Chip { + for (chips) |chip| { + if (chip.device == device and (chip.subsystem == null or chip.subsystem == subsystem)) return chip; + } + return null; +} + +fn poll(_: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { + const used = query(c_longlong, "tpu.runtime.hbm.memory.usage.bytes"); + const total = query(c_longlong, "tpu.runtime.hbm.memory.total.bytes"); + const duty = query(f64, "tpu.runtime.tensorcore.dutycycle.percent"); + for (samples[0..@min(samples.len, max_devices)], 0..) |*sample, id| { + sample.tpu = .{ .util_percent = duty[id], .mem_used_bytes = used[id], .mem_total_bytes = total[id] }; + } +} + +/// One metric of every device, by device id. A failed query leaves them all null. +fn query(comptime T: type, metric: [:0]const u8) [max_devices]?u64 { + var ids: [max_devices]c_longlong = undefined; + var values: [max_devices]T = undefined; + const count = switch (T) { + c_longlong => c.tpu_query_int(address, metric, &ids, &values, max_devices), + f64 => c.tpu_query_double(address, metric, &ids, &values, max_devices), + else => @compileError("a metric is an int or a double"), + }; + const n = std.math.cast(usize, count) orelse 0; + return byId(T, ids[0..n], values[0..n]); +} + +fn byId(comptime T: type, ids: []const c_longlong, values: []const T) [max_devices]?u64 { + var out: [max_devices]?u64 = @splat(null); + for (ids, values) |id, value| { + const index = std.math.cast(usize, id) orelse continue; + if (index >= max_devices) continue; + out[index] = switch (T) { + c_longlong => std.math.cast(u64, value), + f64 => if (std.math.isFinite(value)) std.math.lossyCast(u64, @round(value)) else null, + else => @compileError("a metric is an int or a double"), + }; + } + return out; +} + +fn processes(platform: *smi.Platform, _: std.mem.Allocator, samples: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { + const self: *Tpu = @alignCast(@fieldParentPtr("platform", platform)); + var holders: smi.proc.FdHolders = .init(self.io, &.{ "/dev/accel", "/dev/vfio/" }); + defer holders.deinit(); + while (holders.next()) |holder| try addHolder(out, samples, self.devices_per_chip, holder.pid, holder.rest); +} + +/// Lists a process that holds the node of a chip on every device of that chip, once. +fn addHolder(out: *smi.ProcessList, samples: []const smi.Sample, devices_per_chip: u32, pid: u32, node: []const u8) !void { + const chip = std.fmt.parseInt(u16, node, 10) catch return; + for (0..devices_per_chip) |core| { + const device = @as(usize, chip) * devices_per_chip + core; + if (device >= samples.len) return; + for (out.rows.items) |row| { + if (row.pid == pid and row.device_idx == device) break; + } else { + const tpu = samples[device].tpu; + try out.append(.{ + .pid = pid, + .device_idx = @intCast(device), + .dev_util_percent = if (tpu.util_percent) |percent| std.math.cast(u16, percent) else null, + .dev_mem_kib = if (tpu.mem_used_bytes) |bytes| bytes / 1024 else null, + }); + } + } +} + +test chipOf { + try std.testing.expectEqualStrings("TPU v3", chipOf(0x0027, 0x004f).?.name); + try std.testing.expectEqualStrings("TPU v5e", chipOf(0x0063, 0x0001).?.name); + try std.testing.expect(chipOf(0x0027, null) == null); + try std.testing.expect(chipOf(0x0001, null) == null); +} + +test byId { + const used = byId(c_longlong, &.{ 1, 0, -1, 64 }, &.{ 2048, -5, 7, 7 }); + try std.testing.expectEqual(2048, used[1]); + try std.testing.expectEqual(null, used[0]); + const duty = byId(f64, &.{ 0, 1 }, &.{ 12.5, std.math.nan(f64) }); + try std.testing.expectEqual(13, duty[0]); + try std.testing.expectEqual(null, duty[1]); +} + +test addHolder { + var list: smi.ProcessList = .init(std.testing.allocator); + defer list.deinit(); + const samples = [_]smi.Sample{ .{ .tpu = .{} }, .{ .tpu = .{} }, .{ .tpu = .{ .util_percent = 40, .mem_used_bytes = 8192 } }, .{ .tpu = .{} } }; + + // Chip 1 of a two-device chip is devices 2 and 3, listed once however many descriptors. + try addHolder(&list, &samples, 2, 7, "1"); + try addHolder(&list, &samples, 2, 7, "1"); + // The vfio container and a chip beyond the devices make no row. + try addHolder(&list, &samples, 2, 7, "vfio"); + try addHolder(&list, &samples, 2, 7, "2"); + + try std.testing.expectEqual(2, list.rows.items.len); + try std.testing.expectEqual(2, list.rows.items[0].device_idx); + try std.testing.expectEqual(40, list.rows.items[0].dev_util_percent); + try std.testing.expectEqual(8, list.rows.items[0].dev_mem_kib); + try std.testing.expectEqual(3, list.rows.items[1].device_idx); + try std.testing.expectEqual(null, list.rows.items[1].dev_mem_kib); } From 2768a5e38f70283caa1762eaa70271aabc680227 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Sat, 14 Nov 2026 11:38:05 +0100 Subject: [PATCH 15/19] bin/zml-smi: add Furiosa FuriosaAI RNGD chips through sysfs, one device per chip, with the paths, units and formulas of furiosa-smi-go 2026.4.0: utilization from the task and total cycle counts of the cores, memory in use from the groups of fused cores, power and temperatures from hwmon, governor, liveness and throttle reasons as codes, and one process row per process and chip from the holders of /dev/rngd. --- bin/zml-smi/BUILD.bazel | 1 + bin/zml-smi/platforms.zig | 1 + bin/zml-smi/platforms/furiosa/BUILD.bazel | 18 + bin/zml-smi/platforms/furiosa/Furiosa.zig | 448 ++++++++++++++++++ bin/zml-smi/smi/model.zig | 26 +- bin/zml-smi/smi/prometheus.zig | 16 +- bin/zml-smi/smi/smi.zig | 1 + bin/zml-smi/tui/BUILD.bazel | 1 + bin/zml-smi/tui/data.zig | 5 + bin/zml-smi/tui/pages/detail.zig | 2 + .../tui/pages/detail/furiosa_detail.zig | 113 +++++ bin/zml-smi/tui/widgets/device_card.zig | 8 + 12 files changed, 638 insertions(+), 2 deletions(-) create mode 100644 bin/zml-smi/platforms/furiosa/BUILD.bazel create mode 100644 bin/zml-smi/platforms/furiosa/Furiosa.zig create mode 100644 bin/zml-smi/tui/pages/detail/furiosa_detail.zig diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index ddc18b9b..14d4fcf7 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -86,6 +86,7 @@ zig_binary( ] + select({ "@llvm//platforms/config:linux_x86_64": [ "//bin/zml-smi/platforms/amdsmi", + "//bin/zml-smi/platforms/furiosa", "//bin/zml-smi/platforms/neuron", "//bin/zml-smi/platforms/nvml", "//bin/zml-smi/platforms/oneapi", diff --git a/bin/zml-smi/platforms.zig b/bin/zml-smi/platforms.zig index d1e2e23a..5b92f1cf 100644 --- a/bin/zml-smi/platforms.zig +++ b/bin/zml-smi/platforms.zig @@ -9,6 +9,7 @@ pub const all = switch (builtin.os.tag) { @import("zml-smi/platforms/oneapi"), @import("zml-smi/platforms/neuron"), @import("zml-smi/platforms/tpu"), + @import("zml-smi/platforms/furiosa"), }, .aarch64 => .{ @import("zml-smi/platforms/nvml"), diff --git a/bin/zml-smi/platforms/furiosa/BUILD.bazel b/bin/zml-smi/platforms/furiosa/BUILD.bazel new file mode 100644 index 00000000..818b2d25 --- /dev/null +++ b/bin/zml-smi/platforms/furiosa/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "furiosa", + import_name = "zml-smi/platforms/furiosa", + main = "Furiosa.zig", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + visibility = ["//bin/zml-smi:__subpackages__"], + deps = ["//bin/zml-smi/smi"], +) + +zig_test( + name = "test", + deps = [":furiosa"], +) diff --git a/bin/zml-smi/platforms/furiosa/Furiosa.zig b/bin/zml-smi/platforms/furiosa/Furiosa.zig new file mode 100644 index 00000000..135b3403 --- /dev/null +++ b/bin/zml-smi/platforms/furiosa/Furiosa.zig @@ -0,0 +1,448 @@ +//! FuriosaAI RNGD chips, through sysfs. Paths, units and formulas follow furiosa-smi-go 2026.4.0. +const std = @import("std"); +const smi = @import("zml-smi"); + +const Furiosa = @This(); + +platform: smi.Platform, +io: std.Io, +/// /sys/class/rngd_mgmt: rngd!npumgmt for chip N, and rngd!npupe for its core C. +class: smi.sysfs.Dir, +chips: []Chip, + +/// The cores (PEs) of a chip. +const cores = 8; +const Cores = std.bit_set.Integer(cores); + +const Chip = struct { + /// N in /dev/rngd/npu. + node: u8, + model: *const Model, + mgmt: smi.sysfs.Dir, + hwmon: ?Hwmon, + /// The cores this host may use: those a node of /dev/rngd names. + usable: Cores, + cycles: [cores]Cycles = @splat(.{}), + /// This round: the utilization of each core, and the memory in use of the group each core heads. + util: [cores]u8 = @splat(0), + mem_used: [cores]u64 = @splat(0), + + fn lessThan(_: void, a: Chip, b: Chip) bool { + if (a.model.device != b.model.device) return a.model.device < b.model.device; + return a.node < b.node; + } + + /// The row of a process: the mean utilization of the cores it holds, and the memory their groups use. + fn row(chip: *const Chip, pid: u32, device: u16, held: Cores, sample: smi.Furiosa) smi.ProcessInfo { + const mine = held.intersectWith(chip.usable); + var util: u64 = 0; + var mem_used: u64 = 0; + var it = mine.iterator(.{}); + while (it.next()) |core| { + util += chip.util[core]; + mem_used += chip.mem_used[core]; + } + return .{ + .pid = pid, + .device_idx = device, + .dev_util_percent = if (sample.util_percent != null) @intCast(util / @max(@as(u64, mine.count()), 1)) else null, + .dev_mem_kib = if (sample.mem_used_bytes != null) mem_used / 1024 else null, + }; + } +}; + +/// The running totals of a core: its cycles, and those it spent on tasks. +const Cycles = struct { all: smi.Counter = .{}, task: smi.Counter = .{} }; + +const Hwmon = struct { + dir: smi.sysfs.Dir, + temperature: ?[]const u8, + ambient: ?[]const u8, + power: ?[]const u8, + + fn open(arena: std.mem.Allocator, device: smi.sysfs.Dir) ?Hwmon { + const dir = smi.hwmon.find(device, null) catch return null; + return .{ + .dir = dir, + .temperature = smi.hwmon.resolve(arena, dir, &.{.{ .prefix = "temp", .label = "PEAK" }}), + .ambient = smi.hwmon.resolve(arena, dir, &.{.{ .prefix = "temp", .label = "AMBIENT" }}), + .power = smi.hwmon.resolve(arena, dir, &.{.{ .prefix = "power", .label = "RMS_TOTAL", .suffix = "average" }}), + }; + } +}; + +/// A device type. Its firmware reserves 512 MiB of its DRAM, and its cores share the rest evenly. +const Model = struct { + device: u16, + name: []const u8, + dram_bytes: u64, + + fn memoryBytes(model: Model) u64 { + return model.dram_bytes - 512 * 1024 * 1024; + } +}; + +const models = [_]Model{ + .{ .device = 0x0001, .name = "RNGD", .dram_bytes = 48 << 30 }, + .{ .device = 0x0002, .name = "RNGD+", .dram_bytes = 72 << 30 }, +}; + +const vendor = 0x1ed2; + +const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit }; + +pub fn detect(io: std.Io) bool { + return smi.sysfs.exists(io, "/dev/rngd"); +} + +pub fn init(arena: std.mem.Allocator, _: std.mem.Allocator, io: std.Io) !Furiosa { + const usable = try usableCores(io); + const class = try smi.sysfs.Dir.open(io, "/sys/class/rngd_mgmt"); + + var chips: std.ArrayList(Chip) = .empty; + var it = class.iterate(); + while (try it.next(io)) |entry| { + const node = mgmtNode(entry.name) orelse continue; + // A chip with no node in /dev/rngd is not this host's. + if (usable[node].count() == 0) continue; + const mgmt = class.openDir(entry.name) catch continue; + const chip = openChip(arena, mgmt, node, usable[node]) catch { + mgmt.close(); + continue; + }; + try chips.append(arena, chip); + } + std.mem.sort(Chip, chips.items, {}, Chip.lessThan); + + const devices = try arena.alloc(smi.Device, chips.items.len); + for (devices, chips.items) |*device, chip| { + var details: std.ArrayList(smi.Device.Detail) = .empty; + if (string(arena, chip.mgmt, "fw_version")) |value| try details.append(arena, .{ .key = "firmware_version", .label = "Firmware", .value = value }); + if (string(arena, chip.mgmt, "device_sn")) |value| try details.append(arena, .{ .key = "serial", .label = "Serial", .value = value }); + device.* = .{ + .target = .furiosa, + .name = chip.model.name, + .driver_version = string(arena, chip.mgmt, "version"), + .details = details.items, + }; + } + + return .{ + .platform = .{ .devices = devices, .two_readings = true, .vtable = &vtable }, + .io = io, + .class = class, + .chips = chips.items, + }; +} + +/// The cores of each chip that the nodes of /dev/rngd name. +fn usableCores(io: std.Io) ![256]Cores { + var usable: [256]Cores = @splat(.empty); + const dev = try smi.sysfs.Dir.open(io, "/dev/rngd"); + defer dev.close(); + var it = dev.iterate(); + while (try it.next(io)) |entry| { + if (entry.kind != .character_device) continue; + const node = Node.parse(entry.name) orelse continue; + usable[node.chip].setUnion(node.cores); + } + return usable; +} + +fn openChip(arena: std.mem.Allocator, mgmt: smi.sysfs.Dir, node: u8, usable: Cores) !Chip { + var buffer: [64]u8 = undefined; + var path: [64]u8 = undefined; + const pci = try smi.sysfs.Dir.open(mgmt.io, try std.fmt.bufPrint(&path, "/sys/bus/pci/devices/{s}", .{try mgmt.read("busname", &buffer)})); + defer pci.close(); + + if (try pci.int(u16, "vendor") != vendor) return error.NotFuriosa; + const device = try pci.int(u16, "device"); + const model = for (&models) |*model| { + if (model.device == device) break model; + } else return error.UnknownDevice; + + return .{ .node = node, .model = model, .mgmt = mgmt, .hwmon = .open(arena, pci), .usable = usable }; +} + +fn deinit(platform: *smi.Platform) void { + const self: *Furiosa = @alignCast(@fieldParentPtr("platform", platform)); + for (self.chips) |chip| { + chip.mgmt.close(); + if (chip.hwmon) |hwmon| hwmon.dir.close(); + } + self.class.close(); +} + +fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { + const self: *Furiosa = @alignCast(@fieldParentPtr("platform", platform)); + + for (self.chips, samples) |*chip, *sample| { + const fields = &sample.furiosa; + fields.mem_total_bytes = chip.model.memoryBytes(); + fields.governor = governor(chip.mgmt); + fields.liveness = liveness(chip.mgmt); + fields.throttle_reasons = throttleReasons(chip.mgmt); + + chip.util = @splat(0); + chip.mem_used = @splat(0); + if (occupancy(chip.mgmt)) |in_use| { + fields.util_percent = self.utilization(chip, in_use); + fields.mem_used_bytes = self.memoryInUse(chip, in_use); + } + + const hwmon = chip.hwmon orelse continue; + // hwmon reports millidegrees and microwatts. + if (hwmon.temperature) |file| fields.temperature = hwmon.dir.scaled(file, .milli, .one) catch null; + if (hwmon.ambient) |file| fields.ambient_temperature = hwmon.dir.scaled(file, .milli, .one) catch null; + if (hwmon.power) |file| fields.power_mw = hwmon.dir.scaled(file, .micro, .milli) catch null; + fields.power_limit_mw = hwmon.dir.scaled("power1_max", .micro, .milli) catch null; + } +} + +/// The mean over the usable cores of the share of its cycles each spent on tasks. +fn utilization(self: *const Furiosa, chip: *Chip, in_use: Cores) ?u64 { + var sum: u64 = 0; + var it = chip.usable.iterator(.{}); + while (it.next()) |core| { + var buffer: [1024]u8 = undefined; + const text = if (in_use.isSet(core)) (self.coreFile(chip.node, core, "metrics", &buffer) catch return null) else null; + // A core not in use reads 0 cycles, as in furiosa-smi-go. + const all = if (text) |metrics| (hexField(metrics, "CycleCount") orelse return null) else 0; + const task = if (text) |metrics| (hexField(metrics, "TaskCycleCount") orelse return null) else 0; + const all_growth = chip.cycles[core].all.advance(self.io, all); + const task_growth = chip.cycles[core].task.advance(self.io, task); + chip.util[core] = if (all_growth != null and task_growth != null) task_growth.?.percentOf(all_growth.?) else 0; + sum += chip.util[core]; + } + return sum / @max(@as(u64, chip.usable.count()), 1); +} + +/// Memory in use, summed over the groups of fused cores. Every core of a group reports the whole +/// group, so a group is read once, from its first core. A core not in use is a group of its own. +fn memoryInUse(self: *const Furiosa, chip: *Chip, in_use: Cores) ?u64 { + const share = chip.model.memoryBytes() / cores; + var used: u64 = 0; + var left = chip.usable.count(); + var it = chip.usable.iterator(.{}); + while (it.next()) |head| { + var size: usize = 1; + var buffer: [1024]u8 = undefined; + const text = if (in_use.isSet(head)) (self.coreFile(chip.node, head, "alloc_status", &buffer) catch return null) else null; + if (text) |status| { + const capacity = (hexField(status, "dram_capacity") orelse return null) + (hexField(status, "shared_capacity") orelse return null); + const usage = (hexField(status, "dram_usage") orelse return null) + (hexField(status, "shared_usage") orelse return null); + if (capacity > 0) { + size = @intCast(@max(capacity / share, 1)); + chip.mem_used[head] = usage; + used += usage; + } + } + if (size > left) return null; + left -= size; + for (1..size) |_| _ = it.next(); + } + return used; +} + +/// A file of a core, or null when the core is not in use: the file is then empty or missing. +fn coreFile(self: *const Furiosa, node: u8, core: usize, name: []const u8, buffer: []u8) !?[]const u8 { + var path: [48]u8 = undefined; + const text = self.class.read(try std.fmt.bufPrint(&path, "rngd!npu{d}pe{d}/{s}", .{ node, core, name }), buffer) catch |err| switch (err) { + error.FileNotFound => return null, + else => |e| return e, + }; + return if (text.len > 0) text else null; +} + +/// The cores in use: one line per core, 1 when it is in use and 0 when it is not. +fn occupancy(mgmt: smi.sysfs.Dir) ?Cores { + var buffer: [64]u8 = undefined; + const text = mgmt.read("pe_occupancy", &buffer) catch return null; + var in_use: Cores = .empty; + var lines = std.mem.splitScalar(u8, text, '\n'); + var core: usize = 0; + while (lines.next()) |line| : (core += 1) { + const state = std.mem.trim(u8, line, &std.ascii.whitespace); + if (std.mem.eql(u8, state, "1")) { + if (core < cores) in_use.set(core); + } else if (!std.mem.eql(u8, state, "0")) return null; + } + return in_use; +} + +/// The governor whose name ends the first line. +fn governor(mgmt: smi.sysfs.Dir) ?u64 { + var buffer: [256]u8 = undefined; + const text = mgmt.read("npu_governor", &buffer) catch return null; + const line = std.mem.trimEnd(u8, std.mem.sliceTo(text, '\n'), &std.ascii.whitespace); + for (std.enums.values(smi.Furiosa.Governor)) |profile| { + if (std.mem.endsWith(u8, line, @tagName(profile))) return @backingInt(profile); + } + return null; +} + +/// Alive when the device state is 0. +fn liveness(mgmt: smi.sysfs.Dir) ?u64 { + var buffer: [64]u8 = undefined; + const state = mgmt.read("device_state", &buffer) catch return null; + return @backingInt(if (std.mem.eql(u8, state, "0")) smi.Furiosa.Liveness.alive else .unknown); +} + +/// The bitmask that ends the file, in hex: "Throttle reason = 0x00000000". +fn throttleReasons(mgmt: smi.sysfs.Dir) ?u64 { + var buffer: [128]u8 = undefined; + const text = mgmt.read("throttle_reason", &buffer) catch return null; + const start = if (std.mem.findLastAny(u8, text, &std.ascii.whitespace)) |space| space + 1 else 0; + return hex(text[start..]); +} + +/// The hex number of `key` in a file of "key: value" lines. +fn hexField(text: []const u8, key: []const u8) ?u64 { + var lines = std.mem.tokenizeScalar(u8, text, '\n'); + while (lines.next()) |line| { + if (smi.sysfs.fieldValue(std.mem.trim(u8, line, " \t"), key)) |value| return hex(value); + } + return null; +} + +/// A number in hex, with or without a 0x prefix. +fn hex(text: []const u8) ?u64 { + const digits = std.mem.cutPrefix(u8, text, "0x") orelse std.mem.cutPrefix(u8, text, "0X") orelse text; + return std.fmt.parseInt(u64, digits, 16) catch null; +} + +fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { + const self: *Furiosa = @alignCast(@fieldParentPtr("platform", platform)); + // The cores each process holds on each chip, from the chip and core nodes it has open. + var held: std.AutoArrayHashMapUnmanaged(struct { u32, u16 }, Cores) = .empty; + var holders: smi.proc.FdHolders = .init(self.io, &.{"/dev/rngd/"}); + defer holders.deinit(); + while (holders.next()) |holder| { + const node = Node.parse(holder.rest) orelse continue; + const device = self.deviceOf(node.chip) orelse continue; + const entry = try held.getOrPutValue(scratch, .{ holder.pid, device }, .empty); + entry.value_ptr.setUnion(node.cores); + } + for (held.keys(), held.values()) |key, set| { + const pid, const device = key; + try out.append(self.chips[device].row(pid, device, set, samples[device].furiosa)); + } +} + +fn deviceOf(self: *const Furiosa, node: u8) ?u16 { + for (self.chips, 0..) |chip, index| { + if (chip.node == node) return @intCast(index); + } + return null; +} + +/// The chip and cores a node of /dev/rngd names: npu, npupe or npupe-. +/// A bare npu is core 0, as furiosa-smi-go reads it. Other nodes, such as npuch0, name no core. +const Node = struct { + chip: u8, + cores: Cores, + + fn parse(name: []const u8) ?Node { + const rest = std.mem.cutPrefix(u8, name, "npu") orelse return null; + const end = std.mem.findNone(u8, rest, "0123456789") orelse rest.len; + const chip = decimal(rest[0..end]) orelse return null; + const range = if (end == rest.len) "0" else std.mem.cutPrefix(u8, rest[end..], "pe") orelse return null; + const first, const last = std.mem.cutScalar(u8, range, '-') orelse .{ range, range }; + const start = decimal(first) orelse return null; + const stop = decimal(last) orelse return null; + if (start > stop or stop >= cores) return null; + var set: Cores = .empty; + set.setRangeValue(.{ .start = start, .end = @as(usize, stop) + 1 }, true); + return .{ .chip = chip, .cores = set }; + } +}; + +/// N in a management node name, rngd!npumgmt. +fn mgmtNode(name: []const u8) ?u8 { + const rest = std.mem.cutPrefix(u8, name, "rngd!npu") orelse return null; + return decimal(std.mem.cutSuffix(u8, rest, "mgmt") orelse return null); +} + +fn decimal(text: []const u8) ?u8 { + if (std.mem.findNone(u8, text, "0123456789") != null) return null; + return std.fmt.parseInt(u8, text, 10) catch null; +} + +/// A string of a sysfs file, copied into `arena`, or null when it is missing or empty. +fn string(arena: std.mem.Allocator, dir: smi.sysfs.Dir, file: []const u8) ?[]const u8 { + var buffer: [128]u8 = undefined; + const text = dir.read(file, &buffer) catch return null; + if (text.len == 0) return null; + return arena.dupe(u8, text) catch null; +} + +test Node { + try std.testing.expectEqual(Node{ .chip = 3, .cores = .{ .mask = 0x0f } }, Node.parse("npu3pe0-3").?); + try std.testing.expectEqual(Node{ .chip = 0, .cores = .{ .mask = 0x20 } }, Node.parse("npu0pe5").?); + try std.testing.expectEqual(Node{ .chip = 12, .cores = .{ .mask = 0x01 } }, Node.parse("npu12").?); + for ([_][]const u8{ "npu0ch0", "npu0ch0r", "npu0mgmt", "npu0bar4", "npu0dmar", "npu0pe3-1", "npu0pe8", "npu0pe", "npu" }) |name| { + try std.testing.expectEqual(null, Node.parse(name)); + } + try std.testing.expectEqual(7, mgmtNode("rngd!npu7mgmt")); + try std.testing.expectEqual(null, mgmtNode("rngd!npu7pe0")); +} + +test "utilization and memory of fused cores" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + const gib: u64 = 1 << 30; + const share = models[0].memoryBytes() / cores; + + // Cores 0 to 3 are one group, core 5 is alone, and core 6 was released after the occupancy was read: + // the files of a core not in use are empty. + try tmp.dir.createDirPath(io, "rngd!npu0mgmt"); + try tmp.dir.createDirPath(io, "rngd!npu0pe6"); + try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0pe6/metrics", .data = "" }); + try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0pe6/alloc_status", .data = "" }); + try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0mgmt/pe_occupancy", .data = "1\n1\n1\n1\n0\n1\n1\n0\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0mgmt/npu_governor", .data = "powersave\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0mgmt/device_state", .data = "0\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "rngd!npu0mgmt/throttle_reason", .data = "Throttle reason = 0x00000003\n" }); + var text: [128]u8 = undefined; + var path: [32]u8 = undefined; + for ([_]u8{ 0, 1, 2, 3, 5 }) |core| { + const fused = core < 4; + try tmp.dir.createDirPath(io, try std.fmt.bufPrint(&path, "rngd!npu0pe{d}", .{core})); + try tmp.dir.writeFile(io, .{ + .sub_path = try std.fmt.bufPrint(&path, "rngd!npu0pe{d}/alloc_status", .{core}), + .data = try std.fmt.bufPrint(&text, "dram_capacity: 0x{x}\ndram_usage: 0x{x}\nshared_capacity: 0x0\nshared_usage: 0x0\n", .{ + if (fused) 4 * share else share, if (fused) 3 * gib else gib, + }), + }); + } + + const class: smi.sysfs.Dir = .{ .handle = tmp.dir, .io = io }; + var chips = [_]Chip{.{ .node = 0, .model = &models[0], .mgmt = try class.openDir("rngd!npu0mgmt"), .hwmon = null, .usable = .full }}; + defer chips[0].mgmt.close(); + var furiosa: Furiosa = .{ .platform = .{ .devices = &.{.{ .target = .furiosa }}, .vtable = &vtable }, .io = io, .class = class, .chips = &chips }; + + // Between the polls, the group spends half its cycles on tasks and core 5 all of them. + for ([_]u64{ 0, 37 }, [_]u64{ 1000, 2000 }) |util, cycles| { + for ([_]u8{ 0, 1, 2, 3, 5 }) |core| { + const task = if (core == 5) cycles else cycles / 2 + 250; + try tmp.dir.writeFile(io, .{ + .sub_path = try std.fmt.bufPrint(&path, "rngd!npu0pe{d}/metrics", .{core}), + .data = try std.fmt.bufPrint(&text, "CycleCount: 0x{x}\nTaskCycleCount: 0x{x}\n", .{ cycles, task }), + }); + } + var samples = [_]smi.Sample{.init(.furiosa)}; + poll(&furiosa.platform, std.testing.allocator, &samples); + const chip = samples[0].furiosa; + try std.testing.expectEqual(util, chip.util_percent); + try std.testing.expectEqual(4 * gib, chip.mem_used_bytes); + try std.testing.expectEqual(@backingInt(smi.Furiosa.Governor.powersave), chip.governor); + try std.testing.expectEqual(@backingInt(smi.Furiosa.Liveness.alive), chip.liveness); + try std.testing.expectEqual(0b11, chip.throttle_reasons); + } + + // A process that holds the group sees its share. + const row = chips[0].row(7, 0, .{ .mask = 0x0f }, .{ .util_percent = 37, .mem_used_bytes = 4 * gib }); + try std.testing.expectEqual(50, row.dev_util_percent); + try std.testing.expectEqual(3 * gib / 1024, row.dev_mem_kib); +} diff --git a/bin/zml-smi/smi/model.zig b/bin/zml-smi/smi/model.zig index b6caf35f..9bff2415 100644 --- a/bin/zml-smi/smi/model.zig +++ b/bin/zml-smi/smi/model.zig @@ -1,6 +1,6 @@ const std = @import("std"); -pub const Target = enum { cuda, rocm, oneapi, neuron, tpu }; +pub const Target = enum { cuda, rocm, oneapi, neuron, tpu, furiosa }; /// What does not change while we run. Its strings live as long as the program. pub const Device = struct { @@ -33,6 +33,7 @@ pub const Sample = union(Target) { oneapi: Gpu, neuron: Neuron, tpu: Tpu, + furiosa: Furiosa, /// A sample of the given target with every field null. pub fn init(target: Target) Sample { @@ -98,6 +99,29 @@ pub const Tpu = struct { mem_total_bytes: ?u64 = null, }; +/// One FuriosaAI RNGD chip. +pub const Furiosa = struct { + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + power_mw: ?u64 = null, + power_limit_mw: ?u64 = null, + /// Degrees Celsius, at the hottest point of the SoC. + temperature: ?u64 = null, + /// Degrees Celsius. + ambient_temperature: ?u64 = null, + /// A `Governor`. + governor: ?u64 = null, + /// A `Liveness`. + liveness: ?u64 = null, + /// A bitmask whose bits are `ThrottleReason`s. + throttle_reasons: ?u64 = null, + + pub const Governor = enum(u64) { performance, powersave }; + pub const Liveness = enum(u64) { unknown, alive }; + pub const ThrottleReason = enum(u6) { idle, thermal_slowdown, app_power_cap, app_clock_cap, hw_clock_cap, hw_bus_limit, hw_power_cap, gov_clock_cap, other }; +}; + pub const ProcessInfo = struct { pid: u32 = 0, /// The global index of the device, once the collector has the row. diff --git a/bin/zml-smi/smi/prometheus.zig b/bin/zml-smi/smi/prometheus.zig index 6cff0cfd..787f4c72 100644 --- a/bin/zml-smi/smi/prometheus.zig +++ b/bin/zml-smi/smi/prometheus.zig @@ -62,6 +62,19 @@ const tpu: Metrics(smi.Tpu) = .{ .mem_total_bytes = memory_total, }; +const furiosa: Metrics(smi.Furiosa) = .{ + .util_percent = utilization, + .mem_used_bytes = memory_used, + .mem_total_bytes = memory_total, + .power_mw = gpu.power_mw, + .power_limit_mw = gpu.power_limit_mw, + .temperature = gpu.temperature, + .ambient_temperature = .{ .name = "zml_device_ambient_temperature_celsius", .help = "Ambient temperature in celsius" }, + .governor = .{ .name = "zml_device_governor", .help = "Governor profile: 0 performance, 1 powersave" }, + .liveness = .{ .name = "zml_device_liveness", .help = "Liveness: 1 alive, 0 unknown" }, + .throttle_reasons = .{ .name = "zml_device_throttle_reasons", .help = "Active throttle reasons as a bitmask" }, +}; + const host: Metrics(smi.HostData) = .{ .hostname = null, .kernel = null, @@ -80,6 +93,7 @@ fn metricsOf(comptime T: type) Metrics(T) { smi.Gpu => gpu, smi.Neuron => neuron, smi.Tpu => tpu, + smi.Furiosa => furiosa, else => @compileError("no metrics for " ++ @typeName(T)), }; } @@ -89,7 +103,7 @@ fn metricsOf(comptime T: type) Metrics(T) { const families: []const Metric = families: { @setEvalBranchQuota(10_000); var list: []const Metric = &.{}; - for (.{ smi.Tpu, smi.Gpu, smi.Neuron }) |T| { + for (.{ smi.Tpu, smi.Gpu, smi.Neuron, smi.Furiosa }) |T| { const metrics = metricsOf(T); next: for (@typeInfo(T).@"struct".field_names) |field| { const metric = @field(metrics, field) orelse continue; diff --git a/bin/zml-smi/smi/smi.zig b/bin/zml-smi/smi/smi.zig index d800803a..d41c9075 100644 --- a/bin/zml-smi/smi/smi.zig +++ b/bin/zml-smi/smi/smi.zig @@ -9,6 +9,7 @@ pub const Row = model.Row; pub const Gpu = model.Gpu; pub const Neuron = model.Neuron; pub const Tpu = model.Tpu; +pub const Furiosa = model.Furiosa; pub const ProcessInfo = model.ProcessInfo; pub const HostData = model.HostData; pub const Platform = model.Platform; diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index 8d16df45..1dc399de 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -13,6 +13,7 @@ zig_library( "lib/utils.zig", "pages/detail.zig", "pages/detail/detail_common.zig", + "pages/detail/furiosa_detail.zig", "pages/detail/gpu_detail.zig", "pages/detail/neuron_detail.zig", "pages/detail/tpu_detail.zig", diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index c90838b5..12e87688 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -102,6 +102,11 @@ pub const SystemState = struct { self.record(&self.history.temp[i], 0); self.record(&self.history.power[i], 0); }, + .furiosa => |chip| { + self.recordUsage(i, chip.util_percent, chip.mem_used_bytes, chip.mem_total_bytes); + self.record(&self.history.temp[i], chip.temperature orelse 0); + self.record(&self.history.power[i], chip.power_mw orelse 0); + }, } } } diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig index f5d50c67..9b0fdd15 100644 --- a/bin/zml-smi/tui/pages/detail.zig +++ b/bin/zml-smi/tui/pages/detail.zig @@ -7,6 +7,7 @@ const ProcessTable = @import("../widgets/process_table.zig"); const gpu_detail = @import("detail/gpu_detail.zig"); const neuron_detail = @import("detail/neuron_detail.zig"); const tpu_detail = @import("detail/tpu_detail.zig"); +const furiosa_detail = @import("detail/furiosa_detail.zig"); const Detail = @This(); @@ -24,5 +25,6 @@ pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error! .cuda, .rocm, .oneapi => |gpu| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, device, gpu, wgt), .neuron => |core| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, device, core, wgt), .tpu => |chip| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, device, chip, wgt), + .furiosa => |chip| try furiosa_detail.draw(self.state, self.process_table, ctx, w, id, device, chip, wgt), }; } diff --git a/bin/zml-smi/tui/pages/detail/furiosa_detail.zig b/bin/zml-smi/tui/pages/detail/furiosa_detail.zig new file mode 100644 index 00000000..877a4f0d --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/furiosa_detail.zig @@ -0,0 +1,113 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const smi = @import("zml-smi"); +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const Chart = @import("../../widgets/chart.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const MetricCard = @import("../../widgets/metric_card.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u16, + device: smi.Device, + chip: smi.Furiosa, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + const header = try common.headerText(ctx.arena, id, device); + + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "NPU Utilization", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{chip.util_percent orelse 0}), + }); + + const mem_used = chip.mem_used_bytes orelse 0; + const mem_total = chip.mem_total_bytes orelse 0; + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "DRAM {d} / {d} MB", .{ utils.bytesToMb(chip.mem_used_bytes), utils.bytesToMb(chip.mem_total_bytes) }), + }); + + const temp_chart = try common.historyChart(ctx, state, id, state.history.temp, .{ + .title = "Temperature", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C", .{chip.temperature orelse 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "Ambient {d}\u{00b0}C", .{chip.ambient_temperature orelse 0}), + .y_min = 20, + .y_max = 100, + .y_unit = "\u{00b0}C", + }); + + const power_limit_w = (chip.power_limit_mw orelse 0) / 1000; + const power_raw = state.history.power[id].items; + const power_watts = try ctx.arena.alloc(u64, power_raw.len); + for (power_raw, power_watts) |mw, *watts| watts.* = mw / 1000; + const y_max = if (power_limit_w > 0) power_limit_w else 1; + const power_chart: Chart = .{ + .title = "Power", + .data = try utils.normalizeRange(ctx.arena, power_watts, 0, y_max), + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}W", .{(chip.power_mw orelse 0) / 1000}), + .info_line = if (power_limit_w > 0) try std.fmt.allocPrint(ctx.arena, "Limit {d}W", .{power_limit_w}) else "N/A", + .y_min = 0, + .y_max = @intCast(@min(y_max, std.math.maxInt(u32))), + .y_unit = "W", + .chart_height = 8, + .tui_refresh_rate = state.tui_refresh_rate, + }; + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart), ui.widget(&temp_chart), ui.widget(&power_chart) }, + .min_child_width = 60, + .gap = 1, + }; + + const state_card: MetricCard = .{ + .title = "State", + .lines = &.{ + .{ .label = "Liveness ", .value = try code(ctx.arena, smi.Furiosa.Liveness, chip.liveness) }, + .{ .label = "Governor ", .value = try code(ctx.arena, smi.Furiosa.Governor, chip.governor) }, + .{ .label = "Throttle ", .value = try flags(ctx.arena, smi.Furiosa.ThrottleReason, chip.throttle_reasons) }, + }, + }; + + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(&state_card), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} + +/// The name of a code, or its number when it has none. +fn code(arena: std.mem.Allocator, comptime E: type, value: ?u64) std.mem.Allocator.Error![]const u8 { + const number = value orelse return " N/A"; + if (std.enums.fromInt(E, number)) |known| return std.fmt.allocPrint(arena, " {t}", .{known}); + return std.fmt.allocPrint(arena, " {d}", .{number}); +} + +/// The names of the bits set in a bitmask, or their positions when they have none. +fn flags(arena: std.mem.Allocator, comptime E: type, value: ?u64) std.mem.Allocator.Error![]const u8 { + const mask = value orelse return " N/A"; + if (mask == 0) return " none"; + var text: std.Io.Writer.Allocating = .init(arena); + for (0..64) |bit| { + if (mask >> @intCast(bit) & 1 == 0) continue; + const separator = if (text.written().len == 0) " " else ", "; + if (std.enums.fromInt(E, bit)) |flag| { + text.writer.print("{s}{t}", .{ separator, flag }) catch return error.OutOfMemory; + } else { + text.writer.print("{s}bit {d}", .{ separator, bit }) catch return error.OutOfMemory; + } + } + return text.written(); +} diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig index ffd81565..b3aa77f2 100644 --- a/bin/zml-smi/tui/widgets/device_card.zig +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -76,6 +76,14 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx .mem_used = chip.mem_used_bytes, .mem_total = chip.mem_total_bytes, }, + .furiosa => |chip| .{ + .label = "NPU", + .util_label = "NPU", + .util_percent = chip.util_percent, + .mem_used = chip.mem_used_bytes, + .mem_total = chip.mem_total_bytes, + .suffix = try gpuSuffix(ctx.arena, .{ .temperature = chip.temperature, .power_mw = chip.power_mw, .power_limit_mw = chip.power_limit_mw }), + }, }; const util_percent: u8 = @intCast(@min(usage.util_percent orelse 0, 100)); const mem_used = usage.mem_used orelse 0; From 25ccb38e79dd4291cbbf26f05f595fd43746f06f Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Sat, 14 Nov 2026 11:33:03 +0100 Subject: [PATCH 16/19] bin/zml-smi: let a platform fill host metrics A platform with a host entry fills CPU and system power in its own task. The host step copies what it measured, starting each round from what the host read once, so a value not measured this round is null. --- bin/zml-smi/smi/collector.zig | 35 +++++++++++++++++++++++++++++++--- bin/zml-smi/smi/model.zig | 6 ++++++ bin/zml-smi/smi/prometheus.zig | 2 ++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/bin/zml-smi/smi/collector.zig b/bin/zml-smi/smi/collector.zig index 3cd2053b..450913d0 100644 --- a/bin/zml-smi/smi/collector.zig +++ b/bin/zml-smi/smi/collector.zig @@ -9,6 +9,8 @@ pub const Collector = struct { /// Every process list merged into one, with host details. Written by the host step. processes: smi.DoubleBuffer(smi.ProcessList), host: smi.DoubleBuffer(smi.HostData), + /// What the host reader read once. Each host step starts from it, so a value a platform did not measure is null. + host_once: smi.HostData, /// A source derives values from two polls, so a one-shot print runs two rounds. two_readings: bool = false, @@ -32,7 +34,12 @@ pub const Collector = struct { scratch: std.heap.ArenaAllocator, }; - const Side = struct { samples: []smi.Sample, processes: smi.ProcessList }; + const Side = struct { + samples: []smi.Sample, + processes: smi.ProcessList, + /// What the platform measured of the host. + host: smi.HostData = .{}, + }; pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, options: Options) Collector { var host: smi.HostData = .{}; @@ -40,6 +47,7 @@ pub const Collector = struct { return .{ .processes = .{ .values = .{ .init(gpa), .init(gpa) } }, .host = .{ .values = .{ host, host } }, + .host_once = host, .host_reader = host_reader, .host_scratch = .init(gpa), .arena = arena, @@ -153,6 +161,10 @@ pub const Collector = struct { back.processes.clear(); } } + if (platform.vtable.host) |measureHost| { + back.host = .{}; + measureHost(platform, &back.host); + } source.sides.swap(self.io); } @@ -174,8 +186,16 @@ pub const Collector = struct { _ = self.host_scratch.reset(.retain_capacity); const host = self.host.back(); - host.* = self.host.front().*; + host.* = self.host_once; self.host_reader.sample(host); + for (self.sources.items) |source| { + if (source.platform.vtable.host == null) continue; + const measured = source.sides.lock(self.io).host; + source.sides.unlock(self.io); + inline for (@typeInfo(smi.HostData).@"struct".field_names) |field| { + if (@field(measured, field)) |value| @field(host, field) = value; + } + } self.processes.swap(self.io); self.host.swap(self.io); @@ -186,7 +206,7 @@ const Fake = struct { platform: smi.Platform, polls: u64 = 0, - const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes }; + const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .host = host }; fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { const self: *Fake = @alignCast(@fieldParentPtr("platform", platform)); @@ -203,6 +223,12 @@ const Fake = struct { try out.append(.{ .pid = 7, .device_idx = @intCast(self.platform.devices.len) }); try out.append(.{ .pid = 42, .device_idx = @intCast(self.platform.devices.len - 1), .remote = true, .username = try out.dupe("alice") }); } + + /// Measures the CPU power on odd polls only. + fn host(platform: *smi.Platform, data: *smi.HostData) void { + const self: *Fake = @alignCast(@fieldParentPtr("platform", platform)); + if (self.polls % 2 == 1) data.cpu_power_mw = self.polls; + } }; test "collector rounds" { @@ -239,10 +265,13 @@ test "collector rounds" { try std.testing.expectEqual(2, merged.rows.items[1].device_idx); try std.testing.expectEqualStrings("alice", merged.rows.items[1].username); } + try std.testing.expectEqual(1, collector.host.read(io).cpu_power_mw); // `processes` fails: no rows this round, not the previous ones. collector.round(); try std.testing.expectEqual(2, collector.sample(@fromBackingInt(0)).cuda.util_percent); + // Host values a platform did not measure this round are null, not the previous ones. + try std.testing.expectEqual(null, collector.host.read(io).cpu_power_mw); const merged = collector.processes.lock(io); defer collector.processes.unlock(io); try std.testing.expectEqual(0, merged.rows.items.len); diff --git a/bin/zml-smi/smi/model.zig b/bin/zml-smi/smi/model.zig index 9bff2415..ebabe360 100644 --- a/bin/zml-smi/smi/model.zig +++ b/bin/zml-smi/smi/model.zig @@ -148,6 +148,9 @@ pub const HostData = struct { load_5: ?f32 = null, load_15: ?f32 = null, uptime_seconds: ?u64 = null, + /// Measured by a platform, through `Platform.VTable.host`. + cpu_power_mw: ?u64 = null, + system_power_mw: ?u64 = null, }; /// A source of devices: a local platform or a remote host. @@ -165,6 +168,9 @@ pub const Platform = struct { /// Called right after `poll`, with the samples it just filled. A row's device is its position in `devices`. processes: ?*const fn (platform: *Platform, scratch: std.mem.Allocator, samples: []const Sample, out: *ProcessList) ProcessList.Error!void = null, deinit: ?*const fn (platform: *Platform) void = null, + /// For a platform that also measures the host. Called right after `processes`, in the same task, + /// with host data whose fields start null: it fills those it read. + host: ?*const fn (platform: *Platform, data: *HostData) void = null, }; }; diff --git a/bin/zml-smi/smi/prometheus.zig b/bin/zml-smi/smi/prometheus.zig index 787f4c72..05713dcf 100644 --- a/bin/zml-smi/smi/prometheus.zig +++ b/bin/zml-smi/smi/prometheus.zig @@ -86,6 +86,8 @@ const host: Metrics(smi.HostData) = .{ .load_5 = .{ .name = "zml_host_load_5", .help = "5-minute load average", .decimals = 2 }, .load_15 = .{ .name = "zml_host_load_15", .help = "15-minute load average", .decimals = 2 }, .uptime_seconds = .{ .name = "zml_host_uptime_seconds", .help = "System uptime in seconds" }, + .cpu_power_mw = .{ .name = "zml_host_cpu_power_watts", .help = "CPU power draw in watts", .decimals = 3 }, + .system_power_mw = .{ .name = "zml_host_system_power_watts", .help = "System power draw in watts", .decimals = 3 }, }; fn metricsOf(comptime T: type) Metrics(T) { From 202887603e541c23a57a686f6fcd905177e099cd Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Sat, 14 Nov 2026 11:34:28 +0100 Subject: [PATCH 17/19] bin/zml-smi: list processes on macOS The macOS host adds user, command, CPU and RSS to the local rows, from libproc. The user and the executable path of every process can be read; the CPU and RSS of another user's process read 0 unless zml-smi runs as root. CPU share is CPU time over elapsed time times the number of CPUs, in tenths of a percent, as on Linux. --- bin/zml-smi/smi/host/macos.zig | 140 +++++++++++++++++++++++++++++++-- bin/zml-smi/smi/smi.zig | 1 + 2 files changed, 135 insertions(+), 6 deletions(-) diff --git a/bin/zml-smi/smi/host/macos.zig b/bin/zml-smi/smi/host/macos.zig index 487078f1..8fcd1199 100644 --- a/bin/zml-smi/smi/host/macos.zig +++ b/bin/zml-smi/smi/host/macos.zig @@ -1,21 +1,33 @@ -//! The host, from sysctl and the Mach host statistics. +//! The host, from sysctl, the Mach host statistics and libproc. const std = @import("std"); const smi = @import("../smi.zig"); const c = std.c; pub const Host = struct { + io: std.Io, + /// CPU nanoseconds of each process. + cpu_time: smi.Counters(u32), + cpus: u64, + /// Converts the CPU time libproc reports, in Mach ticks, to nanoseconds. + timebase: c.mach_timebase_info_data, + /// Reads what does not change: hostname, kernel, CPU, core count and total memory. - pub fn init(arena: std.mem.Allocator, _: std.mem.Allocator, _: std.Io, data: *smi.HostData) Host { + pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, data: *smi.HostData) Host { data.hostname = sysctlString(arena, "kern.hostname"); data.kernel = sysctlString(arena, "kern.osrelease"); data.cpu_name = sysctlString(arena, "machdep.cpu.brand_string"); data.cpu_cores = if (sysctl(u32, "hw.logicalcpu")) |cores| cores else null; data.mem_total_kib = if (sysctl(u64, "hw.memsize")) |bytes| bytes / 1024 else null; - return .{}; + + var timebase: c.mach_timebase_info_data = .{ .numer = 1, .denom = 1 }; + if (c.mach_timebase_info(&timebase) != KERN_SUCCESS or timebase.denom == 0) timebase = .{ .numer = 1, .denom = 1 }; + return .{ .io = io, .cpu_time = .init(gpa), .cpus = @max(1, data.cpu_cores orelse 1), .timebase = timebase }; } - pub fn deinit(_: *Host) void {} + pub fn deinit(self: *Host) void { + self.cpu_time.deinit(); + } /// Overwrites what changes: available memory, load and uptime. A read that fails gives null. pub fn sample(_: *Host, data: *smi.HostData) void { @@ -34,10 +46,100 @@ pub const Host = struct { } } - /// The processes all come from remote hosts, which filled their rows. - pub fn enrich(_: *Host, _: std.mem.Allocator, _: std.mem.Allocator, _: []smi.ProcessInfo) void {} + /// Adds user, command, CPU and RSS to the local rows. Each process is read once, + /// however many rows it has. `strings` holds the strings of the rows. + /// The CPU and RSS of another user's process read 0, unless zml-smi runs as root. + pub fn enrich(self: *Host, strings: std.mem.Allocator, scratch: std.mem.Allocator, rows: []smi.ProcessInfo) void { + defer self.cpu_time.sweep(); + + var first_row: std.AutoHashMapUnmanaged(u32, usize) = .empty; + for (rows, 0..) |*row, i| { + // A remote host already filled its rows. + if (row.remote) continue; + const entry = first_row.getOrPut(scratch, row.pid) catch continue; + if (entry.found_existing) { + const done = rows[entry.value_ptr.*]; + row.uid, row.username, row.comm, row.cpu_percent, row.rss_kib = .{ done.uid, done.username, done.comm, done.cpu_percent, done.rss_kib }; + continue; + } + entry.value_ptr.* = i; + self.read(strings, row); + } + } + + fn read(self: *Host, strings: std.mem.Allocator, row: *smi.ProcessInfo) void { + const pid = std.math.cast(c_int, row.pid) orelse return; + + var short: ProcBsdShortInfo = undefined; + if (proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 0, &short, @sizeOf(ProcBsdShortInfo)) == @sizeOf(ProcBsdShortInfo)) { + row.uid = short.uid; + if (c.getpwuid(row.uid)) |user| { + if (user.name) |name| row.username = strings.dupe(u8, std.mem.span(name)) catch ""; + } + } + + var path: [PROC_PIDPATHINFO_MAXSIZE]u8 = undefined; + const length = proc_pidpath(pid, &path, path.len); + if (length > 0) row.comm = strings.dupe(u8, path[0..@intCast(length)]) catch ""; + + var task: ProcTaskInfo = undefined; + if (proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &task, @sizeOf(ProcTaskInfo)) != @sizeOf(ProcTaskInfo)) return; + row.rss_kib = task.resident_size / 1024; + const ticks = @as(u128, task.total_user) + task.total_system; + const nanoseconds = std.math.lossyCast(u64, ticks * self.timebase.numer / self.timebase.denom); + const growth = (self.cpu_time.advance(row.pid, self.io, nanoseconds) catch return) orelse return; + // Tenths of a percent of every CPU, as on Linux. + const whole = @as(u128, @intCast(growth.elapsed.nanoseconds)) * self.cpus; + row.cpu_percent = @intCast(@min(@as(u128, growth.amount) * 1000 / whole, std.math.maxInt(u16))); + } }; +const PROC_PIDTASKINFO: c_int = 4; +const PROC_PIDT_SHORTBSDINFO: c_int = 13; +const PROC_PIDPATHINFO_MAXSIZE = 4096; + +/// struct proc_bsdshortinfo, which libproc gives for every process. +const ProcBsdShortInfo = extern struct { + pid: u32, + ppid: u32, + pgid: u32, + status: u32, + comm: [16]u8, + flags: u32, + uid: c.uid_t, + gid: c.gid_t, + ruid: c.uid_t, + rgid: c.gid_t, + svuid: c.uid_t, + svgid: c.gid_t, + rfu: u32, +}; + +/// struct proc_taskinfo, which libproc gives for the user's own processes, or any as root. +const ProcTaskInfo = extern struct { + virtual_size: u64, + resident_size: u64, + total_user: u64, + total_system: u64, + threads_user: u64, + threads_system: u64, + policy: i32, + faults: i32, + pageins: i32, + cow_faults: i32, + messages_sent: i32, + messages_received: i32, + syscalls_mach: i32, + syscalls_unix: i32, + csw: i32, + threadnum: i32, + numrunning: i32, + priority: i32, +}; + +extern "c" fn proc_pidinfo(pid: c_int, flavor: c_int, arg: u64, buffer: *anyopaque, size: c_int) c_int; +extern "c" fn proc_pidpath(pid: c_int, buffer: [*]u8, size: u32) c_int; + const vm_statistics64 = extern struct { free_count: c_uint, active_count: c_uint, @@ -97,3 +199,29 @@ fn sysctlString(arena: std.mem.Allocator, name: [*:0]const u8) ?[]const u8 { if (c.sysctlbyname(name, &buffer, &len, null, 0) != 0) return null; return arena.dupe(u8, std.mem.sliceTo(buffer[0..len], 0)) catch null; } + +test "enrich" { + const io = std.testing.io; + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + var data: smi.HostData = .{}; + var host: Host = .init(arena.allocator(), std.testing.allocator, io, &data); + defer host.deinit(); + + // This process on two devices, launchd, and a row a remote host filled. + const self: u32 = @intCast(c.getpid()); + var rows = [_]smi.ProcessInfo{ .{ .pid = self }, .{ .pid = 1 }, .{ .pid = self, .device_idx = 1 }, .{ .pid = 1, .remote = true } }; + host.enrich(arena.allocator(), arena.allocator(), &rows); + try std.testing.expectEqual(c.getuid(), rows[0].uid); + try std.testing.expect(rows[0].rss_kib > 0); + try std.testing.expectEqualStrings(rows[0].comm, rows[2].comm); + try std.testing.expectEqualStrings("root", rows[1].username); + try std.testing.expectEqualStrings("/sbin/launchd", rows[1].comm); + try std.testing.expectEqualStrings("", rows[3].comm); + + // CPU share needs a second reading. + const start: std.Io.Timestamp = .now(io, .awake); + while (start.untilNow(io, .awake).toMilliseconds() < 20) {} + host.enrich(arena.allocator(), arena.allocator(), &rows); + try std.testing.expect(rows[0].cpu_percent > 0); +} diff --git a/bin/zml-smi/smi/smi.zig b/bin/zml-smi/smi/smi.zig index d41c9075..d8c1cb69 100644 --- a/bin/zml-smi/smi/smi.zig +++ b/bin/zml-smi/smi/smi.zig @@ -53,4 +53,5 @@ test { _ = @import("sandbox.zig"); _ = sysfs; _ = units; + _ = Host; } From 44f13f51f690e3cf0e82118012c857a5fb61a6b7 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Sat, 14 Nov 2026 12:13:53 +0100 Subject: [PATCH 18/19] bin/zml-smi: add Metal Apple GPUs on macOS, with their own struct and target. Utilization and memory in use come from the accelerator's PerformanceStatistics in the IORegistry, and memory total is the host's. Power, the Neural Engine's power and the graphics clock come from one IOReport delta per poll, so Metal sets two_readings. Temperature, fans and the system power come from the SMC. Processes are the GPU's clients, with their GPU time. Metal also measures the host: CPU power from the same IOReport delta, from the PMP histograms once the Energy Model reads 0 for the CPU, as on macOS 27, and system power from the SMC. CoreFoundation, IOKit and IOReport are opened by install name, since the hermetic toolchain has nothing to link against. The IOReport, SMC and GPU time readers are ported from mactop v2.1.6 (MIT). --- bin/zml-smi/BUILD.bazel | 3 + bin/zml-smi/platforms.zig | 6 + bin/zml-smi/platforms/metal/BUILD.bazel | 19 + bin/zml-smi/platforms/metal/Metal.zig | 643 ++++++++++++++++++ bin/zml-smi/platforms/metal/Smc.zig | 196 ++++++ bin/zml-smi/platforms/metal/iokit.zig | 53 ++ bin/zml-smi/smi/model.zig | 19 +- bin/zml-smi/smi/prometheus.zig | 18 +- bin/zml-smi/smi/smi.zig | 1 + bin/zml-smi/tui/BUILD.bazel | 1 + bin/zml-smi/tui/data.zig | 5 + bin/zml-smi/tui/pages/detail.zig | 2 + bin/zml-smi/tui/pages/detail/metal_detail.zig | 103 +++ bin/zml-smi/tui/widgets/device_card.zig | 9 + 14 files changed, 1075 insertions(+), 3 deletions(-) create mode 100644 bin/zml-smi/platforms/metal/BUILD.bazel create mode 100644 bin/zml-smi/platforms/metal/Metal.zig create mode 100644 bin/zml-smi/platforms/metal/Smc.zig create mode 100644 bin/zml-smi/platforms/metal/iokit.zig create mode 100644 bin/zml-smi/tui/pages/detail/metal_detail.zig diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index 14d4fcf7..acfc51e9 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -95,6 +95,9 @@ zig_binary( "@llvm//platforms/config:linux_aarch64": [ "//bin/zml-smi/platforms/nvml", ], + "@llvm//platforms/config:macos_aarch64": [ + "//bin/zml-smi/platforms/metal", + ], "//conditions:default": [], }), ) diff --git a/bin/zml-smi/platforms.zig b/bin/zml-smi/platforms.zig index 5b92f1cf..e21c8a19 100644 --- a/bin/zml-smi/platforms.zig +++ b/bin/zml-smi/platforms.zig @@ -16,5 +16,11 @@ pub const all = switch (builtin.os.tag) { }, else => .{}, }, + .macos => switch (builtin.cpu.arch) { + .aarch64 => .{ + @import("zml-smi/platforms/metal"), + }, + else => .{}, + }, else => .{}, }; diff --git a/bin/zml-smi/platforms/metal/BUILD.bazel b/bin/zml-smi/platforms/metal/BUILD.bazel new file mode 100644 index 00000000..5e1c68f8 --- /dev/null +++ b/bin/zml-smi/platforms/metal/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "metal", + srcs = [ + "Smc.zig", + "iokit.zig", + ], + import_name = "zml-smi/platforms/metal", + main = "Metal.zig", + target_compatible_with = ["@platforms//os:macos"], + visibility = ["//bin/zml-smi:__subpackages__"], + deps = ["//bin/zml-smi/smi"], +) + +zig_test( + name = "test", + deps = [":metal"], +) diff --git a/bin/zml-smi/platforms/metal/Metal.zig b/bin/zml-smi/platforms/metal/Metal.zig new file mode 100644 index 00000000..e9d5eb8b --- /dev/null +++ b/bin/zml-smi/platforms/metal/Metal.zig @@ -0,0 +1,643 @@ +//! Apple GPUs: utilization and memory from the IORegistry, power and clock from IOReport, +//! temperature and fans from the SMC. It also measures the power of the CPU and of the whole machine. +//! The IOReport readers and the GPU time of each client are ported from mactop v2.1.6 (8dbfaae): +//! internal/app/ioreport.m, internal/app/pmp_names.h and internal/app/native_stats.go. + +// MIT License +// +// Copyright (c) 2024-2026 Carsen Klock +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +const std = @import("std"); +const smi = @import("zml-smi"); +const iokit = @import("iokit.zig"); +const Smc = @import("Smc.zig"); + +const Metal = @This(); +const log = std.log.scoped(.@"zml-smi/metal"); + +platform: smi.Platform, +io: std.Io, +cf: CoreFoundation, +io_kit: iokit.Api, +ioreport: IoReport, +keys: std.EnumArray(Key, *const String), +accelerator: iokit.Object, +smc: ?Smc, +/// The GPU shares the host's memory. +mem_total_bytes: ?u64, +/// The frequency of each active GPU performance state, in MHz, in order. +frequencies: []const u64, +channels: *Dictionary, +subscription: *const Subscription, +/// The sample the next poll makes its delta against. +last: ?Reading = null, +/// macOS 27 reads 0 for the CPU in the Energy Model. Once it does while the PMP +/// histograms show power, they give the CPU power for the rest of the run. +cpu_from_pmp: bool = false, +/// What this round measured of the host, for `host`. +cpu_power_mw: ?u64 = null, +system_power_mw: ?u64 = null, +/// GPU nanoseconds of each process. +busy: smi.Counters(u32), + +const Reading = struct { sample: *const Dictionary, at: std.Io.Timestamp }; + +const vtable: smi.Platform.VTable = .{ .poll = poll, .processes = processes, .deinit = deinit, .host = host }; + +/// The keys Metal looks up, named by their text. Made once in `init`. +const Key = enum { + IOReportChannels, + PerformanceStatistics, + @"Device Utilization %", + @"In use system memory", + IOUserClientCreator, + AppUsage, + accumulatedGPUTime, + model, + IOSourceVersion, + @"voltage-states9-sram", + @"voltage-states9", +}; + +/// Every Apple silicon Mac has a GPU, which `init` finds. +pub fn detect(_: std.Io) bool { + return true; +} + +pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io) !Metal { + var self: Metal = .{ + .platform = undefined, + .io = io, + .cf = try smi.dynlib.open(CoreFoundation, &.{"/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation"}), + .io_kit = try iokit.open(), + .ioreport = try smi.dynlib.open(IoReport, &.{"/usr/lib/libIOReport.dylib"}), + .keys = undefined, + .accelerator = 0, + .smc = null, + .mem_total_bytes = memoryTotal(), + .frequencies = &.{}, + .channels = undefined, + .subscription = undefined, + .busy = .init(gpa), + }; + + self.accelerator = self.io_kit.IOServiceGetMatchingService(iokit.main_port, self.io_kit.IOServiceMatching("AGXAccelerator")); + if (self.accelerator == 0) { + log.warn("no Apple GPU", .{}); + return error.GpuNotFound; + } + errdefer _ = self.io_kit.IOObjectRelease(self.accelerator); + + for (std.enums.values(Key), 0..) |key, made| { + self.keys.set(key, self.cf.CFStringCreateWithCString(null, @tagName(key), utf8) orelse { + for (std.enums.values(Key)[0..made]) |done| self.cf.CFRelease(self.keys.get(done)); + return error.OutOfMemory; + }); + } + errdefer for (self.keys.values) |string| self.cf.CFRelease(string); + + self.channels = try self.copyChannels(); + errdefer self.cf.CFRelease(self.channels); + var subscribed: ?*Dictionary = null; + self.subscription = self.ioreport.IOReportCreateSubscription(null, self.channels, &subscribed, 0, null) orelse { + log.warn("IOReport refused the subscription", .{}); + return error.IoReportUnavailable; + }; + if (subscribed) |channels| self.cf.CFRelease(channels); + errdefer self.cf.CFRelease(self.subscription); + + self.frequencies = try self.gpuFrequencies(arena); + self.smc = Smc.open(arena, self.io_kit) catch |err| smc: { + log.warn("no SMC, so no temperature, fan or system power: {t}", .{err}); + break :smc null; + }; + errdefer if (self.smc) |smc| smc.close(); + + const devices = try arena.alloc(smi.Device, 1); + devices[0] = .{ + .target = .metal, + .name = try self.propertyText(arena, .model), + .driver_version = try self.propertyText(arena, .IOSourceVersion), + }; + self.platform = .{ .devices = devices, .two_readings = true, .vtable = &vtable }; + return self; +} + +fn deinit(platform: *smi.Platform) void { + const self: *Metal = @alignCast(@fieldParentPtr("platform", platform)); + if (self.last) |last| self.cf.CFRelease(last.sample); + self.cf.CFRelease(self.subscription); + self.cf.CFRelease(self.channels); + for (self.keys.values) |string| self.cf.CFRelease(string); + if (self.smc) |smc| smc.close(); + _ = self.io_kit.IOObjectRelease(self.accelerator); + self.busy.deinit(); +} + +fn poll(platform: *smi.Platform, _: std.mem.Allocator, samples: []smi.Sample) void { + const self: *Metal = @alignCast(@fieldParentPtr("platform", platform)); + const gpu = &samples[0].metal; + + gpu.mem_total_bytes = self.mem_total_bytes; + gpu.clock_graphics_max_mhz = if (self.frequencies.len > 0) std.mem.max(u64, self.frequencies) else null; + if (self.property(self.accelerator, .PerformanceStatistics)) |statistics| { + defer self.cf.CFRelease(statistics); + if (self.as(Dictionary, statistics)) |values| { + gpu.util_percent = self.number(self.cf.CFDictionaryGetValue(values, self.keys.get(.@"Device Utilization %"))); + gpu.mem_used_bytes = self.number(self.cf.CFDictionaryGetValue(values, self.keys.get(.@"In use system memory"))); + } + } + + self.cpu_power_mw = null; + self.readReport(gpu); + + self.system_power_mw = null; + if (self.smc) |smc| { + gpu.temperature = smc.temperature(); + gpu.fan_speed_percent = smc.fanPercent(); + self.system_power_mw = smc.systemPower(); + } +} + +fn host(platform: *smi.Platform, data: *smi.HostData) void { + const self: *Metal = @alignCast(@fieldParentPtr("platform", platform)); + data.cpu_power_mw = self.cpu_power_mw; + data.system_power_mw = self.system_power_mw; +} + +/// One row per process with GPU time, which has no GPU memory of its own. +fn processes(platform: *smi.Platform, scratch: std.mem.Allocator, samples: []const smi.Sample, out: *smi.ProcessList) smi.ProcessList.Error!void { + const self: *Metal = @alignCast(@fieldParentPtr("platform", platform)); + defer self.busy.sweep(); + const children = self.io_kit.IORegistryEntryGetChildIterator.get(.{ self.accelerator, "IOService" }) catch return error.Unavailable; + defer _ = self.io_kit.IOObjectRelease(children); + + // A process may have several clients. + var times: std.AutoArrayHashMapUnmanaged(u32, u64) = .empty; + while (true) { + const client = self.io_kit.IOIteratorNext(children); + if (client == 0) break; + defer _ = self.io_kit.IOObjectRelease(client); + if (self.io_kit.IOObjectConformsTo(client, "AGXDeviceUserClient") == 0) continue; + const pid = self.clientPid(client) orelse continue; + const time = try times.getOrPutValue(scratch, pid, 0); + time.value_ptr.* +|= self.gpuTime(client); + } + + const growths = try scratch.alloc(?smi.Growth, times.count()); + var total: u64 = 0; + for (times.keys(), times.values(), growths) |pid, nanoseconds, *growth| { + growth.* = if (nanoseconds == 0) null else try self.busy.advance(pid, self.io, nanoseconds); + if (growth.*) |busy| total +|= busy.amount; + } + const device = samples[0].metal.util_percent; + for (times.keys(), times.values(), growths) |pid, nanoseconds, growth| { + if (nanoseconds == 0) continue; + try out.append(.{ .pid = pid, .dev_util_percent = if (growth) |busy| share(busy, total, device) else null }); + } +} + +/// A process's part of the device's utilization, in proportion to its GPU time, as mactop does: +/// the GPU time of the clients adds up to less than the device's utilization. +/// Its own share of the elapsed time when the device's utilization is unknown or 0. +fn share(busy: smi.Growth, total: u64, device: ?u64) u16 { + const percent = device orelse 0; + if (percent == 0 or total == 0) return busy.percentOfTime(); + return @intCast(@as(u128, @min(percent, 100)) * busy.amount / total); +} + +/// The process that opened a GPU client, from its creator: "pid 682, WindowServer". +fn clientPid(self: *Metal, client: iokit.Object) ?u32 { + const creator = self.property(client, .IOUserClientCreator) orelse return null; + defer self.cf.CFRelease(creator); + var buffer: [128]u8 = undefined; + const creator_text = self.text(self.as(String, creator), &buffer); + if (!std.mem.startsWith(u8, creator_text, "pid ")) return null; + const end = std.mem.indexOfScalar(u8, creator_text, ',') orelse return null; + return std.fmt.parseInt(u32, creator_text[4..end], 10) catch null; +} + +/// The GPU nanoseconds of a client, summed over the APIs it used. +fn gpuTime(self: *Metal, client: iokit.Object) u64 { + const usage = self.property(client, .AppUsage) orelse return 0; + defer self.cf.CFRelease(usage); + const uses = self.as(Array, usage) orelse return 0; + var total: u64 = 0; + for (0..@intCast(@max(0, self.cf.CFArrayGetCount(uses)))) |index| { + const use = self.as(Dictionary, self.cf.CFArrayGetValueAtIndex(uses, @intCast(index))) orelse continue; + total +|= self.number(self.cf.CFDictionaryGetValue(use, self.keys.get(.accumulatedGPUTime))) orelse 0; + } + return total; +} + +/// Takes a sample and reads its delta against the last one: GPU, ANE and CPU power, and the GPU clock. +fn readReport(self: *Metal, gpu: *smi.Metal) void { + const sample = self.ioreport.IOReportCreateSamples(self.subscription, self.channels, null) orelse return; + const reading: Reading = .{ .sample = sample, .at = .now(self.io, .awake) }; + const last = self.last orelse { + self.last = reading; + return; + }; + self.last = reading; + defer self.cf.CFRelease(last.sample); + + const delta = self.ioreport.IOReportCreateSamplesDelta(last.sample, sample, null) orelse return; + defer self.cf.CFRelease(delta); + const elapsed = last.at.durationTo(reading.at); + if (elapsed.nanoseconds <= 0) return; + const channels = self.as(Array, self.cf.CFDictionaryGetValue(delta, self.keys.get(.IOReportChannels))) orelse return; + + var energy: std.EnumArray(Energy, u64) = .initFill(0); + var pmp_cpu_mw: u64 = 0; + for (0..@intCast(@max(0, self.cf.CFArrayGetCount(channels)))) |index| { + const channel = self.as(Dictionary, self.cf.CFArrayGetValueAtIndex(channels, @intCast(index))) orelse continue; + var group_buffer: [64]u8 = undefined; + var name_buffer: [128]u8 = undefined; + const group = self.text(self.ioreport.IOReportChannelGetGroup(channel), &group_buffer); + const name = self.text(self.ioreport.IOReportChannelGetChannelName(channel), &name_buffer); + + if (std.mem.eql(u8, group, "Energy Model")) { + const kind = Energy.of(name) orelse continue; + // Negative is an invalid value. + const amount = std.math.cast(u64, self.ioreport.IOReportSimpleGetIntegerValue(channel, 0)) orelse continue; + var unit_buffer: [16]u8 = undefined; + const unit = self.text(self.ioreport.IOReportChannelGetUnitLabel(channel), &unit_buffer); + energy.getPtr(kind).* +|= milliwatts(amount, unit, elapsed); + } else if (std.mem.eql(u8, group, "GPU Stats") and std.mem.eql(u8, name, "GPUPH")) { + gpu.clock_graphics_mhz = self.clock(channel); + } else if (isPmpGroup(group) and isPmpCpuPower(name)) { + pmp_cpu_mw +|= self.histogramMilliwatts(channel); + } + } + + // Whole-CPU totals, else the clusters: an OS with both would count them twice. + const cpu = if (energy.get(.cpu) > 0) energy.get(.cpu) else energy.get(.cpu_cluster); + if (cpu == 0 and pmp_cpu_mw > 0) self.cpu_from_pmp = true; + self.cpu_power_mw = if (self.cpu_from_pmp) pmp_cpu_mw else cpu; + gpu.power_mw = if (energy.get(.gpu) > 0) energy.get(.gpu) else energy.get(.gpu_alias); + gpu.ane_power_mw = if (energy.get(.ane_block) > 0) energy.get(.ane_block) else energy.get(.ane); +} + +/// The Energy Model channels Metal reads, by name. Where an OS has two names for one +/// thing, the first of each pair is read, and the second only when the first reads 0. +const Energy = enum { + cpu, + cpu_cluster, + gpu, + gpu_alias, + ane_block, + ane, + + fn of(name: []const u8) ?Energy { + const clusters = [_][]const u8{ "ECPU Energy", "PCPU Energy", "MCPU Energy", "eCPUs Energy", "pCPUs Energy", "mCPUs Energy" }; + for (clusters) |cluster| { + if (std.mem.indexOf(u8, name, cluster) != null) return .cpu_cluster; + } + if (std.mem.indexOf(u8, name, "CPU Energy") != null) return .cpu; + if (std.mem.eql(u8, name, "GPU Energy")) return .gpu; + if (std.mem.eql(u8, name, "GPU")) return .gpu_alias; + for ([_][]const u8{ "ANE", "NPU", "Neural", "ane" }) |engine| { + if (std.mem.indexOf(u8, name, engine) == null) continue; + return if (std.mem.indexOf(u8, name, "Energy") != null) .ane else .ane_block; + } + return null; + } +}; + +/// Energy over the time it took, in milliwatts. mactop reads a unit that is not mJ or nJ as µJ. +fn milliwatts(amount: u64, unit: []const u8, elapsed: std.Io.Duration) u64 { + const symbol = unit[0 .. std.mem.indexOfScalar(u8, unit, ' ') orelse unit.len]; + const prefix: smi.units.Prefix = if (std.mem.eql(u8, symbol, "mJ")) .milli else if (std.mem.eql(u8, symbol, "nJ")) .nano else .micro; + const growth: smi.Growth = .{ .amount = amount, .elapsed = elapsed }; + return smi.units.scale(growth.perSecond(), prefix, .milli); +} + +/// The mean frequency of the active GPU performance states, weighted by the time spent in each. +/// 0 when the GPU was off or idle all along. +fn clock(self: *Metal, channel: *const Dictionary) ?u64 { + if (self.frequencies.len == 0) return null; + var active: u64 = 0; + var weighted: u128 = 0; + var state_index: usize = 0; + for (0..@intCast(@max(0, self.ioreport.IOReportStateGetCount(channel)))) |state| { + var buffer: [32]u8 = undefined; + const name = self.text(self.ioreport.IOReportStateGetNameForIndex(channel, @intCast(state)), &buffer); + if (std.mem.eql(u8, name, "OFF") or std.mem.eql(u8, name, "IDLE") or std.mem.eql(u8, name, "DOWN")) continue; + const residency = std.math.cast(u64, self.ioreport.IOReportStateGetResidency(channel, @intCast(state))) orelse 0; + active += residency; + if (state_index < self.frequencies.len) weighted += @as(u128, self.frequencies[state_index]) * residency; + state_index += 1; + } + return if (active > 0) @intCast(weighted / active) else 0; +} + +/// The power of a CPU cluster from its histogram, whose states are named by watts (" 2W", " 0.250W"): +/// the mean of the states weighted by the time spent in each, the lowest reading 0, where an idle cluster sits. +fn histogramMilliwatts(self: *Metal, channel: *const Dictionary) u64 { + var total: u64 = 0; + var weighted: f64 = 0; + for (0..@intCast(@max(0, self.ioreport.IOReportStateGetCount(channel)))) |state| { + const residency = std.math.cast(u64, self.ioreport.IOReportStateGetResidency(channel, @intCast(state))) orelse 0; + total += residency; + if (state == 0) continue; + var buffer: [32]u8 = undefined; + weighted += stateWatts(self.text(self.ioreport.IOReportStateGetNameForIndex(channel, @intCast(state)), &buffer)) * @as(f64, @floatFromInt(residency)); + } + if (total == 0) return 0; + return @intFromFloat(@round(weighted / @as(f64, @floatFromInt(total)) * 1000)); +} + +fn stateWatts(name: []const u8) f64 { + return std.fmt.parseFloat(f64, std.mem.trimEnd(u8, std.mem.trim(u8, name, " "), "W")) catch 0; +} + +/// "PMP" on older chips, "PMP0", "PMP1"... one per die on an M5 Max. +fn isPmpGroup(group: []const u8) bool { + if (!std.mem.startsWith(u8, group, "PMP")) return false; + for (group[3..]) |char| if (!std.ascii.isDigit(char)) return false; + return true; +} + +/// A CPU cluster's power histogram: "PACC", "MACC1", "PACC0 SRAM", but not "AGX", the GPU. +fn isPmpCpuPower(name: []const u8) bool { + if (name.len < 4 or !std.mem.eql(u8, name[1..4], "ACC")) return false; + var rest = name[4..]; + while (rest.len > 0 and std.ascii.isDigit(rest[0])) rest = rest[1..]; + return rest.len == 0 or std.mem.eql(u8, rest, " SRAM"); +} + +/// The channels Metal subscribes to: the Energy Model, the GPU performance states, and the +/// power histograms of PMP, the CPU clusters' among them. +fn copyChannels(self: *Metal) !*Dictionary { + const energy = self.channelsIn("Energy Model", null) orelse { + log.warn("IOReport has no Energy Model", .{}); + return error.IoReportUnavailable; + }; + defer self.cf.CFRelease(energy); + const channels = self.cf.CFDictionaryCreateMutableCopy(null, 0, energy) orelse return error.OutOfMemory; + + if (self.channelsIn("GPU Stats", "GPU Performance States")) |gpu| { + defer self.cf.CFRelease(gpu); + self.ioreport.IOReportMergeChannels(channels, gpu, null); + } + // The dies are numbered contiguously, so the first without channels ends the search. + for (0..9) |die| { + var buffer: [8]u8 = undefined; + const group = if (die == 0) "PMP" else std.fmt.bufPrintSentinel(&buffer, "PMP{d}", .{die - 1}, 0) catch unreachable; + const pmp = self.channelsIn(group, "Energy") orelse if (die == 0) continue else break; + defer self.cf.CFRelease(pmp); + self.ioreport.IOReportMergeChannels(channels, pmp, null); + } + return channels; +} + +/// The channels of an IOReport group, or null when it has none. +fn channelsIn(self: *Metal, group: [:0]const u8, subgroup: ?[:0]const u8) ?*const Dictionary { + const group_string = self.cf.CFStringCreateWithCString(null, group, utf8) orelse return null; + defer self.cf.CFRelease(group_string); + const subgroup_string = if (subgroup) |name| (self.cf.CFStringCreateWithCString(null, name, utf8) orelse return null) else null; + defer if (subgroup_string) |string| self.cf.CFRelease(string); + + const channels = self.ioreport.IOReportCopyChannelsInGroup(group_string, subgroup_string, 0, 0, 0) orelse return null; + const list = self.as(Array, self.cf.CFDictionaryGetValue(channels, self.keys.get(.IOReportChannels))); + if (list == null or self.cf.CFArrayGetCount(list.?) == 0) { + self.cf.CFRelease(channels); + return null; + } + return channels; +} + +/// The frequency of each GPU performance state, from the power manager's voltage states: +/// pairs of a frequency and a voltage, in Hz up to the M4 and for the M5 GPU, in kHz for the M5 CPU. +fn gpuFrequencies(self: *Metal, arena: std.mem.Allocator) ![]const u64 { + const manager = self.io_kit.IOServiceGetMatchingService(iokit.main_port, self.io_kit.IOServiceNameMatching("pmgr")); + if (manager == 0) return &.{}; + defer _ = self.io_kit.IOObjectRelease(manager); + + for ([_]Key{ .@"voltage-states9-sram", .@"voltage-states9" }) |key| { + const states = self.property(manager, key) orelse continue; + defer self.cf.CFRelease(states); + const data = self.as(Data, states) orelse continue; + const bytes = self.cf.CFDataGetBytePtr(data)[0..@intCast(@max(0, self.cf.CFDataGetLength(data)))]; + + var frequencies: std.ArrayList(u64) = .empty; + var pairs = std.mem.window(u8, bytes, 8, 8); + while (pairs.next()) |pair| { + if (pair.len < 4) break; + const frequency = std.mem.readInt(u32, pair[0..4], .little); + const mhz = if (frequency >= 100_000_000) frequency / 1_000_000 else if (frequency >= 100_000) frequency / 1000 else 0; + if (mhz > 0) try frequencies.append(arena, mhz); + } + return frequencies.items; + } + log.warn("no GPU frequencies, so no clock", .{}); + return &.{}; +} + +/// A string property of the GPU, copied into `arena`. +fn propertyText(self: *Metal, arena: std.mem.Allocator, key: Key) !?[]const u8 { + const value = self.property(self.accelerator, key) orelse return null; + defer self.cf.CFRelease(value); + var buffer: [128]u8 = undefined; + const value_text = self.text(self.as(String, value), &buffer); + return if (value_text.len > 0) try arena.dupe(u8, value_text) else null; +} + +/// A property of a registry entry, which the caller releases. +fn property(self: *Metal, entry: iokit.Object, key: Key) ?*const anyopaque { + return self.io_kit.IORegistryEntryCreateCFProperty(entry, self.keys.get(key), null, 0); +} + +/// `object` as a T, or null when it is another kind of object. +fn as(self: *Metal, comptime T: type, object: ?*const anyopaque) ?*const T { + const value = object orelse return null; + const id = switch (T) { + String => self.cf.CFStringGetTypeID(), + Number => self.cf.CFNumberGetTypeID(), + Array => self.cf.CFArrayGetTypeID(), + Dictionary => self.cf.CFDictionaryGetTypeID(), + Data => self.cf.CFDataGetTypeID(), + else => @compileError("not a CoreFoundation type"), + }; + return if (self.cf.CFGetTypeID(value) == id) @ptrCast(value) else null; +} + +fn number(self: *Metal, object: ?*const anyopaque) ?u64 { + const value = self.as(Number, object) orelse return null; + var result: i64 = 0; + if (self.cf.CFNumberGetValue(value, number_sint64, &result) == 0) return null; + return std.math.cast(u64, result); +} + +/// The text of a string, in `buffer`. Empty when there is none or it does not fit. +fn text(self: *Metal, string: ?*const String, buffer: []u8) []const u8 { + if (self.cf.CFStringGetCString(string orelse return "", buffer.ptr, @intCast(buffer.len), utf8) == 0) return ""; + return std.mem.sliceTo(buffer, 0); +} + +fn memoryTotal() ?u64 { + var bytes: u64 = 0; + var size: usize = @sizeOf(u64); + if (std.c.sysctlbyname("hw.memsize", &bytes, &size, null, 0) != 0) return null; + return bytes; +} + +const String = opaque {}; +const Number = opaque {}; +const Array = opaque {}; +const Dictionary = opaque {}; +const Data = opaque {}; +const Subscription = opaque {}; + +/// kCFStringEncodingUTF8. +const utf8: u32 = 0x08000100; +/// kCFNumberSInt64Type. +const number_sint64: isize = 4; + +const CoreFoundation = struct { + CFRelease: *const @TypeOf(decls.CFRelease), + CFGetTypeID: *const @TypeOf(decls.CFGetTypeID), + CFStringGetTypeID: *const @TypeOf(decls.CFStringGetTypeID), + CFNumberGetTypeID: *const @TypeOf(decls.CFNumberGetTypeID), + CFArrayGetTypeID: *const @TypeOf(decls.CFArrayGetTypeID), + CFDictionaryGetTypeID: *const @TypeOf(decls.CFDictionaryGetTypeID), + CFDataGetTypeID: *const @TypeOf(decls.CFDataGetTypeID), + CFStringCreateWithCString: *const @TypeOf(decls.CFStringCreateWithCString), + CFStringGetCString: *const @TypeOf(decls.CFStringGetCString), + CFNumberGetValue: *const @TypeOf(decls.CFNumberGetValue), + CFArrayGetCount: *const @TypeOf(decls.CFArrayGetCount), + CFArrayGetValueAtIndex: *const @TypeOf(decls.CFArrayGetValueAtIndex), + CFDictionaryGetValue: *const @TypeOf(decls.CFDictionaryGetValue), + CFDictionaryCreateMutableCopy: *const @TypeOf(decls.CFDictionaryCreateMutableCopy), + CFDataGetLength: *const @TypeOf(decls.CFDataGetLength), + CFDataGetBytePtr: *const @TypeOf(decls.CFDataGetBytePtr), +}; + +/// The private library behind powermetrics. +const IoReport = struct { + IOReportCopyChannelsInGroup: *const @TypeOf(decls.IOReportCopyChannelsInGroup), + IOReportMergeChannels: *const @TypeOf(decls.IOReportMergeChannels), + IOReportCreateSubscription: *const @TypeOf(decls.IOReportCreateSubscription), + IOReportCreateSamples: *const @TypeOf(decls.IOReportCreateSamples), + IOReportCreateSamplesDelta: *const @TypeOf(decls.IOReportCreateSamplesDelta), + IOReportChannelGetGroup: *const @TypeOf(decls.IOReportChannelGetGroup), + IOReportChannelGetChannelName: *const @TypeOf(decls.IOReportChannelGetChannelName), + IOReportChannelGetUnitLabel: *const @TypeOf(decls.IOReportChannelGetUnitLabel), + IOReportSimpleGetIntegerValue: *const @TypeOf(decls.IOReportSimpleGetIntegerValue), + IOReportStateGetCount: *const @TypeOf(decls.IOReportStateGetCount), + IOReportStateGetNameForIndex: *const @TypeOf(decls.IOReportStateGetNameForIndex), + IOReportStateGetResidency: *const @TypeOf(decls.IOReportStateGetResidency), +}; + +/// Never called directly: `CoreFoundation` and `IoReport` take their types. +/// The functions named Create or Copy return an object the caller releases; Get functions do not. +const decls = struct { + extern fn CFRelease(object: *const anyopaque) void; + extern fn CFGetTypeID(object: *const anyopaque) usize; + extern fn CFStringGetTypeID() usize; + extern fn CFNumberGetTypeID() usize; + extern fn CFArrayGetTypeID() usize; + extern fn CFDictionaryGetTypeID() usize; + extern fn CFDataGetTypeID() usize; + extern fn CFStringCreateWithCString(allocator: ?*const anyopaque, text: [*:0]const u8, encoding: u32) ?*const String; + extern fn CFStringGetCString(string: *const String, buffer: [*]u8, size: isize, encoding: u32) u8; + extern fn CFNumberGetValue(number: *const Number, kind: isize, value: *anyopaque) u8; + extern fn CFArrayGetCount(array: *const Array) isize; + extern fn CFArrayGetValueAtIndex(array: *const Array, index: isize) ?*const anyopaque; + extern fn CFDictionaryGetValue(dictionary: *const Dictionary, key: *const anyopaque) ?*const anyopaque; + extern fn CFDictionaryCreateMutableCopy(allocator: ?*const anyopaque, capacity: isize, dictionary: *const Dictionary) ?*Dictionary; + extern fn CFDataGetLength(data: *const Data) isize; + extern fn CFDataGetBytePtr(data: *const Data) [*]const u8; + + extern fn IOReportCopyChannelsInGroup(group: ?*const String, subgroup: ?*const String, a: u64, b: u64, c: u64) ?*const Dictionary; + /// Adds the channels of `from` to `into`. + extern fn IOReportMergeChannels(into: *Dictionary, from: *const Dictionary, unused: ?*const anyopaque) void; + extern fn IOReportCreateSubscription(a: ?*const anyopaque, channels: *Dictionary, subscribed: *?*Dictionary, d: u64, e: ?*const anyopaque) ?*const Subscription; + extern fn IOReportCreateSamples(subscription: *const Subscription, channels: *Dictionary, unused: ?*const anyopaque) ?*const Dictionary; + extern fn IOReportCreateSamplesDelta(previous: *const Dictionary, current: *const Dictionary, unused: ?*const anyopaque) ?*const Dictionary; + extern fn IOReportChannelGetGroup(channel: *const Dictionary) ?*const String; + extern fn IOReportChannelGetChannelName(channel: *const Dictionary) ?*const String; + extern fn IOReportChannelGetUnitLabel(channel: *const Dictionary) ?*const String; + extern fn IOReportSimpleGetIntegerValue(channel: *const Dictionary, index: i32) i64; + extern fn IOReportStateGetCount(channel: *const Dictionary) i32; + extern fn IOReportStateGetNameForIndex(channel: *const Dictionary, index: i32) ?*const String; + extern fn IOReportStateGetResidency(channel: *const Dictionary, index: i32) i64; +}; + +test "channel names" { + try std.testing.expectEqual(.cpu, Energy.of("CPU Energy")); + try std.testing.expectEqual(.cpu, Energy.of("DIE_0_CPU Energy")); + try std.testing.expectEqual(.cpu_cluster, Energy.of("PCPU Energy")); + try std.testing.expectEqual(.gpu, Energy.of("GPU Energy")); + try std.testing.expectEqual(null, Energy.of("GPU0")); + try std.testing.expectEqual(.ane_block, Energy.of("ANE0")); + try std.testing.expectEqual(.ane, Energy.of("ANE Energy")); + try std.testing.expectEqual(null, Energy.of("PACC_0")); + + try std.testing.expect(isPmpGroup("PMP") and isPmpGroup("PMP1") and !isPmpGroup("PMPX")); + try std.testing.expect(isPmpCpuPower("PACC") and isPmpCpuPower("MACC1") and isPmpCpuPower("PACC0 SRAM")); + try std.testing.expect(!isPmpCpuPower("AGX") and !isPmpCpuPower("AGX AFR") and !isPmpCpuPower("PACC SRAM2")); + try std.testing.expectEqual(0.25, stateWatts(" 0.250W")); + try std.testing.expectEqual(64, stateWatts(" 64W")); + + // 1500 mJ in 500 ms. + try std.testing.expectEqual(3000, milliwatts(1500, "mJ", .fromMilliseconds(500))); + try std.testing.expectEqual(450, milliwatts(450_000_000, "nJ", .fromSeconds(1))); +} + +test share { + const second: std.Io.Duration = .fromSeconds(1); + // Two processes with 300 ms and 100 ms of GPU time share a device at 80%. + try std.testing.expectEqual(60, share(.{ .amount = 300_000_000, .elapsed = second }, 400_000_000, 80)); + try std.testing.expectEqual(20, share(.{ .amount = 100_000_000, .elapsed = second }, 400_000_000, 80)); + try std.testing.expectEqual(30, share(.{ .amount = 300_000_000, .elapsed = second }, 400_000_000, null)); +} + +test "a GPU of this Mac" { + const io = std.testing.io; + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + var metal = Metal.init(arena.allocator(), std.testing.allocator, io) catch return error.SkipZigTest; + defer deinit(&metal.platform); + + var scratch: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer scratch.deinit(); + var list: smi.ProcessList = .init(std.testing.allocator); + defer list.deinit(); + var data: smi.HostData = .{}; + + var samples: [1]smi.Sample = .{.init(.metal)}; + for (0..2) |round| { + samples = .{.init(.metal)}; + list.clear(); + if (round > 0) try io.sleep(.fromMilliseconds(100), .awake); + poll(&metal.platform, scratch.allocator(), &samples); + try processes(&metal.platform, scratch.allocator(), &samples, &list); + host(&metal.platform, &data); + } + const gpu = samples[0].metal; + try std.testing.expect(gpu.util_percent.? <= 100); + try std.testing.expect(gpu.mem_used_bytes.? < gpu.mem_total_bytes.?); + try std.testing.expect(gpu.clock_graphics_mhz.? <= gpu.clock_graphics_max_mhz.?); + try std.testing.expect(gpu.power_mw != null and data.cpu_power_mw != null); + // WindowServer draws the screen with the GPU. + try std.testing.expect(list.rows.items.len > 0); +} diff --git a/bin/zml-smi/platforms/metal/Smc.zig b/bin/zml-smi/platforms/metal/Smc.zig new file mode 100644 index 00000000..0e23923e --- /dev/null +++ b/bin/zml-smi/platforms/metal/Smc.zig @@ -0,0 +1,196 @@ +//! The System Management Controller of an Apple silicon Mac: GPU temperature, fans and system power. +//! Ported from mactop v2.1.6 (8dbfaae): internal/app/smc.c and the SMC readers of internal/app/ioreport.m. + +// MIT License +// +// Copyright (c) 2024-2026 Carsen Klock +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +const std = @import("std"); +const iokit = @import("iokit.zig"); + +const Smc = @This(); + +api: iokit.Api, +connection: iokit.Object, +/// The GPU sensors. +temperatures: []const Key, +fans: []const Fan, +/// Whole-machine power: the system total (laptops), the DC-in total, then the DC-in rail, +/// which is the power supply's output on a Mac Studio. The first that reads a value wins. +power: []const Key, + +/// A key and the type and size of its value. +const Key = struct { code: u32, info: Info }; + +const Fan = struct { speed: Key, min_rpm: f64, max_rpm: f64 }; + +const Info = extern struct { size: u32 = 0, type: u32 = 0, attributes: u8 = 0 }; + +/// SMCKeyData_t: what each call sends and receives. +const Call = extern struct { + key: u32 = 0, + version: extern struct { major: u8 = 0, minor: u8 = 0, build: u8 = 0, reserved: u8 = 0, release: u16 = 0 } = .{}, + power_limits: extern struct { version: u16 = 0, length: u16 = 0, cpu: u32 = 0, gpu: u32 = 0, memory: u32 = 0 } = .{}, + info: Info = .{}, + result: u8 = 0, + status: u8 = 0, + command: Command = .read_bytes, + index: u32 = 0, + bytes: [32]u8 = @splat(0), + + const Command = enum(u8) { read_bytes = 5, read_index = 8, read_info = 9 }; +}; + +comptime { + std.debug.assert(@sizeOf(Call) == 80); +} + +/// A GPU sensor at or below this has not started: it reads about 2 °C right after boot. +const min_celsius = 10; +const max_celsius = 150; +const max_watts = 1000; + +pub fn open(arena: std.mem.Allocator, api: iokit.Api) !Smc { + const service = api.IOServiceGetMatchingService(iokit.main_port, api.IOServiceMatching("AppleSMC")); + if (service == 0) return error.SmcNotFound; + defer _ = api.IOObjectRelease(service); + + var smc: Smc = .{ + .api = api, + .connection = try api.IOServiceOpen.get(.{ service, std.c.mach_task_self(), 0 }), + .temperatures = &.{}, + .fans = &.{}, + .power = &.{}, + }; + errdefer smc.close(); + smc.temperatures = try smc.gpuSensors(arena); + smc.fans = try smc.findFans(arena); + smc.power = try smc.powerKeys(arena); + return smc; +} + +pub fn close(self: Smc) void { + _ = self.api.IOServiceClose(self.connection); +} + +/// The hottest GPU sensor, in degrees Celsius. +pub fn temperature(self: Smc) ?u64 { + var hottest: ?f64 = null; + for (self.temperatures) |sensor| { + const celsius = self.read(sensor) catch continue; + if (celsius <= min_celsius or celsius > max_celsius) continue; + hottest = @max(hottest orelse celsius, celsius); + } + return if (hottest) |celsius| @intFromFloat(@round(celsius)) else null; +} + +/// The fastest fan, as a share of its top speed. +pub fn fanPercent(self: Smc) ?u64 { + var fastest: ?u64 = null; + for (self.fans) |fan| { + const rpm = self.read(fan.speed) catch continue; + // A fan that cannot stop and reads 0 hides its speed from this user. + if (rpm <= 0 and fan.min_rpm > 0) continue; + const percent: u64 = @intFromFloat(@round(std.math.clamp(rpm / fan.max_rpm * 100, 0, 100))); + fastest = @max(fastest orelse percent, percent); + } + return fastest; +} + +/// The power of the whole machine, in milliwatts. +pub fn systemPower(self: Smc) ?u64 { + for (self.power) |key| { + const watts = self.read(key) catch continue; + if (watts > 0 and watts <= max_watts) return @intFromFloat(@round(watts * 1000)); + } + return null; +} + +/// The GPU sensors, "Tg" keys that hold a float. Finding them reads the name of every key, about 3800 on an M5 Max. +fn gpuSensors(self: Smc, arena: std.mem.Allocator) ![]const Key { + const count = self.read(self.find(code("#KEY".*)) catch return &.{}) catch return &.{}; + var sensors: std.ArrayList(Key) = .empty; + for (0..@intFromFloat(count)) |index| { + const name = (self.call(.{ .command = .read_index, .index = @intCast(index) }) catch continue).key; + if (name >> 16 != code("Tg\x00\x00".*) >> 16) continue; + const key = self.find(name) catch continue; + if (key.info.type == code("flt ".*)) try sensors.append(arena, key); + } + return sensors.items; +} + +fn findFans(self: Smc, arena: std.mem.Allocator) ![]const Fan { + const count = self.read(self.find(code("FNum".*)) catch return &.{}) catch return &.{}; + var fans: std.ArrayList(Fan) = .empty; + for (0..@intFromFloat(@min(count, 10))) |index| { + const max_rpm = self.read(self.find(fanKey(index, "Mx".*)) catch continue) catch continue; + if (max_rpm <= 0) continue; + try fans.append(arena, .{ + .speed = self.find(fanKey(index, "Ac".*)) catch continue, + .min_rpm = if (self.find(fanKey(index, "Mn".*))) |key| self.read(key) catch 0 else |_| 0, + .max_rpm = max_rpm, + }); + } + return fans.items; +} + +fn powerKeys(self: Smc, arena: std.mem.Allocator) ![]const Key { + var keys: std.ArrayList(Key) = .empty; + for ([_][4]u8{ "PSTR".*, "PDTR".*, "PD0R".* }) |name| { + const key = self.find(code(name)) catch continue; + if (key.info.type == code("flt ".*)) try keys.append(arena, key); + } + return keys.items; +} + +fn call(self: Smc, input: Call) !Call { + var output: Call = .{}; + var size: usize = @sizeOf(Call); + try self.api.IOConnectCallStructMethod.call(.{ self.connection, 2, &input, @sizeOf(Call), &output, &size }); + if (output.result != 0) return error.SmcError; + return output; +} + +fn find(self: Smc, name: u32) !Key { + return .{ .code = name, .info = (try self.call(.{ .key = name, .command = .read_info })).info }; +} + +/// The value of a key that holds a float or an unsigned integer. +fn read(self: Smc, key: Key) !f64 { + const bytes = (try self.call(.{ .key = key.code, .info = .{ .size = key.info.size }, .command = .read_bytes })).bytes; + return switch (key.info.type) { + // Native byte order, unlike the integers. + code("flt ".*) => @as(f32, @bitCast(bytes[0..4].*)), + code("ui8 ".*) => @floatFromInt(bytes[0]), + code("ui16".*) => @floatFromInt(std.mem.readInt(u16, bytes[0..2], .big)), + code("ui32".*) => @floatFromInt(std.mem.readInt(u32, bytes[0..4], .big)), + else => error.UnknownType, + }; +} + +fn code(name: [4]u8) u32 { + return std.mem.readInt(u32, &name, .big); +} + +/// "F0Ac": the actual speed of fan 0. +fn fanKey(index: usize, suffix: [2]u8) u32 { + return code(.{ 'F', '0' + @as(u8, @intCast(index)), suffix[0], suffix[1] }); +} diff --git a/bin/zml-smi/platforms/metal/iokit.zig b/bin/zml-smi/platforms/metal/iokit.zig new file mode 100644 index 00000000..6081c56e --- /dev/null +++ b/bin/zml-smi/platforms/metal/iokit.zig @@ -0,0 +1,53 @@ +//! The IOKit functions Metal calls. The hermetic toolchain has no IOKit to link +//! against and no headers for it, so they are declared here and resolved by install name. +const std = @import("std"); +const smi = @import("zml-smi"); + +/// An io_object_t: a service, an iterator or a connection. 0 is none. +pub const Object = std.c.mach_port_t; + +/// kIOMainPortDefault. +pub const main_port: std.c.mach_port_t = 0; + +pub const Api = struct { + IOServiceMatching: *const @TypeOf(decls.IOServiceMatching), + IOServiceNameMatching: *const @TypeOf(decls.IOServiceNameMatching), + IOServiceGetMatchingService: *const @TypeOf(decls.IOServiceGetMatchingService), + IOServiceOpen: Checked(decls.IOServiceOpen), + IOServiceClose: *const @TypeOf(decls.IOServiceClose), + IOConnectCallStructMethod: Checked(decls.IOConnectCallStructMethod), + IORegistryEntryGetChildIterator: Checked(decls.IORegistryEntryGetChildIterator), + IORegistryEntryCreateCFProperty: *const @TypeOf(decls.IORegistryEntryCreateCFProperty), + IOIteratorNext: *const @TypeOf(decls.IOIteratorNext), + IOObjectConformsTo: *const @TypeOf(decls.IOObjectConformsTo), + IOObjectRelease: *const @TypeOf(decls.IOObjectRelease), +}; + +pub fn open() !Api { + return smi.dynlib.open(Api, &.{"/System/Library/Frameworks/IOKit.framework/IOKit"}); +} + +fn Checked(comptime function: anytype) type { + return smi.dynlib.Checked(function, check); +} + +fn check(status: std.c.kern_return_t) error{IoKitError}!void { + if (status != 0) return error.IoKitError; +} + +/// Never called directly: `Api` takes their types. +const decls = struct { + /// Returns a matching dictionary, which the lookup consumes. + extern fn IOServiceMatching(name: [*:0]const u8) ?*anyopaque; + extern fn IOServiceNameMatching(name: [*:0]const u8) ?*anyopaque; + extern fn IOServiceGetMatchingService(port: std.c.mach_port_t, matching: ?*anyopaque) Object; + extern fn IOServiceOpen(service: Object, task: std.c.mach_port_t, kind: u32, connection: *Object) std.c.kern_return_t; + extern fn IOServiceClose(connection: Object) std.c.kern_return_t; + extern fn IOConnectCallStructMethod(connection: Object, selector: u32, input: *const anyopaque, input_size: usize, output: *anyopaque, output_size: *usize) std.c.kern_return_t; + extern fn IORegistryEntryGetChildIterator(entry: Object, plane: [*:0]const u8, iterator: *Object) std.c.kern_return_t; + /// Returns a CoreFoundation object the caller releases. + extern fn IORegistryEntryCreateCFProperty(entry: Object, key: *const anyopaque, allocator: ?*const anyopaque, options: u32) ?*const anyopaque; + extern fn IOIteratorNext(iterator: Object) Object; + extern fn IOObjectConformsTo(object: Object, class: [*:0]const u8) c_int; + extern fn IOObjectRelease(object: Object) std.c.kern_return_t; +}; diff --git a/bin/zml-smi/smi/model.zig b/bin/zml-smi/smi/model.zig index ebabe360..caba0d73 100644 --- a/bin/zml-smi/smi/model.zig +++ b/bin/zml-smi/smi/model.zig @@ -1,6 +1,6 @@ const std = @import("std"); -pub const Target = enum { cuda, rocm, oneapi, neuron, tpu, furiosa }; +pub const Target = enum { cuda, rocm, oneapi, neuron, tpu, furiosa, metal }; /// What does not change while we run. Its strings live as long as the program. pub const Device = struct { @@ -34,6 +34,7 @@ pub const Sample = union(Target) { neuron: Neuron, tpu: Tpu, furiosa: Furiosa, + metal: Metal, /// A sample of the given target with every field null. pub fn init(target: Target) Sample { @@ -122,6 +123,22 @@ pub const Furiosa = struct { pub const ThrottleReason = enum(u6) { idle, thermal_slowdown, app_power_cap, app_clock_cap, hw_clock_cap, hw_bus_limit, hw_power_cap, gov_clock_cap, other }; }; +/// An Apple GPU. Its memory is the host's. +pub const Metal = struct { + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + power_mw: ?u64 = null, + /// Degrees Celsius, of the hottest GPU sensor. + temperature: ?u64 = null, + /// The fastest fan. + fan_speed_percent: ?u64 = null, + clock_graphics_mhz: ?u64 = null, + clock_graphics_max_mhz: ?u64 = null, + /// The Neural Engine. + ane_power_mw: ?u64 = null, +}; + pub const ProcessInfo = struct { pid: u32 = 0, /// The global index of the device, once the collector has the row. diff --git a/bin/zml-smi/smi/prometheus.zig b/bin/zml-smi/smi/prometheus.zig index 05713dcf..03f9d62c 100644 --- a/bin/zml-smi/smi/prometheus.zig +++ b/bin/zml-smi/smi/prometheus.zig @@ -75,6 +75,18 @@ const furiosa: Metrics(smi.Furiosa) = .{ .throttle_reasons = .{ .name = "zml_device_throttle_reasons", .help = "Active throttle reasons as a bitmask" }, }; +const metal: Metrics(smi.Metal) = .{ + .util_percent = utilization, + .mem_used_bytes = memory_used, + .mem_total_bytes = memory_total, + .power_mw = gpu.power_mw, + .temperature = gpu.temperature, + .fan_speed_percent = gpu.fan_speed_percent, + .clock_graphics_mhz = gpu.clock_graphics_mhz, + .clock_graphics_max_mhz = null, + .ane_power_mw = .{ .name = "zml_device_ane_power_watts", .help = "Neural Engine power draw in watts", .decimals = 3 }, +}; + const host: Metrics(smi.HostData) = .{ .hostname = null, .kernel = null, @@ -96,6 +108,7 @@ fn metricsOf(comptime T: type) Metrics(T) { smi.Neuron => neuron, smi.Tpu => tpu, smi.Furiosa => furiosa, + smi.Metal => metal, else => @compileError("no metrics for " ++ @typeName(T)), }; } @@ -105,7 +118,7 @@ fn metricsOf(comptime T: type) Metrics(T) { const families: []const Metric = families: { @setEvalBranchQuota(10_000); var list: []const Metric = &.{}; - for (.{ smi.Tpu, smi.Gpu, smi.Neuron, smi.Furiosa }) |T| { + for (.{ smi.Tpu, smi.Gpu, smi.Neuron, smi.Furiosa, smi.Metal }) |T| { const metrics = metricsOf(T); next: for (@typeInfo(T).@"struct".field_names) |field| { const metric = @field(metrics, field) orelse continue; @@ -124,7 +137,8 @@ const families: []const Metric = families: { /// The field of `T` written as `family`, or null. fn fieldOf(comptime T: type, comptime family: Metric) ?[]const u8 { - @setEvalBranchQuota(10_000); + // Every call made while `write` is analyzed counts against one quota. + @setEvalBranchQuota(100_000); const metrics = metricsOf(T); for (@typeInfo(T).@"struct".field_names) |field| { const metric = @field(metrics, field) orelse continue; diff --git a/bin/zml-smi/smi/smi.zig b/bin/zml-smi/smi/smi.zig index d8c1cb69..858859ab 100644 --- a/bin/zml-smi/smi/smi.zig +++ b/bin/zml-smi/smi/smi.zig @@ -10,6 +10,7 @@ pub const Gpu = model.Gpu; pub const Neuron = model.Neuron; pub const Tpu = model.Tpu; pub const Furiosa = model.Furiosa; +pub const Metal = model.Metal; pub const ProcessInfo = model.ProcessInfo; pub const HostData = model.HostData; pub const Platform = model.Platform; diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index 1dc399de..47c513e7 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -15,6 +15,7 @@ zig_library( "pages/detail/detail_common.zig", "pages/detail/furiosa_detail.zig", "pages/detail/gpu_detail.zig", + "pages/detail/metal_detail.zig", "pages/detail/neuron_detail.zig", "pages/detail/tpu_detail.zig", "pages/overview.zig", diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index 12e87688..79a4e62c 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -107,6 +107,11 @@ pub const SystemState = struct { self.record(&self.history.temp[i], chip.temperature orelse 0); self.record(&self.history.power[i], chip.power_mw orelse 0); }, + .metal => |gpu| { + self.recordUsage(i, gpu.util_percent, gpu.mem_used_bytes, gpu.mem_total_bytes); + self.record(&self.history.temp[i], gpu.temperature orelse 0); + self.record(&self.history.power[i], gpu.power_mw orelse 0); + }, } } } diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig index 9b0fdd15..03b6a615 100644 --- a/bin/zml-smi/tui/pages/detail.zig +++ b/bin/zml-smi/tui/pages/detail.zig @@ -8,6 +8,7 @@ const gpu_detail = @import("detail/gpu_detail.zig"); const neuron_detail = @import("detail/neuron_detail.zig"); const tpu_detail = @import("detail/tpu_detail.zig"); const furiosa_detail = @import("detail/furiosa_detail.zig"); +const metal_detail = @import("detail/metal_detail.zig"); const Detail = @This(); @@ -26,5 +27,6 @@ pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error! .neuron => |core| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, device, core, wgt), .tpu => |chip| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, device, chip, wgt), .furiosa => |chip| try furiosa_detail.draw(self.state, self.process_table, ctx, w, id, device, chip, wgt), + .metal => |gpu| try metal_detail.draw(self.state, self.process_table, ctx, w, id, device, gpu, wgt), }; } diff --git a/bin/zml-smi/tui/pages/detail/metal_detail.zig b/bin/zml-smi/tui/pages/detail/metal_detail.zig new file mode 100644 index 00000000..5ccb1cb0 --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/metal_detail.zig @@ -0,0 +1,103 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const smi = @import("zml-smi"); +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const Chart = @import("../../widgets/chart.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const MetricCard = @import("../../widgets/metric_card.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u16, + device: smi.Device, + gpu: smi.Metal, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + const header = try common.headerText(ctx.arena, id, device); + + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "GPU Utilization", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{gpu.util_percent orelse 0}), + }); + const mem_total = gpu.mem_total_bytes orelse 0; + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) (gpu.mem_used_bytes orelse 0) * 100 / mem_total else 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "In use {d} / {d} MB unified", .{ + utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), + }), + }); + const temp_chart = try common.historyChart(ctx, state, id, state.history.temp, .{ + .title = "Temperature", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C", .{gpu.temperature orelse 0}), + .info_line = if (gpu.fan_speed_percent) |fan| try std.fmt.allocPrint(ctx.arena, "Fan {d}%", .{fan}) else "Fan N/A", + .y_min = 20, + .y_max = 100, + .y_unit = "\u{00b0}C", + }); + + // No power limit: the scale follows the highest power in the history, in whole watts. + const power_mw = state.history.power[id].items; + const highest = if (power_mw.len > 0) std.mem.max(u64, power_mw) else 0; + const y_max: u32 = std.math.lossyCast(u32, @max(2, (highest + 999) / 1000)); + const power_chart: Chart = .{ + .title = "Power", + .data = try utils.normalizeRange(ctx.arena, power_mw, 0, @as(u64, y_max) * 1000), + .value_label = try watts(ctx.arena, gpu.power_mw), + .info_line = try std.fmt.allocPrint(ctx.arena, "ANE {s}", .{try watts(ctx.arena, gpu.ane_power_mw)}), + .y_min = 0, + .y_max = y_max, + .y_unit = "W", + .chart_height = 8, + .tui_refresh_rate = state.tui_refresh_rate, + }; + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart), ui.widget(&temp_chart), ui.widget(&power_chart) }, + .min_child_width = 60, + .gap = 1, + }; + + const clocks_card: MetricCard = .{ .title = "Clocks", .lines = &.{ + .{ .label = "Graphics ", .value = if (gpu.clock_graphics_mhz == null and gpu.clock_graphics_max_mhz == null) + " N/A" + else + try std.fmt.allocPrint(ctx.arena, " {d} / {d} MHz", .{ gpu.clock_graphics_mhz orelse 0, gpu.clock_graphics_max_mhz orelse 0 }) }, + } }; + // The CPU and the whole machine, which the GPU's platform measures. + const host = state.host(); + const power_card: MetricCard = .{ .title = "Power", .lines = &.{ + .{ .label = "GPU ", .value = try std.fmt.allocPrint(ctx.arena, " {s}", .{try watts(ctx.arena, gpu.power_mw)}) }, + .{ .label = "ANE ", .value = try std.fmt.allocPrint(ctx.arena, " {s}", .{try watts(ctx.arena, gpu.ane_power_mw)}) }, + .{ .label = "CPU ", .value = try std.fmt.allocPrint(ctx.arena, " {s}", .{try watts(ctx.arena, host.cpu_power_mw)}) }, + .{ .label = "System ", .value = try std.fmt.allocPrint(ctx.arena, " {s}", .{try watts(ctx.arena, host.system_power_mw)}) }, + } }; + const cards_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&clocks_card), ui.widget(&power_card) }, + .min_child_width = 30, + }; + + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(&cards_flow), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} + +/// "12.3W", or "N/A". +fn watts(arena: std.mem.Allocator, milliwatts: ?u64) std.mem.Allocator.Error![]const u8 { + const mw = milliwatts orelse return "N/A"; + return std.fmt.allocPrint(arena, "{d}.{d}W", .{ mw / 1000, mw % 1000 / 100 }); +} diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig index b3aa77f2..5db21cb4 100644 --- a/bin/zml-smi/tui/widgets/device_card.zig +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -84,6 +84,15 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx .mem_total = chip.mem_total_bytes, .suffix = try gpuSuffix(ctx.arena, .{ .temperature = chip.temperature, .power_mw = chip.power_mw, .power_limit_mw = chip.power_limit_mw }), }, + .metal => |gpu| .{ + .label = "GPU", + .util_label = "GPU", + .util_percent = gpu.util_percent, + .mem_used = gpu.mem_used_bytes, + .mem_total = gpu.mem_total_bytes, + // An Apple GPU idles below a watt. + .suffix = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C | {d}.{d} W", .{ gpu.temperature orelse 0, (gpu.power_mw orelse 0) / 1000, (gpu.power_mw orelse 0) % 1000 / 100 }), + }, }; const util_percent: u8 = @intCast(@min(usage.util_percent orelse 0, 100)); const mem_used = usage.mem_used orelse 0; From e207f3f3de5e1457ed8456c396dd5711ab8b5a0e Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Mon, 16 Nov 2026 16:32:25 +0100 Subject: [PATCH 19/19] refactor: replace cut function with std.mem.cutPrefix for clarity (#1088) --- bin/zml-smi/smi/fdinfo.zig | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/bin/zml-smi/smi/fdinfo.zig b/bin/zml-smi/smi/fdinfo.zig index a98f83c4..be79558d 100644 --- a/bin/zml-smi/smi/fdinfo.zig +++ b/bin/zml-smi/smi/fdinfo.zig @@ -62,17 +62,17 @@ pub fn parse(text: []const u8) Fdinfo { info.client_id = sysfs.firstNumber(value); } else if (std.mem.startsWith(u8, key, "drm-engine-capacity-")) { continue; - } else if (cut(key, "drm-engine-")) |class| { + } else if (std.mem.cutPrefix(u8, key, "drm-engine-")) |class| { if (info.engine(class)) |engine| engine.busy_ns = sysfs.firstNumber(value); - } else if (cut(key, "drm-total-cycles-")) |class| { + } else if (std.mem.cutPrefix(u8, key, "drm-total-cycles-")) |class| { if (info.engine(class)) |engine| engine.total_cycles = sysfs.firstNumber(value); - } else if (cut(key, "drm-cycles-")) |class| { + } else if (std.mem.cutPrefix(u8, key, "drm-cycles-")) |class| { if (info.engine(class)) |engine| engine.cycles = sysfs.firstNumber(value); - } else if (cut(key, "drm-total-")) |name| { + } else if (std.mem.cutPrefix(u8, key, "drm-total-")) |name| { if (info.region(name)) |region| region.total = bytes(value); - } else if (cut(key, "drm-resident-")) |name| { + } else if (std.mem.cutPrefix(u8, key, "drm-resident-")) |name| { if (info.region(name)) |region| region.resident = bytes(value); - } else if (cut(key, "drm-memory-")) |name| { + } else if (std.mem.cutPrefix(u8, key, "drm-memory-")) |name| { // The older amdgpu name of resident memory. if (info.region(name)) |region| region.resident = bytes(value); } else if (std.mem.endsWith(u8, key, " mem")) { @@ -83,10 +83,6 @@ pub fn parse(text: []const u8) Fdinfo { return info; } -fn cut(key: []const u8, prefix: []const u8) ?[]const u8 { - return if (std.mem.startsWith(u8, key, prefix)) key[prefix.len..] else null; -} - /// The entry called `name`, added when missing. Null past the capacity. fn slot(comptime T: type, buffer: []T, count: *u8, comptime key: []const u8, name: []const u8) ?*T { for (buffer[0..count.*]) |*item| {