From bca55296bf12097e25744e393581c56aa610a024 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 10 Nov 2026 23:53:49 +0100 Subject: [PATCH 1/2] zml-smi: ignore fdinfo reads that race with a closed fd procfs looks an fdinfo file's fd up again on every read, so a file that opened fine fails to read with ENOENT once that fd is closed, or ENOENT or ESRCH once its process exits. zml-smi's own poller threads open and close files all the time, so scanning /proc/self/fdinfo hit this about twice a second. The error was already ignored, but std.Io treats that errno as unexpected and, in Debug builds, prints a stack trace over the `zml-smi --top` TUI. Read fdinfo files with a new sysfs.readProcFile, which uses the raw read syscall and reports ENOENT and ESRCH as error.FileNotFound. Use it in the oneAPI and AMD fdinfo readers. Co-Authored-By: Claude Opus 5.5 --- bin/zml-smi/platforms/amdsmi/BUILD.bazel | 1 + bin/zml-smi/platforms/amdsmi/process.zig | 12 ++++------ bin/zml-smi/platforms/oneapi/monitor.zig | 2 +- bin/zml-smi/utils/sysfs.zig | 28 ++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel index dac6b935..4db41bf5 100644 --- a/bin/zml-smi/platforms/amdsmi/BUILD.bazel +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -16,6 +16,7 @@ zig_library( "//bin/zml-smi/utils:double_buffer", "//bin/zml-smi/utils:dynlib", "//bin/zml-smi/utils:sandbox", + "//bin/zml-smi/utils:sysfs", "//stdx", "@amdrocm-amdsmi//:amdsmi", ], diff --git a/bin/zml-smi/platforms/amdsmi/process.zig b/bin/zml-smi/platforms/amdsmi/process.zig index 8d1fdd01..cedea9bb 100644 --- a/bin/zml-smi/platforms/amdsmi/process.zig +++ b/bin/zml-smi/platforms/amdsmi/process.zig @@ -4,6 +4,7 @@ const AmdSmi = @import("amdsmi.zig"); const pi = @import("zml-smi/info").process_info; const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); const Collector = @import("zml-smi/collector").Collector; +const smi_sysfs = @import("zml-smi/sysfs"); const bdf_len = "0000:00:00.0".len; @@ -96,19 +97,14 @@ fn readProcessGpuVram(io: std.Io, pid: u32, pci_slot: *const [bdf_len]u8) u64 { } fn parseFdinfoVram(io: std.Io, path: []const u8, pci_slot: *const [bdf_len]u8) u64 { - var file = std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }) catch return 0; - defer file.close(io); - var read_buf: [4096]u8 = undefined; - var reader = file.reader(io, &read_buf); + const data = smi_sysfs.readProcFile(io, path, &read_buf) catch return 0; var found_pci = false; var vram_kib: ?u64 = null; - while (true) { - const line = reader.interface.takeDelimiterExclusive('\n') catch break; - reader.interface.toss(1); - + var lines = std.mem.splitScalar(u8, data, '\n'); + while (lines.next()) |line| { if (std.mem.indexOf(u8, line, pci_slot) != null) found_pci = true; // Support both new and old fdinfo keys (see amdgpu_fdinfo.c) diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig index dd5f97a9..55dbae1b 100644 --- a/bin/zml-smi/platforms/oneapi/monitor.zig +++ b/bin/zml-smi/platforms/oneapi/monitor.zig @@ -528,7 +528,7 @@ const ParsedFdinfo = struct { inline fn parseFdinfoFile(io: std.Io, path: []const u8, timestamp_ns: u64) !ParsedFdinfo { var read_buf: [8192]u8 = undefined; - const data = std.Io.Dir.readFile(.cwd(), io, path, &read_buf) catch return error.FileUnavailable; + const data = smi_sysfs.readProcFile(io, path, &read_buf) catch return error.FileUnavailable; var parsed: ParsedFdinfo = .{}; diff --git a/bin/zml-smi/utils/sysfs.zig b/bin/zml-smi/utils/sysfs.zig index 636883b2..49d1f0e2 100644 --- a/bin/zml-smi/utils/sysfs.zig +++ b/bin/zml-smi/utils/sysfs.zig @@ -54,6 +54,34 @@ fn findField(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptim } } +/// Reads a procfs file into `buf`, truncated to `buf.len`. +/// +/// procfs looks the process (and, for fdinfo, the fd) up again on every read, +/// so a file that opened fine fails to read with ENOENT or ESRCH once its +/// process exits or closes that fd. zml-smi hits this constantly on its own +/// fdinfo, whose fds its poller threads open and close. std.Io does not expect +/// these errnos and prints a stack trace in Debug builds, which garbles the +/// TUI, so this reads with plain syscalls and returns error.FileNotFound. +pub fn readProcFile(io: std.Io, path: []const u8, buf: []u8) ![]u8 { + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var len: usize = 0; + while (len < buf.len) { + const rc = std.posix.system.read(file.handle, buf[len..].ptr, buf.len - len); + switch (std.posix.errno(rc)) { + .SUCCESS => { + if (rc == 0) break; + len += @intCast(rc); + }, + .INTR => {}, + .NOENT, .SRCH => return error.FileNotFound, + else => return error.Unexpected, + } + } + return buf[0..len]; +} + pub fn readFirstLine(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]u8 { var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); defer file.close(io); From 2b05533c77516032d41fbd3c42428775ff0cc3ea Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 10 Nov 2026 23:53:49 +0100 Subject: [PATCH 2/2] intel fp8 local --- platforms/oneapi/oneapi.bzl | 5 +++-- zml/Compiler.zig | 6 +++++- zml/nn.zig | 39 ++++++++++++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 6016e389..47316107 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" +_PLUGIN_ONEAPI_URL = "file:///home/brabier/github/zml/zml-oneapi-plugins/zml-oneapi-linux-amd64-20261111-ctxperdev.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "5c6dd37e83798b5e93fceeff68eb06dc52d3cf67fb5edde5bfaaf0d2f69bbe23" ONEAPI_VERSION = "2026.1" @@ -248,6 +248,7 @@ filegroup( name = "mkl_core_runtime", srcs = [ "{ONEAPI_MKL_LIB}/libmkl_core.so.3", + "{ONEAPI_MKL_LIB}/libmkl_def.so.3", "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 93eafa8d..719d7ee3 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -998,7 +998,11 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load try setXlaOverrideFlag(overrides_map, "xla_gpu_collective_permute_connected_components", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLASLT,CUSTOM_CALL,FUSION", upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); + // CONCURRENT orders buffer assignment by data dependencies only, so every + // dequantized FP8 weight (oneAPI has no scaled dot) gets its own buffer: + // 22.7 GiB of temps for Qwen3.8-27B-FP8 prefill. CONCURRENT_REGIONS + // follows the schedule and keeps concurrency for small kernels only. + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT_REGIONS", upb_arena); // Grouped GEMM still needs a oneDNN lowering. try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_ragged_dot_grouped_gemm", false, upb_arena); diff --git a/zml/nn.zig b/zml/nn.zig index a2b582ee..db827913 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -66,12 +66,49 @@ pub const Linear = struct { const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag, self.tag) else self.weight; const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; - const acc = scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag, .{ .rhs_scale_swizzled = q.swizzled_scales }); + const acc = if (lhs_scale == null and !q.swizzled_scales and dequantizesBlockFp8InGraph(q.scheme, weight, scales)) + lhs.dot(dequantizeBlockFp8(weight, scales), self.tag).convert(output_dtype) + else + scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag, .{ .rhs_scale_swizzled = q.swizzled_scales }); const y = applyGlobalScale(acc, input_global_scale, weight_global_scale).convert(output_dtype); return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; } }; +/// oneAPI has no scaled dot: XLA would dequantize block FP8 weights with a generic f8e4m3fn +/// conversion that is slow on Intel GPUs (Qwen3.8-27B-FP8 decoded at 18 tok/s on two B70s, +/// 32 tok/s with `dequantizeBlockFp8`). +fn dequantizesBlockFp8InGraph(scheme: Quantization.Scheme, weight: Tensor, scales: Tensor) bool { + if (zml.Compiler.current().platform.target != .oneapi) return false; + if (scheme != .fp8_block128 and scheme != .fp8_block32) return false; + if (weight.dtype() != .f8e4m3fn or weight.rank() != 2 or scales.rank() != 2) return false; + if (scales.dtype() != .bf16 and scales.dtype() != .f32) return false; + for (0..2) |axis| { + if (@mod(weight.dim(axis), scales.dim(axis)) != 0) return false; + } + return true; +} + +/// Dequantizes block FP8 to bf16 with integer ops. Moving the E4M3FN bits into an f16 +/// (s eeee mmm -> s 0eeee mmm0000000) gives exactly value / 256, subnormals included, and +/// f16 -> f32 is a hardware conversion. 0x7f and 0xff, NaN in E4M3FN, become +-480. +fn dequantizeBlockFp8(values: Tensor, scales: Tensor) Tensor { + const bits = values.bitCast(.u8).convert(.u16); + const sign = bits.logical(.AND, .scalar(0x80, .u16)).shiftLeft(.scalar(8, .u16)); + const magnitude = bits.logical(.AND, .scalar(0x7f, .u16)).shiftLeft(.scalar(7, .u16)); + const unscaled = sign.logical(.OR, magnitude).bitCast(.f16).convert(.f32); + + // Scales are positional ([out / block, k / block]); loaders may tag them differently. + var expanded = scales.withTags(values.shape()).convert(.f32).scale(256.0); + for (0..values.rank()) |axis| { + const block = @divExact(values.dim(axis), scales.dim(axis)); + const shape = expanded.shape().insert(axis + 1, .{block}); + const axes = Shape.range(shape.rank(), .i64).remove(axis + 1); + expanded = expanded.broadcast(shape, axes.dims()).reshape(expanded.shape().setDim(axis, values.dim(axis))); + } + return unscaled.mul(expanded).convert(.bf16); +} + pub fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); }