From 08391af750dc6be08146ced32ef523b944017485 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 8 Oct 2026 12:33:34 +0200 Subject: [PATCH] zml/Compiler: remove conditionals from command buffer (#913) Due to recent changes in XLA (introduced in https://github.com/openxla/xla/commit/3cf18a01c7), the way child are counted to decide if a conditional is expressed as a graph conditional or kept outside has been changed. With the new corrected way, the body of the conditional used in the paged attention StableHLO backend was recorded as a child graph. The problem is that when we are sharded, there are collectives inside this child graph and XLA adds event wait/record around the actual collective kernel. This is explicitly forbidden because https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html#conditional-node-body-graph-requirements states: > The graph can only contain kernel nodes, empty nodes, memcpy nodes, memset nodes, child graph nodes, and conditional nodes. The temporary solution is to remove `CONDITIONAL` from the `xla_gpu_enable_command_buffer` XLA flag. In practice this reverts to the old behavior we had when the counting was done differently. Long term, I don't have a solution yet and. There is an environment variable that solves it [NCCL_GRAPH_MIXING_SUPPORT](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html#nccl-graph-mixing-support) but I don't know if it's safe to disable it. Maybe we could also check if the events introduced by NCCL are actually required. --- AGENTS.md | 4 ++++ zml/Compiler.zig | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e3f5ef1e..fe0f5d9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,3 +29,7 @@ Tests are generally inline Zig `test "..." {}` blocks near the code they cover. ## Commit & Pull Request Guidelines Recent history uses scoped, imperative commit subjects such as `zml/tensor: add Tensor.onMemory()` or `workspace: use latest version of upstreamable rules_zig`. Keep subjects concise and mention the affected area first. Pull requests should include a short description, relevant linked issue, platform impact if any, and the exact `bazel build` or `bazel test` commands run. Include screenshots only for docs or UI-visible changes. + +## Quirks + +If during build, you encounter an `AccessDenied` error, the solution is not to use the `--repo_env` bazel argument to override the zig cache. The solution is to run `sudo chmod -R 777`on `/tmp/zig-cache` and `/var/tmp/zig-cache`. This is due to the fact that all users share the same zig cache directory, and bazel runs as a different user than the one who created the cache. diff --git a/zml/Compiler.zig b/zml/Compiler.zig index a76a828e..bb0d21a7 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -937,7 +937,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings // if VMM reclaims their backing under memory pressure, causing silent corruption. // Enable user buffers to assign color 1 and receive the reclaim exemption: @@ -955,7 +955,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load // Only capture all-reduce and all-gather collectives in HIP graphs; other collective types execute outside command buffers since they are unsupported in HIP graphs. try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_collectives_command_buffer_filter", "ALLREDUCE,ALLGATHER", upb_arena); // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings // if VMM reclaims their backing under memory pressure, causing silent corruption. // Enable user buffers to assign color 1 and receive the reclaim exemption: