diff --git a/AGENTS.md b/AGENTS.md index e3f5ef1e..fe0f5d9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,3 +29,7 @@ Tests are generally inline Zig `test "..." {}` blocks near the code they cover. ## Commit & Pull Request Guidelines Recent history uses scoped, imperative commit subjects such as `zml/tensor: add Tensor.onMemory()` or `workspace: use latest version of upstreamable rules_zig`. Keep subjects concise and mention the affected area first. Pull requests should include a short description, relevant linked issue, platform impact if any, and the exact `bazel build` or `bazel test` commands run. Include screenshots only for docs or UI-visible changes. + +## Quirks + +If during build, you encounter an `AccessDenied` error, the solution is not to use the `--repo_env` bazel argument to override the zig cache. The solution is to run `sudo chmod -R 777`on `/tmp/zig-cache` and `/var/tmp/zig-cache`. This is due to the fact that all users share the same zig cache directory, and bazel runs as a different user than the one who created the cache. diff --git a/zml/Compiler.zig b/zml/Compiler.zig index a76a828e..bb0d21a7 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -937,7 +937,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings // if VMM reclaims their backing under memory pressure, causing silent corruption. // Enable user buffers to assign color 1 and receive the reclaim exemption: @@ -955,7 +955,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load // Only capture all-reduce and all-gather collectives in HIP graphs; other collective types execute outside command buffers since they are unsupported in HIP graphs. try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_collectives_command_buffer_filter", "ALLREDUCE,ALLGATHER", upb_arena); // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings // if VMM reclaims their backing under memory pressure, causing silent corruption. // Enable user buffers to assign color 1 and receive the reclaim exemption: